1use core::str;
8use std::collections::HashMap;
9use std::pin::Pin;
10use std::sync::Arc;
11
12use arrow::{
13 array::ArrayRef,
14 buffer::Buffer,
15 datatypes::Schema,
16 datatypes::SchemaRef,
17 ipc::{self, reader, writer},
18 record_batch::RecordBatch,
19};
20use arrow_flight::{
21 Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightEndpoint, FlightInfo,
22 HandshakeRequest, HandshakeResponse, IpcMessage, PollInfo, PutResult, SchemaAsIpc,
23 SchemaResult, Ticket, flight_descriptor::DescriptorType, flight_service_server::FlightService,
24 flight_service_server::FlightServiceServer,
25};
26use futures::{Stream, StreamExt, channel::mpsc, sink::SinkExt};
27use tokio::sync::Mutex;
28use tonic::{Request, Response, Status, Streaming, transport::Server};
29
30type TonicStream<T> = Pin<Box<dyn Stream<Item = T> + Send + Sync + 'static>>;
31
32type Error = Box<dyn std::error::Error + Send + Sync + 'static>;
33type Result<T = (), E = Error> = std::result::Result<T, E>;
34
35pub async fn scenario_setup(port: u16) -> Result {
37 let addr = super::listen_on(port).await?;
38 let resolved_port = addr.port();
39
40 let service = FlightServiceImpl {
41 server_location: format!("grpc+tcp://localhost:{resolved_port}"),
45 ..Default::default()
46 };
47 let svc = FlightServiceServer::new(service);
48
49 let server = Server::builder().add_service(svc).serve(addr);
50
51 println!("Server listening on localhost:{}", addr.port());
53 server.await?;
54 Ok(())
55}
56
57#[derive(Debug, Clone)]
58struct IntegrationDataset {
59 schema: Schema,
60 chunks: Vec<RecordBatch>,
61}
62
63#[derive(Clone, Default)]
65pub struct FlightServiceImpl {
66 server_location: String,
67 uploaded_chunks: Arc<Mutex<HashMap<String, IntegrationDataset>>>,
68}
69
70impl FlightServiceImpl {
71 fn endpoint_from_path(&self, path: &str) -> FlightEndpoint {
72 super::endpoint(path, &self.server_location)
73 }
74}
75
76#[tonic::async_trait]
77impl FlightService for FlightServiceImpl {
78 type HandshakeStream = TonicStream<Result<HandshakeResponse, Status>>;
79 type ListFlightsStream = TonicStream<Result<FlightInfo, Status>>;
80 type DoGetStream = TonicStream<Result<FlightData, Status>>;
81 type DoPutStream = TonicStream<Result<PutResult, Status>>;
82 type DoActionStream = TonicStream<Result<arrow_flight::Result, Status>>;
83 type ListActionsStream = TonicStream<Result<ActionType, Status>>;
84 type DoExchangeStream = TonicStream<Result<FlightData, Status>>;
85
86 async fn get_schema(
87 &self,
88 _request: Request<FlightDescriptor>,
89 ) -> Result<Response<SchemaResult>, Status> {
90 Err(Status::unimplemented("Not yet implemented"))
91 }
92
93 async fn do_get(
94 &self,
95 request: Request<Ticket>,
96 ) -> Result<Response<Self::DoGetStream>, Status> {
97 let ticket = request.into_inner();
98
99 let key = str::from_utf8(&ticket.ticket)
100 .map_err(|e| Status::invalid_argument(format!("Invalid ticket: {e:?}")))?;
101
102 let uploaded_chunks = self.uploaded_chunks.lock().await;
103
104 let flight = uploaded_chunks
105 .get(key)
106 .ok_or_else(|| Status::not_found(format!("Could not find flight. {key}")))?;
107
108 let options = arrow::ipc::writer::IpcWriteOptions::default();
109 let mut dictionary_tracker = writer::DictionaryTracker::new(false);
110 let data_gen = writer::IpcDataGenerator::default();
111 let data = IpcMessage(
112 data_gen
113 .schema_to_bytes_with_dictionary_tracker(
114 &flight.schema,
115 &mut dictionary_tracker,
116 &options,
117 )
118 .ipc_message
119 .into(),
120 );
121 let schema_flight_data = FlightData {
122 data_header: data.0,
123 ..Default::default()
124 };
125
126 let schema = std::iter::once(Ok(schema_flight_data));
127
128 let batches = flight
129 .chunks
130 .iter()
131 .enumerate()
132 .flat_map(|(counter, batch)| {
133 let (encoded_dictionaries, encoded_batch) = data_gen
134 .encode(
135 batch,
136 &mut dictionary_tracker,
137 &options,
138 &mut Default::default(),
139 )
140 .expect("DictionaryTracker configured above to not error on replacement");
141
142 let dictionary_flight_data = encoded_dictionaries.into_iter().map(Into::into);
143 let mut batch_flight_data: FlightData = encoded_batch.into();
144
145 let metadata = counter.to_string().into();
147 batch_flight_data.app_metadata = metadata;
148
149 dictionary_flight_data
150 .chain(std::iter::once(batch_flight_data))
151 .map(Ok)
152 });
153
154 let output = futures::stream::iter(schema.chain(batches).collect::<Vec<_>>());
155
156 Ok(Response::new(Box::pin(output) as Self::DoGetStream))
157 }
158
159 async fn handshake(
160 &self,
161 _request: Request<Streaming<HandshakeRequest>>,
162 ) -> Result<Response<Self::HandshakeStream>, Status> {
163 Err(Status::unimplemented("Not yet implemented"))
164 }
165
166 async fn list_flights(
167 &self,
168 _request: Request<Criteria>,
169 ) -> Result<Response<Self::ListFlightsStream>, Status> {
170 Err(Status::unimplemented("Not yet implemented"))
171 }
172
173 async fn get_flight_info(
174 &self,
175 request: Request<FlightDescriptor>,
176 ) -> Result<Response<FlightInfo>, Status> {
177 let descriptor = request.into_inner();
178
179 match descriptor.r#type {
180 t if t == DescriptorType::Path as i32 => {
181 let path = &descriptor.path;
182 if path.is_empty() {
183 return Err(Status::invalid_argument("Invalid path"));
184 }
185
186 let uploaded_chunks = self.uploaded_chunks.lock().await;
187 let flight = uploaded_chunks.get(&path[0]).ok_or_else(|| {
188 Status::not_found(format!("Could not find flight. {}", path[0]))
189 })?;
190
191 let endpoint = self.endpoint_from_path(&path[0]);
192
193 let total_records: usize = flight.chunks.iter().map(|chunk| chunk.num_rows()).sum();
194
195 let options = arrow::ipc::writer::IpcWriteOptions::default();
196 let message = SchemaAsIpc::new(&flight.schema, &options)
197 .try_into()
198 .expect(
199 "Could not generate schema bytes from schema stored by a DoPut; \
200 this should be impossible",
201 );
202 let IpcMessage(schema) = message;
203
204 let info = FlightInfo {
205 schema,
206 flight_descriptor: Some(descriptor.clone()),
207 endpoint: vec![endpoint],
208 total_records: total_records as i64,
209 total_bytes: -1,
210 ordered: false,
211 app_metadata: vec![].into(),
212 };
213
214 Ok(Response::new(info))
215 }
216 other => Err(Status::unimplemented(format!("Request type: {other}"))),
217 }
218 }
219
220 async fn poll_flight_info(
221 &self,
222 _request: Request<FlightDescriptor>,
223 ) -> Result<Response<PollInfo>, Status> {
224 Err(Status::unimplemented("Not yet implemented"))
225 }
226
227 async fn do_put(
228 &self,
229 request: Request<Streaming<FlightData>>,
230 ) -> Result<Response<Self::DoPutStream>, Status> {
231 let mut input_stream = request.into_inner();
232 let flight_data = input_stream
233 .message()
234 .await?
235 .ok_or_else(|| Status::invalid_argument("Must send some FlightData"))?;
236
237 let descriptor = flight_data
238 .flight_descriptor
239 .clone()
240 .ok_or_else(|| Status::invalid_argument("Must have a descriptor"))?;
241
242 if descriptor.r#type != DescriptorType::Path as i32 || descriptor.path.is_empty() {
243 return Err(Status::invalid_argument("Must specify a path"));
244 }
245
246 let key = descriptor.path[0].clone();
247
248 let schema = Schema::try_from(&flight_data)
249 .map_err(|e| Status::invalid_argument(format!("Invalid schema: {e:?}")))?;
250 let schema_ref = Arc::new(schema.clone());
251
252 let (response_tx, response_rx) = mpsc::channel(10);
253
254 let uploaded_chunks = self.uploaded_chunks.clone();
255
256 tokio::spawn(async {
257 let mut error_tx = response_tx.clone();
258 if let Err(e) = save_uploaded_chunks(
259 uploaded_chunks,
260 schema_ref,
261 input_stream,
262 response_tx,
263 schema,
264 key,
265 )
266 .await
267 {
268 error_tx.send(Err(e)).await.expect("Error sending error")
269 }
270 });
271
272 Ok(Response::new(Box::pin(response_rx) as Self::DoPutStream))
273 }
274
275 async fn do_action(
276 &self,
277 _request: Request<Action>,
278 ) -> Result<Response<Self::DoActionStream>, Status> {
279 Err(Status::unimplemented("Not yet implemented"))
280 }
281
282 async fn list_actions(
283 &self,
284 _request: Request<Empty>,
285 ) -> Result<Response<Self::ListActionsStream>, Status> {
286 Err(Status::unimplemented("Not yet implemented"))
287 }
288
289 async fn do_exchange(
290 &self,
291 _request: Request<Streaming<FlightData>>,
292 ) -> Result<Response<Self::DoExchangeStream>, Status> {
293 Err(Status::unimplemented("Not yet implemented"))
294 }
295}
296
297async fn send_app_metadata(
298 tx: &mut mpsc::Sender<Result<PutResult, Status>>,
299 app_metadata: &[u8],
300) -> Result<(), Status> {
301 tx.send(Ok(PutResult {
302 app_metadata: app_metadata.to_vec().into(),
303 }))
304 .await
305 .map_err(|e| Status::internal(format!("Could not send PutResult: {e:?}")))
306}
307
308fn record_batch_from_message(
309 message: ipc::Message<'_>,
310 data_body: &Buffer,
311 schema_ref: SchemaRef,
312 dictionaries_by_id: &HashMap<i64, ArrayRef>,
313) -> Result<RecordBatch, Status> {
314 let ipc_batch = message
315 .header_as_record_batch()
316 .ok_or_else(|| Status::internal("Could not parse message header as record batch"))?;
317
318 let arrow_batch_result = reader::read_record_batch(
319 data_body,
320 ipc_batch,
321 schema_ref,
322 dictionaries_by_id,
323 None,
324 &message.version(),
325 );
326
327 arrow_batch_result
328 .map_err(|e| Status::internal(format!("Could not convert to RecordBatch: {e:?}")))
329}
330
331fn dictionary_from_message(
332 message: ipc::Message<'_>,
333 data_body: &Buffer,
334 schema_ref: SchemaRef,
335 dictionaries_by_id: &mut HashMap<i64, ArrayRef>,
336) -> Result<(), Status> {
337 let ipc_batch = message
338 .header_as_dictionary_batch()
339 .ok_or_else(|| Status::internal("Could not parse message header as dictionary batch"))?;
340
341 let dictionary_batch_result = reader::read_dictionary(
342 data_body,
343 ipc_batch,
344 &schema_ref,
345 dictionaries_by_id,
346 &message.version(),
347 );
348 dictionary_batch_result
349 .map_err(|e| Status::internal(format!("Could not convert to Dictionary: {e:?}")))
350}
351
352async fn save_uploaded_chunks(
353 uploaded_chunks: Arc<Mutex<HashMap<String, IntegrationDataset>>>,
354 schema_ref: Arc<Schema>,
355 mut input_stream: Streaming<FlightData>,
356 mut response_tx: mpsc::Sender<Result<PutResult, Status>>,
357 schema: Schema,
358 key: String,
359) -> Result<(), Status> {
360 let mut chunks = vec![];
361 let mut uploaded_chunks = uploaded_chunks.lock().await;
362
363 let mut dictionaries_by_id = HashMap::new();
364
365 while let Some(Ok(data)) = input_stream.next().await {
366 let message = arrow::ipc::root_as_message(&data.data_header[..])
367 .map_err(|e| Status::internal(format!("Could not parse message: {e:?}")))?;
368
369 match message.header_type() {
370 ipc::MessageHeader::Schema => {
371 return Err(Status::internal(
372 "Not expecting a schema when messages are read",
373 ));
374 }
375 ipc::MessageHeader::RecordBatch => {
376 send_app_metadata(&mut response_tx, &data.app_metadata).await?;
377
378 let batch = record_batch_from_message(
379 message,
380 &Buffer::from(data.data_body.as_ref()),
381 schema_ref.clone(),
382 &dictionaries_by_id,
383 )?;
384
385 chunks.push(batch);
386 }
387 ipc::MessageHeader::DictionaryBatch => {
388 dictionary_from_message(
389 message,
390 &Buffer::from(data.data_body.as_ref()),
391 schema_ref.clone(),
392 &mut dictionaries_by_id,
393 )?;
394 }
395 t => {
396 return Err(Status::internal(format!(
397 "Reading types other than record batches not yet supported, \
398 unable to read {t:?}"
399 )));
400 }
401 }
402 }
403
404 let dataset = IntegrationDataset { schema, chunks };
405 uploaded_chunks.insert(key, dataset);
406
407 Ok(())
408}