Skip to main content

arrow_integration_testing/flight_server_scenarios/
integration_test.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Integration tests for the Flight server.
6
7use core::str;
8use std::collections::HashMap;
9use std::pin::Pin;
10use std::sync::Arc;
11
12use arrow::{
13    array::ArrayRef,
14    buffer::Buffer,
15    datatypes::Schema,
16    datatypes::SchemaRef,
17    ipc::{self, reader, writer},
18    record_batch::RecordBatch,
19};
20use arrow_flight::{
21    Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightEndpoint, FlightInfo,
22    HandshakeRequest, HandshakeResponse, IpcMessage, PollInfo, PutResult, SchemaAsIpc,
23    SchemaResult, Ticket, flight_descriptor::DescriptorType, flight_service_server::FlightService,
24    flight_service_server::FlightServiceServer,
25};
26use futures::{Stream, StreamExt, channel::mpsc, sink::SinkExt};
27use tokio::sync::Mutex;
28use tonic::{Request, Response, Status, Streaming, transport::Server};
29
30type TonicStream<T> = Pin<Box<dyn Stream<Item = T> + Send + Sync + 'static>>;
31
32type Error = Box<dyn std::error::Error + Send + Sync + 'static>;
33type Result<T = (), E = Error> = std::result::Result<T, E>;
34
35/// Run a scenario that tests integration testing.
36pub async fn scenario_setup(port: u16) -> Result {
37    let addr = super::listen_on(port).await?;
38    let resolved_port = addr.port();
39
40    let service = FlightServiceImpl {
41        // See https://github.com/apache/arrow-rs/issues/6577
42        // C# had trouble resolving addressed like 0.0.0.0:port
43        // server_location: format!("grpc+tcp://{addr}"),
44        server_location: format!("grpc+tcp://localhost:{resolved_port}"),
45        ..Default::default()
46    };
47    let svc = FlightServiceServer::new(service);
48
49    let server = Server::builder().add_service(svc).serve(addr);
50
51    // NOTE: Log output used in tests to signal server is ready
52    println!("Server listening on localhost:{}", addr.port());
53    server.await?;
54    Ok(())
55}
56
57#[derive(Debug, Clone)]
58struct IntegrationDataset {
59    schema: Schema,
60    chunks: Vec<RecordBatch>,
61}
62
63/// Flight service implementation for integration testing
64#[derive(Clone, Default)]
65pub struct FlightServiceImpl {
66    server_location: String,
67    uploaded_chunks: Arc<Mutex<HashMap<String, IntegrationDataset>>>,
68}
69
70impl FlightServiceImpl {
71    fn endpoint_from_path(&self, path: &str) -> FlightEndpoint {
72        super::endpoint(path, &self.server_location)
73    }
74}
75
76#[tonic::async_trait]
77impl FlightService for FlightServiceImpl {
78    type HandshakeStream = TonicStream<Result<HandshakeResponse, Status>>;
79    type ListFlightsStream = TonicStream<Result<FlightInfo, Status>>;
80    type DoGetStream = TonicStream<Result<FlightData, Status>>;
81    type DoPutStream = TonicStream<Result<PutResult, Status>>;
82    type DoActionStream = TonicStream<Result<arrow_flight::Result, Status>>;
83    type ListActionsStream = TonicStream<Result<ActionType, Status>>;
84    type DoExchangeStream = TonicStream<Result<FlightData, Status>>;
85
86    async fn get_schema(
87        &self,
88        _request: Request<FlightDescriptor>,
89    ) -> Result<Response<SchemaResult>, Status> {
90        Err(Status::unimplemented("Not yet implemented"))
91    }
92
93    async fn do_get(
94        &self,
95        request: Request<Ticket>,
96    ) -> Result<Response<Self::DoGetStream>, Status> {
97        let ticket = request.into_inner();
98
99        let key = str::from_utf8(&ticket.ticket)
100            .map_err(|e| Status::invalid_argument(format!("Invalid ticket: {e:?}")))?;
101
102        let uploaded_chunks = self.uploaded_chunks.lock().await;
103
104        let flight = uploaded_chunks
105            .get(key)
106            .ok_or_else(|| Status::not_found(format!("Could not find flight. {key}")))?;
107
108        let options = arrow::ipc::writer::IpcWriteOptions::default();
109        let mut dictionary_tracker = writer::DictionaryTracker::new(false);
110        let data_gen = writer::IpcDataGenerator::default();
111        let data = IpcMessage(
112            data_gen
113                .schema_to_bytes_with_dictionary_tracker(
114                    &flight.schema,
115                    &mut dictionary_tracker,
116                    &options,
117                )
118                .ipc_message
119                .into(),
120        );
121        let schema_flight_data = FlightData {
122            data_header: data.0,
123            ..Default::default()
124        };
125
126        let schema = std::iter::once(Ok(schema_flight_data));
127
128        let batches = flight
129            .chunks
130            .iter()
131            .enumerate()
132            .flat_map(|(counter, batch)| {
133                let (encoded_dictionaries, encoded_batch) = data_gen
134                    .encode(
135                        batch,
136                        &mut dictionary_tracker,
137                        &options,
138                        &mut Default::default(),
139                    )
140                    .expect("DictionaryTracker configured above to not error on replacement");
141
142                let dictionary_flight_data = encoded_dictionaries.into_iter().map(Into::into);
143                let mut batch_flight_data: FlightData = encoded_batch.into();
144
145                // Only the record batch's FlightData gets app_metadata
146                let metadata = counter.to_string().into();
147                batch_flight_data.app_metadata = metadata;
148
149                dictionary_flight_data
150                    .chain(std::iter::once(batch_flight_data))
151                    .map(Ok)
152            });
153
154        let output = futures::stream::iter(schema.chain(batches).collect::<Vec<_>>());
155
156        Ok(Response::new(Box::pin(output) as Self::DoGetStream))
157    }
158
159    async fn handshake(
160        &self,
161        _request: Request<Streaming<HandshakeRequest>>,
162    ) -> Result<Response<Self::HandshakeStream>, Status> {
163        Err(Status::unimplemented("Not yet implemented"))
164    }
165
166    async fn list_flights(
167        &self,
168        _request: Request<Criteria>,
169    ) -> Result<Response<Self::ListFlightsStream>, Status> {
170        Err(Status::unimplemented("Not yet implemented"))
171    }
172
173    async fn get_flight_info(
174        &self,
175        request: Request<FlightDescriptor>,
176    ) -> Result<Response<FlightInfo>, Status> {
177        let descriptor = request.into_inner();
178
179        match descriptor.r#type {
180            t if t == DescriptorType::Path as i32 => {
181                let path = &descriptor.path;
182                if path.is_empty() {
183                    return Err(Status::invalid_argument("Invalid path"));
184                }
185
186                let uploaded_chunks = self.uploaded_chunks.lock().await;
187                let flight = uploaded_chunks.get(&path[0]).ok_or_else(|| {
188                    Status::not_found(format!("Could not find flight. {}", path[0]))
189                })?;
190
191                let endpoint = self.endpoint_from_path(&path[0]);
192
193                let total_records: usize = flight.chunks.iter().map(|chunk| chunk.num_rows()).sum();
194
195                let options = arrow::ipc::writer::IpcWriteOptions::default();
196                let message = SchemaAsIpc::new(&flight.schema, &options)
197                    .try_into()
198                    .expect(
199                        "Could not generate schema bytes from schema stored by a DoPut; \
200                         this should be impossible",
201                    );
202                let IpcMessage(schema) = message;
203
204                let info = FlightInfo {
205                    schema,
206                    flight_descriptor: Some(descriptor.clone()),
207                    endpoint: vec![endpoint],
208                    total_records: total_records as i64,
209                    total_bytes: -1,
210                    ordered: false,
211                    app_metadata: vec![].into(),
212                };
213
214                Ok(Response::new(info))
215            }
216            other => Err(Status::unimplemented(format!("Request type: {other}"))),
217        }
218    }
219
220    async fn poll_flight_info(
221        &self,
222        _request: Request<FlightDescriptor>,
223    ) -> Result<Response<PollInfo>, Status> {
224        Err(Status::unimplemented("Not yet implemented"))
225    }
226
227    async fn do_put(
228        &self,
229        request: Request<Streaming<FlightData>>,
230    ) -> Result<Response<Self::DoPutStream>, Status> {
231        let mut input_stream = request.into_inner();
232        let flight_data = input_stream
233            .message()
234            .await?
235            .ok_or_else(|| Status::invalid_argument("Must send some FlightData"))?;
236
237        let descriptor = flight_data
238            .flight_descriptor
239            .clone()
240            .ok_or_else(|| Status::invalid_argument("Must have a descriptor"))?;
241
242        if descriptor.r#type != DescriptorType::Path as i32 || descriptor.path.is_empty() {
243            return Err(Status::invalid_argument("Must specify a path"));
244        }
245
246        let key = descriptor.path[0].clone();
247
248        let schema = Schema::try_from(&flight_data)
249            .map_err(|e| Status::invalid_argument(format!("Invalid schema: {e:?}")))?;
250        let schema_ref = Arc::new(schema.clone());
251
252        let (response_tx, response_rx) = mpsc::channel(10);
253
254        let uploaded_chunks = self.uploaded_chunks.clone();
255
256        tokio::spawn(async {
257            let mut error_tx = response_tx.clone();
258            if let Err(e) = save_uploaded_chunks(
259                uploaded_chunks,
260                schema_ref,
261                input_stream,
262                response_tx,
263                schema,
264                key,
265            )
266            .await
267            {
268                error_tx.send(Err(e)).await.expect("Error sending error")
269            }
270        });
271
272        Ok(Response::new(Box::pin(response_rx) as Self::DoPutStream))
273    }
274
275    async fn do_action(
276        &self,
277        _request: Request<Action>,
278    ) -> Result<Response<Self::DoActionStream>, Status> {
279        Err(Status::unimplemented("Not yet implemented"))
280    }
281
282    async fn list_actions(
283        &self,
284        _request: Request<Empty>,
285    ) -> Result<Response<Self::ListActionsStream>, Status> {
286        Err(Status::unimplemented("Not yet implemented"))
287    }
288
289    async fn do_exchange(
290        &self,
291        _request: Request<Streaming<FlightData>>,
292    ) -> Result<Response<Self::DoExchangeStream>, Status> {
293        Err(Status::unimplemented("Not yet implemented"))
294    }
295}
296
297async fn send_app_metadata(
298    tx: &mut mpsc::Sender<Result<PutResult, Status>>,
299    app_metadata: &[u8],
300) -> Result<(), Status> {
301    tx.send(Ok(PutResult {
302        app_metadata: app_metadata.to_vec().into(),
303    }))
304    .await
305    .map_err(|e| Status::internal(format!("Could not send PutResult: {e:?}")))
306}
307
308fn record_batch_from_message(
309    message: ipc::Message<'_>,
310    data_body: &Buffer,
311    schema_ref: SchemaRef,
312    dictionaries_by_id: &HashMap<i64, ArrayRef>,
313) -> Result<RecordBatch, Status> {
314    let ipc_batch = message
315        .header_as_record_batch()
316        .ok_or_else(|| Status::internal("Could not parse message header as record batch"))?;
317
318    let arrow_batch_result = reader::read_record_batch(
319        data_body,
320        ipc_batch,
321        schema_ref,
322        dictionaries_by_id,
323        None,
324        &message.version(),
325    );
326
327    arrow_batch_result
328        .map_err(|e| Status::internal(format!("Could not convert to RecordBatch: {e:?}")))
329}
330
331fn dictionary_from_message(
332    message: ipc::Message<'_>,
333    data_body: &Buffer,
334    schema_ref: SchemaRef,
335    dictionaries_by_id: &mut HashMap<i64, ArrayRef>,
336) -> Result<(), Status> {
337    let ipc_batch = message
338        .header_as_dictionary_batch()
339        .ok_or_else(|| Status::internal("Could not parse message header as dictionary batch"))?;
340
341    let dictionary_batch_result = reader::read_dictionary(
342        data_body,
343        ipc_batch,
344        &schema_ref,
345        dictionaries_by_id,
346        &message.version(),
347    );
348    dictionary_batch_result
349        .map_err(|e| Status::internal(format!("Could not convert to Dictionary: {e:?}")))
350}
351
352async fn save_uploaded_chunks(
353    uploaded_chunks: Arc<Mutex<HashMap<String, IntegrationDataset>>>,
354    schema_ref: Arc<Schema>,
355    mut input_stream: Streaming<FlightData>,
356    mut response_tx: mpsc::Sender<Result<PutResult, Status>>,
357    schema: Schema,
358    key: String,
359) -> Result<(), Status> {
360    let mut chunks = vec![];
361    let mut uploaded_chunks = uploaded_chunks.lock().await;
362
363    let mut dictionaries_by_id = HashMap::new();
364
365    while let Some(Ok(data)) = input_stream.next().await {
366        let message = arrow::ipc::root_as_message(&data.data_header[..])
367            .map_err(|e| Status::internal(format!("Could not parse message: {e:?}")))?;
368
369        match message.header_type() {
370            ipc::MessageHeader::Schema => {
371                return Err(Status::internal(
372                    "Not expecting a schema when messages are read",
373                ));
374            }
375            ipc::MessageHeader::RecordBatch => {
376                send_app_metadata(&mut response_tx, &data.app_metadata).await?;
377
378                let batch = record_batch_from_message(
379                    message,
380                    &Buffer::from(data.data_body.as_ref()),
381                    schema_ref.clone(),
382                    &dictionaries_by_id,
383                )?;
384
385                chunks.push(batch);
386            }
387            ipc::MessageHeader::DictionaryBatch => {
388                dictionary_from_message(
389                    message,
390                    &Buffer::from(data.data_body.as_ref()),
391                    schema_ref.clone(),
392                    &mut dictionaries_by_id,
393                )?;
394            }
395            t => {
396                return Err(Status::internal(format!(
397                    "Reading types other than record batches not yet supported, \
398                                              unable to read {t:?}"
399                )));
400            }
401        }
402    }
403
404    let dataset = IntegrationDataset { schema, chunks };
405    uploaded_chunks.insert(key, dataset);
406
407    Ok(())
408}