Skip to main content

arrow_integration_testing/flight_client_scenarios/
integration_test.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Integration tests for the Flight client.
6
7use crate::open_json_file;
8use std::collections::HashMap;
9
10use arrow::{
11    array::ArrayRef,
12    buffer::Buffer,
13    datatypes::SchemaRef,
14    ipc::{
15        self, reader,
16        writer::{self, IpcWriteContext},
17    },
18    record_batch::RecordBatch,
19};
20use arrow_flight::{
21    FlightData, FlightDescriptor, IpcMessage, Location, Ticket, flight_descriptor::DescriptorType,
22    flight_service_client::FlightServiceClient, utils::flight_data_to_arrow_batch,
23};
24use futures::{StreamExt, channel::mpsc, sink::SinkExt, stream};
25use tonic::{Request, Streaming, transport::Endpoint};
26
27use arrow::datatypes::Schema;
28use std::sync::Arc;
29
30type Error = Box<dyn std::error::Error + Send + Sync + 'static>;
31type Result<T = (), E = Error> = std::result::Result<T, E>;
32
33type Client = FlightServiceClient<tonic::transport::Channel>;
34
35/// Run a scenario that uploads data to a Flight server and then downloads it back
36pub async fn run_scenario(host: &str, port: u16, path: &str) -> Result {
37    let url = format!("http://{host}:{port}");
38
39    let endpoint = Endpoint::new(url)?;
40    let channel = endpoint.connect().await?;
41    let client = FlightServiceClient::new(channel);
42
43    let json_file = open_json_file(path)?;
44
45    let batches = json_file.read_batches()?;
46    let schema = Arc::new(json_file.schema);
47
48    let mut descriptor = FlightDescriptor::default();
49    descriptor.set_type(DescriptorType::Path);
50    descriptor.path = vec![path.to_string()];
51
52    upload_data(client.clone(), schema, descriptor.clone(), batches.clone()).await?;
53    verify_data(client, descriptor, &batches).await?;
54
55    Ok(())
56}
57
58async fn upload_data(
59    mut client: Client,
60    schema: SchemaRef,
61    descriptor: FlightDescriptor,
62    original_data: Vec<RecordBatch>,
63) -> Result {
64    let (mut upload_tx, upload_rx) = mpsc::channel(10);
65
66    let options = arrow::ipc::writer::IpcWriteOptions::default();
67    let mut dict_tracker = writer::DictionaryTracker::new(false);
68    let data_gen = writer::IpcDataGenerator::default();
69    let data = IpcMessage(
70        data_gen
71            .schema_to_bytes_with_dictionary_tracker(&schema, &mut dict_tracker, &options)
72            .ipc_message
73            .into(),
74    );
75    let mut schema_flight_data = FlightData {
76        data_header: data.0,
77        ..Default::default()
78    };
79    // arrow_flight::utils::flight_data_from_arrow_schema(&schema, &options);
80    schema_flight_data.flight_descriptor = Some(descriptor.clone());
81    upload_tx.send(schema_flight_data).await?;
82
83    let mut original_data_iter = original_data.iter().enumerate();
84
85    let mut ipc_write_context = IpcWriteContext::default();
86
87    if let Some((counter, first_batch)) = original_data_iter.next() {
88        let metadata = counter.to_string().into_bytes();
89        // Preload the first batch into the channel before starting the request
90        send_batch(
91            &mut upload_tx,
92            &metadata,
93            first_batch,
94            &options,
95            &mut dict_tracker,
96            &mut ipc_write_context,
97        )
98        .await?;
99
100        let outer = client.do_put(Request::new(upload_rx)).await?;
101        let mut inner = outer.into_inner();
102
103        let r = inner
104            .next()
105            .await
106            .expect("No response received")
107            .expect("Invalid response received");
108        assert_eq!(metadata, r.app_metadata);
109
110        // Stream the rest of the batches
111        for (counter, batch) in original_data_iter {
112            let metadata = counter.to_string().into_bytes();
113            send_batch(
114                &mut upload_tx,
115                &metadata,
116                batch,
117                &options,
118                &mut dict_tracker,
119                &mut ipc_write_context,
120            )
121            .await?;
122
123            let r = inner
124                .next()
125                .await
126                .expect("No response received")
127                .expect("Invalid response received");
128            assert_eq!(metadata, r.app_metadata);
129        }
130        drop(upload_tx);
131        assert!(
132            inner.next().await.is_none(),
133            "Should not receive more results"
134        );
135    } else {
136        drop(upload_tx);
137        client.do_put(Request::new(upload_rx)).await?;
138    }
139
140    Ok(())
141}
142
143async fn send_batch(
144    upload_tx: &mut mpsc::Sender<FlightData>,
145    metadata: &[u8],
146    batch: &RecordBatch,
147    options: &writer::IpcWriteOptions,
148    dictionary_tracker: &mut writer::DictionaryTracker,
149    ipc_write_context: &mut IpcWriteContext,
150) -> Result {
151    let data_gen = writer::IpcDataGenerator::default();
152
153    let (encoded_dictionaries, encoded_batch) = data_gen
154        .encode(batch, dictionary_tracker, options, ipc_write_context)
155        .expect("DictionaryTracker configured above to not error on replacement");
156
157    let dictionary_flight_data: Vec<FlightData> =
158        encoded_dictionaries.into_iter().map(Into::into).collect();
159    let mut batch_flight_data: FlightData = encoded_batch.into();
160
161    upload_tx
162        .send_all(&mut stream::iter(dictionary_flight_data).map(Ok))
163        .await?;
164
165    // Only the record batch's FlightData gets app_metadata
166    batch_flight_data.app_metadata = metadata.to_vec().into();
167    upload_tx.send(batch_flight_data).await?;
168    Ok(())
169}
170
171async fn verify_data(
172    mut client: Client,
173    descriptor: FlightDescriptor,
174    expected_data: &[RecordBatch],
175) -> Result {
176    let resp = client.get_flight_info(Request::new(descriptor)).await?;
177    let info = resp.into_inner();
178
179    assert!(
180        !info.endpoint.is_empty(),
181        "No endpoints returned from Flight server",
182    );
183    for endpoint in info.endpoint {
184        let ticket = endpoint
185            .ticket
186            .expect("No ticket returned from Flight server");
187
188        assert!(
189            !endpoint.location.is_empty(),
190            "No locations returned from Flight server",
191        );
192        for location in endpoint.location {
193            consume_flight_location(location, ticket.clone(), expected_data).await?;
194        }
195    }
196
197    Ok(())
198}
199
200async fn consume_flight_location(
201    location: Location,
202    ticket: Ticket,
203    expected_data: &[RecordBatch],
204) -> Result {
205    let mut location = location;
206    // The other Flight implementations use the `grpc+tcp` scheme, but the Rust http libs
207    // don't recognize this as valid.
208    // more details: https://github.com/apache/arrow-rs/issues/1398
209    location.uri = location.uri.replace("grpc+tcp://", "http://");
210
211    let endpoint = Endpoint::new(location.uri)?;
212    let channel = endpoint.connect().await?;
213    let mut client = FlightServiceClient::new(channel);
214    let resp = client.do_get(ticket).await?;
215    let mut resp = resp.into_inner();
216
217    let flight_schema = receive_schema_flight_data(&mut resp)
218        .await
219        .unwrap_or_else(|| panic!("Failed to receive flight schema"));
220    let actual_schema = Arc::new(flight_schema);
221
222    let mut dictionaries_by_id = HashMap::new();
223
224    for (counter, expected_batch) in expected_data.iter().enumerate() {
225        let data =
226            receive_batch_flight_data(&mut resp, actual_schema.clone(), &mut dictionaries_by_id)
227                .await
228                .unwrap_or_else(|| {
229                    panic!(
230                        "Got fewer batches than expected, received so far: {} expected: {}",
231                        counter,
232                        expected_data.len(),
233                    )
234                });
235
236        let metadata = counter.to_string().into_bytes();
237        assert_eq!(metadata, data.app_metadata);
238
239        let actual_batch =
240            flight_data_to_arrow_batch(&data, actual_schema.clone(), &dictionaries_by_id)
241                .expect("Unable to convert flight data to Arrow batch");
242
243        assert_eq!(actual_schema, actual_batch.schema());
244        assert_eq!(expected_batch.num_columns(), actual_batch.num_columns());
245        assert_eq!(expected_batch.num_rows(), actual_batch.num_rows());
246        let schema = expected_batch.schema();
247        for i in 0..expected_batch.num_columns() {
248            let field = schema.field(i);
249            let field_name = field.name();
250
251            let expected_data = expected_batch.column(i).as_ref();
252            let actual_data = actual_batch.column(i).as_ref();
253
254            assert_eq!(expected_data, actual_data, "Data for field {field_name}");
255        }
256    }
257
258    assert!(
259        resp.next().await.is_none(),
260        "Got more batches than the expected: {}",
261        expected_data.len(),
262    );
263
264    Ok(())
265}
266
267async fn receive_schema_flight_data(resp: &mut Streaming<FlightData>) -> Option<Schema> {
268    let data = resp.next().await?.ok()?;
269    let message =
270        arrow::ipc::root_as_message(&data.data_header[..]).expect("Error parsing message");
271
272    // message header is a Schema, so read it
273    let ipc_schema: ipc::Schema = message
274        .header_as_schema()
275        .expect("Unable to read IPC message as schema");
276    let schema = ipc::convert::try_fb_to_schema(ipc_schema).unwrap();
277
278    Some(schema)
279}
280
281async fn receive_batch_flight_data(
282    resp: &mut Streaming<FlightData>,
283    schema: SchemaRef,
284    dictionaries_by_id: &mut HashMap<i64, ArrayRef>,
285) -> Option<FlightData> {
286    let mut data = resp.next().await?.ok()?;
287    let mut message =
288        arrow::ipc::root_as_message(&data.data_header[..]).expect("Error parsing first message");
289
290    while message.header_type() == ipc::MessageHeader::DictionaryBatch {
291        reader::read_dictionary(
292            &Buffer::from(data.data_body.as_ref()),
293            message
294                .header_as_dictionary_batch()
295                .expect("Error parsing dictionary"),
296            &schema,
297            dictionaries_by_id,
298            &message.version(),
299        )
300        .expect("Error reading dictionary");
301
302        data = resp.next().await?.ok()?;
303        message =
304            arrow::ipc::root_as_message(&data.data_header[..]).expect("Error parsing message");
305    }
306
307    Some(data)
308}