arrow_integration_testing/flight_client_scenarios/
integration_test.rs1use crate::open_json_file;
8use std::collections::HashMap;
9
10use arrow::{
11 array::ArrayRef,
12 buffer::Buffer,
13 datatypes::SchemaRef,
14 ipc::{
15 self, reader,
16 writer::{self, IpcWriteContext},
17 },
18 record_batch::RecordBatch,
19};
20use arrow_flight::{
21 FlightData, FlightDescriptor, IpcMessage, Location, Ticket, flight_descriptor::DescriptorType,
22 flight_service_client::FlightServiceClient, utils::flight_data_to_arrow_batch,
23};
24use futures::{StreamExt, channel::mpsc, sink::SinkExt, stream};
25use tonic::{Request, Streaming, transport::Endpoint};
26
27use arrow::datatypes::Schema;
28use std::sync::Arc;
29
30type Error = Box<dyn std::error::Error + Send + Sync + 'static>;
31type Result<T = (), E = Error> = std::result::Result<T, E>;
32
33type Client = FlightServiceClient<tonic::transport::Channel>;
34
35pub async fn run_scenario(host: &str, port: u16, path: &str) -> Result {
37 let url = format!("http://{host}:{port}");
38
39 let endpoint = Endpoint::new(url)?;
40 let channel = endpoint.connect().await?;
41 let client = FlightServiceClient::new(channel);
42
43 let json_file = open_json_file(path)?;
44
45 let batches = json_file.read_batches()?;
46 let schema = Arc::new(json_file.schema);
47
48 let mut descriptor = FlightDescriptor::default();
49 descriptor.set_type(DescriptorType::Path);
50 descriptor.path = vec![path.to_string()];
51
52 upload_data(client.clone(), schema, descriptor.clone(), batches.clone()).await?;
53 verify_data(client, descriptor, &batches).await?;
54
55 Ok(())
56}
57
58async fn upload_data(
59 mut client: Client,
60 schema: SchemaRef,
61 descriptor: FlightDescriptor,
62 original_data: Vec<RecordBatch>,
63) -> Result {
64 let (mut upload_tx, upload_rx) = mpsc::channel(10);
65
66 let options = arrow::ipc::writer::IpcWriteOptions::default();
67 let mut dict_tracker = writer::DictionaryTracker::new(false);
68 let data_gen = writer::IpcDataGenerator::default();
69 let data = IpcMessage(
70 data_gen
71 .schema_to_bytes_with_dictionary_tracker(&schema, &mut dict_tracker, &options)
72 .ipc_message
73 .into(),
74 );
75 let mut schema_flight_data = FlightData {
76 data_header: data.0,
77 ..Default::default()
78 };
79 schema_flight_data.flight_descriptor = Some(descriptor.clone());
81 upload_tx.send(schema_flight_data).await?;
82
83 let mut original_data_iter = original_data.iter().enumerate();
84
85 let mut ipc_write_context = IpcWriteContext::default();
86
87 if let Some((counter, first_batch)) = original_data_iter.next() {
88 let metadata = counter.to_string().into_bytes();
89 send_batch(
91 &mut upload_tx,
92 &metadata,
93 first_batch,
94 &options,
95 &mut dict_tracker,
96 &mut ipc_write_context,
97 )
98 .await?;
99
100 let outer = client.do_put(Request::new(upload_rx)).await?;
101 let mut inner = outer.into_inner();
102
103 let r = inner
104 .next()
105 .await
106 .expect("No response received")
107 .expect("Invalid response received");
108 assert_eq!(metadata, r.app_metadata);
109
110 for (counter, batch) in original_data_iter {
112 let metadata = counter.to_string().into_bytes();
113 send_batch(
114 &mut upload_tx,
115 &metadata,
116 batch,
117 &options,
118 &mut dict_tracker,
119 &mut ipc_write_context,
120 )
121 .await?;
122
123 let r = inner
124 .next()
125 .await
126 .expect("No response received")
127 .expect("Invalid response received");
128 assert_eq!(metadata, r.app_metadata);
129 }
130 drop(upload_tx);
131 assert!(
132 inner.next().await.is_none(),
133 "Should not receive more results"
134 );
135 } else {
136 drop(upload_tx);
137 client.do_put(Request::new(upload_rx)).await?;
138 }
139
140 Ok(())
141}
142
143async fn send_batch(
144 upload_tx: &mut mpsc::Sender<FlightData>,
145 metadata: &[u8],
146 batch: &RecordBatch,
147 options: &writer::IpcWriteOptions,
148 dictionary_tracker: &mut writer::DictionaryTracker,
149 ipc_write_context: &mut IpcWriteContext,
150) -> Result {
151 let data_gen = writer::IpcDataGenerator::default();
152
153 let (encoded_dictionaries, encoded_batch) = data_gen
154 .encode(batch, dictionary_tracker, options, ipc_write_context)
155 .expect("DictionaryTracker configured above to not error on replacement");
156
157 let dictionary_flight_data: Vec<FlightData> =
158 encoded_dictionaries.into_iter().map(Into::into).collect();
159 let mut batch_flight_data: FlightData = encoded_batch.into();
160
161 upload_tx
162 .send_all(&mut stream::iter(dictionary_flight_data).map(Ok))
163 .await?;
164
165 batch_flight_data.app_metadata = metadata.to_vec().into();
167 upload_tx.send(batch_flight_data).await?;
168 Ok(())
169}
170
171async fn verify_data(
172 mut client: Client,
173 descriptor: FlightDescriptor,
174 expected_data: &[RecordBatch],
175) -> Result {
176 let resp = client.get_flight_info(Request::new(descriptor)).await?;
177 let info = resp.into_inner();
178
179 assert!(
180 !info.endpoint.is_empty(),
181 "No endpoints returned from Flight server",
182 );
183 for endpoint in info.endpoint {
184 let ticket = endpoint
185 .ticket
186 .expect("No ticket returned from Flight server");
187
188 assert!(
189 !endpoint.location.is_empty(),
190 "No locations returned from Flight server",
191 );
192 for location in endpoint.location {
193 consume_flight_location(location, ticket.clone(), expected_data).await?;
194 }
195 }
196
197 Ok(())
198}
199
200async fn consume_flight_location(
201 location: Location,
202 ticket: Ticket,
203 expected_data: &[RecordBatch],
204) -> Result {
205 let mut location = location;
206 location.uri = location.uri.replace("grpc+tcp://", "http://");
210
211 let endpoint = Endpoint::new(location.uri)?;
212 let channel = endpoint.connect().await?;
213 let mut client = FlightServiceClient::new(channel);
214 let resp = client.do_get(ticket).await?;
215 let mut resp = resp.into_inner();
216
217 let flight_schema = receive_schema_flight_data(&mut resp)
218 .await
219 .unwrap_or_else(|| panic!("Failed to receive flight schema"));
220 let actual_schema = Arc::new(flight_schema);
221
222 let mut dictionaries_by_id = HashMap::new();
223
224 for (counter, expected_batch) in expected_data.iter().enumerate() {
225 let data =
226 receive_batch_flight_data(&mut resp, actual_schema.clone(), &mut dictionaries_by_id)
227 .await
228 .unwrap_or_else(|| {
229 panic!(
230 "Got fewer batches than expected, received so far: {} expected: {}",
231 counter,
232 expected_data.len(),
233 )
234 });
235
236 let metadata = counter.to_string().into_bytes();
237 assert_eq!(metadata, data.app_metadata);
238
239 let actual_batch =
240 flight_data_to_arrow_batch(&data, actual_schema.clone(), &dictionaries_by_id)
241 .expect("Unable to convert flight data to Arrow batch");
242
243 assert_eq!(actual_schema, actual_batch.schema());
244 assert_eq!(expected_batch.num_columns(), actual_batch.num_columns());
245 assert_eq!(expected_batch.num_rows(), actual_batch.num_rows());
246 let schema = expected_batch.schema();
247 for i in 0..expected_batch.num_columns() {
248 let field = schema.field(i);
249 let field_name = field.name();
250
251 let expected_data = expected_batch.column(i).as_ref();
252 let actual_data = actual_batch.column(i).as_ref();
253
254 assert_eq!(expected_data, actual_data, "Data for field {field_name}");
255 }
256 }
257
258 assert!(
259 resp.next().await.is_none(),
260 "Got more batches than the expected: {}",
261 expected_data.len(),
262 );
263
264 Ok(())
265}
266
267async fn receive_schema_flight_data(resp: &mut Streaming<FlightData>) -> Option<Schema> {
268 let data = resp.next().await?.ok()?;
269 let message =
270 arrow::ipc::root_as_message(&data.data_header[..]).expect("Error parsing message");
271
272 let ipc_schema: ipc::Schema = message
274 .header_as_schema()
275 .expect("Unable to read IPC message as schema");
276 let schema = ipc::convert::try_fb_to_schema(ipc_schema).unwrap();
277
278 Some(schema)
279}
280
281async fn receive_batch_flight_data(
282 resp: &mut Streaming<FlightData>,
283 schema: SchemaRef,
284 dictionaries_by_id: &mut HashMap<i64, ArrayRef>,
285) -> Option<FlightData> {
286 let mut data = resp.next().await?.ok()?;
287 let mut message =
288 arrow::ipc::root_as_message(&data.data_header[..]).expect("Error parsing first message");
289
290 while message.header_type() == ipc::MessageHeader::DictionaryBatch {
291 reader::read_dictionary(
292 &Buffer::from(data.data_body.as_ref()),
293 message
294 .header_as_dictionary_batch()
295 .expect("Error parsing dictionary"),
296 &schema,
297 dictionaries_by_id,
298 &message.version(),
299 )
300 .expect("Error reading dictionary");
301
302 data = resp.next().await?.ok()?;
303 message =
304 arrow::ipc::root_as_message(&data.data_header[..]).expect("Error parsing message");
305 }
306
307 Some(data)
308}