Skip to main content

arrow_integration_testing/
lib.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Common code used in the integration test binaries
6
7#![warn(missing_docs)]
8use serde_json::Value;
9
10use arrow::array::{Array, StructArray};
11use arrow::datatypes::{DataType, Field, Fields, Schema};
12use arrow::error::{ArrowError, Result};
13use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi_and_data_type};
14use arrow::record_batch::RecordBatch;
15use arrow::util::test_util::arrow_test_data;
16use arrow_integration_test::*;
17use std::collections::HashMap;
18use std::ffi::{CStr, CString, c_char, c_int};
19use std::fs::File;
20use std::io::BufReader;
21use std::iter::zip;
22use std::ptr;
23use std::sync::Arc;
24
25/// The expected username for the basic auth integration test.
26pub const AUTH_USERNAME: &str = "arrow";
27/// The expected password for the basic auth integration test.
28pub const AUTH_PASSWORD: &str = "flight";
29
30pub mod flight_client_scenarios;
31pub mod flight_server_scenarios;
32
33/// An Arrow file in JSON format
34pub struct ArrowFile {
35    /// The schema of the file
36    pub schema: Schema,
37    // we can evolve this into a concrete Arrow type
38    // this is temporarily not being read from
39    dictionaries: HashMap<i64, ArrowJsonDictionaryBatch>,
40    arrow_json: Value,
41}
42
43impl ArrowFile {
44    /// Read a single [RecordBatch] from the file
45    pub fn read_batch(&self, batch_num: usize) -> Result<RecordBatch> {
46        let b = self.arrow_json["batches"].get(batch_num).ok_or_else(|| {
47            ArrowError::ParseError(format!("Arrow JSON has no batch {batch_num}"))
48        })?;
49        self.batch_from_json(b)
50    }
51
52    /// Read all [RecordBatch]es from the file
53    pub fn read_batches(&self) -> Result<Vec<RecordBatch>> {
54        self.arrow_json["batches"]
55            .as_array()
56            .ok_or_else(|| ArrowError::ParseError("Arrow JSON has no 'batches' array".to_string()))?
57            .iter()
58            .map(|b| self.batch_from_json(b))
59            .collect()
60    }
61
62    fn batch_from_json(&self, batch: &Value) -> Result<RecordBatch> {
63        let json_batch: ArrowJsonBatch = serde_json::from_value(batch.clone())
64            .map_err(|err| ArrowError::ParseError(format!("Invalid Arrow JSON batch: {err}")))?;
65        record_batch_from_json(&self.schema, json_batch, Some(&self.dictionaries))
66    }
67}
68
69/// Canonicalize the names of map fields in a schema
70pub fn canonicalize_schema(schema: &Schema) -> Schema {
71    let fields = schema
72        .fields()
73        .iter()
74        .map(|field| match field.data_type() {
75            DataType::Map(child_field, sorted) => match child_field.data_type() {
76                DataType::Struct(fields) if fields.len() == 2 => {
77                    let first_field = &fields[0];
78                    let key_field = Arc::new(Field::new(
79                        Field::MAP_KEY_FIELD_DEFAULT_NAME,
80                        first_field.data_type().clone(),
81                        false,
82                    ));
83                    let second_field = &fields[1];
84                    let value_field = Arc::new(Field::new(
85                        Field::MAP_VALUE_FIELD_DEFAULT_NAME,
86                        second_field.data_type().clone(),
87                        second_field.is_nullable(),
88                    ));
89
90                    let fields = Fields::from([key_field, value_field]);
91                    let struct_type = DataType::Struct(fields);
92                    let child_field =
93                        Field::new(Field::MAP_ENTRIES_FIELD_DEFAULT_NAME, struct_type, false);
94
95                    Arc::new(Field::new(
96                        field.name().as_str(),
97                        DataType::Map(Arc::new(child_field), *sorted),
98                        field.is_nullable(),
99                    ))
100                }
101                _ => panic!("The child field of Map type should be Struct type with 2 fields."),
102            },
103            _ => field.clone(),
104        })
105        .collect::<Fields>();
106
107    Schema::new(fields).with_metadata(schema.metadata().clone())
108}
109
110/// Read an Arrow file in JSON format
111pub fn open_json_file(json_name: &str) -> Result<ArrowFile> {
112    let json_file = File::open(json_name)?;
113    let reader = BufReader::new(json_file);
114    let arrow_json: Value = serde_json::from_reader(reader)
115        .map_err(|err| ArrowError::ParseError(format!("Invalid Arrow JSON: {err}")))?;
116    let schema = schema_from_json(&arrow_json["schema"])?;
117    // read dictionaries
118    let mut dictionaries = HashMap::new();
119    if let Some(dicts) = arrow_json.get("dictionaries") {
120        let dicts = dicts.as_array().ok_or_else(|| {
121            ArrowError::ParseError("Arrow JSON 'dictionaries' is not an array".to_string())
122        })?;
123        for d in dicts {
124            let json_dict: ArrowJsonDictionaryBatch =
125                serde_json::from_value(d.clone()).map_err(|err| {
126                    ArrowError::ParseError(format!("Invalid Arrow JSON dictionary: {err}"))
127                })?;
128            // TODO: convert to a concrete Arrow type
129            dictionaries.insert(json_dict.id, json_dict);
130        }
131    }
132    Ok(ArrowFile {
133        schema,
134        dictionaries,
135        arrow_json,
136    })
137}
138
139/// Read gzipped JSON test file
140///
141/// For example given the input:
142/// version = `0.17.1`
143/// path = `generated_union`
144///
145/// Returns the contents of
146/// `arrow-ipc-stream/integration/0.17.1/generated_union.json.gz`
147pub fn read_gzip_json(version: &str, path: &str) -> ArrowJson {
148    use flate2::read::GzDecoder;
149    use std::io::Read;
150
151    let testdata = arrow_test_data();
152    let file = File::open(format!(
153        "{testdata}/arrow-ipc-stream/integration/{version}/{path}.json.gz"
154    ))
155    .unwrap();
156    let mut gz = GzDecoder::new(&file);
157    let mut s = String::new();
158    gz.read_to_string(&mut s).unwrap();
159    // convert to Arrow JSON
160    let arrow_json: ArrowJson = serde_json::from_str(&s).unwrap();
161    arrow_json
162}
163
164/// C Data Integration entrypoint to export the schema from a JSON file
165fn cdata_integration_export_schema_from_json(
166    c_json_name: *const c_char,
167    out: *mut FFI_ArrowSchema,
168) -> Result<()> {
169    let json_name = unsafe { CStr::from_ptr(c_json_name) };
170    let f = open_json_file(json_name.to_str()?)?;
171    let c_schema = FFI_ArrowSchema::try_from(&f.schema)?;
172    // Move exported schema into output struct
173    unsafe { ptr::write(out, c_schema) };
174    Ok(())
175}
176
177/// C Data Integration entrypoint to export a batch from a JSON file
178fn cdata_integration_export_batch_from_json(
179    c_json_name: *const c_char,
180    batch_num: c_int,
181    out: *mut FFI_ArrowArray,
182) -> Result<()> {
183    let json_name = unsafe { CStr::from_ptr(c_json_name) };
184    let b = open_json_file(json_name.to_str()?)?.read_batch(batch_num.try_into().unwrap())?;
185    let a = StructArray::from(b).into_data();
186    let c_array = FFI_ArrowArray::new(&a);
187    // Move exported array into output struct
188    unsafe { ptr::write(out, c_array) };
189    Ok(())
190}
191
192fn cdata_integration_import_schema_and_compare_to_json(
193    c_json_name: *const c_char,
194    c_schema: *mut FFI_ArrowSchema,
195) -> Result<()> {
196    let json_name = unsafe { CStr::from_ptr(c_json_name) };
197    let json_schema = open_json_file(json_name.to_str()?)?.schema;
198
199    // The source ArrowSchema will be released when this is dropped
200    let imported_schema = unsafe { FFI_ArrowSchema::from_raw(c_schema) };
201    let imported_schema = Schema::try_from(&imported_schema)?;
202
203    // compare schemas
204    if canonicalize_schema(&json_schema) != canonicalize_schema(&imported_schema) {
205        return Err(ArrowError::ComputeError(format!(
206            "Schemas do not match.\n- JSON: {json_schema:?}\n- Imported: {imported_schema:?}",
207        )));
208    }
209    Ok(())
210}
211
212fn compare_batches(a: &RecordBatch, b: &RecordBatch) -> Result<()> {
213    if a.num_columns() != b.num_columns() {
214        return Err(ArrowError::InvalidArgumentError(
215            "batches do not have the same number of columns".to_string(),
216        ));
217    }
218    for (a_column, b_column) in zip(a.columns(), b.columns()) {
219        if a_column != b_column {
220            return Err(ArrowError::InvalidArgumentError(
221                "batch columns are not the same".to_string(),
222            ));
223        }
224    }
225    Ok(())
226}
227
228fn cdata_integration_import_batch_and_compare_to_json(
229    c_json_name: *const c_char,
230    batch_num: c_int,
231    c_array: *mut FFI_ArrowArray,
232) -> Result<()> {
233    let json_name = unsafe { CStr::from_ptr(c_json_name) };
234    let json_batch =
235        open_json_file(json_name.to_str()?)?.read_batch(batch_num.try_into().unwrap())?;
236    let schema = json_batch.schema();
237
238    let data_type_for_import = DataType::Struct(schema.fields.clone());
239    let imported_array = unsafe { FFI_ArrowArray::from_raw(c_array) };
240    let imported_array = unsafe { from_ffi_and_data_type(imported_array, data_type_for_import) }?;
241    imported_array.validate_full()?;
242    let imported_batch = RecordBatch::from(StructArray::from(imported_array));
243
244    compare_batches(&json_batch, &imported_batch)
245}
246
247// If Result is an error, then export a const char* to its string display, otherwise NULL
248fn result_to_c_error<T, E: std::fmt::Display>(result: &std::result::Result<T, E>) -> *mut c_char {
249    match result {
250        Ok(_) => ptr::null_mut(),
251        Err(e) => CString::new(format!("{e}")).unwrap().into_raw(),
252    }
253}
254
255/// Release a const char* exported by result_to_c_error()
256///
257/// # Safety
258///
259/// The pointer is assumed to have been obtained using CString::into_raw.
260#[unsafe(no_mangle)]
261pub unsafe extern "C" fn arrow_rs_free_error(c_error: *mut c_char) {
262    if !c_error.is_null() {
263        drop(unsafe { CString::from_raw(c_error) });
264    }
265}
266
267/// A C-ABI for exporting an Arrow schema from a JSON file
268#[unsafe(no_mangle)]
269pub extern "C" fn arrow_rs_cdata_integration_export_schema_from_json(
270    c_json_name: *const c_char,
271    out: *mut FFI_ArrowSchema,
272) -> *mut c_char {
273    let r = cdata_integration_export_schema_from_json(c_json_name, out);
274    result_to_c_error(&r)
275}
276
277/// A C-ABI to compare an Arrow schema against a JSON file
278#[unsafe(no_mangle)]
279pub extern "C" fn arrow_rs_cdata_integration_import_schema_and_compare_to_json(
280    c_json_name: *const c_char,
281    c_schema: *mut FFI_ArrowSchema,
282) -> *mut c_char {
283    let r = cdata_integration_import_schema_and_compare_to_json(c_json_name, c_schema);
284    result_to_c_error(&r)
285}
286
287/// A C-ABI for exporting a RecordBatch from a JSON file
288#[unsafe(no_mangle)]
289pub extern "C" fn arrow_rs_cdata_integration_export_batch_from_json(
290    c_json_name: *const c_char,
291    batch_num: c_int,
292    out: *mut FFI_ArrowArray,
293) -> *mut c_char {
294    let r = cdata_integration_export_batch_from_json(c_json_name, batch_num, out);
295    result_to_c_error(&r)
296}
297
298/// A C-ABI to compare a RecordBatch against a JSON file
299#[unsafe(no_mangle)]
300pub extern "C" fn arrow_rs_cdata_integration_import_batch_and_compare_to_json(
301    c_json_name: *const c_char,
302    batch_num: c_int,
303    c_array: *mut FFI_ArrowArray,
304) -> *mut c_char {
305    let r = cdata_integration_import_batch_and_compare_to_json(c_json_name, batch_num, c_array);
306    result_to_c_error(&r)
307}