Skip to main content

arrow_json_integration_test/
arrow-json-integration-test.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Converts between the Arrow IPC file format and the integration test JSON format.
6
7use arrow::error::{ArrowError, Result};
8use arrow::ipc::reader::FileReader;
9use arrow::ipc::writer::FileWriter;
10use arrow_integration_test::*;
11use arrow_integration_testing::{canonicalize_schema, open_json_file};
12use clap::Parser;
13use std::fs::File;
14
15#[derive(clap::ValueEnum, Debug, Clone)]
16#[clap(rename_all = "SCREAMING_SNAKE_CASE")]
17enum Mode {
18    ArrowToJson,
19    JsonToArrow,
20    Validate,
21}
22
23#[derive(Debug, Parser)]
24#[clap(author, version, about("rust arrow-json-integration-test"), long_about = None)]
25struct Args {
26    #[clap(short, long)]
27    integration: bool,
28    #[clap(short, long, help("Path to ARROW file"))]
29    arrow: String,
30    #[clap(short, long, help("Path to JSON file"))]
31    json: String,
32    #[clap(
33        value_enum,
34        short,
35        long,
36        default_value = "VALIDATE",
37        help = "Mode of integration testing tool"
38    )]
39    mode: Mode,
40    #[clap(short, long)]
41    verbose: bool,
42}
43
44fn main() -> Result<()> {
45    let args = Args::parse();
46    let arrow_file = args.arrow;
47    let json_file = args.json;
48    let verbose = args.verbose;
49    match args.mode {
50        Mode::JsonToArrow => json_to_arrow(&json_file, &arrow_file, verbose),
51        Mode::ArrowToJson => arrow_to_json(&arrow_file, &json_file, verbose),
52        Mode::Validate => validate(&arrow_file, &json_file, verbose),
53    }
54}
55
56fn json_to_arrow(json_name: &str, arrow_name: &str, verbose: bool) -> Result<()> {
57    if verbose {
58        eprintln!("Converting {json_name} to {arrow_name}");
59    }
60
61    let json_file = open_json_file(json_name)?;
62
63    let arrow_file = File::create(arrow_name)?;
64    let mut writer = FileWriter::try_new(arrow_file, &json_file.schema)?;
65
66    for b in json_file.read_batches()? {
67        writer.write(&b)?;
68    }
69
70    writer.finish()?;
71
72    Ok(())
73}
74
75fn arrow_to_json(arrow_name: &str, json_name: &str, verbose: bool) -> Result<()> {
76    if verbose {
77        eprintln!("Converting {arrow_name} to {json_name}");
78    }
79
80    let arrow_file = File::open(arrow_name)?;
81    let reader = FileReader::try_new(arrow_file, None)?;
82
83    let mut fields: Vec<ArrowJsonField> = vec![];
84    for f in reader.schema().fields() {
85        fields.push(ArrowJsonField::from(f));
86    }
87    let schema = ArrowJsonSchema {
88        fields,
89        metadata: None,
90    };
91
92    let batches = reader
93        .map(|batch| Ok(ArrowJsonBatch::from_batch(&batch?)))
94        .collect::<Result<Vec<_>>>()?;
95
96    let arrow_json = ArrowJson {
97        schema,
98        batches,
99        dictionaries: None,
100    };
101
102    let json_file = File::create(json_name)?;
103    serde_json::to_writer(&json_file, &arrow_json).unwrap();
104
105    Ok(())
106}
107
108fn validate(arrow_name: &str, json_name: &str, verbose: bool) -> Result<()> {
109    if verbose {
110        eprintln!("Validating {arrow_name} and {json_name}");
111    }
112
113    // open JSON file
114    let json_file = open_json_file(json_name)?;
115
116    // open Arrow file
117    let arrow_file = File::open(arrow_name)?;
118    let mut arrow_reader = FileReader::try_new(arrow_file, None)?;
119    let arrow_schema = arrow_reader.schema().as_ref().to_owned();
120
121    // compare schemas
122    if canonicalize_schema(&json_file.schema) != canonicalize_schema(&arrow_schema) {
123        return Err(ArrowError::ComputeError(format!(
124            "Schemas do not match. JSON: {:?}. Arrow: {:?}",
125            json_file.schema, arrow_schema
126        )));
127    }
128
129    let json_batches = json_file.read_batches()?;
130
131    // compare number of batches
132    assert!(
133        json_batches.len() == arrow_reader.num_batches(),
134        "JSON batches and Arrow batches are unequal"
135    );
136
137    if verbose {
138        eprintln!(
139            "Schemas match. JSON file has {} batches.",
140            json_batches.len()
141        );
142    }
143
144    for json_batch in json_batches {
145        if let Some(Ok(arrow_batch)) = arrow_reader.next() {
146            // compare batches
147            let num_columns = arrow_batch.num_columns();
148            assert_eq!(num_columns, json_batch.num_columns());
149            assert_eq!(arrow_batch.num_rows(), json_batch.num_rows());
150
151            for i in 0..num_columns {
152                assert_eq!(
153                    arrow_batch.column(i).as_ref(),
154                    json_batch.column(i).as_ref(),
155                    "Arrow and JSON batch columns not the same"
156                );
157            }
158        } else {
159            return Err(ArrowError::ComputeError(
160                "no more arrow batches left".to_owned(),
161            ));
162        }
163    }
164
165    if arrow_reader.next().is_some() {
166        return Err(ArrowError::ComputeError(
167            "no more json batches left".to_owned(),
168        ));
169    }
170
171    Ok(())
172}