arrow_json_integration_test/
arrow-json-integration-test.rs1use arrow::error::{ArrowError, Result};
8use arrow::ipc::reader::FileReader;
9use arrow::ipc::writer::FileWriter;
10use arrow_integration_test::*;
11use arrow_integration_testing::{canonicalize_schema, open_json_file};
12use clap::Parser;
13use std::fs::File;
14
15#[derive(clap::ValueEnum, Debug, Clone)]
16#[clap(rename_all = "SCREAMING_SNAKE_CASE")]
17enum Mode {
18 ArrowToJson,
19 JsonToArrow,
20 Validate,
21}
22
23#[derive(Debug, Parser)]
24#[clap(author, version, about("rust arrow-json-integration-test"), long_about = None)]
25struct Args {
26 #[clap(short, long)]
27 integration: bool,
28 #[clap(short, long, help("Path to ARROW file"))]
29 arrow: String,
30 #[clap(short, long, help("Path to JSON file"))]
31 json: String,
32 #[clap(
33 value_enum,
34 short,
35 long,
36 default_value = "VALIDATE",
37 help = "Mode of integration testing tool"
38 )]
39 mode: Mode,
40 #[clap(short, long)]
41 verbose: bool,
42}
43
44fn main() -> Result<()> {
45 let args = Args::parse();
46 let arrow_file = args.arrow;
47 let json_file = args.json;
48 let verbose = args.verbose;
49 match args.mode {
50 Mode::JsonToArrow => json_to_arrow(&json_file, &arrow_file, verbose),
51 Mode::ArrowToJson => arrow_to_json(&arrow_file, &json_file, verbose),
52 Mode::Validate => validate(&arrow_file, &json_file, verbose),
53 }
54}
55
56fn json_to_arrow(json_name: &str, arrow_name: &str, verbose: bool) -> Result<()> {
57 if verbose {
58 eprintln!("Converting {json_name} to {arrow_name}");
59 }
60
61 let json_file = open_json_file(json_name)?;
62
63 let arrow_file = File::create(arrow_name)?;
64 let mut writer = FileWriter::try_new(arrow_file, &json_file.schema)?;
65
66 for b in json_file.read_batches()? {
67 writer.write(&b)?;
68 }
69
70 writer.finish()?;
71
72 Ok(())
73}
74
75fn arrow_to_json(arrow_name: &str, json_name: &str, verbose: bool) -> Result<()> {
76 if verbose {
77 eprintln!("Converting {arrow_name} to {json_name}");
78 }
79
80 let arrow_file = File::open(arrow_name)?;
81 let reader = FileReader::try_new(arrow_file, None)?;
82
83 let mut fields: Vec<ArrowJsonField> = vec![];
84 for f in reader.schema().fields() {
85 fields.push(ArrowJsonField::from(f));
86 }
87 let schema = ArrowJsonSchema {
88 fields,
89 metadata: None,
90 };
91
92 let batches = reader
93 .map(|batch| Ok(ArrowJsonBatch::from_batch(&batch?)))
94 .collect::<Result<Vec<_>>>()?;
95
96 let arrow_json = ArrowJson {
97 schema,
98 batches,
99 dictionaries: None,
100 };
101
102 let json_file = File::create(json_name)?;
103 serde_json::to_writer(&json_file, &arrow_json).unwrap();
104
105 Ok(())
106}
107
108fn validate(arrow_name: &str, json_name: &str, verbose: bool) -> Result<()> {
109 if verbose {
110 eprintln!("Validating {arrow_name} and {json_name}");
111 }
112
113 let json_file = open_json_file(json_name)?;
115
116 let arrow_file = File::open(arrow_name)?;
118 let mut arrow_reader = FileReader::try_new(arrow_file, None)?;
119 let arrow_schema = arrow_reader.schema().as_ref().to_owned();
120
121 if canonicalize_schema(&json_file.schema) != canonicalize_schema(&arrow_schema) {
123 return Err(ArrowError::ComputeError(format!(
124 "Schemas do not match. JSON: {:?}. Arrow: {:?}",
125 json_file.schema, arrow_schema
126 )));
127 }
128
129 let json_batches = json_file.read_batches()?;
130
131 assert!(
133 json_batches.len() == arrow_reader.num_batches(),
134 "JSON batches and Arrow batches are unequal"
135 );
136
137 if verbose {
138 eprintln!(
139 "Schemas match. JSON file has {} batches.",
140 json_batches.len()
141 );
142 }
143
144 for json_batch in json_batches {
145 if let Some(Ok(arrow_batch)) = arrow_reader.next() {
146 let num_columns = arrow_batch.num_columns();
148 assert_eq!(num_columns, json_batch.num_columns());
149 assert_eq!(arrow_batch.num_rows(), json_batch.num_rows());
150
151 for i in 0..num_columns {
152 assert_eq!(
153 arrow_batch.column(i).as_ref(),
154 json_batch.column(i).as_ref(),
155 "Arrow and JSON batch columns not the same"
156 );
157 }
158 } else {
159 return Err(ArrowError::ComputeError(
160 "no more arrow batches left".to_owned(),
161 ));
162 }
163 }
164
165 if arrow_reader.next().is_some() {
166 return Err(ArrowError::ComputeError(
167 "no more json batches left".to_owned(),
168 ));
169 }
170
171 Ok(())
172}