arrow_json/
lib.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Transfer data between the Arrow memory format and JSON line-delimited records.
19//!
20//! See the module level documentation for the
21//! [`reader`] and [`writer`] for usage examples.
22//!
23//! # Binary Data Encoding
24//!
25//! As per [RFC7159] JSON cannot encode arbitrary binary data. This crate works around that
26//! limitation by encoding/decoding binary data as a [hexadecimal] string (i.e.
27//! [`Base16` encoding]).
28//!
29//! Note that `Base16` only has 50% space efficiency (i.e., the encoded data is twice as large
30//! as the original). If that is an issue, there are two alternatives:
31//!
32//! 1. Provide a custom encoder and/or decoder. See the [Customizing the encoder] section of the
33//!    writer documentation and the [Customizing the decoder] section of the reader documentation.
34//! 2. Convert binary data to/from a different encoding format such as `Base64` before
35//!    writing / after reading, as shown in the following example.
36//!
37//! [Customizing the encoder]: writer#customizing-the-encoder
38//! [Customizing the decoder]: reader#customizing-the-decoder
39//!
40//! ## `Base64` Encoding Example
41//!
42//! [`Base64`] is a common [binary-to-text encoding] scheme with a space efficiency of 75%. The
43//! following example shows how to use the [`arrow_cast`] crate to encode binary data to `Base64`
44//! before converting it to JSON and how to decode it back.
45//!
46//! ```
47//! # use std::io::Cursor;
48//! # use std::sync::Arc;
49//! # use arrow_array::{BinaryArray, RecordBatch, StringArray};
50//! # use arrow_array::cast::AsArray;
51//! use arrow_cast::base64::{b64_decode, b64_encode, BASE64_STANDARD};
52//! # use arrow_json::{LineDelimitedWriter, ReaderBuilder};
53//! #
54//! // The data we want to write
55//! let input = BinaryArray::from(vec![b"\xDE\x00\xFF".as_ref()]);
56//!
57//! // Base64 encode it to a string
58//! let encoded: StringArray = b64_encode(&BASE64_STANDARD, &input);
59//!
60//! // Write the StringArray to JSON
61//! let batch = RecordBatch::try_from_iter([("col", Arc::new(encoded) as _)]).unwrap();
62//! let mut buf = Vec::with_capacity(1024);
63//! let mut writer = LineDelimitedWriter::new(&mut buf);
64//! writer.write(&batch).unwrap();
65//! writer.finish().unwrap();
66//!
67//! // Read the JSON data
68//! let cursor = Cursor::new(buf);
69//! let mut reader = ReaderBuilder::new(batch.schema()).build(cursor).unwrap();
70//! let batch = reader.next().unwrap().unwrap();
71//!
72//! // Reverse the base64 encoding
73//! let col: BinaryArray = batch.column(0).as_string::<i32>().clone().into();
74//! let output = b64_decode(&BASE64_STANDARD, &col).unwrap();
75//!
76//! assert_eq!(input, output);
77//! ```
78//!
79//! [RFC7159]: https://datatracker.ietf.org/doc/html/rfc7159#section-8.1
80//! [binary-to-text encoding]: https://en.wikipedia.org/wiki/Binary-to-text_encoding
81//! [hexadecimal]: https://en.wikipedia.org/wiki/Hexadecimal
82//! [`Base16` encoding]: https://en.wikipedia.org/wiki/Base16#Base16
83//! [`Base64`]: https://en.wikipedia.org/wiki/Base64
84//!
85//! # Platform Support
86//!
87//! Only little-endian platforms are officially supported and tested in CI.
88//! Big-endian platforms are not tested in CI and may not work correctly.
89//! Fixes for big-endian platforms are welcome and handled on a best-effort basis,
90//! but compatibility is not guaranteed.
91
92#![doc(
93    html_logo_url = "https://arrow.apache.org/img/arrow-logo_chevrons_black-txt_white-bg.svg",
94    html_favicon_url = "https://arrow.apache.org/img/arrow-logo_chevrons_black-txt_transparent-bg.svg"
95)]
96#![cfg_attr(docsrs, feature(doc_cfg))]
97#![deny(rustdoc::broken_intra_doc_links)]
98#![warn(missing_docs)]
99
100pub mod reader;
101pub mod writer;
102
103pub use self::reader::{
104    ArrayDecoder, DecoderContext, DecoderFactory, Reader, ReaderBuilder, Tape, TapeElement,
105};
106pub use self::writer::{
107    ArrayWriter, Encoder, EncoderFactory, EncoderOptions, LineDelimitedWriter, Writer,
108    WriterBuilder,
109};
110use half::f16;
111use serde_json::{Number, Value};
112
113/// Specifies what is considered valid JSON when reading or writing
114/// RecordBatches or StructArrays.
115///
116/// This enum controls which form(s) the Reader will accept and which form the
117/// Writer will produce. For example, if the RecordBatch Schema is
118/// `[("a", Int32), ("r", Struct("b": Boolean, "c" Utf8))]`
119/// then a Reader with [`StructMode::ObjectOnly`] would read rows of the form
120/// `{"a": 1, "r": {"b": true, "c": "cat"}}` while with [`StructMode::ListOnly`]
121/// would read rows of the form `[1, [true, "cat"]]`. A Writer would produce
122/// rows formatted similarly.
123///
124/// The list encoding is more compact if the schema is known, and is used by
125/// tools such as [Presto] and [Trino].
126///
127/// When reading objects, the order of the key does not matter. When reading
128/// lists, the entries must be the same number and in the same order as the
129/// struct fields. Map columns are not affected by this option.
130///
131/// [Presto]: https://prestodb.io/docs/current/develop/client-protocol.html#important-queryresults-attributes
132/// [Trino]: https://trino.io/docs/current/develop/client-protocol.html#important-queryresults-attributes
133#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
134pub enum StructMode {
135    #[default]
136    /// Encode/decode structs as objects (e.g., {"a": 1, "b": "c"})
137    ObjectOnly,
138    /// Encode/decode structs as lists (e.g., [1, "c"])
139    ListOnly,
140}
141
142/// Trait declaring any type that is serializable to JSON. This includes all primitive types (bool, i32, etc.).
143pub trait JsonSerializable: 'static {
144    /// Converts self into json value if its possible
145    fn into_json_value(self) -> Option<Value>;
146}
147
148macro_rules! json_serializable {
149    ($t:ty) => {
150        impl JsonSerializable for $t {
151            fn into_json_value(self) -> Option<Value> {
152                Some(self.into())
153            }
154        }
155    };
156}
157
158json_serializable!(bool);
159json_serializable!(u8);
160json_serializable!(u16);
161json_serializable!(u32);
162json_serializable!(u64);
163json_serializable!(i8);
164json_serializable!(i16);
165json_serializable!(i32);
166json_serializable!(i64);
167
168impl JsonSerializable for i128 {
169    fn into_json_value(self) -> Option<Value> {
170        // Serialize as string to avoid issues with arbitrary_precision serde_json feature
171        // - https://github.com/serde-rs/json/issues/559
172        // - https://github.com/serde-rs/json/issues/845
173        // - https://github.com/serde-rs/json/issues/846
174        Some(self.to_string().into())
175    }
176}
177
178impl JsonSerializable for f16 {
179    fn into_json_value(self) -> Option<Value> {
180        Number::from_f64(f64::round(f64::from(self) * 1000.0) / 1000.0).map(Value::Number)
181    }
182}
183
184impl JsonSerializable for f32 {
185    fn into_json_value(self) -> Option<Value> {
186        Number::from_f64(f64::round(self as f64 * 1000.0) / 1000.0).map(Value::Number)
187    }
188}
189
190impl JsonSerializable for f64 {
191    fn into_json_value(self) -> Option<Value> {
192        Number::from_f64(self).map(Value::Number)
193    }
194}
195
196#[cfg(test)]
197mod tests {
198    use super::*;
199    use crate::writer::JsonArray;
200    use crate::writer::LineDelimited;
201    use arrow_array::{
202        ArrayRef, GenericBinaryArray, GenericByteViewArray, GenericListViewArray, RecordBatch,
203        RecordBatchWriter, builder::FixedSizeBinaryBuilder, types::BinaryViewType,
204    };
205    use arrow_schema::{DataType, Field, Fields, Schema};
206    use serde_json::Value::{Bool, Number as VNumber, String as VString};
207    use std::io::Cursor;
208    use std::sync::Arc;
209
210    #[test]
211    fn test_arrow_native_type_to_json() {
212        assert_eq!(Some(Bool(true)), true.into_json_value());
213        assert_eq!(Some(VNumber(Number::from(1))), 1i8.into_json_value());
214        assert_eq!(Some(VNumber(Number::from(1))), 1i16.into_json_value());
215        assert_eq!(Some(VNumber(Number::from(1))), 1i32.into_json_value());
216        assert_eq!(Some(VNumber(Number::from(1))), 1i64.into_json_value());
217        assert_eq!(Some(VString("1".to_string())), 1i128.into_json_value());
218        assert_eq!(Some(VNumber(Number::from(1))), 1u8.into_json_value());
219        assert_eq!(Some(VNumber(Number::from(1))), 1u16.into_json_value());
220        assert_eq!(Some(VNumber(Number::from(1))), 1u32.into_json_value());
221        assert_eq!(Some(VNumber(Number::from(1))), 1u64.into_json_value());
222        assert_eq!(
223            Some(VNumber(Number::from_f64(0.01f64).unwrap())),
224            0.01.into_json_value()
225        );
226        assert_eq!(
227            Some(VNumber(Number::from_f64(0.01f64).unwrap())),
228            0.01f64.into_json_value()
229        );
230        assert_eq!(None, f32::NAN.into_json_value());
231    }
232
233    #[test]
234    fn test_json_roundtrip_structs() {
235        let schema = Arc::new(Schema::new(vec![
236            Field::new(
237                "c1",
238                DataType::Struct(Fields::from(vec![
239                    Field::new("c11", DataType::Int32, true),
240                    Field::new(
241                        "c12",
242                        DataType::Struct(vec![Field::new("c121", DataType::Utf8, false)].into()),
243                        false,
244                    ),
245                ])),
246                false,
247            ),
248            Field::new("c2", DataType::Utf8, false),
249        ]));
250
251        {
252            let object_input = r#"{"c1":{"c11":1,"c12":{"c121":"e"}},"c2":"a"}
253{"c1":{"c12":{"c121":"f"}},"c2":"b"}
254{"c1":{"c11":5,"c12":{"c121":"g"}},"c2":"c"}
255"#
256            .as_bytes();
257            let object_reader = ReaderBuilder::new(schema.clone())
258                .with_struct_mode(StructMode::ObjectOnly)
259                .build(object_input)
260                .unwrap();
261
262            let mut object_output: Vec<u8> = Vec::new();
263            let mut object_writer = WriterBuilder::new()
264                .with_struct_mode(StructMode::ObjectOnly)
265                .build::<_, LineDelimited>(&mut object_output);
266            for batch_res in object_reader {
267                object_writer.write(&batch_res.unwrap()).unwrap();
268            }
269            assert_eq!(object_input, &object_output);
270        }
271
272        {
273            let list_input = r#"[[1,["e"]],"a"]
274[[null,["f"]],"b"]
275[[5,["g"]],"c"]
276"#
277            .as_bytes();
278            let list_reader = ReaderBuilder::new(schema.clone())
279                .with_struct_mode(StructMode::ListOnly)
280                .build(list_input)
281                .unwrap();
282
283            let mut list_output: Vec<u8> = Vec::new();
284            let mut list_writer = WriterBuilder::new()
285                .with_struct_mode(StructMode::ListOnly)
286                .build::<_, LineDelimited>(&mut list_output);
287            for batch_res in list_reader {
288                list_writer.write(&batch_res.unwrap()).unwrap();
289            }
290            assert_eq!(list_input, &list_output);
291        }
292    }
293
294    #[test]
295    #[expect(invalid_from_utf8)]
296    fn test_json_roundtrip_binary() {
297        let not_utf8: &[u8] = b"Not UTF8 \xa0\xa1!";
298        assert!(str::from_utf8(not_utf8).is_err());
299
300        let values: &[Option<&[u8]>] = &[
301            Some(b"Bob Thompson" as &[u8]),
302            None,
303            Some(b"Troy McClure" as &[u8]),
304            Some(not_utf8),
305        ];
306        // Binary:
307        assert_binary_json(Arc::new(GenericBinaryArray::<i32>::from_iter(values)));
308
309        // LargeBinary:
310        assert_binary_json(Arc::new(GenericBinaryArray::<i64>::from_iter(values)));
311
312        // FixedSizeBinary:
313        assert_binary_json(build_array_fixed_size_binary(12, values));
314
315        // BinaryView:
316        assert_binary_json(Arc::new(GenericByteViewArray::<BinaryViewType>::from_iter(
317            values,
318        )));
319    }
320
321    fn build_array_fixed_size_binary(byte_width: i32, values: &[Option<&[u8]>]) -> ArrayRef {
322        let mut builder = FixedSizeBinaryBuilder::new(byte_width);
323        for value in values {
324            match value {
325                Some(v) => builder.append_value(v).unwrap(),
326                None => builder.append_null(),
327            }
328        }
329        Arc::new(builder.finish())
330    }
331
332    fn assert_binary_json(array: ArrayRef) {
333        // encode and check JSON with and without explicit nulls
334        assert_binary_json_with_writer(
335            array.clone(),
336            WriterBuilder::new().with_explicit_nulls(true),
337        );
338        assert_binary_json_with_writer(array, WriterBuilder::new().with_explicit_nulls(false));
339    }
340
341    fn assert_binary_json_with_writer(array: ArrayRef, builder: WriterBuilder) {
342        let batch = RecordBatch::try_from_iter([("bytes", array)]).unwrap();
343
344        let mut buf = Vec::new();
345        let json_value: Value = {
346            let mut writer = builder.build::<_, JsonArray>(&mut buf);
347            writer.write(&batch).unwrap();
348            writer.close().unwrap();
349            serde_json::from_slice(&buf).unwrap()
350        };
351
352        let json_array = json_value.as_array().unwrap();
353
354        let decoded = {
355            let mut decoder = ReaderBuilder::new(batch.schema().clone())
356                .build_decoder()
357                .unwrap();
358            decoder.serialize(json_array).unwrap();
359            decoder.flush().unwrap().unwrap()
360        };
361
362        assert_eq!(batch, decoded);
363    }
364
365    fn assert_list_view_roundtrip<O: arrow_array::OffsetSizeTrait>() {
366        let flat_field = Arc::new(Field::new("item", DataType::Int32, true));
367        let flat_dt = GenericListViewArray::<O>::DATA_TYPE_CONSTRUCTOR(flat_field);
368
369        let nested_inner = Arc::new(Field::new("item", DataType::Int32, false));
370        let nested_inner_dt = GenericListViewArray::<O>::DATA_TYPE_CONSTRUCTOR(nested_inner);
371        let nested_outer = Arc::new(Field::new("item", nested_inner_dt, true));
372        let nested_dt = GenericListViewArray::<O>::DATA_TYPE_CONSTRUCTOR(nested_outer);
373
374        let schema = Arc::new(Schema::new(vec![
375            Field::new("flat", flat_dt, true),
376            Field::new("nested", nested_dt, true),
377        ]));
378
379        let input = r#"{"flat":[1,2,3],"nested":[[1,2],[3]]}
380{"flat":[4,null]}
381{}
382{"flat":[6],"nested":[[4,5,6]]}
383{"flat":[]}
384"#
385        .as_bytes();
386
387        let batches: Vec<RecordBatch> = ReaderBuilder::new(schema.clone())
388            .with_batch_size(1024)
389            .build(Cursor::new(input))
390            .unwrap()
391            .collect::<Result<Vec<_>, _>>()
392            .unwrap();
393
394        let mut output = Vec::new();
395        let mut writer = WriterBuilder::new().build::<_, LineDelimited>(&mut output);
396        for batch in &batches {
397            writer.write(batch).unwrap();
398        }
399        writer.finish().unwrap();
400
401        assert_eq!(input, &output);
402    }
403
404    #[test]
405    fn test_json_roundtrip_list_view() {
406        assert_list_view_roundtrip::<i32>();
407        assert_list_view_roundtrip::<i64>();
408    }
409
410    #[test]
411    fn test_json_roundtrip_fixed_size_list() {
412        let inner = Arc::new(Field::new("item", DataType::Int32, true));
413        let schema = Arc::new(Schema::new(vec![
414            Field::new("flat", DataType::FixedSizeList(inner.clone(), 3), true),
415            Field::new(
416                "nested",
417                DataType::FixedSizeList(
418                    Arc::new(Field::new("item", DataType::FixedSizeList(inner, 2), true)),
419                    2,
420                ),
421                true,
422            ),
423        ]));
424
425        let input = r#"{"flat":[1,2,3],"nested":[[1,2],[3,4]]}
426{"flat":[4,null,5]}
427{"flat":[6,7,8],"nested":[[null,5],[6,null]]}
428"#
429        .as_bytes();
430
431        let batches: Vec<RecordBatch> = ReaderBuilder::new(schema.clone())
432            .with_batch_size(1024)
433            .build(Cursor::new(input))
434            .unwrap()
435            .collect::<Result<Vec<_>, _>>()
436            .unwrap();
437
438        let mut output = Vec::new();
439        let mut writer = WriterBuilder::new().build::<_, LineDelimited>(&mut output);
440        for batch in &batches {
441            writer.write(batch).unwrap();
442        }
443        writer.finish().unwrap();
444
445        assert_eq!(input, &output);
446    }
447}
arrow_json/lib.rs

arrow_json/
lib.rs
