arrow_json/
lib.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Transfer data between the Arrow memory format and JSON line-delimited records.
19//!
20//! See the module level documentation for the
21//! [`reader`] and [`writer`] for usage examples.
22//!
23//! # Binary Data Encoding
24//!
25//! As per [RFC7159] JSON cannot encode arbitrary binary data. This crate works around that
26//! limitation by encoding/decoding binary data as a [hexadecimal] string (i.e.
27//! [`Base16` encoding]).
28//!
29//! Note that `Base16` only has 50% space efficiency (i.e., the encoded data is twice as large
30//! as the original). If that is an issue, there are two alternatives:
31//!
32//! 1. Provide a custom encoder. See the [Customizing the encoder] section of the writer documentation.
33//! 2. Convert binary data to/from a different encoding format such as `Base64` before
34//!    writing / after reading, as shown in the following example.
35//!
36//! [Customizing the encoder]: writer#customizing-the-encoder
37//!
38//! ## `Base64` Encoding Example
39//!
40//! [`Base64`] is a common [binary-to-text encoding] scheme with a space efficiency of 75%. The
41//! following example shows how to use the [`arrow_cast`] crate to encode binary data to `Base64`
42//! before converting it to JSON and how to decode it back.
43//!
44//! ```
45//! # use std::io::Cursor;
46//! # use std::sync::Arc;
47//! # use arrow_array::{BinaryArray, RecordBatch, StringArray};
48//! # use arrow_array::cast::AsArray;
49//! use arrow_cast::base64::{b64_decode, b64_encode, BASE64_STANDARD};
50//! # use arrow_json::{LineDelimitedWriter, ReaderBuilder};
51//! #
52//! // The data we want to write
53//! let input = BinaryArray::from(vec![b"\xDE\x00\xFF".as_ref()]);
54//!
55//! // Base64 encode it to a string
56//! let encoded: StringArray = b64_encode(&BASE64_STANDARD, &input);
57//!
58//! // Write the StringArray to JSON
59//! let batch = RecordBatch::try_from_iter([("col", Arc::new(encoded) as _)]).unwrap();
60//! let mut buf = Vec::with_capacity(1024);
61//! let mut writer = LineDelimitedWriter::new(&mut buf);
62//! writer.write(&batch).unwrap();
63//! writer.finish().unwrap();
64//!
65//! // Read the JSON data
66//! let cursor = Cursor::new(buf);
67//! let mut reader = ReaderBuilder::new(batch.schema()).build(cursor).unwrap();
68//! let batch = reader.next().unwrap().unwrap();
69//!
70//! // Reverse the base64 encoding
71//! let col: BinaryArray = batch.column(0).as_string::<i32>().clone().into();
72//! let output = b64_decode(&BASE64_STANDARD, &col).unwrap();
73//!
74//! assert_eq!(input, output);
75//! ```
76//!
77//! [RFC7159]: https://datatracker.ietf.org/doc/html/rfc7159#section-8.1
78//! [binary-to-text encoding]: https://en.wikipedia.org/wiki/Binary-to-text_encoding
79//! [hexadecimal]: https://en.wikipedia.org/wiki/Hexadecimal
80//! [`Base16` encoding]: https://en.wikipedia.org/wiki/Base16#Base16
81//! [`Base64`]: https://en.wikipedia.org/wiki/Base64
82
83#![doc(
84    html_logo_url = "https://arrow.apache.org/img/arrow-logo_chevrons_black-txt_white-bg.svg",
85    html_favicon_url = "https://arrow.apache.org/img/arrow-logo_chevrons_black-txt_transparent-bg.svg"
86)]
87#![cfg_attr(docsrs, feature(doc_cfg))]
88#![deny(rustdoc::broken_intra_doc_links)]
89#![warn(missing_docs)]
90
91pub mod reader;
92pub mod writer;
93
94pub use self::reader::{
95    ArrayDecoder, DecoderContext, DecoderFactory, Reader, ReaderBuilder, Tape, TapeElement,
96};
97pub use self::writer::{
98    ArrayWriter, Encoder, EncoderFactory, EncoderOptions, LineDelimitedWriter, Writer,
99    WriterBuilder,
100};
101use half::f16;
102use serde_json::{Number, Value};
103
104/// Specifies what is considered valid JSON when reading or writing
105/// RecordBatches or StructArrays.
106///
107/// This enum controls which form(s) the Reader will accept and which form the
108/// Writer will produce. For example, if the RecordBatch Schema is
109/// `[("a", Int32), ("r", Struct("b": Boolean, "c" Utf8))]`
110/// then a Reader with [`StructMode::ObjectOnly`] would read rows of the form
111/// `{"a": 1, "r": {"b": true, "c": "cat"}}` while with [`StructMode::ListOnly`]
112/// would read rows of the form `[1, [true, "cat"]]`. A Writer would produce
113/// rows formatted similarly.
114///
115/// The list encoding is more compact if the schema is known, and is used by
116/// tools such as [Presto] and [Trino].
117///
118/// When reading objects, the order of the key does not matter. When reading
119/// lists, the entries must be the same number and in the same order as the
120/// struct fields. Map columns are not affected by this option.
121///
122/// [Presto]: https://prestodb.io/docs/current/develop/client-protocol.html#important-queryresults-attributes
123/// [Trino]: https://trino.io/docs/current/develop/client-protocol.html#important-queryresults-attributes
124#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
125pub enum StructMode {
126    #[default]
127    /// Encode/decode structs as objects (e.g., {"a": 1, "b": "c"})
128    ObjectOnly,
129    /// Encode/decode structs as lists (e.g., [1, "c"])
130    ListOnly,
131}
132
133/// Trait declaring any type that is serializable to JSON. This includes all primitive types (bool, i32, etc.).
134pub trait JsonSerializable: 'static {
135    /// Converts self into json value if its possible
136    fn into_json_value(self) -> Option<Value>;
137}
138
139macro_rules! json_serializable {
140    ($t:ty) => {
141        impl JsonSerializable for $t {
142            fn into_json_value(self) -> Option<Value> {
143                Some(self.into())
144            }
145        }
146    };
147}
148
149json_serializable!(bool);
150json_serializable!(u8);
151json_serializable!(u16);
152json_serializable!(u32);
153json_serializable!(u64);
154json_serializable!(i8);
155json_serializable!(i16);
156json_serializable!(i32);
157json_serializable!(i64);
158
159impl JsonSerializable for i128 {
160    fn into_json_value(self) -> Option<Value> {
161        // Serialize as string to avoid issues with arbitrary_precision serde_json feature
162        // - https://github.com/serde-rs/json/issues/559
163        // - https://github.com/serde-rs/json/issues/845
164        // - https://github.com/serde-rs/json/issues/846
165        Some(self.to_string().into())
166    }
167}
168
169impl JsonSerializable for f16 {
170    fn into_json_value(self) -> Option<Value> {
171        Number::from_f64(f64::round(f64::from(self) * 1000.0) / 1000.0).map(Value::Number)
172    }
173}
174
175impl JsonSerializable for f32 {
176    fn into_json_value(self) -> Option<Value> {
177        Number::from_f64(f64::round(self as f64 * 1000.0) / 1000.0).map(Value::Number)
178    }
179}
180
181impl JsonSerializable for f64 {
182    fn into_json_value(self) -> Option<Value> {
183        Number::from_f64(self).map(Value::Number)
184    }
185}
186
187#[cfg(test)]
188mod tests {
189    use super::*;
190    use crate::writer::JsonArray;
191    use crate::writer::LineDelimited;
192    use arrow_array::{
193        ArrayRef, GenericBinaryArray, GenericByteViewArray, GenericListViewArray, RecordBatch,
194        RecordBatchWriter, builder::FixedSizeBinaryBuilder, types::BinaryViewType,
195    };
196    use arrow_schema::{DataType, Field, Fields, Schema};
197    use serde_json::Value::{Bool, Number as VNumber, String as VString};
198    use std::io::Cursor;
199    use std::sync::Arc;
200
201    #[test]
202    fn test_arrow_native_type_to_json() {
203        assert_eq!(Some(Bool(true)), true.into_json_value());
204        assert_eq!(Some(VNumber(Number::from(1))), 1i8.into_json_value());
205        assert_eq!(Some(VNumber(Number::from(1))), 1i16.into_json_value());
206        assert_eq!(Some(VNumber(Number::from(1))), 1i32.into_json_value());
207        assert_eq!(Some(VNumber(Number::from(1))), 1i64.into_json_value());
208        assert_eq!(Some(VString("1".to_string())), 1i128.into_json_value());
209        assert_eq!(Some(VNumber(Number::from(1))), 1u8.into_json_value());
210        assert_eq!(Some(VNumber(Number::from(1))), 1u16.into_json_value());
211        assert_eq!(Some(VNumber(Number::from(1))), 1u32.into_json_value());
212        assert_eq!(Some(VNumber(Number::from(1))), 1u64.into_json_value());
213        assert_eq!(
214            Some(VNumber(Number::from_f64(0.01f64).unwrap())),
215            0.01.into_json_value()
216        );
217        assert_eq!(
218            Some(VNumber(Number::from_f64(0.01f64).unwrap())),
219            0.01f64.into_json_value()
220        );
221        assert_eq!(None, f32::NAN.into_json_value());
222    }
223
224    #[test]
225    fn test_json_roundtrip_structs() {
226        let schema = Arc::new(Schema::new(vec![
227            Field::new(
228                "c1",
229                DataType::Struct(Fields::from(vec![
230                    Field::new("c11", DataType::Int32, true),
231                    Field::new(
232                        "c12",
233                        DataType::Struct(vec![Field::new("c121", DataType::Utf8, false)].into()),
234                        false,
235                    ),
236                ])),
237                false,
238            ),
239            Field::new("c2", DataType::Utf8, false),
240        ]));
241
242        {
243            let object_input = r#"{"c1":{"c11":1,"c12":{"c121":"e"}},"c2":"a"}
244{"c1":{"c12":{"c121":"f"}},"c2":"b"}
245{"c1":{"c11":5,"c12":{"c121":"g"}},"c2":"c"}
246"#
247            .as_bytes();
248            let object_reader = ReaderBuilder::new(schema.clone())
249                .with_struct_mode(StructMode::ObjectOnly)
250                .build(object_input)
251                .unwrap();
252
253            let mut object_output: Vec<u8> = Vec::new();
254            let mut object_writer = WriterBuilder::new()
255                .with_struct_mode(StructMode::ObjectOnly)
256                .build::<_, LineDelimited>(&mut object_output);
257            for batch_res in object_reader {
258                object_writer.write(&batch_res.unwrap()).unwrap();
259            }
260            assert_eq!(object_input, &object_output);
261        }
262
263        {
264            let list_input = r#"[[1,["e"]],"a"]
265[[null,["f"]],"b"]
266[[5,["g"]],"c"]
267"#
268            .as_bytes();
269            let list_reader = ReaderBuilder::new(schema.clone())
270                .with_struct_mode(StructMode::ListOnly)
271                .build(list_input)
272                .unwrap();
273
274            let mut list_output: Vec<u8> = Vec::new();
275            let mut list_writer = WriterBuilder::new()
276                .with_struct_mode(StructMode::ListOnly)
277                .build::<_, LineDelimited>(&mut list_output);
278            for batch_res in list_reader {
279                list_writer.write(&batch_res.unwrap()).unwrap();
280            }
281            assert_eq!(list_input, &list_output);
282        }
283    }
284
285    #[test]
286    #[expect(invalid_from_utf8)]
287    fn test_json_roundtrip_binary() {
288        let not_utf8: &[u8] = b"Not UTF8 \xa0\xa1!";
289        assert!(str::from_utf8(not_utf8).is_err());
290
291        let values: &[Option<&[u8]>] = &[
292            Some(b"Bob Thompson" as &[u8]),
293            None,
294            Some(b"Troy McClure" as &[u8]),
295            Some(not_utf8),
296        ];
297        // Binary:
298        assert_binary_json(Arc::new(GenericBinaryArray::<i32>::from_iter(values)));
299
300        // LargeBinary:
301        assert_binary_json(Arc::new(GenericBinaryArray::<i64>::from_iter(values)));
302
303        // FixedSizeBinary:
304        assert_binary_json(build_array_fixed_size_binary(12, values));
305
306        // BinaryView:
307        assert_binary_json(Arc::new(GenericByteViewArray::<BinaryViewType>::from_iter(
308            values,
309        )));
310    }
311
312    fn build_array_fixed_size_binary(byte_width: i32, values: &[Option<&[u8]>]) -> ArrayRef {
313        let mut builder = FixedSizeBinaryBuilder::new(byte_width);
314        for value in values {
315            match value {
316                Some(v) => builder.append_value(v).unwrap(),
317                None => builder.append_null(),
318            }
319        }
320        Arc::new(builder.finish())
321    }
322
323    fn assert_binary_json(array: ArrayRef) {
324        // encode and check JSON with and without explicit nulls
325        assert_binary_json_with_writer(
326            array.clone(),
327            WriterBuilder::new().with_explicit_nulls(true),
328        );
329        assert_binary_json_with_writer(array, WriterBuilder::new().with_explicit_nulls(false));
330    }
331
332    fn assert_binary_json_with_writer(array: ArrayRef, builder: WriterBuilder) {
333        let batch = RecordBatch::try_from_iter([("bytes", array)]).unwrap();
334
335        let mut buf = Vec::new();
336        let json_value: Value = {
337            let mut writer = builder.build::<_, JsonArray>(&mut buf);
338            writer.write(&batch).unwrap();
339            writer.close().unwrap();
340            serde_json::from_slice(&buf).unwrap()
341        };
342
343        let json_array = json_value.as_array().unwrap();
344
345        let decoded = {
346            let mut decoder = ReaderBuilder::new(batch.schema().clone())
347                .build_decoder()
348                .unwrap();
349            decoder.serialize(json_array).unwrap();
350            decoder.flush().unwrap().unwrap()
351        };
352
353        assert_eq!(batch, decoded);
354    }
355
356    fn assert_list_view_roundtrip<O: arrow_array::OffsetSizeTrait>() {
357        let flat_field = Arc::new(Field::new("item", DataType::Int32, true));
358        let flat_dt = GenericListViewArray::<O>::DATA_TYPE_CONSTRUCTOR(flat_field);
359
360        let nested_inner = Arc::new(Field::new("item", DataType::Int32, false));
361        let nested_inner_dt = GenericListViewArray::<O>::DATA_TYPE_CONSTRUCTOR(nested_inner);
362        let nested_outer = Arc::new(Field::new("item", nested_inner_dt, true));
363        let nested_dt = GenericListViewArray::<O>::DATA_TYPE_CONSTRUCTOR(nested_outer);
364
365        let schema = Arc::new(Schema::new(vec![
366            Field::new("flat", flat_dt, true),
367            Field::new("nested", nested_dt, true),
368        ]));
369
370        let input = r#"{"flat":[1,2,3],"nested":[[1,2],[3]]}
371{"flat":[4,null]}
372{}
373{"flat":[6],"nested":[[4,5,6]]}
374{"flat":[]}
375"#
376        .as_bytes();
377
378        let batches: Vec<RecordBatch> = ReaderBuilder::new(schema.clone())
379            .with_batch_size(1024)
380            .build(Cursor::new(input))
381            .unwrap()
382            .collect::<Result<Vec<_>, _>>()
383            .unwrap();
384
385        let mut output = Vec::new();
386        let mut writer = WriterBuilder::new().build::<_, LineDelimited>(&mut output);
387        for batch in &batches {
388            writer.write(batch).unwrap();
389        }
390        writer.finish().unwrap();
391
392        assert_eq!(input, &output);
393    }
394
395    #[test]
396    fn test_json_roundtrip_list_view() {
397        assert_list_view_roundtrip::<i32>();
398        assert_list_view_roundtrip::<i64>();
399    }
400
401    #[test]
402    fn test_json_roundtrip_fixed_size_list() {
403        let inner = Arc::new(Field::new("item", DataType::Int32, true));
404        let schema = Arc::new(Schema::new(vec![
405            Field::new("flat", DataType::FixedSizeList(inner.clone(), 3), true),
406            Field::new(
407                "nested",
408                DataType::FixedSizeList(
409                    Arc::new(Field::new("item", DataType::FixedSizeList(inner, 2), true)),
410                    2,
411                ),
412                true,
413            ),
414        ]));
415
416        let input = r#"{"flat":[1,2,3],"nested":[[1,2],[3,4]]}
417{"flat":[4,null,5]}
418{"flat":[6,7,8],"nested":[[null,5],[6,null]]}
419"#
420        .as_bytes();
421
422        let batches: Vec<RecordBatch> = ReaderBuilder::new(schema.clone())
423            .with_batch_size(1024)
424            .build(Cursor::new(input))
425            .unwrap()
426            .collect::<Result<Vec<_>, _>>()
427            .unwrap();
428
429        let mut output = Vec::new();
430        let mut writer = WriterBuilder::new().build::<_, LineDelimited>(&mut output);
431        for batch in &batches {
432            writer.write(batch).unwrap();
433        }
434        writer.finish().unwrap();
435
436        assert_eq!(input, &output);
437    }
438}
arrow_json/lib.rs

arrow_json/
lib.rs
