Skip to main content

arrow_json/writer/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! # JSON Writer
6//!
7//! This JSON writer converts Arrow [`RecordBatch`]es into arrays of
8//! JSON objects or JSON formatted byte streams.
9//!
10//! ## Writing JSON formatted byte streams
11//!
12//! To serialize [`RecordBatch`]es into line-delimited JSON bytes, use
13//! [`LineDelimitedWriter`]:
14//!
15//! ```
16//! # use std::sync::Arc;
17//! # use arrow_array::{Int32Array, RecordBatch};
18//! # use arrow_schema::{DataType, Field, Schema};
19//!
20//! let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
21//! let a = Int32Array::from(vec![1, 2, 3]);
22//! let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a)]).unwrap();
23//!
24//! // Write the record batch out as JSON
25//! let buf = Vec::new();
26//! let mut writer = arrow_json::LineDelimitedWriter::new(buf);
27//! writer.write_batches(&vec![&batch]).unwrap();
28//! writer.finish().unwrap();
29//!
30//! // Get the underlying buffer back,
31//! let buf = writer.into_inner();
32//! assert_eq!(r#"{"a":1}
33//! {"a":2}
34//! {"a":3}
35//!"#, String::from_utf8(buf).unwrap())
36//! ```
37//!
38//! To serialize [`RecordBatch`]es into a well formed JSON array, use
39//! [`ArrayWriter`]:
40//!
41//! ```
42//! # use std::sync::Arc;
43//! # use arrow_array::{Int32Array, RecordBatch};
44//! use arrow_schema::{DataType, Field, Schema};
45//!
46//! let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
47//! let a = Int32Array::from(vec![1, 2, 3]);
48//! let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a)]).unwrap();
49//!
50//! // Write the record batch out as a JSON array
51//! let buf = Vec::new();
52//! let mut writer = arrow_json::ArrayWriter::new(buf);
53//! writer.write_batches(&vec![&batch]).unwrap();
54//! writer.finish().unwrap();
55//!
56//! // Get the underlying buffer back,
57//! let buf = writer.into_inner();
58//! assert_eq!(r#"[{"a":1},{"a":2},{"a":3}]"#, String::from_utf8(buf).unwrap())
59//! ```
60//!
61//! [`LineDelimitedWriter`] and [`ArrayWriter`] will omit writing keys with null values.
62//! In order to explicitly write null values for keys, configure a custom [`Writer`] by
63//! using a [`WriterBuilder`] to construct a [`Writer`].
64//!
65//! ## Writing to [serde_json] JSON Objects
66//!
67//! To serialize [`RecordBatch`]es into an array of
68//! [JSON](https://docs.serde.rs/serde_json/) objects you can reparse the resulting JSON string.
69//! Note that this is less efficient than using the `Writer` API.
70//!
71//! ```
72//! # use std::sync::Arc;
73//! # use arrow_array::{Int32Array, RecordBatch};
74//! # use arrow_schema::{DataType, Field, Schema};
75//! let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
76//! let a = Int32Array::from(vec![1, 2, 3]);
77//! let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a)]).unwrap();
78//!
79//! // Write the record batch out as json bytes (string)
80//! let buf = Vec::new();
81//! let mut writer = arrow_json::ArrayWriter::new(buf);
82//! writer.write_batches(&vec![&batch]).unwrap();
83//! writer.finish().unwrap();
84//! let json_data = writer.into_inner();
85//!
86//! // Parse the string using serde_json
87//! use serde_json::{Map, Value};
88//! let json_rows: Vec<Map<String, Value>> = serde_json::from_reader(json_data.as_slice()).unwrap();
89//! assert_eq!(
90//!     serde_json::Value::Object(json_rows[1].clone()),
91//!     serde_json::json!({"a": 2}),
92//! );
93//! ```
94//!
95//! ## Customizing the encoder
96//!
97//! The output produced for each data type can be customized using
98//! [`WriterBuilder::with_encoder_factory`]. For example, you can override the
99//! default hex encoding of binary data to use `Base64` instead, or provide
100//! encoders for types with no built-in encoding, such as unions.
101//! See the example on [`EncoderFactory`].
102mod encoder;
103
104use std::{fmt::Debug, io::Write, sync::Arc};
105
106use crate::StructMode;
107use arrow_array::*;
108use arrow_schema::*;
109
110pub use encoder::{Encoder, EncoderFactory, EncoderOptions, NullableEncoder, make_encoder};
111
112/// This trait defines how to format a sequence of JSON objects to a
113/// byte stream.
114pub trait JsonFormat: Debug + Default {
115    #[inline]
116    /// write any bytes needed at the start of the file to the writer
117    fn start_stream<W: Write>(&self, _writer: &mut W) -> Result<(), ArrowError> {
118        Ok(())
119    }
120
121    #[inline]
122    /// write any bytes needed for the start of each row
123    fn start_row<W: Write>(&self, _writer: &mut W, _is_first_row: bool) -> Result<(), ArrowError> {
124        Ok(())
125    }
126
127    #[inline]
128    /// write any bytes needed for the end of each row
129    fn end_row<W: Write>(&self, _writer: &mut W) -> Result<(), ArrowError> {
130        Ok(())
131    }
132
133    /// write any bytes needed for the start of each row
134    fn end_stream<W: Write>(&self, _writer: &mut W) -> Result<(), ArrowError> {
135        Ok(())
136    }
137}
138
139/// Produces JSON output with one record per line.
140///
141/// For example:
142///
143/// ```json
144/// {"foo":1}
145/// {"bar":1}
146///
147/// ```
148#[derive(Debug, Default)]
149pub struct LineDelimited {}
150
151impl JsonFormat for LineDelimited {
152    fn end_row<W: Write>(&self, writer: &mut W) -> Result<(), ArrowError> {
153        writer.write_all(b"\n")?;
154        Ok(())
155    }
156}
157
158/// Produces JSON output as a single JSON array.
159///
160/// For example:
161///
162/// ```json
163/// [{"foo":1},{"bar":1}]
164/// ```
165#[derive(Debug, Default)]
166pub struct JsonArray {}
167
168impl JsonFormat for JsonArray {
169    fn start_stream<W: Write>(&self, writer: &mut W) -> Result<(), ArrowError> {
170        writer.write_all(b"[")?;
171        Ok(())
172    }
173
174    fn start_row<W: Write>(&self, writer: &mut W, is_first_row: bool) -> Result<(), ArrowError> {
175        if !is_first_row {
176            writer.write_all(b",")?;
177        }
178        Ok(())
179    }
180
181    fn end_stream<W: Write>(&self, writer: &mut W) -> Result<(), ArrowError> {
182        writer.write_all(b"]")?;
183        Ok(())
184    }
185}
186
187/// A JSON writer which serializes [`RecordBatch`]es to newline delimited JSON objects.
188pub type LineDelimitedWriter<W> = Writer<W, LineDelimited>;
189
190/// A JSON writer which serializes [`RecordBatch`]es to JSON arrays.
191pub type ArrayWriter<W> = Writer<W, JsonArray>;
192
193/// JSON writer builder.
194#[derive(Debug, Clone, Default)]
195pub struct WriterBuilder(EncoderOptions);
196
197impl WriterBuilder {
198    /// Create a new builder for configuring JSON writing options.
199    ///
200    /// # Example
201    ///
202    /// ```
203    /// # use arrow_json::{Writer, WriterBuilder};
204    /// # use arrow_json::writer::LineDelimited;
205    /// # use std::fs::File;
206    ///
207    /// fn example() -> Writer<File, LineDelimited> {
208    ///     let file = File::create("target/out.json").unwrap();
209    ///
210    ///     // create a builder that keeps keys with null values
211    ///     let builder = WriterBuilder::new().with_explicit_nulls(true);
212    ///     let writer = builder.build::<_, LineDelimited>(file);
213    ///
214    ///     writer
215    /// }
216    /// ```
217    pub fn new() -> Self {
218        Self::default()
219    }
220
221    /// Returns `true` if this writer is configured to keep keys with null values.
222    pub fn explicit_nulls(&self) -> bool {
223        self.0.explicit_nulls()
224    }
225
226    /// Set whether to keep keys with null values, or to omit writing them.
227    ///
228    /// For example, with [`LineDelimited`] format:
229    ///
230    /// Skip nulls (set to `false`):
231    ///
232    /// ```json
233    /// {"foo":1}
234    /// {"foo":1,"bar":2}
235    /// {}
236    /// ```
237    ///
238    /// Keep nulls (set to `true`):
239    ///
240    /// ```json
241    /// {"foo":1,"bar":null}
242    /// {"foo":1,"bar":2}
243    /// {"foo":null,"bar":null}
244    /// ```
245    ///
246    /// Default is to skip nulls (set to `false`). If `struct_mode == ListOnly`,
247    /// nulls will be written explicitly regardless of this setting.
248    pub fn with_explicit_nulls(mut self, explicit_nulls: bool) -> Self {
249        self.0 = self.0.with_explicit_nulls(explicit_nulls);
250        self
251    }
252
253    /// Returns if this writer is configured to write structs as JSON Objects or Arrays.
254    pub fn struct_mode(&self) -> StructMode {
255        self.0.struct_mode()
256    }
257
258    /// Set the [`StructMode`] for the writer, which determines whether structs
259    /// are encoded to JSON as objects or lists. For more details refer to the
260    /// enum documentation. Default is to use `ObjectOnly`. If this is set to
261    /// `ListOnly`, nulls will be written explicitly regardless of the
262    /// `explicit_nulls` setting.
263    pub fn with_struct_mode(mut self, struct_mode: StructMode) -> Self {
264        self.0 = self.0.with_struct_mode(struct_mode);
265        self
266    }
267
268    /// Set an encoder factory to use when creating encoders for writing JSON.
269    ///
270    /// This can be used to override how some types are encoded or to provide
271    /// a fallback for types that are not supported by the default encoder.
272    pub fn with_encoder_factory(mut self, factory: Arc<dyn EncoderFactory>) -> Self {
273        self.0 = self.0.with_encoder_factory(factory);
274        self
275    }
276
277    /// Set the JSON file's date format
278    pub fn with_date_format(mut self, format: String) -> Self {
279        self.0 = self.0.with_date_format(format);
280        self
281    }
282
283    /// Set the JSON file's datetime format
284    pub fn with_datetime_format(mut self, format: String) -> Self {
285        self.0 = self.0.with_datetime_format(format);
286        self
287    }
288
289    /// Set the JSON file's time format
290    pub fn with_time_format(mut self, format: String) -> Self {
291        self.0 = self.0.with_time_format(format);
292        self
293    }
294
295    /// Set the JSON file's timestamp format
296    pub fn with_timestamp_format(mut self, format: String) -> Self {
297        self.0 = self.0.with_timestamp_format(format);
298        self
299    }
300
301    /// Set the JSON file's timestamp tz format
302    pub fn with_timestamp_tz_format(mut self, tz_format: String) -> Self {
303        self.0 = self.0.with_timestamp_tz_format(tz_format);
304        self
305    }
306
307    /// Create a new `Writer` with specified `JsonFormat` and builder options.
308    pub fn build<W, F>(self, writer: W) -> Writer<W, F>
309    where
310        W: Write,
311        F: JsonFormat,
312    {
313        Writer {
314            writer,
315            started: false,
316            finished: false,
317            format: F::default(),
318            options: self.0,
319        }
320    }
321}
322
323/// A JSON writer which serializes [`RecordBatch`]es to a stream of
324/// `u8` encoded JSON objects.
325///
326/// See the module level documentation for detailed usage and examples.
327/// The specific format of the stream is controlled by the [`JsonFormat`]
328/// type parameter.
329///
330/// By default the writer will skip writing keys with null values for
331/// backward compatibility. See [`WriterBuilder`] on how to customize
332/// this behaviour when creating a new writer.
333#[derive(Debug)]
334pub struct Writer<W, F>
335where
336    W: Write,
337    F: JsonFormat,
338{
339    /// Underlying writer to use to write bytes
340    writer: W,
341
342    /// Has the writer output any records yet?
343    started: bool,
344
345    /// Is the writer finished?
346    finished: bool,
347
348    /// Determines how the byte stream is formatted
349    format: F,
350
351    /// Controls how JSON should be encoded, e.g. whether to write explicit nulls or skip them
352    options: EncoderOptions,
353}
354
355impl<W, F> Writer<W, F>
356where
357    W: Write,
358    F: JsonFormat,
359{
360    /// Construct a new writer
361    pub fn new(writer: W) -> Self {
362        Self {
363            writer,
364            started: false,
365            finished: false,
366            format: F::default(),
367            options: EncoderOptions::default(),
368        }
369    }
370
371    /// Serialize `batch` to JSON output
372    pub fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
373        if batch.num_rows() == 0 {
374            return Ok(());
375        }
376
377        // BufWriter uses a buffer size of 8KB
378        // We therefore double this and flush once we have more than 8KB
379        let mut buffer = Vec::with_capacity(16 * 1024);
380
381        let mut is_first_row = !self.started;
382        if !self.started {
383            self.format.start_stream(&mut buffer)?;
384            self.started = true;
385        }
386
387        let array = StructArray::from(batch.clone());
388        let field = Arc::new(Field::new_struct(
389            "",
390            batch.schema().fields().clone(),
391            false,
392        ));
393
394        let mut encoder = make_encoder(&field, &array, &self.options)?;
395
396        // Validate that the root is not nullable
397        assert!(!encoder.has_nulls(), "root cannot be nullable");
398        for idx in 0..batch.num_rows() {
399            self.format.start_row(&mut buffer, is_first_row)?;
400            is_first_row = false;
401
402            encoder.encode(idx, &mut buffer);
403            if buffer.len() > 8 * 1024 {
404                self.writer.write_all(&buffer)?;
405                buffer.clear();
406            }
407            self.format.end_row(&mut buffer)?;
408        }
409
410        if !buffer.is_empty() {
411            self.writer.write_all(&buffer)?;
412        }
413
414        Ok(())
415    }
416
417    /// Serialize `batches` to JSON output
418    pub fn write_batches(&mut self, batches: &[&RecordBatch]) -> Result<(), ArrowError> {
419        for b in batches {
420            self.write(b)?;
421        }
422        Ok(())
423    }
424
425    /// Finishes the output stream. This function must be called after
426    /// all record batches have been produced. (e.g. producing the final `']'` if writing
427    /// arrays.
428    pub fn finish(&mut self) -> Result<(), ArrowError> {
429        if !self.started {
430            self.format.start_stream(&mut self.writer)?;
431            self.started = true;
432        }
433        if !self.finished {
434            self.format.end_stream(&mut self.writer)?;
435            self.finished = true;
436        }
437
438        Ok(())
439    }
440
441    /// Gets a reference to the underlying writer.
442    pub fn get_ref(&self) -> &W {
443        &self.writer
444    }
445
446    /// Gets a mutable reference to the underlying writer.
447    ///
448    /// Writing to the underlying writer must be done with care
449    /// to avoid corrupting the output JSON.
450    pub fn get_mut(&mut self) -> &mut W {
451        &mut self.writer
452    }
453
454    /// Unwraps this `Writer<W>`, returning the underlying writer
455    pub fn into_inner(self) -> W {
456        self.writer
457    }
458}
459
460impl<W, F> RecordBatchWriter for Writer<W, F>
461where
462    W: Write,
463    F: JsonFormat,
464{
465    fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
466        self.write(batch)
467    }
468
469    fn close(mut self) -> Result<(), ArrowError> {
470        self.finish()
471    }
472}
473
474#[cfg(test)]
475mod tests {
476    use core::str;
477    use std::collections::HashMap;
478    use std::fs::{File, read_to_string};
479    use std::io::{BufReader, Seek};
480    use std::sync::Arc;
481
482    use arrow_array::cast::AsArray;
483    use serde_json::{Value, json};
484
485    use super::LineDelimited;
486    use super::{Encoder, WriterBuilder};
487    use arrow_array::builder::*;
488    use arrow_array::types::*;
489    use arrow_buffer::{Buffer, NullBuffer, OffsetBuffer, ScalarBuffer, i256};
490
491    use crate::reader::*;
492
493    use super::*;
494
495    /// Asserts that the NDJSON `input` is semantically identical to `expected`
496    fn assert_json_eq(input: &[u8], expected: &str) {
497        let expected: Vec<Option<Value>> = expected
498            .split('\n')
499            .map(|s| (!s.is_empty()).then(|| serde_json::from_str(s).unwrap()))
500            .collect();
501
502        let actual: Vec<Option<Value>> = input
503            .split(|b| *b == b'\n')
504            .map(|s| (!s.is_empty()).then(|| serde_json::from_slice(s).unwrap()))
505            .collect();
506
507        assert_eq!(actual, expected);
508    }
509
510    #[test]
511    fn write_simple_rows() {
512        let schema = Schema::new(vec![
513            Field::new("c1", DataType::Int32, true),
514            Field::new("c2", DataType::Utf8, true),
515        ]);
516
517        let a = Int32Array::from(vec![Some(1), Some(2), Some(3), None, Some(5)]);
518        let b = StringArray::from(vec![Some("a"), Some("b"), Some("c"), Some("d"), None]);
519
520        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]).unwrap();
521
522        let mut buf = Vec::new();
523        {
524            let mut writer = LineDelimitedWriter::new(&mut buf);
525            writer.write_batches(&[&batch]).unwrap();
526        }
527
528        assert_json_eq(
529            &buf,
530            r#"{"c1":1,"c2":"a"}
531{"c1":2,"c2":"b"}
532{"c1":3,"c2":"c"}
533{"c2":"d"}
534{"c1":5}
535"#,
536        );
537    }
538
539    #[test]
540    fn write_large_utf8_and_utf8_view() {
541        let schema = Schema::new(vec![
542            Field::new("c1", DataType::Utf8, true),
543            Field::new("c2", DataType::LargeUtf8, true),
544            Field::new("c3", DataType::Utf8View, true),
545        ]);
546
547        let a = StringArray::from(vec![Some("a"), None, Some("c"), Some("d"), None]);
548        let b = LargeStringArray::from(vec![Some("a"), Some("b"), None, Some("d"), None]);
549        let c = StringViewArray::from(vec![Some("a"), Some("b"), None, Some("d"), None]);
550
551        let batch = RecordBatch::try_new(
552            Arc::new(schema),
553            vec![Arc::new(a), Arc::new(b), Arc::new(c)],
554        )
555        .unwrap();
556
557        let mut buf = Vec::new();
558        {
559            let mut writer = LineDelimitedWriter::new(&mut buf);
560            writer.write_batches(&[&batch]).unwrap();
561        }
562
563        assert_json_eq(
564            &buf,
565            r#"{"c1":"a","c2":"a","c3":"a"}
566{"c2":"b","c3":"b"}
567{"c1":"c"}
568{"c1":"d","c2":"d","c3":"d"}
569{}
570"#,
571        );
572    }
573
574    #[test]
575    fn write_dictionary() {
576        let schema = Schema::new(vec![
577            Field::new_dictionary("c1", DataType::Int32, DataType::Utf8, true),
578            Field::new_dictionary("c2", DataType::Int8, DataType::Utf8, true),
579        ]);
580
581        let a: DictionaryArray<Int32Type> = vec![
582            Some("cupcakes"),
583            Some("foo"),
584            Some("foo"),
585            None,
586            Some("cupcakes"),
587        ]
588        .into_iter()
589        .collect();
590        let b: DictionaryArray<Int8Type> =
591            vec![Some("sdsd"), Some("sdsd"), None, Some("sd"), Some("sdsd")]
592                .into_iter()
593                .collect();
594
595        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]).unwrap();
596
597        let mut buf = Vec::new();
598        {
599            let mut writer = LineDelimitedWriter::new(&mut buf);
600            writer.write_batches(&[&batch]).unwrap();
601        }
602
603        assert_json_eq(
604            &buf,
605            r#"{"c1":"cupcakes","c2":"sdsd"}
606{"c1":"foo","c2":"sdsd"}
607{"c1":"foo"}
608{"c2":"sd"}
609{"c1":"cupcakes","c2":"sdsd"}
610"#,
611        );
612    }
613
614    #[test]
615    fn write_list_of_dictionary() {
616        let dict_field = Arc::new(Field::new_dictionary(
617            "item",
618            DataType::Int32,
619            DataType::Utf8,
620            true,
621        ));
622        let schema = Schema::new(vec![Field::new_large_list("l", dict_field.clone(), true)]);
623
624        let dict_array: DictionaryArray<Int32Type> =
625            vec![Some("a"), Some("b"), Some("c"), Some("a"), None, Some("c")]
626                .into_iter()
627                .collect();
628        let list_array = LargeListArray::try_new(
629            dict_field,
630            OffsetBuffer::from_lengths([3_usize, 2, 0, 1]),
631            Arc::new(dict_array),
632            Some(NullBuffer::from_iter([true, true, false, true])),
633        )
634        .unwrap();
635
636        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(list_array)]).unwrap();
637
638        let mut buf = Vec::new();
639        {
640            let mut writer = LineDelimitedWriter::new(&mut buf);
641            writer.write_batches(&[&batch]).unwrap();
642        }
643
644        assert_json_eq(
645            &buf,
646            r#"{"l":["a","b","c"]}
647{"l":["a",null]}
648{}
649{"l":["c"]}
650"#,
651        );
652    }
653
654    #[test]
655    fn write_list_of_dictionary_large_values() {
656        let dict_field = Arc::new(Field::new_dictionary(
657            "item",
658            DataType::Int32,
659            DataType::LargeUtf8,
660            true,
661        ));
662        let schema = Schema::new(vec![Field::new_large_list("l", dict_field.clone(), true)]);
663
664        let keys = PrimitiveArray::<Int32Type>::from(vec![
665            Some(0),
666            Some(1),
667            Some(2),
668            Some(0),
669            None,
670            Some(2),
671        ]);
672        let values = LargeStringArray::from(vec!["a", "b", "c"]);
673        let dict_array = DictionaryArray::try_new(keys, Arc::new(values)).unwrap();
674
675        let list_array = LargeListArray::try_new(
676            dict_field,
677            OffsetBuffer::from_lengths([3_usize, 2, 0, 1]),
678            Arc::new(dict_array),
679            Some(NullBuffer::from_iter([true, true, false, true])),
680        )
681        .unwrap();
682
683        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(list_array)]).unwrap();
684
685        let mut buf = Vec::new();
686        {
687            let mut writer = LineDelimitedWriter::new(&mut buf);
688            writer.write_batches(&[&batch]).unwrap();
689        }
690
691        assert_json_eq(
692            &buf,
693            r#"{"l":["a","b","c"]}
694{"l":["a",null]}
695{}
696{"l":["c"]}
697"#,
698        );
699    }
700
701    #[test]
702    fn write_timestamps() {
703        let ts_string = "2018-11-13T17:11:10.011375885995";
704        let ts_nanos = ts_string
705            .parse::<chrono::NaiveDateTime>()
706            .unwrap()
707            .and_utc()
708            .timestamp_nanos_opt()
709            .unwrap();
710        let ts_micros = ts_nanos / 1000;
711        let ts_millis = ts_micros / 1000;
712        let ts_secs = ts_millis / 1000;
713
714        let arr_nanos = TimestampNanosecondArray::from(vec![Some(ts_nanos), None]);
715        let arr_micros = TimestampMicrosecondArray::from(vec![Some(ts_micros), None]);
716        let arr_millis = TimestampMillisecondArray::from(vec![Some(ts_millis), None]);
717        let arr_secs = TimestampSecondArray::from(vec![Some(ts_secs), None]);
718        let arr_names = StringArray::from(vec![Some("a"), Some("b")]);
719
720        let schema = Schema::new(vec![
721            Field::new("nanos", arr_nanos.data_type().clone(), true),
722            Field::new("micros", arr_micros.data_type().clone(), true),
723            Field::new("millis", arr_millis.data_type().clone(), true),
724            Field::new("secs", arr_secs.data_type().clone(), true),
725            Field::new("name", arr_names.data_type().clone(), true),
726        ]);
727        let schema = Arc::new(schema);
728
729        let batch = RecordBatch::try_new(
730            schema,
731            vec![
732                Arc::new(arr_nanos),
733                Arc::new(arr_micros),
734                Arc::new(arr_millis),
735                Arc::new(arr_secs),
736                Arc::new(arr_names),
737            ],
738        )
739        .unwrap();
740
741        let mut buf = Vec::new();
742        {
743            let mut writer = LineDelimitedWriter::new(&mut buf);
744            writer.write_batches(&[&batch]).unwrap();
745        }
746
747        assert_json_eq(
748            &buf,
749            r#"{"micros":"2018-11-13T17:11:10.011375","millis":"2018-11-13T17:11:10.011","name":"a","nanos":"2018-11-13T17:11:10.011375885","secs":"2018-11-13T17:11:10"}
750{"name":"b"}
751"#,
752        );
753
754        let mut buf = Vec::new();
755        {
756            let mut writer = WriterBuilder::new()
757                .with_timestamp_format("%m-%d-%Y".to_string())
758                .build::<_, LineDelimited>(&mut buf);
759            writer.write_batches(&[&batch]).unwrap();
760        }
761
762        assert_json_eq(
763            &buf,
764            r#"{"nanos":"11-13-2018","micros":"11-13-2018","millis":"11-13-2018","secs":"11-13-2018","name":"a"}
765{"name":"b"}
766"#,
767        );
768    }
769
770    #[test]
771    fn write_timestamps_with_tz() {
772        let ts_string = "2018-11-13T17:11:10.011375885995";
773        let ts_nanos = ts_string
774            .parse::<chrono::NaiveDateTime>()
775            .unwrap()
776            .and_utc()
777            .timestamp_nanos_opt()
778            .unwrap();
779        let ts_micros = ts_nanos / 1000;
780        let ts_millis = ts_micros / 1000;
781        let ts_secs = ts_millis / 1000;
782
783        let arr_nanos = TimestampNanosecondArray::from(vec![Some(ts_nanos), None]);
784        let arr_micros = TimestampMicrosecondArray::from(vec![Some(ts_micros), None]);
785        let arr_millis = TimestampMillisecondArray::from(vec![Some(ts_millis), None]);
786        let arr_secs = TimestampSecondArray::from(vec![Some(ts_secs), None]);
787        let arr_names = StringArray::from(vec![Some("a"), Some("b")]);
788
789        let tz = "+00:00";
790
791        let arr_nanos = arr_nanos.with_timezone(tz);
792        let arr_micros = arr_micros.with_timezone(tz);
793        let arr_millis = arr_millis.with_timezone(tz);
794        let arr_secs = arr_secs.with_timezone(tz);
795
796        let schema = Schema::new(vec![
797            Field::new("nanos", arr_nanos.data_type().clone(), true),
798            Field::new("micros", arr_micros.data_type().clone(), true),
799            Field::new("millis", arr_millis.data_type().clone(), true),
800            Field::new("secs", arr_secs.data_type().clone(), true),
801            Field::new("name", arr_names.data_type().clone(), true),
802        ]);
803        let schema = Arc::new(schema);
804
805        let batch = RecordBatch::try_new(
806            schema,
807            vec![
808                Arc::new(arr_nanos),
809                Arc::new(arr_micros),
810                Arc::new(arr_millis),
811                Arc::new(arr_secs),
812                Arc::new(arr_names),
813            ],
814        )
815        .unwrap();
816
817        let mut buf = Vec::new();
818        {
819            let mut writer = LineDelimitedWriter::new(&mut buf);
820            writer.write_batches(&[&batch]).unwrap();
821        }
822
823        assert_json_eq(
824            &buf,
825            r#"{"micros":"2018-11-13T17:11:10.011375Z","millis":"2018-11-13T17:11:10.011Z","name":"a","nanos":"2018-11-13T17:11:10.011375885Z","secs":"2018-11-13T17:11:10Z"}
826{"name":"b"}
827"#,
828        );
829
830        let mut buf = Vec::new();
831        {
832            let mut writer = WriterBuilder::new()
833                .with_timestamp_tz_format("%m-%d-%Y %Z".to_string())
834                .build::<_, LineDelimited>(&mut buf);
835            writer.write_batches(&[&batch]).unwrap();
836        }
837
838        assert_json_eq(
839            &buf,
840            r#"{"nanos":"11-13-2018 +00:00","micros":"11-13-2018 +00:00","millis":"11-13-2018 +00:00","secs":"11-13-2018 +00:00","name":"a"}
841{"name":"b"}
842"#,
843        );
844    }
845
846    #[test]
847    fn write_dates() {
848        let ts_string = "2018-11-13T17:11:10.011375885995";
849        let ts_millis = ts_string
850            .parse::<chrono::NaiveDateTime>()
851            .unwrap()
852            .and_utc()
853            .timestamp_millis();
854
855        let arr_date32 = Date32Array::from(vec![
856            Some(i32::try_from(ts_millis / 1000 / (60 * 60 * 24)).unwrap()),
857            None,
858        ]);
859        let arr_date64 = Date64Array::from(vec![Some(ts_millis), None]);
860        let arr_names = StringArray::from(vec![Some("a"), Some("b")]);
861
862        let schema = Schema::new(vec![
863            Field::new("date32", arr_date32.data_type().clone(), true),
864            Field::new("date64", arr_date64.data_type().clone(), true),
865            Field::new("name", arr_names.data_type().clone(), false),
866        ]);
867        let schema = Arc::new(schema);
868
869        let batch = RecordBatch::try_new(
870            schema,
871            vec![
872                Arc::new(arr_date32),
873                Arc::new(arr_date64),
874                Arc::new(arr_names),
875            ],
876        )
877        .unwrap();
878
879        let mut buf = Vec::new();
880        {
881            let mut writer = LineDelimitedWriter::new(&mut buf);
882            writer.write_batches(&[&batch]).unwrap();
883        }
884
885        assert_json_eq(
886            &buf,
887            r#"{"date32":"2018-11-13","date64":"2018-11-13T17:11:10.011","name":"a"}
888{"name":"b"}
889"#,
890        );
891
892        let mut buf = Vec::new();
893        {
894            let mut writer = WriterBuilder::new()
895                .with_date_format("%m-%d-%Y".to_string())
896                .with_datetime_format("%m-%d-%Y %Mmin %Ssec %Hhour".to_string())
897                .build::<_, LineDelimited>(&mut buf);
898            writer.write_batches(&[&batch]).unwrap();
899        }
900
901        assert_json_eq(
902            &buf,
903            r#"{"date32":"11-13-2018","date64":"11-13-2018 11min 10sec 17hour","name":"a"}
904{"name":"b"}
905"#,
906        );
907    }
908
909    #[test]
910    fn write_times() {
911        let arr_time32sec = Time32SecondArray::from(vec![Some(120), None]);
912        let arr_time32msec = Time32MillisecondArray::from(vec![Some(120), None]);
913        let arr_time64usec = Time64MicrosecondArray::from(vec![Some(120), None]);
914        let arr_time64nsec = Time64NanosecondArray::from(vec![Some(120), None]);
915        let arr_names = StringArray::from(vec![Some("a"), Some("b")]);
916
917        let schema = Schema::new(vec![
918            Field::new("time32sec", arr_time32sec.data_type().clone(), true),
919            Field::new("time32msec", arr_time32msec.data_type().clone(), true),
920            Field::new("time64usec", arr_time64usec.data_type().clone(), true),
921            Field::new("time64nsec", arr_time64nsec.data_type().clone(), true),
922            Field::new("name", arr_names.data_type().clone(), true),
923        ]);
924        let schema = Arc::new(schema);
925
926        let batch = RecordBatch::try_new(
927            schema,
928            vec![
929                Arc::new(arr_time32sec),
930                Arc::new(arr_time32msec),
931                Arc::new(arr_time64usec),
932                Arc::new(arr_time64nsec),
933                Arc::new(arr_names),
934            ],
935        )
936        .unwrap();
937
938        let mut buf = Vec::new();
939        {
940            let mut writer = LineDelimitedWriter::new(&mut buf);
941            writer.write_batches(&[&batch]).unwrap();
942        }
943
944        assert_json_eq(
945            &buf,
946            r#"{"time32sec":"00:02:00","time32msec":"00:00:00.120","time64usec":"00:00:00.000120","time64nsec":"00:00:00.000000120","name":"a"}
947{"name":"b"}
948"#,
949        );
950
951        let mut buf = Vec::new();
952        {
953            let mut writer = WriterBuilder::new()
954                .with_time_format("%H-%M-%S %f".to_string())
955                .build::<_, LineDelimited>(&mut buf);
956            writer.write_batches(&[&batch]).unwrap();
957        }
958
959        assert_json_eq(
960            &buf,
961            r#"{"time32sec":"00-02-00 000000000","time32msec":"00-00-00 120000000","time64usec":"00-00-00 000120000","time64nsec":"00-00-00 000000120","name":"a"}
962{"name":"b"}
963"#,
964        );
965    }
966
967    #[test]
968    fn write_durations() {
969        let arr_durationsec = DurationSecondArray::from(vec![Some(120), None]);
970        let arr_durationmsec = DurationMillisecondArray::from(vec![Some(120), None]);
971        let arr_durationusec = DurationMicrosecondArray::from(vec![Some(120), None]);
972        let arr_durationnsec = DurationNanosecondArray::from(vec![Some(120), None]);
973        let arr_names = StringArray::from(vec![Some("a"), Some("b")]);
974
975        let schema = Schema::new(vec![
976            Field::new("duration_sec", arr_durationsec.data_type().clone(), true),
977            Field::new("duration_msec", arr_durationmsec.data_type().clone(), true),
978            Field::new("duration_usec", arr_durationusec.data_type().clone(), true),
979            Field::new("duration_nsec", arr_durationnsec.data_type().clone(), true),
980            Field::new("name", arr_names.data_type().clone(), true),
981        ]);
982        let schema = Arc::new(schema);
983
984        let batch = RecordBatch::try_new(
985            schema,
986            vec![
987                Arc::new(arr_durationsec),
988                Arc::new(arr_durationmsec),
989                Arc::new(arr_durationusec),
990                Arc::new(arr_durationnsec),
991                Arc::new(arr_names),
992            ],
993        )
994        .unwrap();
995
996        let mut buf = Vec::new();
997        {
998            let mut writer = LineDelimitedWriter::new(&mut buf);
999            writer.write_batches(&[&batch]).unwrap();
1000        }
1001
1002        assert_json_eq(
1003            &buf,
1004            r#"{"duration_sec":"PT120S","duration_msec":"PT0.12S","duration_usec":"PT0.00012S","duration_nsec":"PT0.00000012S","name":"a"}
1005{"name":"b"}
1006"#,
1007        );
1008    }
1009
1010    #[test]
1011    fn write_nested_structs() {
1012        let schema = Schema::new(vec![
1013            Field::new(
1014                "c1",
1015                DataType::Struct(Fields::from(vec![
1016                    Field::new("c11", DataType::Int32, true),
1017                    Field::new(
1018                        "c12",
1019                        DataType::Struct(vec![Field::new("c121", DataType::Utf8, false)].into()),
1020                        false,
1021                    ),
1022                ])),
1023                false,
1024            ),
1025            Field::new("c2", DataType::Utf8, false),
1026        ]);
1027
1028        let c1 = StructArray::from(vec![
1029            (
1030                Arc::new(Field::new("c11", DataType::Int32, true)),
1031                Arc::new(Int32Array::from(vec![Some(1), None, Some(5)])) as ArrayRef,
1032            ),
1033            (
1034                Arc::new(Field::new(
1035                    "c12",
1036                    DataType::Struct(vec![Field::new("c121", DataType::Utf8, false)].into()),
1037                    false,
1038                )),
1039                Arc::new(StructArray::from(vec![(
1040                    Arc::new(Field::new("c121", DataType::Utf8, false)),
1041                    Arc::new(StringArray::from(vec![Some("e"), Some("f"), Some("g")])) as ArrayRef,
1042                )])) as ArrayRef,
1043            ),
1044        ]);
1045        let c2 = StringArray::from(vec![Some("a"), Some("b"), Some("c")]);
1046
1047        let batch =
1048            RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c1), Arc::new(c2)]).unwrap();
1049
1050        let mut buf = Vec::new();
1051        {
1052            let mut writer = LineDelimitedWriter::new(&mut buf);
1053            writer.write_batches(&[&batch]).unwrap();
1054        }
1055
1056        assert_json_eq(
1057            &buf,
1058            r#"{"c1":{"c11":1,"c12":{"c121":"e"}},"c2":"a"}
1059{"c1":{"c12":{"c121":"f"}},"c2":"b"}
1060{"c1":{"c11":5,"c12":{"c121":"g"}},"c2":"c"}
1061"#,
1062        );
1063    }
1064
1065    #[test]
1066    fn write_struct_with_list_field() {
1067        let field_c_list = Arc::new(Field::new("c_list", DataType::Utf8, false));
1068        let field_c1 = Field::new("c1", DataType::List(field_c_list.clone()), false);
1069        let field_c2 = Field::new("c2", DataType::Int32, false);
1070        let schema = Schema::new(vec![field_c1.clone(), field_c2]);
1071
1072        let a_values = StringArray::from(vec!["a", "a1", "b", "c", "d", "e"]);
1073        // list column rows: ["a", "a1"], ["b"], ["c"], ["d"], ["e"]
1074        let a = ListArray::new(
1075            field_c_list,
1076            OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 2, 3, 4, 5, 6])),
1077            Arc::new(a_values),
1078            None,
1079        );
1080
1081        let b = Int32Array::from(vec![1, 2, 3, 4, 5]);
1082
1083        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]).unwrap();
1084
1085        let mut buf = Vec::new();
1086        {
1087            let mut writer = LineDelimitedWriter::new(&mut buf);
1088            writer.write_batches(&[&batch]).unwrap();
1089        }
1090
1091        assert_json_eq(
1092            &buf,
1093            r#"{"c1":["a","a1"],"c2":1}
1094{"c1":["b"],"c2":2}
1095{"c1":["c"],"c2":3}
1096{"c1":["d"],"c2":4}
1097{"c1":["e"],"c2":5}
1098"#,
1099        );
1100    }
1101
1102    #[test]
1103    fn write_nested_list() {
1104        let field_b = Arc::new(Field::new("b", DataType::Int32, false));
1105        let field_a = Arc::new(Field::new("a", DataType::List(field_b.clone()), false));
1106        let field_c1 = Field::new("c1", DataType::List(field_a.clone()), false);
1107        let field_c2 = Field::new("c2", DataType::Utf8, true);
1108        let schema = Schema::new(vec![field_c1.clone(), field_c2]);
1109
1110        // list column rows: [[1, 2], [3]], [], [[4, 5, 6]]
1111        let a_values = Int32Array::from(vec![1, 2, 3, 4, 5, 6]);
1112
1113        let a_list = ListArray::new(
1114            field_b,
1115            OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 2, 3, 6])),
1116            Arc::new(a_values),
1117            None,
1118        );
1119
1120        let c1 = ListArray::new(
1121            field_a,
1122            OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 2, 2, 3])),
1123            Arc::new(a_list),
1124            None,
1125        );
1126        let c2 = StringArray::from(vec![Some("foo"), Some("bar"), None]);
1127
1128        let batch =
1129            RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c1), Arc::new(c2)]).unwrap();
1130
1131        let mut buf = Vec::new();
1132        {
1133            let mut writer = LineDelimitedWriter::new(&mut buf);
1134            writer.write_batches(&[&batch]).unwrap();
1135        }
1136
1137        assert_json_eq(
1138            &buf,
1139            r#"{"c1":[[1,2],[3]],"c2":"foo"}
1140{"c1":[],"c2":"bar"}
1141{"c1":[[4,5,6]]}
1142"#,
1143        );
1144    }
1145
1146    #[test]
1147    fn write_list_of_struct() {
1148        let field_c1 = Field::new(
1149            "c1",
1150            DataType::List(Arc::new(Field::new(
1151                "s",
1152                DataType::Struct(Fields::from(vec![
1153                    Field::new("c11", DataType::Int32, true),
1154                    Field::new(
1155                        "c12",
1156                        DataType::Struct(vec![Field::new("c121", DataType::Utf8, false)].into()),
1157                        false,
1158                    ),
1159                ])),
1160                false,
1161            ))),
1162            true,
1163        );
1164        let field_c2 = Field::new("c2", DataType::Int32, false);
1165        let schema = Schema::new(vec![field_c1.clone(), field_c2]);
1166
1167        let struct_values = StructArray::from(vec![
1168            (
1169                Arc::new(Field::new("c11", DataType::Int32, true)),
1170                Arc::new(Int32Array::from(vec![Some(1), None, Some(5)])) as ArrayRef,
1171            ),
1172            (
1173                Arc::new(Field::new(
1174                    "c12",
1175                    DataType::Struct(vec![Field::new("c121", DataType::Utf8, false)].into()),
1176                    false,
1177                )),
1178                Arc::new(StructArray::from(vec![(
1179                    Arc::new(Field::new("c121", DataType::Utf8, false)),
1180                    Arc::new(StringArray::from(vec![Some("e"), Some("f"), Some("g")])) as ArrayRef,
1181                )])) as ArrayRef,
1182            ),
1183        ]);
1184
1185        // list column rows (c1):
1186        // [{"c11": 1, "c12": {"c121": "e"}}, {"c12": {"c121": "f"}}],
1187        // null,
1188        // [{"c11": 5, "c12": {"c121": "g"}}]
1189        let c1_inner = match field_c1.data_type() {
1190            DataType::List(f) => f.clone(),
1191            _ => unreachable!(),
1192        };
1193        let c1 = ListArray::new(
1194            c1_inner,
1195            OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 2, 2, 3])),
1196            Arc::new(struct_values),
1197            Some(NullBuffer::from(vec![true, false, true])),
1198        );
1199
1200        let c2 = Int32Array::from(vec![1, 2, 3]);
1201
1202        let batch =
1203            RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c1), Arc::new(c2)]).unwrap();
1204
1205        let mut buf = Vec::new();
1206        {
1207            let mut writer = LineDelimitedWriter::new(&mut buf);
1208            writer.write_batches(&[&batch]).unwrap();
1209        }
1210
1211        assert_json_eq(
1212            &buf,
1213            r#"{"c1":[{"c11":1,"c12":{"c121":"e"}},{"c12":{"c121":"f"}}],"c2":1}
1214{"c2":2}
1215{"c1":[{"c11":5,"c12":{"c121":"g"}}],"c2":3}
1216"#,
1217        );
1218    }
1219
1220    fn assert_write_list_view<O: OffsetSizeTrait>() {
1221        let field = Arc::new(Field::new("item", DataType::Int32, true));
1222        let data_type = GenericListViewArray::<O>::DATA_TYPE_CONSTRUCTOR(field.clone());
1223        let schema = Schema::new(vec![Field::new("lv", data_type, true)]);
1224
1225        // rows: [1, 2, 3], [4, null], null, [6]
1226        let values = Int32Array::from(vec![Some(1), Some(2), Some(3), Some(4), None, Some(6)]);
1227        let offsets = [0, 3, 0, 5]
1228            .iter()
1229            .map(|&v| O::from_usize(v).unwrap())
1230            .collect::<Vec<_>>();
1231        let sizes = [3, 2, 0, 1]
1232            .iter()
1233            .map(|&v| O::from_usize(v).unwrap())
1234            .collect::<Vec<_>>();
1235        let list_view = GenericListViewArray::<O>::try_new(
1236            field,
1237            ScalarBuffer::from(offsets),
1238            ScalarBuffer::from(sizes),
1239            Arc::new(values),
1240            Some(NullBuffer::from_iter([true, true, false, true])),
1241        )
1242        .unwrap();
1243
1244        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(list_view)]).unwrap();
1245
1246        let mut buf = Vec::new();
1247        {
1248            let mut writer = LineDelimitedWriter::new(&mut buf);
1249            writer.write_batches(&[&batch]).unwrap();
1250        }
1251
1252        assert_json_eq(
1253            &buf,
1254            r#"{"lv":[1,2,3]}
1255{"lv":[4,null]}
1256{}
1257{"lv":[6]}
1258"#,
1259        );
1260    }
1261
1262    #[test]
1263    fn write_list_view() {
1264        assert_write_list_view::<i32>();
1265        assert_write_list_view::<i64>();
1266    }
1267
1268    fn test_write_for_file(test_file: &str, remove_nulls: bool) {
1269        let file = File::open(test_file).unwrap();
1270        let mut reader = BufReader::new(file);
1271        let (schema, _) = infer_json_schema(&mut reader, None).unwrap();
1272        reader.rewind().unwrap();
1273
1274        let builder = ReaderBuilder::new(Arc::new(schema)).with_batch_size(1024);
1275        let mut reader = builder.build(reader).unwrap();
1276        let batch = reader.next().unwrap().unwrap();
1277
1278        let mut buf = Vec::new();
1279        {
1280            if remove_nulls {
1281                let mut writer = LineDelimitedWriter::new(&mut buf);
1282                writer.write_batches(&[&batch]).unwrap();
1283            } else {
1284                let mut writer = WriterBuilder::new()
1285                    .with_explicit_nulls(true)
1286                    .build::<_, LineDelimited>(&mut buf);
1287                writer.write_batches(&[&batch]).unwrap();
1288            }
1289        }
1290
1291        let result = str::from_utf8(&buf).unwrap();
1292        let expected = read_to_string(test_file).unwrap();
1293        for (r, e) in result.lines().zip(expected.lines()) {
1294            let mut expected_json = serde_json::from_str::<Value>(e).unwrap();
1295            if remove_nulls {
1296                // remove null value from object to make comparison consistent:
1297                if let Value::Object(obj) = expected_json {
1298                    expected_json =
1299                        Value::Object(obj.into_iter().filter(|(_, v)| *v != Value::Null).collect());
1300                }
1301            }
1302            assert_eq!(serde_json::from_str::<Value>(r).unwrap(), expected_json,);
1303        }
1304    }
1305
1306    #[test]
1307    fn write_basic_rows() {
1308        test_write_for_file("test/data/basic.json", true);
1309    }
1310
1311    #[test]
1312    fn write_arrays() {
1313        test_write_for_file("test/data/arrays.json", true);
1314    }
1315
1316    #[test]
1317    fn write_basic_nulls() {
1318        test_write_for_file("test/data/basic_nulls.json", true);
1319    }
1320
1321    #[test]
1322    fn write_nested_with_nulls() {
1323        test_write_for_file("test/data/nested_with_nulls.json", false);
1324    }
1325
1326    #[test]
1327    fn json_line_writer_empty() {
1328        let mut writer = LineDelimitedWriter::new(vec![] as Vec<u8>);
1329        writer.finish().unwrap();
1330        assert_eq!(str::from_utf8(&writer.into_inner()).unwrap(), "");
1331    }
1332
1333    #[test]
1334    fn json_array_writer_empty() {
1335        let mut writer = ArrayWriter::new(vec![] as Vec<u8>);
1336        writer.finish().unwrap();
1337        assert_eq!(str::from_utf8(&writer.into_inner()).unwrap(), "[]");
1338    }
1339
1340    #[test]
1341    fn json_line_writer_empty_batch() {
1342        let mut writer = LineDelimitedWriter::new(vec![] as Vec<u8>);
1343
1344        let array = Int32Array::from(Vec::<i32>::new());
1345        let schema = Schema::new(vec![Field::new("c", DataType::Int32, true)]);
1346        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap();
1347
1348        writer.write(&batch).unwrap();
1349        writer.finish().unwrap();
1350        assert_eq!(str::from_utf8(&writer.into_inner()).unwrap(), "");
1351    }
1352
1353    #[test]
1354    fn json_array_writer_empty_batch() {
1355        let mut writer = ArrayWriter::new(vec![] as Vec<u8>);
1356
1357        let array = Int32Array::from(Vec::<i32>::new());
1358        let schema = Schema::new(vec![Field::new("c", DataType::Int32, true)]);
1359        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap();
1360
1361        writer.write(&batch).unwrap();
1362        writer.finish().unwrap();
1363        assert_eq!(str::from_utf8(&writer.into_inner()).unwrap(), "[]");
1364    }
1365
1366    #[test]
1367    fn json_struct_array_nulls() {
1368        let inner = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
1369            Some(vec![Some(1), Some(2)]),
1370            Some(vec![None]),
1371            Some(vec![]),
1372            Some(vec![Some(3), None]), // masked for a
1373            Some(vec![Some(4), Some(5)]),
1374            None, // masked for a
1375            None,
1376        ]);
1377
1378        let field = Arc::new(Field::new("list", inner.data_type().clone(), true));
1379        let array = Arc::new(inner) as ArrayRef;
1380        let struct_array_a = StructArray::from((
1381            vec![(field.clone(), array.clone())],
1382            Buffer::from([0b01010111]),
1383        ));
1384        let struct_array_b = StructArray::from(vec![(field, array)]);
1385
1386        let schema = Schema::new(vec![
1387            Field::new_struct("a", struct_array_a.fields().clone(), true),
1388            Field::new_struct("b", struct_array_b.fields().clone(), true),
1389        ]);
1390
1391        let batch = RecordBatch::try_new(
1392            Arc::new(schema),
1393            vec![Arc::new(struct_array_a), Arc::new(struct_array_b)],
1394        )
1395        .unwrap();
1396
1397        let mut buf = Vec::new();
1398        {
1399            let mut writer = LineDelimitedWriter::new(&mut buf);
1400            writer.write_batches(&[&batch]).unwrap();
1401        }
1402
1403        assert_json_eq(
1404            &buf,
1405            r#"{"a":{"list":[1,2]},"b":{"list":[1,2]}}
1406{"a":{"list":[null]},"b":{"list":[null]}}
1407{"a":{"list":[]},"b":{"list":[]}}
1408{"b":{"list":[3,null]}}
1409{"a":{"list":[4,5]},"b":{"list":[4,5]}}
1410{"b":{}}
1411{"a":{},"b":{}}
1412"#,
1413        );
1414    }
1415
1416    fn run_json_writer_map_with_keys(keys_array: ArrayRef) {
1417        let values_array = super::Int64Array::from(vec![10, 20, 30, 40, 50]);
1418
1419        let keys_field = Arc::new(Field::new(
1420            Field::MAP_KEY_FIELD_DEFAULT_NAME,
1421            keys_array.data_type().clone(),
1422            false,
1423        ));
1424        let values_field = Arc::new(Field::new(
1425            Field::MAP_VALUE_FIELD_DEFAULT_NAME,
1426            DataType::Int64,
1427            false,
1428        ));
1429        let entry_struct = StructArray::from(vec![
1430            (keys_field, keys_array.clone()),
1431            (values_field, Arc::new(values_array) as ArrayRef),
1432        ]);
1433
1434        let entries_field = Arc::new(Field::new(
1435            Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
1436            entry_struct.data_type().clone(),
1437            false,
1438        ));
1439
1440        // [{"foo": 10}, null, {}, {"bar": 20, "baz": 30, "qux": 40}, {"quux": 50}, {}]
1441        let map = MapArray::new(
1442            entries_field.clone(),
1443            OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 1, 1, 1, 4, 5, 5])),
1444            entry_struct,
1445            Some(NullBuffer::from(vec![true, false, true, true, true, true])),
1446            false,
1447        );
1448
1449        let map_field = Field::new("map", DataType::Map(entries_field, false), true);
1450        let schema = Arc::new(Schema::new(vec![map_field]));
1451
1452        let batch = RecordBatch::try_new(schema, vec![Arc::new(map)]).unwrap();
1453
1454        let mut buf = Vec::new();
1455        {
1456            let mut writer = LineDelimitedWriter::new(&mut buf);
1457            writer.write_batches(&[&batch]).unwrap();
1458        }
1459
1460        assert_json_eq(
1461            &buf,
1462            r#"{"map":{"foo":10}}
1463{}
1464{"map":{}}
1465{"map":{"bar":20,"baz":30,"qux":40}}
1466{"map":{"quux":50}}
1467{"map":{}}
1468"#,
1469        );
1470    }
1471
1472    #[test]
1473    fn json_writer_map() {
1474        // Utf8 (StringArray)
1475        let keys_utf8 = super::StringArray::from(vec!["foo", "bar", "baz", "qux", "quux"]);
1476        run_json_writer_map_with_keys(Arc::new(keys_utf8) as ArrayRef);
1477
1478        // LargeUtf8 (LargeStringArray)
1479        let keys_large = super::LargeStringArray::from(vec!["foo", "bar", "baz", "qux", "quux"]);
1480        run_json_writer_map_with_keys(Arc::new(keys_large) as ArrayRef);
1481
1482        // Utf8View (StringViewArray)
1483        let keys_view = super::StringViewArray::from(vec!["foo", "bar", "baz", "qux", "quux"]);
1484        run_json_writer_map_with_keys(Arc::new(keys_view) as ArrayRef);
1485    }
1486
1487    #[test]
1488    fn test_write_single_batch() {
1489        let test_file = "test/data/basic.json";
1490        let file = File::open(test_file).unwrap();
1491        let mut reader = BufReader::new(file);
1492        let (schema, _) = infer_json_schema(&mut reader, None).unwrap();
1493        reader.rewind().unwrap();
1494
1495        let builder = ReaderBuilder::new(Arc::new(schema)).with_batch_size(1024);
1496        let mut reader = builder.build(reader).unwrap();
1497        let batch = reader.next().unwrap().unwrap();
1498
1499        let mut buf = Vec::new();
1500        {
1501            let mut writer = LineDelimitedWriter::new(&mut buf);
1502            writer.write(&batch).unwrap();
1503        }
1504
1505        let result = str::from_utf8(&buf).unwrap();
1506        let expected = read_to_string(test_file).unwrap();
1507        for (r, e) in result.lines().zip(expected.lines()) {
1508            let mut expected_json = serde_json::from_str::<Value>(e).unwrap();
1509            // remove null value from object to make comparison consistent:
1510            if let Value::Object(obj) = expected_json {
1511                expected_json =
1512                    Value::Object(obj.into_iter().filter(|(_, v)| *v != Value::Null).collect());
1513            }
1514            assert_eq!(serde_json::from_str::<Value>(r).unwrap(), expected_json,);
1515        }
1516    }
1517
1518    #[test]
1519    #[cfg_attr(miri, ignore)] // Unsupported inline assembly
1520    fn test_write_multi_batches() {
1521        let test_file = "test/data/basic.json";
1522
1523        let schema = SchemaRef::new(Schema::new(vec![
1524            Field::new("a", DataType::Int64, true),
1525            Field::new("b", DataType::Float64, true),
1526            Field::new("c", DataType::Boolean, true),
1527            Field::new("d", DataType::Utf8, true),
1528            Field::new("e", DataType::Utf8, true),
1529            Field::new("f", DataType::Utf8, true),
1530            Field::new("g", DataType::Timestamp(TimeUnit::Millisecond, None), true),
1531            Field::new("h", DataType::Float16, true),
1532        ]));
1533
1534        let mut reader = ReaderBuilder::new(schema.clone())
1535            .build(BufReader::new(File::open(test_file).unwrap()))
1536            .unwrap();
1537        let batch = reader.next().unwrap().unwrap();
1538
1539        // test batches = an empty batch + 2 same batches, finally result should be eq to 2 same batches
1540        let batches = [&RecordBatch::new_empty(schema), &batch, &batch];
1541
1542        let mut buf = Vec::new();
1543        {
1544            let mut writer = LineDelimitedWriter::new(&mut buf);
1545            writer.write_batches(&batches).unwrap();
1546        }
1547
1548        let result = str::from_utf8(&buf).unwrap();
1549        let expected = read_to_string(test_file).unwrap();
1550        // result is eq to 2 same batches
1551        let expected = format!("{expected}\n{expected}");
1552        for (r, e) in result.lines().zip(expected.lines()) {
1553            let mut expected_json = serde_json::from_str::<Value>(e).unwrap();
1554            // remove null value from object to make comparison consistent:
1555            if let Value::Object(obj) = expected_json {
1556                expected_json =
1557                    Value::Object(obj.into_iter().filter(|(_, v)| *v != Value::Null).collect());
1558            }
1559            assert_eq!(serde_json::from_str::<Value>(r).unwrap(), expected_json,);
1560        }
1561    }
1562
1563    #[test]
1564    fn test_writer_explicit_nulls() -> Result<(), ArrowError> {
1565        fn nested_list() -> (Arc<ListArray>, Arc<Field>) {
1566            let array = Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
1567                Some(vec![None, None, None]),
1568                Some(vec![Some(1), Some(2), Some(3)]),
1569                None,
1570                Some(vec![None, None, None]),
1571            ]));
1572            let field = Arc::new(Field::new("list", array.data_type().clone(), true));
1573            // [{"list":[null,null,null]},{"list":[1,2,3]},{"list":null},{"list":[null,null,null]}]
1574            (array, field)
1575        }
1576
1577        fn nested_dict() -> (Arc<DictionaryArray<Int32Type>>, Arc<Field>) {
1578            let array = Arc::new(DictionaryArray::from_iter(vec![
1579                Some("cupcakes"),
1580                None,
1581                Some("bear"),
1582                Some("kuma"),
1583            ]));
1584            let field = Arc::new(Field::new("dict", array.data_type().clone(), true));
1585            // [{"dict":"cupcakes"},{"dict":null},{"dict":"bear"},{"dict":"kuma"}]
1586            (array, field)
1587        }
1588
1589        fn nested_map() -> (Arc<MapArray>, Arc<Field>) {
1590            let string_builder = StringBuilder::new();
1591            let int_builder = Int64Builder::new();
1592            let mut builder = MapBuilder::new(None, string_builder, int_builder);
1593
1594            // [{"foo": 10}, null, {}, {"bar": 20, "baz": 30, "qux": 40}]
1595            builder.keys().append_value("foo");
1596            builder.values().append_value(10);
1597            builder.append(true).unwrap();
1598
1599            builder.append(false).unwrap();
1600
1601            builder.append(true).unwrap();
1602
1603            builder.keys().append_value("bar");
1604            builder.values().append_value(20);
1605            builder.keys().append_value("baz");
1606            builder.values().append_value(30);
1607            builder.keys().append_value("qux");
1608            builder.values().append_value(40);
1609            builder.append(true).unwrap();
1610
1611            let array = Arc::new(builder.finish());
1612            let field = Arc::new(Field::new("map", array.data_type().clone(), true));
1613            (array, field)
1614        }
1615
1616        fn root_list() -> (Arc<ListArray>, Field) {
1617            let struct_array = StructArray::from(vec![
1618                (
1619                    Arc::new(Field::new("utf8", DataType::Utf8, true)),
1620                    Arc::new(StringArray::from(vec![Some("a"), Some("b"), None, None])) as ArrayRef,
1621                ),
1622                (
1623                    Arc::new(Field::new("int32", DataType::Int32, true)),
1624                    Arc::new(Int32Array::from(vec![Some(1), None, Some(5), None])) as ArrayRef,
1625                ),
1626            ]);
1627
1628            let values_field =
1629                Arc::new(Field::new("struct", struct_array.data_type().clone(), true));
1630            let field = Field::new_list("list", values_field.as_ref().clone(), true);
1631
1632            // [{"list":[{"int32":1,"utf8":"a"},{"int32":null,"utf8":"b"}]},{"list":null},{"list":[{int32":5,"utf8":null}]},{"list":null}]
1633            let array = Arc::new(ListArray::new(
1634                values_field,
1635                OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 2, 2, 3, 3])),
1636                Arc::new(struct_array),
1637                Some(NullBuffer::from(vec![true, false, true, false])),
1638            ));
1639            (array, field)
1640        }
1641
1642        let (nested_list_array, nested_list_field) = nested_list();
1643        let (nested_dict_array, nested_dict_field) = nested_dict();
1644        let (nested_map_array, nested_map_field) = nested_map();
1645        let (root_list_array, root_list_field) = root_list();
1646
1647        let schema = Schema::new(vec![
1648            Field::new("date", DataType::Date32, true),
1649            Field::new("null", DataType::Null, true),
1650            Field::new_struct(
1651                "struct",
1652                vec![
1653                    Arc::new(Field::new("utf8", DataType::Utf8, true)),
1654                    nested_list_field.clone(),
1655                    nested_dict_field.clone(),
1656                    nested_map_field.clone(),
1657                ],
1658                true,
1659            ),
1660            root_list_field,
1661        ]);
1662
1663        let arr_date32 = Date32Array::from(vec![Some(0), None, Some(1), None]);
1664        let arr_null = NullArray::new(4);
1665        let arr_struct = StructArray::from(vec![
1666            // [{"utf8":"a"},{"utf8":null},{"utf8":null},{"utf8":"b"}]
1667            (
1668                Arc::new(Field::new("utf8", DataType::Utf8, true)),
1669                Arc::new(StringArray::from(vec![Some("a"), None, None, Some("b")])) as ArrayRef,
1670            ),
1671            // [{"list":[null,null,null]},{"list":[1,2,3]},{"list":null},{"list":[null,null,null]}]
1672            (nested_list_field, nested_list_array as ArrayRef),
1673            // [{"dict":"cupcakes"},{"dict":null},{"dict":"bear"},{"dict":"kuma"}]
1674            (nested_dict_field, nested_dict_array as ArrayRef),
1675            // [{"foo": 10}, null, {}, {"bar": 20, "baz": 30, "qux": 40}]
1676            (nested_map_field, nested_map_array as ArrayRef),
1677        ]);
1678
1679        let batch = RecordBatch::try_new(
1680            Arc::new(schema),
1681            vec![
1682                // [{"date":"1970-01-01"},{"date":null},{"date":"1970-01-02"},{"date":null}]
1683                Arc::new(arr_date32),
1684                // [{"null":null},{"null":null},{"null":null},{"null":null}]
1685                Arc::new(arr_null),
1686                Arc::new(arr_struct),
1687                // [{"list":[{"int32":1,"utf8":"a"},{"int32":null,"utf8":"b"}]},{"list":null},{"list":[{int32":5,"utf8":null}]},{"list":null}]
1688                root_list_array,
1689            ],
1690        )?;
1691
1692        let mut buf = Vec::new();
1693        {
1694            let mut writer = WriterBuilder::new()
1695                .with_explicit_nulls(true)
1696                .build::<_, JsonArray>(&mut buf);
1697            writer.write_batches(&[&batch])?;
1698            writer.finish()?;
1699        }
1700
1701        let actual = serde_json::from_slice::<Vec<Value>>(&buf).unwrap();
1702        let expected = serde_json::from_value::<Vec<Value>>(json!([
1703          {
1704            "date": "1970-01-01",
1705            "list": [
1706              {
1707                "int32": 1,
1708                "utf8": "a"
1709              },
1710              {
1711                "int32": null,
1712                "utf8": "b"
1713              }
1714            ],
1715            "null": null,
1716            "struct": {
1717              "dict": "cupcakes",
1718              "list": [
1719                null,
1720                null,
1721                null
1722              ],
1723              "map": {
1724                "foo": 10
1725              },
1726              "utf8": "a"
1727            }
1728          },
1729          {
1730            "date": null,
1731            "list": null,
1732            "null": null,
1733            "struct": {
1734              "dict": null,
1735              "list": [
1736                1,
1737                2,
1738                3
1739              ],
1740              "map": null,
1741              "utf8": null
1742            }
1743          },
1744          {
1745            "date": "1970-01-02",
1746            "list": [
1747              {
1748                "int32": 5,
1749                "utf8": null
1750              }
1751            ],
1752            "null": null,
1753            "struct": {
1754              "dict": "bear",
1755              "list": null,
1756              "map": {},
1757              "utf8": null
1758            }
1759          },
1760          {
1761            "date": null,
1762            "list": null,
1763            "null": null,
1764            "struct": {
1765              "dict": "kuma",
1766              "list": [
1767                null,
1768                null,
1769                null
1770              ],
1771              "map": {
1772                "bar": 20,
1773                "baz": 30,
1774                "qux": 40
1775              },
1776              "utf8": "b"
1777            }
1778          }
1779        ]))
1780        .unwrap();
1781
1782        assert_eq!(actual, expected);
1783
1784        Ok(())
1785    }
1786
1787    fn build_array_binary<O: OffsetSizeTrait>(values: &[Option<&[u8]>]) -> RecordBatch {
1788        let schema = SchemaRef::new(Schema::new(vec![Field::new(
1789            "bytes",
1790            GenericBinaryType::<O>::DATA_TYPE,
1791            true,
1792        )]));
1793        let mut builder = GenericByteBuilder::<GenericBinaryType<O>>::new();
1794        for value in values {
1795            match value {
1796                Some(v) => builder.append_value(v),
1797                None => builder.append_null(),
1798            }
1799        }
1800        let array = Arc::new(builder.finish()) as ArrayRef;
1801        RecordBatch::try_new(schema, vec![array]).unwrap()
1802    }
1803
1804    fn build_array_binary_view(values: &[Option<&[u8]>]) -> RecordBatch {
1805        let schema = SchemaRef::new(Schema::new(vec![Field::new(
1806            "bytes",
1807            DataType::BinaryView,
1808            true,
1809        )]));
1810        let mut builder = BinaryViewBuilder::new();
1811        for value in values {
1812            match value {
1813                Some(v) => builder.append_value(v),
1814                None => builder.append_null(),
1815            }
1816        }
1817        let array = Arc::new(builder.finish()) as ArrayRef;
1818        RecordBatch::try_new(schema, vec![array]).unwrap()
1819    }
1820
1821    fn assert_binary_json(batch: &RecordBatch) {
1822        // encode and check JSON with explicit nulls:
1823        {
1824            let mut buf = Vec::new();
1825            let json_value: Value = {
1826                let mut writer = WriterBuilder::new()
1827                    .with_explicit_nulls(true)
1828                    .build::<_, JsonArray>(&mut buf);
1829                writer.write(batch).unwrap();
1830                writer.close().unwrap();
1831                serde_json::from_slice(&buf).unwrap()
1832            };
1833
1834            assert_eq!(
1835                json!([
1836                    {
1837                        "bytes": "426f622054686f6d70736f6e"
1838                    },
1839                    {
1840                        "bytes": null // the explicit null
1841                    },
1842                    {
1843                        "bytes": "54726f79204d63436c757265"
1844                    }
1845                ]),
1846                json_value,
1847            );
1848        }
1849
1850        // encode and check JSON with no explicit nulls:
1851        {
1852            let mut buf = Vec::new();
1853            let json_value: Value = {
1854                // explicit nulls are off by default, so we don't need
1855                // to set that when creating the writer:
1856                let mut writer = ArrayWriter::new(&mut buf);
1857                writer.write(batch).unwrap();
1858                writer.close().unwrap();
1859                serde_json::from_slice(&buf).unwrap()
1860            };
1861
1862            assert_eq!(
1863                json!([
1864                    { "bytes": "426f622054686f6d70736f6e" },
1865                    {},
1866                    { "bytes": "54726f79204d63436c757265" }
1867                ]),
1868                json_value
1869            );
1870        }
1871    }
1872
1873    #[test]
1874    fn test_writer_binary() {
1875        let values: [Option<&[u8]>; 3] = [
1876            Some(b"Bob Thompson" as &[u8]),
1877            None,
1878            Some(b"Troy McClure" as &[u8]),
1879        ];
1880        // Binary:
1881        {
1882            let batch = build_array_binary::<i32>(&values);
1883            assert_binary_json(&batch);
1884        }
1885        // LargeBinary:
1886        {
1887            let batch = build_array_binary::<i64>(&values);
1888            assert_binary_json(&batch);
1889        }
1890        {
1891            let batch = build_array_binary_view(&values);
1892            assert_binary_json(&batch);
1893        }
1894    }
1895
1896    #[test]
1897    fn test_writer_fixed_size_binary() {
1898        // set up schema:
1899        let size = 11;
1900        let schema = SchemaRef::new(Schema::new(vec![Field::new(
1901            "bytes",
1902            DataType::FixedSizeBinary(size),
1903            true,
1904        )]));
1905
1906        // build record batch:
1907        let mut builder = FixedSizeBinaryBuilder::new(size);
1908        let values = [Some(b"hello world"), None, Some(b"summer rain")];
1909        for value in values {
1910            match value {
1911                Some(v) => builder.append_value(v).unwrap(),
1912                None => builder.append_null(),
1913            }
1914        }
1915        let array = Arc::new(builder.finish()) as ArrayRef;
1916        let batch = RecordBatch::try_new(schema, vec![array]).unwrap();
1917
1918        // encode and check JSON with explicit nulls:
1919        {
1920            let mut buf = Vec::new();
1921            let json_value: Value = {
1922                let mut writer = WriterBuilder::new()
1923                    .with_explicit_nulls(true)
1924                    .build::<_, JsonArray>(&mut buf);
1925                writer.write(&batch).unwrap();
1926                writer.close().unwrap();
1927                serde_json::from_slice(&buf).unwrap()
1928            };
1929
1930            assert_eq!(
1931                json!([
1932                    {
1933                        "bytes": "68656c6c6f20776f726c64"
1934                    },
1935                    {
1936                        "bytes": null // the explicit null
1937                    },
1938                    {
1939                        "bytes": "73756d6d6572207261696e"
1940                    }
1941                ]),
1942                json_value,
1943            );
1944        }
1945        // encode and check JSON with no explicit nulls:
1946        {
1947            let mut buf = Vec::new();
1948            let json_value: Value = {
1949                // explicit nulls are off by default, so we don't need
1950                // to set that when creating the writer:
1951                let mut writer = ArrayWriter::new(&mut buf);
1952                writer.write(&batch).unwrap();
1953                writer.close().unwrap();
1954                serde_json::from_slice(&buf).unwrap()
1955            };
1956
1957            assert_eq!(
1958                json!([
1959                    {
1960                        "bytes": "68656c6c6f20776f726c64"
1961                    },
1962                    {}, // empty because nulls are omitted
1963                    {
1964                        "bytes": "73756d6d6572207261696e"
1965                    }
1966                ]),
1967                json_value,
1968            );
1969        }
1970    }
1971
1972    #[test]
1973    fn test_writer_fixed_size_list() {
1974        let size = 3;
1975        let field = FieldRef::new(Field::new_list_field(DataType::Int32, true));
1976        let schema = SchemaRef::new(Schema::new(vec![Field::new(
1977            "list",
1978            DataType::FixedSizeList(field, size),
1979            true,
1980        )]));
1981
1982        let values_builder = Int32Builder::new();
1983        let mut list_builder = FixedSizeListBuilder::new(values_builder, size);
1984        let lists = [
1985            Some([Some(1), Some(2), None]),
1986            Some([Some(3), None, Some(4)]),
1987            Some([None, Some(5), Some(6)]),
1988            None,
1989        ];
1990        for list in lists {
1991            match list {
1992                Some(l) => {
1993                    for value in l {
1994                        match value {
1995                            Some(v) => list_builder.values().append_value(v),
1996                            None => list_builder.values().append_null(),
1997                        }
1998                    }
1999                    list_builder.append(true);
2000                }
2001                None => {
2002                    for _ in 0..size {
2003                        list_builder.values().append_null();
2004                    }
2005                    list_builder.append(false);
2006                }
2007            }
2008        }
2009        let array = Arc::new(list_builder.finish()) as ArrayRef;
2010        let batch = RecordBatch::try_new(schema, vec![array]).unwrap();
2011
2012        //encode and check JSON with explicit nulls:
2013        {
2014            let json_value: Value = {
2015                let mut buf = Vec::new();
2016                let mut writer = WriterBuilder::new()
2017                    .with_explicit_nulls(true)
2018                    .build::<_, JsonArray>(&mut buf);
2019                writer.write(&batch).unwrap();
2020                writer.close().unwrap();
2021                serde_json::from_slice(&buf).unwrap()
2022            };
2023            assert_eq!(
2024                json!([
2025                    {"list": [1, 2, null]},
2026                    {"list": [3, null, 4]},
2027                    {"list": [null, 5, 6]},
2028                    {"list": null},
2029                ]),
2030                json_value
2031            );
2032        }
2033        // encode and check JSON with no explicit nulls:
2034        {
2035            let json_value: Value = {
2036                let mut buf = Vec::new();
2037                let mut writer = ArrayWriter::new(&mut buf);
2038                writer.write(&batch).unwrap();
2039                writer.close().unwrap();
2040                serde_json::from_slice(&buf).unwrap()
2041            };
2042            assert_eq!(
2043                json!([
2044                    {"list": [1, 2, null]},
2045                    {"list": [3, null, 4]},
2046                    {"list": [null, 5, 6]},
2047                    {}, // empty because nulls are omitted
2048                ]),
2049                json_value
2050            );
2051        }
2052    }
2053
2054    #[test]
2055    fn test_writer_null_dict() {
2056        let keys = Int32Array::from_iter(vec![Some(0), None, Some(1)]);
2057        let values = Arc::new(StringArray::from_iter(vec![Some("a"), None]));
2058        let dict = DictionaryArray::new(keys, values);
2059
2060        let schema = SchemaRef::new(Schema::new(vec![Field::new(
2061            "my_dict",
2062            DataType::Dictionary(DataType::Int32.into(), DataType::Utf8.into()),
2063            true,
2064        )]));
2065
2066        let array = Arc::new(dict) as ArrayRef;
2067        let batch = RecordBatch::try_new(schema, vec![array]).unwrap();
2068
2069        let mut json = Vec::new();
2070        let write_builder = WriterBuilder::new().with_explicit_nulls(true);
2071        let mut writer = write_builder.build::<_, JsonArray>(&mut json);
2072        writer.write(&batch).unwrap();
2073        writer.close().unwrap();
2074
2075        let json_str = str::from_utf8(&json).unwrap();
2076        assert_eq!(
2077            json_str,
2078            r#"[{"my_dict":"a"},{"my_dict":null},{"my_dict":null}]"#
2079        )
2080    }
2081
2082    #[test]
2083    fn test_decimal32_encoder() {
2084        let array = Decimal32Array::from_iter_values([1234, 5678, 9012])
2085            .with_precision_and_scale(8, 2)
2086            .unwrap();
2087        let field = Arc::new(Field::new("decimal", array.data_type().clone(), true));
2088        let schema = Schema::new(vec![field]);
2089        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap();
2090
2091        let mut buf = Vec::new();
2092        {
2093            let mut writer = LineDelimitedWriter::new(&mut buf);
2094            writer.write_batches(&[&batch]).unwrap();
2095        }
2096
2097        assert_json_eq(
2098            &buf,
2099            r#"{"decimal":12.34}
2100{"decimal":56.78}
2101{"decimal":90.12}
2102"#,
2103        );
2104    }
2105
2106    #[test]
2107    fn test_decimal64_encoder() {
2108        let array = Decimal64Array::from_iter_values([1234, 5678, 9012])
2109            .with_precision_and_scale(10, 2)
2110            .unwrap();
2111        let field = Arc::new(Field::new("decimal", array.data_type().clone(), true));
2112        let schema = Schema::new(vec![field]);
2113        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap();
2114
2115        let mut buf = Vec::new();
2116        {
2117            let mut writer = LineDelimitedWriter::new(&mut buf);
2118            writer.write_batches(&[&batch]).unwrap();
2119        }
2120
2121        assert_json_eq(
2122            &buf,
2123            r#"{"decimal":12.34}
2124{"decimal":56.78}
2125{"decimal":90.12}
2126"#,
2127        );
2128    }
2129
2130    #[test]
2131    fn test_decimal128_encoder() {
2132        let array = Decimal128Array::from_iter_values([1234, 5678, 9012])
2133            .with_precision_and_scale(10, 2)
2134            .unwrap();
2135        let field = Arc::new(Field::new("decimal", array.data_type().clone(), true));
2136        let schema = Schema::new(vec![field]);
2137        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap();
2138
2139        let mut buf = Vec::new();
2140        {
2141            let mut writer = LineDelimitedWriter::new(&mut buf);
2142            writer.write_batches(&[&batch]).unwrap();
2143        }
2144
2145        assert_json_eq(
2146            &buf,
2147            r#"{"decimal":12.34}
2148{"decimal":56.78}
2149{"decimal":90.12}
2150"#,
2151        );
2152    }
2153
2154    #[test]
2155    fn test_decimal256_encoder() {
2156        let array = Decimal256Array::from_iter_values([
2157            i256::from(123400),
2158            i256::from(567800),
2159            i256::from(901200),
2160        ])
2161        .with_precision_and_scale(10, 4)
2162        .unwrap();
2163        let field = Arc::new(Field::new("decimal", array.data_type().clone(), true));
2164        let schema = Schema::new(vec![field]);
2165        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap();
2166
2167        let mut buf = Vec::new();
2168        {
2169            let mut writer = LineDelimitedWriter::new(&mut buf);
2170            writer.write_batches(&[&batch]).unwrap();
2171        }
2172
2173        assert_json_eq(
2174            &buf,
2175            r#"{"decimal":12.3400}
2176{"decimal":56.7800}
2177{"decimal":90.1200}
2178"#,
2179        );
2180    }
2181
2182    #[test]
2183    fn test_decimal_encoder_with_nulls() {
2184        let array = Decimal128Array::from_iter([Some(1234), None, Some(5678)])
2185            .with_precision_and_scale(10, 2)
2186            .unwrap();
2187        let field = Arc::new(Field::new("decimal", array.data_type().clone(), true));
2188        let schema = Schema::new(vec![field]);
2189        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap();
2190
2191        let mut buf = Vec::new();
2192        {
2193            let mut writer = LineDelimitedWriter::new(&mut buf);
2194            writer.write_batches(&[&batch]).unwrap();
2195        }
2196
2197        assert_json_eq(
2198            &buf,
2199            r#"{"decimal":12.34}
2200{}
2201{"decimal":56.78}
2202"#,
2203        );
2204    }
2205
2206    #[test]
2207    fn test_decimal_encoder_negative_scale() {
2208        // https://github.com/apache/arrow-rs/issues/10865
2209        let array = Decimal128Array::from_iter([Some(0), Some(12), Some(-12)])
2210            .with_precision_and_scale(10, -2)
2211            .unwrap();
2212        let field = Arc::new(Field::new("decimal", array.data_type().clone(), true));
2213        let schema = Schema::new(vec![field]);
2214        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap();
2215
2216        let mut buf = Vec::new();
2217        {
2218            let mut writer = LineDelimitedWriter::new(&mut buf);
2219            writer.write_batches(&[&batch]).unwrap();
2220        }
2221
2222        assert_json_eq(
2223            &buf,
2224            r#"{"decimal":0}
2225{"decimal":1200}
2226{"decimal":-1200}
2227"#,
2228        );
2229    }
2230
2231    #[test]
2232    fn write_structs_as_list() {
2233        let schema = Schema::new(vec![
2234            Field::new(
2235                "c1",
2236                DataType::Struct(Fields::from(vec![
2237                    Field::new("c11", DataType::Int32, true),
2238                    Field::new(
2239                        "c12",
2240                        DataType::Struct(vec![Field::new("c121", DataType::Utf8, false)].into()),
2241                        false,
2242                    ),
2243                ])),
2244                false,
2245            ),
2246            Field::new("c2", DataType::Utf8, false),
2247        ]);
2248
2249        let c1 = StructArray::from(vec![
2250            (
2251                Arc::new(Field::new("c11", DataType::Int32, true)),
2252                Arc::new(Int32Array::from(vec![Some(1), None, Some(5)])) as ArrayRef,
2253            ),
2254            (
2255                Arc::new(Field::new(
2256                    "c12",
2257                    DataType::Struct(vec![Field::new("c121", DataType::Utf8, false)].into()),
2258                    false,
2259                )),
2260                Arc::new(StructArray::from(vec![(
2261                    Arc::new(Field::new("c121", DataType::Utf8, false)),
2262                    Arc::new(StringArray::from(vec![Some("e"), Some("f"), Some("g")])) as ArrayRef,
2263                )])) as ArrayRef,
2264            ),
2265        ]);
2266        let c2 = StringArray::from(vec![Some("a"), Some("b"), Some("c")]);
2267
2268        let batch =
2269            RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c1), Arc::new(c2)]).unwrap();
2270
2271        let expected = r#"[[1,["e"]],"a"]
2272[[null,["f"]],"b"]
2273[[5,["g"]],"c"]
2274"#;
2275
2276        let mut buf = Vec::new();
2277        {
2278            let builder = WriterBuilder::new()
2279                .with_explicit_nulls(true)
2280                .with_struct_mode(StructMode::ListOnly);
2281            let mut writer = builder.build::<_, LineDelimited>(&mut buf);
2282            writer.write_batches(&[&batch]).unwrap();
2283        }
2284        assert_json_eq(&buf, expected);
2285
2286        let mut buf = Vec::new();
2287        {
2288            let builder = WriterBuilder::new()
2289                .with_explicit_nulls(false)
2290                .with_struct_mode(StructMode::ListOnly);
2291            let mut writer = builder.build::<_, LineDelimited>(&mut buf);
2292            writer.write_batches(&[&batch]).unwrap();
2293        }
2294        assert_json_eq(&buf, expected);
2295    }
2296
2297    fn make_fallback_encoder_test_data() -> (RecordBatch, Arc<dyn EncoderFactory>) {
2298        // Note: this is not intended to be an efficient implementation.
2299        // Just a simple example to demonstrate how to implement a custom encoder.
2300        #[derive(Debug)]
2301        enum UnionValue {
2302            Int32(i32),
2303            String(String),
2304        }
2305
2306        #[derive(Debug)]
2307        struct UnionEncoder {
2308            array: Vec<Option<UnionValue>>,
2309        }
2310
2311        impl Encoder for UnionEncoder {
2312            fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
2313                match &self.array[idx] {
2314                    None => out.extend_from_slice(b"null"),
2315                    Some(UnionValue::Int32(v)) => out.extend_from_slice(v.to_string().as_bytes()),
2316                    Some(UnionValue::String(v)) => {
2317                        out.extend_from_slice(format!("\"{v}\"").as_bytes())
2318                    }
2319                }
2320            }
2321        }
2322
2323        #[derive(Debug)]
2324        struct UnionEncoderFactory;
2325
2326        impl EncoderFactory for UnionEncoderFactory {
2327            fn make_default_encoder<'a>(
2328                &self,
2329                _field: &'a FieldRef,
2330                array: &'a dyn Array,
2331                _options: &'a EncoderOptions,
2332            ) -> Result<Option<NullableEncoder<'a>>, ArrowError> {
2333                let data_type = array.data_type();
2334                let DataType::Union(fields, UnionMode::Sparse) = data_type else {
2335                    return Ok(None);
2336                };
2337                // check that the fields are supported
2338                let fields = fields.iter().map(|(_, f)| f).collect::<Vec<_>>();
2339                for f in &fields {
2340                    match f.data_type() {
2341                        DataType::Null => {}
2342                        DataType::Int32 => {}
2343                        DataType::Utf8 => {}
2344                        _ => return Ok(None),
2345                    }
2346                }
2347                let (_, type_ids, _, buffers) = array.as_union().clone().into_parts();
2348                let mut values = Vec::with_capacity(type_ids.len());
2349                for idx in 0..type_ids.len() {
2350                    let type_id = type_ids[idx];
2351                    let field = &fields[type_id as usize];
2352                    let value = match field.data_type() {
2353                        DataType::Null => None,
2354                        DataType::Int32 => Some(UnionValue::Int32(
2355                            buffers[type_id as usize]
2356                                .as_primitive::<Int32Type>()
2357                                .value(idx),
2358                        )),
2359                        DataType::Utf8 => Some(UnionValue::String(
2360                            buffers[type_id as usize]
2361                                .as_string::<i32>()
2362                                .value(idx)
2363                                .to_string(),
2364                        )),
2365                        _ => unreachable!(),
2366                    };
2367                    values.push(value);
2368                }
2369                let array_encoder =
2370                    Box::new(UnionEncoder { array: values }) as Box<dyn Encoder + 'a>;
2371                let nulls = array.nulls().cloned();
2372                Ok(Some(NullableEncoder::new(array_encoder, nulls)))
2373            }
2374        }
2375
2376        let int_array = Int32Array::from(vec![Some(1), None, None]);
2377        let string_array = StringArray::from(vec![None, Some("a"), None]);
2378        let null_array = NullArray::new(3);
2379        let type_ids = [0_i8, 1, 2].into_iter().collect::<ScalarBuffer<i8>>();
2380
2381        let union_fields = [
2382            (0, Arc::new(Field::new("A", DataType::Int32, false))),
2383            (1, Arc::new(Field::new("B", DataType::Utf8, false))),
2384            (2, Arc::new(Field::new("C", DataType::Null, false))),
2385        ]
2386        .into_iter()
2387        .collect::<UnionFields>();
2388
2389        let children = vec![
2390            Arc::new(int_array) as Arc<dyn Array>,
2391            Arc::new(string_array),
2392            Arc::new(null_array),
2393        ];
2394
2395        let array = UnionArray::try_new(union_fields.clone(), type_ids, None, children).unwrap();
2396
2397        let float_array = Float64Array::from(vec![Some(1.0), None, Some(3.4)]);
2398
2399        let fields = vec![
2400            Field::new(
2401                "union",
2402                DataType::Union(union_fields, UnionMode::Sparse),
2403                true,
2404            ),
2405            Field::new("float", DataType::Float64, true),
2406        ];
2407
2408        let batch = RecordBatch::try_new(
2409            Arc::new(Schema::new(fields)),
2410            vec![
2411                Arc::new(array) as Arc<dyn Array>,
2412                Arc::new(float_array) as Arc<dyn Array>,
2413            ],
2414        )
2415        .unwrap();
2416
2417        (batch, Arc::new(UnionEncoderFactory))
2418    }
2419
2420    #[test]
2421    fn test_fallback_encoder_factory_line_delimited_implicit_nulls() {
2422        let (batch, encoder_factory) = make_fallback_encoder_test_data();
2423
2424        let mut buf = Vec::new();
2425        {
2426            let mut writer = WriterBuilder::new()
2427                .with_encoder_factory(encoder_factory)
2428                .with_explicit_nulls(false)
2429                .build::<_, LineDelimited>(&mut buf);
2430            writer.write_batches(&[&batch]).unwrap();
2431            writer.finish().unwrap();
2432        }
2433
2434        println!("{}", str::from_utf8(&buf).unwrap());
2435
2436        assert_json_eq(
2437            &buf,
2438            r#"{"union":1,"float":1.0}
2439{"union":"a"}
2440{"union":null,"float":3.4}
2441"#,
2442        );
2443    }
2444
2445    #[test]
2446    fn test_fallback_encoder_factory_line_delimited_explicit_nulls() {
2447        let (batch, encoder_factory) = make_fallback_encoder_test_data();
2448
2449        let mut buf = Vec::new();
2450        {
2451            let mut writer = WriterBuilder::new()
2452                .with_encoder_factory(encoder_factory)
2453                .with_explicit_nulls(true)
2454                .build::<_, LineDelimited>(&mut buf);
2455            writer.write_batches(&[&batch]).unwrap();
2456            writer.finish().unwrap();
2457        }
2458
2459        assert_json_eq(
2460            &buf,
2461            r#"{"union":1,"float":1.0}
2462{"union":"a","float":null}
2463{"union":null,"float":3.4}
2464"#,
2465        );
2466    }
2467
2468    #[test]
2469    fn test_fallback_encoder_factory_array_implicit_nulls() {
2470        let (batch, encoder_factory) = make_fallback_encoder_test_data();
2471
2472        let json_value: Value = {
2473            let mut buf = Vec::new();
2474            let mut writer = WriterBuilder::new()
2475                .with_encoder_factory(encoder_factory)
2476                .build::<_, JsonArray>(&mut buf);
2477            writer.write_batches(&[&batch]).unwrap();
2478            writer.finish().unwrap();
2479            serde_json::from_slice(&buf).unwrap()
2480        };
2481
2482        let expected = json!([
2483            {"union":1,"float":1.0},
2484            {"union":"a"},
2485            {"float":3.4,"union":null},
2486        ]);
2487
2488        assert_eq!(json_value, expected);
2489    }
2490
2491    #[test]
2492    fn test_fallback_encoder_factory_array_explicit_nulls() {
2493        let (batch, encoder_factory) = make_fallback_encoder_test_data();
2494
2495        let json_value: Value = {
2496            let mut buf = Vec::new();
2497            let mut writer = WriterBuilder::new()
2498                .with_encoder_factory(encoder_factory)
2499                .with_explicit_nulls(true)
2500                .build::<_, JsonArray>(&mut buf);
2501            writer.write_batches(&[&batch]).unwrap();
2502            writer.finish().unwrap();
2503            serde_json::from_slice(&buf).unwrap()
2504        };
2505
2506        let expected = json!([
2507            {"union":1,"float":1.0},
2508            {"union":"a", "float": null},
2509            {"union":null,"float":3.4},
2510        ]);
2511
2512        assert_eq!(json_value, expected);
2513    }
2514
2515    #[test]
2516    fn test_default_encoder_byte_array() {
2517        struct IntArrayBinaryEncoder<B> {
2518            array: B,
2519        }
2520
2521        impl<'a, B> Encoder for IntArrayBinaryEncoder<B>
2522        where
2523            B: ArrayAccessor<Item = &'a [u8]>,
2524        {
2525            fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
2526                out.push(b'[');
2527                let child = self.array.value(idx);
2528                for (idx, byte) in child.iter().enumerate() {
2529                    write!(out, "{byte}").unwrap();
2530                    if idx < child.len() - 1 {
2531                        out.push(b',');
2532                    }
2533                }
2534                out.push(b']');
2535            }
2536        }
2537
2538        #[derive(Debug)]
2539        struct IntArrayBinaryEncoderFactory;
2540
2541        impl EncoderFactory for IntArrayBinaryEncoderFactory {
2542            fn make_default_encoder<'a>(
2543                &self,
2544                _field: &'a FieldRef,
2545                array: &'a dyn Array,
2546                _options: &'a EncoderOptions,
2547            ) -> Result<Option<NullableEncoder<'a>>, ArrowError> {
2548                match array.data_type() {
2549                    DataType::Binary => {
2550                        let array = array.as_binary::<i32>();
2551                        let encoder = IntArrayBinaryEncoder { array };
2552                        let array_encoder = Box::new(encoder) as Box<dyn Encoder + 'a>;
2553                        let nulls = array.nulls().cloned();
2554                        Ok(Some(NullableEncoder::new(array_encoder, nulls)))
2555                    }
2556                    _ => Ok(None),
2557                }
2558            }
2559        }
2560
2561        let binary_array = BinaryArray::from_opt_vec(vec![Some(b"a"), None, Some(b"b")]);
2562        let float_array = Float64Array::from(vec![Some(1.0), Some(2.3), None]);
2563        let fields = vec![
2564            Field::new("bytes", DataType::Binary, true),
2565            Field::new("float", DataType::Float64, true),
2566        ];
2567        let batch = RecordBatch::try_new(
2568            Arc::new(Schema::new(fields)),
2569            vec![
2570                Arc::new(binary_array) as Arc<dyn Array>,
2571                Arc::new(float_array) as Arc<dyn Array>,
2572            ],
2573        )
2574        .unwrap();
2575
2576        let json_value: Value = {
2577            let mut buf = Vec::new();
2578            let mut writer = WriterBuilder::new()
2579                .with_encoder_factory(Arc::new(IntArrayBinaryEncoderFactory))
2580                .build::<_, JsonArray>(&mut buf);
2581            writer.write_batches(&[&batch]).unwrap();
2582            writer.finish().unwrap();
2583            serde_json::from_slice(&buf).unwrap()
2584        };
2585
2586        let expected = json!([
2587            {"bytes": [97], "float": 1.0},
2588            {"float": 2.3},
2589            {"bytes": [98]},
2590        ]);
2591
2592        assert_eq!(json_value, expected);
2593    }
2594
2595    #[test]
2596    fn test_encoder_factory_customize_dictionary() {
2597        // Test that we can customize the encoding of T even when it shows up as Dictionary<_, T>.
2598
2599        // No particular reason to choose this example.
2600        // Just trying to add some variety to the test cases and demonstrate use cases of the encoder factory.
2601        struct PaddedInt32Encoder {
2602            array: Int32Array,
2603        }
2604
2605        impl Encoder for PaddedInt32Encoder {
2606            fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
2607                let value = self.array.value(idx);
2608                write!(out, "\"{value:0>8}\"").unwrap();
2609            }
2610        }
2611
2612        #[derive(Debug)]
2613        struct CustomEncoderFactory;
2614
2615        impl EncoderFactory for CustomEncoderFactory {
2616            fn make_default_encoder<'a>(
2617                &self,
2618                field: &'a FieldRef,
2619                array: &'a dyn Array,
2620                _options: &'a EncoderOptions,
2621            ) -> Result<Option<NullableEncoder<'a>>, ArrowError> {
2622                // The point here is:
2623                // 1. You can use information from Field to determine how to do the encoding.
2624                // 2. For dictionary arrays the Field is always the outer field but the array may be the keys or values array
2625                //    and thus the data type of `field` may not match the data type of `array`.
2626                let padded = field.metadata().get("padded").is_some_and(|v| v == "true");
2627                match (array.data_type(), padded) {
2628                    (DataType::Int32, true) => {
2629                        let array = array.as_primitive::<Int32Type>();
2630                        let nulls = array.nulls().cloned();
2631                        let encoder = PaddedInt32Encoder {
2632                            array: array.clone(),
2633                        };
2634                        let array_encoder = Box::new(encoder) as Box<dyn Encoder + 'a>;
2635                        Ok(Some(NullableEncoder::new(array_encoder, nulls)))
2636                    }
2637                    _ => Ok(None),
2638                }
2639            }
2640        }
2641
2642        let to_json = |batch| {
2643            let mut buf = Vec::new();
2644            let mut writer = WriterBuilder::new()
2645                .with_encoder_factory(Arc::new(CustomEncoderFactory))
2646                .build::<_, JsonArray>(&mut buf);
2647            writer.write_batches(&[batch]).unwrap();
2648            writer.finish().unwrap();
2649            serde_json::from_slice::<Value>(&buf).unwrap()
2650        };
2651
2652        // Control case: no dictionary wrapping works as expected.
2653        let array = Int32Array::from(vec![Some(1), None, Some(2)]);
2654        let field = Arc::new(Field::new("int", DataType::Int32, true).with_metadata(
2655            HashMap::from_iter(vec![("padded".to_string(), "true".to_string())]),
2656        ));
2657        let batch = RecordBatch::try_new(
2658            Arc::new(Schema::new(vec![field.clone()])),
2659            vec![Arc::new(array)],
2660        )
2661        .unwrap();
2662
2663        let json_value = to_json(&batch);
2664
2665        let expected = json!([
2666            {"int": "00000001"},
2667            {},
2668            {"int": "00000002"},
2669        ]);
2670
2671        assert_eq!(json_value, expected);
2672
2673        // Now make a dictionary batch
2674        let mut array_builder = PrimitiveDictionaryBuilder::<UInt16Type, Int32Type>::new();
2675        array_builder.append_value(1);
2676        array_builder.append_null();
2677        array_builder.append_value(1);
2678        let array = array_builder.finish();
2679        let field = Field::new(
2680            "int",
2681            DataType::Dictionary(Box::new(DataType::UInt16), Box::new(DataType::Int32)),
2682            true,
2683        )
2684        .with_metadata(HashMap::from_iter(vec![(
2685            "padded".to_string(),
2686            "true".to_string(),
2687        )]));
2688        let batch = RecordBatch::try_new(Arc::new(Schema::new(vec![field])), vec![Arc::new(array)])
2689            .unwrap();
2690
2691        let json_value = to_json(&batch);
2692
2693        let expected = json!([
2694            {"int": "00000001"},
2695            {},
2696            {"int": "00000001"},
2697        ]);
2698
2699        assert_eq!(json_value, expected);
2700    }
2701
2702    #[test]
2703    fn test_write_run_end_encoded() {
2704        let run_ends = Int32Array::from(vec![2, 5, 6]);
2705        let values = StringArray::from(vec![Some("a"), Some("b"), None]);
2706        let ree = RunArray::<Int32Type>::try_new(&run_ends, &values).unwrap();
2707
2708        let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
2709            "c1",
2710            ree.data_type().clone(),
2711            true,
2712        )]));
2713
2714        let batch = RecordBatch::try_new(schema, vec![Arc::new(ree)]).unwrap();
2715
2716        let mut buf = Vec::new();
2717        {
2718            let mut writer = LineDelimitedWriter::new(&mut buf);
2719            writer.write_batches(&[&batch]).unwrap();
2720        }
2721
2722        assert_json_eq(
2723            &buf,
2724            r#"{"c1":"a"}
2725{"c1":"a"}
2726{"c1":"b"}
2727{"c1":"b"}
2728{"c1":"b"}
2729{}
2730"#,
2731        );
2732    }
2733
2734    #[test]
2735    fn test_write_run_end_encoded_int_values() {
2736        let run_ends = Int32Array::from(vec![3, 5]);
2737        let values = Int32Array::from(vec![10, 20]);
2738        let ree = RunArray::<Int32Type>::try_new(&run_ends, &values).unwrap();
2739
2740        let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
2741            "n",
2742            ree.data_type().clone(),
2743            true,
2744        )]));
2745
2746        let batch = RecordBatch::try_new(schema, vec![Arc::new(ree)]).unwrap();
2747
2748        let json_value: Value = {
2749            let mut buf = Vec::new();
2750            let mut writer = WriterBuilder::new().build::<_, JsonArray>(&mut buf);
2751            writer.write_batches(&[&batch]).unwrap();
2752            writer.finish().unwrap();
2753            serde_json::from_slice(&buf).unwrap()
2754        };
2755
2756        let expected = json!([
2757            {"n": 10},
2758            {"n": 10},
2759            {"n": 10},
2760            {"n": 20},
2761            {"n": 20},
2762        ]);
2763
2764        assert_eq!(json_value, expected);
2765    }
2766
2767    #[test]
2768    fn test_run_end_encoded_roundtrip() {
2769        let run_ends = Int32Array::from(vec![3, 5, 7]);
2770        let values = StringArray::from(vec![Some("a"), None, Some("b")]);
2771        let ree = RunArray::<Int32Type>::try_new(&run_ends, &values).unwrap();
2772
2773        let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
2774            "c",
2775            ree.data_type().clone(),
2776            true,
2777        )]));
2778        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(ree)]).unwrap();
2779
2780        let mut buf = Vec::new();
2781        {
2782            let mut writer = super::LineDelimitedWriter::new(&mut buf);
2783            writer.write_batches(&[&batch]).unwrap();
2784        }
2785
2786        let batches: Vec<RecordBatch> = ReaderBuilder::new(schema)
2787            .with_batch_size(1024)
2788            .build(std::io::Cursor::new(&buf))
2789            .unwrap()
2790            .collect::<Result<Vec<_>, _>>()
2791            .unwrap();
2792        assert_eq!(batches.len(), 1);
2793
2794        let col = batches[0].column(0);
2795        let run_array = col.as_run::<Int32Type>();
2796
2797        assert_eq!(run_array.len(), 7);
2798        assert_eq!(run_array.run_ends().values(), &[3, 5, 7]);
2799
2800        let values = run_array.values().as_string::<i32>();
2801        assert_eq!(values.len(), 3);
2802        assert_eq!(values.value(0), "a");
2803        assert!(values.is_null(1));
2804        assert_eq!(values.value(2), "b");
2805    }
2806}