1mod encoder;
103
104use std::{fmt::Debug, io::Write, sync::Arc};
105
106use crate::StructMode;
107use arrow_array::*;
108use arrow_schema::*;
109
110pub use encoder::{Encoder, EncoderFactory, EncoderOptions, NullableEncoder, make_encoder};
111
112pub trait JsonFormat: Debug + Default {
115 #[inline]
116 fn start_stream<W: Write>(&self, _writer: &mut W) -> Result<(), ArrowError> {
118 Ok(())
119 }
120
121 #[inline]
122 fn start_row<W: Write>(&self, _writer: &mut W, _is_first_row: bool) -> Result<(), ArrowError> {
124 Ok(())
125 }
126
127 #[inline]
128 fn end_row<W: Write>(&self, _writer: &mut W) -> Result<(), ArrowError> {
130 Ok(())
131 }
132
133 fn end_stream<W: Write>(&self, _writer: &mut W) -> Result<(), ArrowError> {
135 Ok(())
136 }
137}
138
139#[derive(Debug, Default)]
149pub struct LineDelimited {}
150
151impl JsonFormat for LineDelimited {
152 fn end_row<W: Write>(&self, writer: &mut W) -> Result<(), ArrowError> {
153 writer.write_all(b"\n")?;
154 Ok(())
155 }
156}
157
158#[derive(Debug, Default)]
166pub struct JsonArray {}
167
168impl JsonFormat for JsonArray {
169 fn start_stream<W: Write>(&self, writer: &mut W) -> Result<(), ArrowError> {
170 writer.write_all(b"[")?;
171 Ok(())
172 }
173
174 fn start_row<W: Write>(&self, writer: &mut W, is_first_row: bool) -> Result<(), ArrowError> {
175 if !is_first_row {
176 writer.write_all(b",")?;
177 }
178 Ok(())
179 }
180
181 fn end_stream<W: Write>(&self, writer: &mut W) -> Result<(), ArrowError> {
182 writer.write_all(b"]")?;
183 Ok(())
184 }
185}
186
187pub type LineDelimitedWriter<W> = Writer<W, LineDelimited>;
189
190pub type ArrayWriter<W> = Writer<W, JsonArray>;
192
193#[derive(Debug, Clone, Default)]
195pub struct WriterBuilder(EncoderOptions);
196
197impl WriterBuilder {
198 pub fn new() -> Self {
218 Self::default()
219 }
220
221 pub fn explicit_nulls(&self) -> bool {
223 self.0.explicit_nulls()
224 }
225
226 pub fn with_explicit_nulls(mut self, explicit_nulls: bool) -> Self {
249 self.0 = self.0.with_explicit_nulls(explicit_nulls);
250 self
251 }
252
253 pub fn struct_mode(&self) -> StructMode {
255 self.0.struct_mode()
256 }
257
258 pub fn with_struct_mode(mut self, struct_mode: StructMode) -> Self {
264 self.0 = self.0.with_struct_mode(struct_mode);
265 self
266 }
267
268 pub fn with_encoder_factory(mut self, factory: Arc<dyn EncoderFactory>) -> Self {
273 self.0 = self.0.with_encoder_factory(factory);
274 self
275 }
276
277 pub fn with_date_format(mut self, format: String) -> Self {
279 self.0 = self.0.with_date_format(format);
280 self
281 }
282
283 pub fn with_datetime_format(mut self, format: String) -> Self {
285 self.0 = self.0.with_datetime_format(format);
286 self
287 }
288
289 pub fn with_time_format(mut self, format: String) -> Self {
291 self.0 = self.0.with_time_format(format);
292 self
293 }
294
295 pub fn with_timestamp_format(mut self, format: String) -> Self {
297 self.0 = self.0.with_timestamp_format(format);
298 self
299 }
300
301 pub fn with_timestamp_tz_format(mut self, tz_format: String) -> Self {
303 self.0 = self.0.with_timestamp_tz_format(tz_format);
304 self
305 }
306
307 pub fn build<W, F>(self, writer: W) -> Writer<W, F>
309 where
310 W: Write,
311 F: JsonFormat,
312 {
313 Writer {
314 writer,
315 started: false,
316 finished: false,
317 format: F::default(),
318 options: self.0,
319 }
320 }
321}
322
323#[derive(Debug)]
334pub struct Writer<W, F>
335where
336 W: Write,
337 F: JsonFormat,
338{
339 writer: W,
341
342 started: bool,
344
345 finished: bool,
347
348 format: F,
350
351 options: EncoderOptions,
353}
354
355impl<W, F> Writer<W, F>
356where
357 W: Write,
358 F: JsonFormat,
359{
360 pub fn new(writer: W) -> Self {
362 Self {
363 writer,
364 started: false,
365 finished: false,
366 format: F::default(),
367 options: EncoderOptions::default(),
368 }
369 }
370
371 pub fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
373 if batch.num_rows() == 0 {
374 return Ok(());
375 }
376
377 let mut buffer = Vec::with_capacity(16 * 1024);
380
381 let mut is_first_row = !self.started;
382 if !self.started {
383 self.format.start_stream(&mut buffer)?;
384 self.started = true;
385 }
386
387 let array = StructArray::from(batch.clone());
388 let field = Arc::new(Field::new_struct(
389 "",
390 batch.schema().fields().clone(),
391 false,
392 ));
393
394 let mut encoder = make_encoder(&field, &array, &self.options)?;
395
396 assert!(!encoder.has_nulls(), "root cannot be nullable");
398 for idx in 0..batch.num_rows() {
399 self.format.start_row(&mut buffer, is_first_row)?;
400 is_first_row = false;
401
402 encoder.encode(idx, &mut buffer);
403 if buffer.len() > 8 * 1024 {
404 self.writer.write_all(&buffer)?;
405 buffer.clear();
406 }
407 self.format.end_row(&mut buffer)?;
408 }
409
410 if !buffer.is_empty() {
411 self.writer.write_all(&buffer)?;
412 }
413
414 Ok(())
415 }
416
417 pub fn write_batches(&mut self, batches: &[&RecordBatch]) -> Result<(), ArrowError> {
419 for b in batches {
420 self.write(b)?;
421 }
422 Ok(())
423 }
424
425 pub fn finish(&mut self) -> Result<(), ArrowError> {
429 if !self.started {
430 self.format.start_stream(&mut self.writer)?;
431 self.started = true;
432 }
433 if !self.finished {
434 self.format.end_stream(&mut self.writer)?;
435 self.finished = true;
436 }
437
438 Ok(())
439 }
440
441 pub fn get_ref(&self) -> &W {
443 &self.writer
444 }
445
446 pub fn get_mut(&mut self) -> &mut W {
451 &mut self.writer
452 }
453
454 pub fn into_inner(self) -> W {
456 self.writer
457 }
458}
459
460impl<W, F> RecordBatchWriter for Writer<W, F>
461where
462 W: Write,
463 F: JsonFormat,
464{
465 fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
466 self.write(batch)
467 }
468
469 fn close(mut self) -> Result<(), ArrowError> {
470 self.finish()
471 }
472}
473
474#[cfg(test)]
475mod tests {
476 use core::str;
477 use std::collections::HashMap;
478 use std::fs::{File, read_to_string};
479 use std::io::{BufReader, Seek};
480 use std::sync::Arc;
481
482 use arrow_array::cast::AsArray;
483 use serde_json::{Value, json};
484
485 use super::LineDelimited;
486 use super::{Encoder, WriterBuilder};
487 use arrow_array::builder::*;
488 use arrow_array::types::*;
489 use arrow_buffer::{Buffer, NullBuffer, OffsetBuffer, ScalarBuffer, i256};
490
491 use crate::reader::*;
492
493 use super::*;
494
495 fn assert_json_eq(input: &[u8], expected: &str) {
497 let expected: Vec<Option<Value>> = expected
498 .split('\n')
499 .map(|s| (!s.is_empty()).then(|| serde_json::from_str(s).unwrap()))
500 .collect();
501
502 let actual: Vec<Option<Value>> = input
503 .split(|b| *b == b'\n')
504 .map(|s| (!s.is_empty()).then(|| serde_json::from_slice(s).unwrap()))
505 .collect();
506
507 assert_eq!(actual, expected);
508 }
509
510 #[test]
511 fn write_simple_rows() {
512 let schema = Schema::new(vec![
513 Field::new("c1", DataType::Int32, true),
514 Field::new("c2", DataType::Utf8, true),
515 ]);
516
517 let a = Int32Array::from(vec![Some(1), Some(2), Some(3), None, Some(5)]);
518 let b = StringArray::from(vec![Some("a"), Some("b"), Some("c"), Some("d"), None]);
519
520 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]).unwrap();
521
522 let mut buf = Vec::new();
523 {
524 let mut writer = LineDelimitedWriter::new(&mut buf);
525 writer.write_batches(&[&batch]).unwrap();
526 }
527
528 assert_json_eq(
529 &buf,
530 r#"{"c1":1,"c2":"a"}
531{"c1":2,"c2":"b"}
532{"c1":3,"c2":"c"}
533{"c2":"d"}
534{"c1":5}
535"#,
536 );
537 }
538
539 #[test]
540 fn write_large_utf8_and_utf8_view() {
541 let schema = Schema::new(vec![
542 Field::new("c1", DataType::Utf8, true),
543 Field::new("c2", DataType::LargeUtf8, true),
544 Field::new("c3", DataType::Utf8View, true),
545 ]);
546
547 let a = StringArray::from(vec![Some("a"), None, Some("c"), Some("d"), None]);
548 let b = LargeStringArray::from(vec![Some("a"), Some("b"), None, Some("d"), None]);
549 let c = StringViewArray::from(vec![Some("a"), Some("b"), None, Some("d"), None]);
550
551 let batch = RecordBatch::try_new(
552 Arc::new(schema),
553 vec![Arc::new(a), Arc::new(b), Arc::new(c)],
554 )
555 .unwrap();
556
557 let mut buf = Vec::new();
558 {
559 let mut writer = LineDelimitedWriter::new(&mut buf);
560 writer.write_batches(&[&batch]).unwrap();
561 }
562
563 assert_json_eq(
564 &buf,
565 r#"{"c1":"a","c2":"a","c3":"a"}
566{"c2":"b","c3":"b"}
567{"c1":"c"}
568{"c1":"d","c2":"d","c3":"d"}
569{}
570"#,
571 );
572 }
573
574 #[test]
575 fn write_dictionary() {
576 let schema = Schema::new(vec![
577 Field::new_dictionary("c1", DataType::Int32, DataType::Utf8, true),
578 Field::new_dictionary("c2", DataType::Int8, DataType::Utf8, true),
579 ]);
580
581 let a: DictionaryArray<Int32Type> = vec![
582 Some("cupcakes"),
583 Some("foo"),
584 Some("foo"),
585 None,
586 Some("cupcakes"),
587 ]
588 .into_iter()
589 .collect();
590 let b: DictionaryArray<Int8Type> =
591 vec![Some("sdsd"), Some("sdsd"), None, Some("sd"), Some("sdsd")]
592 .into_iter()
593 .collect();
594
595 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]).unwrap();
596
597 let mut buf = Vec::new();
598 {
599 let mut writer = LineDelimitedWriter::new(&mut buf);
600 writer.write_batches(&[&batch]).unwrap();
601 }
602
603 assert_json_eq(
604 &buf,
605 r#"{"c1":"cupcakes","c2":"sdsd"}
606{"c1":"foo","c2":"sdsd"}
607{"c1":"foo"}
608{"c2":"sd"}
609{"c1":"cupcakes","c2":"sdsd"}
610"#,
611 );
612 }
613
614 #[test]
615 fn write_list_of_dictionary() {
616 let dict_field = Arc::new(Field::new_dictionary(
617 "item",
618 DataType::Int32,
619 DataType::Utf8,
620 true,
621 ));
622 let schema = Schema::new(vec![Field::new_large_list("l", dict_field.clone(), true)]);
623
624 let dict_array: DictionaryArray<Int32Type> =
625 vec![Some("a"), Some("b"), Some("c"), Some("a"), None, Some("c")]
626 .into_iter()
627 .collect();
628 let list_array = LargeListArray::try_new(
629 dict_field,
630 OffsetBuffer::from_lengths([3_usize, 2, 0, 1]),
631 Arc::new(dict_array),
632 Some(NullBuffer::from_iter([true, true, false, true])),
633 )
634 .unwrap();
635
636 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(list_array)]).unwrap();
637
638 let mut buf = Vec::new();
639 {
640 let mut writer = LineDelimitedWriter::new(&mut buf);
641 writer.write_batches(&[&batch]).unwrap();
642 }
643
644 assert_json_eq(
645 &buf,
646 r#"{"l":["a","b","c"]}
647{"l":["a",null]}
648{}
649{"l":["c"]}
650"#,
651 );
652 }
653
654 #[test]
655 fn write_list_of_dictionary_large_values() {
656 let dict_field = Arc::new(Field::new_dictionary(
657 "item",
658 DataType::Int32,
659 DataType::LargeUtf8,
660 true,
661 ));
662 let schema = Schema::new(vec![Field::new_large_list("l", dict_field.clone(), true)]);
663
664 let keys = PrimitiveArray::<Int32Type>::from(vec![
665 Some(0),
666 Some(1),
667 Some(2),
668 Some(0),
669 None,
670 Some(2),
671 ]);
672 let values = LargeStringArray::from(vec!["a", "b", "c"]);
673 let dict_array = DictionaryArray::try_new(keys, Arc::new(values)).unwrap();
674
675 let list_array = LargeListArray::try_new(
676 dict_field,
677 OffsetBuffer::from_lengths([3_usize, 2, 0, 1]),
678 Arc::new(dict_array),
679 Some(NullBuffer::from_iter([true, true, false, true])),
680 )
681 .unwrap();
682
683 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(list_array)]).unwrap();
684
685 let mut buf = Vec::new();
686 {
687 let mut writer = LineDelimitedWriter::new(&mut buf);
688 writer.write_batches(&[&batch]).unwrap();
689 }
690
691 assert_json_eq(
692 &buf,
693 r#"{"l":["a","b","c"]}
694{"l":["a",null]}
695{}
696{"l":["c"]}
697"#,
698 );
699 }
700
701 #[test]
702 fn write_timestamps() {
703 let ts_string = "2018-11-13T17:11:10.011375885995";
704 let ts_nanos = ts_string
705 .parse::<chrono::NaiveDateTime>()
706 .unwrap()
707 .and_utc()
708 .timestamp_nanos_opt()
709 .unwrap();
710 let ts_micros = ts_nanos / 1000;
711 let ts_millis = ts_micros / 1000;
712 let ts_secs = ts_millis / 1000;
713
714 let arr_nanos = TimestampNanosecondArray::from(vec![Some(ts_nanos), None]);
715 let arr_micros = TimestampMicrosecondArray::from(vec![Some(ts_micros), None]);
716 let arr_millis = TimestampMillisecondArray::from(vec![Some(ts_millis), None]);
717 let arr_secs = TimestampSecondArray::from(vec![Some(ts_secs), None]);
718 let arr_names = StringArray::from(vec![Some("a"), Some("b")]);
719
720 let schema = Schema::new(vec![
721 Field::new("nanos", arr_nanos.data_type().clone(), true),
722 Field::new("micros", arr_micros.data_type().clone(), true),
723 Field::new("millis", arr_millis.data_type().clone(), true),
724 Field::new("secs", arr_secs.data_type().clone(), true),
725 Field::new("name", arr_names.data_type().clone(), true),
726 ]);
727 let schema = Arc::new(schema);
728
729 let batch = RecordBatch::try_new(
730 schema,
731 vec![
732 Arc::new(arr_nanos),
733 Arc::new(arr_micros),
734 Arc::new(arr_millis),
735 Arc::new(arr_secs),
736 Arc::new(arr_names),
737 ],
738 )
739 .unwrap();
740
741 let mut buf = Vec::new();
742 {
743 let mut writer = LineDelimitedWriter::new(&mut buf);
744 writer.write_batches(&[&batch]).unwrap();
745 }
746
747 assert_json_eq(
748 &buf,
749 r#"{"micros":"2018-11-13T17:11:10.011375","millis":"2018-11-13T17:11:10.011","name":"a","nanos":"2018-11-13T17:11:10.011375885","secs":"2018-11-13T17:11:10"}
750{"name":"b"}
751"#,
752 );
753
754 let mut buf = Vec::new();
755 {
756 let mut writer = WriterBuilder::new()
757 .with_timestamp_format("%m-%d-%Y".to_string())
758 .build::<_, LineDelimited>(&mut buf);
759 writer.write_batches(&[&batch]).unwrap();
760 }
761
762 assert_json_eq(
763 &buf,
764 r#"{"nanos":"11-13-2018","micros":"11-13-2018","millis":"11-13-2018","secs":"11-13-2018","name":"a"}
765{"name":"b"}
766"#,
767 );
768 }
769
770 #[test]
771 fn write_timestamps_with_tz() {
772 let ts_string = "2018-11-13T17:11:10.011375885995";
773 let ts_nanos = ts_string
774 .parse::<chrono::NaiveDateTime>()
775 .unwrap()
776 .and_utc()
777 .timestamp_nanos_opt()
778 .unwrap();
779 let ts_micros = ts_nanos / 1000;
780 let ts_millis = ts_micros / 1000;
781 let ts_secs = ts_millis / 1000;
782
783 let arr_nanos = TimestampNanosecondArray::from(vec![Some(ts_nanos), None]);
784 let arr_micros = TimestampMicrosecondArray::from(vec![Some(ts_micros), None]);
785 let arr_millis = TimestampMillisecondArray::from(vec![Some(ts_millis), None]);
786 let arr_secs = TimestampSecondArray::from(vec![Some(ts_secs), None]);
787 let arr_names = StringArray::from(vec![Some("a"), Some("b")]);
788
789 let tz = "+00:00";
790
791 let arr_nanos = arr_nanos.with_timezone(tz);
792 let arr_micros = arr_micros.with_timezone(tz);
793 let arr_millis = arr_millis.with_timezone(tz);
794 let arr_secs = arr_secs.with_timezone(tz);
795
796 let schema = Schema::new(vec![
797 Field::new("nanos", arr_nanos.data_type().clone(), true),
798 Field::new("micros", arr_micros.data_type().clone(), true),
799 Field::new("millis", arr_millis.data_type().clone(), true),
800 Field::new("secs", arr_secs.data_type().clone(), true),
801 Field::new("name", arr_names.data_type().clone(), true),
802 ]);
803 let schema = Arc::new(schema);
804
805 let batch = RecordBatch::try_new(
806 schema,
807 vec![
808 Arc::new(arr_nanos),
809 Arc::new(arr_micros),
810 Arc::new(arr_millis),
811 Arc::new(arr_secs),
812 Arc::new(arr_names),
813 ],
814 )
815 .unwrap();
816
817 let mut buf = Vec::new();
818 {
819 let mut writer = LineDelimitedWriter::new(&mut buf);
820 writer.write_batches(&[&batch]).unwrap();
821 }
822
823 assert_json_eq(
824 &buf,
825 r#"{"micros":"2018-11-13T17:11:10.011375Z","millis":"2018-11-13T17:11:10.011Z","name":"a","nanos":"2018-11-13T17:11:10.011375885Z","secs":"2018-11-13T17:11:10Z"}
826{"name":"b"}
827"#,
828 );
829
830 let mut buf = Vec::new();
831 {
832 let mut writer = WriterBuilder::new()
833 .with_timestamp_tz_format("%m-%d-%Y %Z".to_string())
834 .build::<_, LineDelimited>(&mut buf);
835 writer.write_batches(&[&batch]).unwrap();
836 }
837
838 assert_json_eq(
839 &buf,
840 r#"{"nanos":"11-13-2018 +00:00","micros":"11-13-2018 +00:00","millis":"11-13-2018 +00:00","secs":"11-13-2018 +00:00","name":"a"}
841{"name":"b"}
842"#,
843 );
844 }
845
846 #[test]
847 fn write_dates() {
848 let ts_string = "2018-11-13T17:11:10.011375885995";
849 let ts_millis = ts_string
850 .parse::<chrono::NaiveDateTime>()
851 .unwrap()
852 .and_utc()
853 .timestamp_millis();
854
855 let arr_date32 = Date32Array::from(vec![
856 Some(i32::try_from(ts_millis / 1000 / (60 * 60 * 24)).unwrap()),
857 None,
858 ]);
859 let arr_date64 = Date64Array::from(vec![Some(ts_millis), None]);
860 let arr_names = StringArray::from(vec![Some("a"), Some("b")]);
861
862 let schema = Schema::new(vec![
863 Field::new("date32", arr_date32.data_type().clone(), true),
864 Field::new("date64", arr_date64.data_type().clone(), true),
865 Field::new("name", arr_names.data_type().clone(), false),
866 ]);
867 let schema = Arc::new(schema);
868
869 let batch = RecordBatch::try_new(
870 schema,
871 vec![
872 Arc::new(arr_date32),
873 Arc::new(arr_date64),
874 Arc::new(arr_names),
875 ],
876 )
877 .unwrap();
878
879 let mut buf = Vec::new();
880 {
881 let mut writer = LineDelimitedWriter::new(&mut buf);
882 writer.write_batches(&[&batch]).unwrap();
883 }
884
885 assert_json_eq(
886 &buf,
887 r#"{"date32":"2018-11-13","date64":"2018-11-13T17:11:10.011","name":"a"}
888{"name":"b"}
889"#,
890 );
891
892 let mut buf = Vec::new();
893 {
894 let mut writer = WriterBuilder::new()
895 .with_date_format("%m-%d-%Y".to_string())
896 .with_datetime_format("%m-%d-%Y %Mmin %Ssec %Hhour".to_string())
897 .build::<_, LineDelimited>(&mut buf);
898 writer.write_batches(&[&batch]).unwrap();
899 }
900
901 assert_json_eq(
902 &buf,
903 r#"{"date32":"11-13-2018","date64":"11-13-2018 11min 10sec 17hour","name":"a"}
904{"name":"b"}
905"#,
906 );
907 }
908
909 #[test]
910 fn write_times() {
911 let arr_time32sec = Time32SecondArray::from(vec![Some(120), None]);
912 let arr_time32msec = Time32MillisecondArray::from(vec![Some(120), None]);
913 let arr_time64usec = Time64MicrosecondArray::from(vec![Some(120), None]);
914 let arr_time64nsec = Time64NanosecondArray::from(vec![Some(120), None]);
915 let arr_names = StringArray::from(vec![Some("a"), Some("b")]);
916
917 let schema = Schema::new(vec![
918 Field::new("time32sec", arr_time32sec.data_type().clone(), true),
919 Field::new("time32msec", arr_time32msec.data_type().clone(), true),
920 Field::new("time64usec", arr_time64usec.data_type().clone(), true),
921 Field::new("time64nsec", arr_time64nsec.data_type().clone(), true),
922 Field::new("name", arr_names.data_type().clone(), true),
923 ]);
924 let schema = Arc::new(schema);
925
926 let batch = RecordBatch::try_new(
927 schema,
928 vec![
929 Arc::new(arr_time32sec),
930 Arc::new(arr_time32msec),
931 Arc::new(arr_time64usec),
932 Arc::new(arr_time64nsec),
933 Arc::new(arr_names),
934 ],
935 )
936 .unwrap();
937
938 let mut buf = Vec::new();
939 {
940 let mut writer = LineDelimitedWriter::new(&mut buf);
941 writer.write_batches(&[&batch]).unwrap();
942 }
943
944 assert_json_eq(
945 &buf,
946 r#"{"time32sec":"00:02:00","time32msec":"00:00:00.120","time64usec":"00:00:00.000120","time64nsec":"00:00:00.000000120","name":"a"}
947{"name":"b"}
948"#,
949 );
950
951 let mut buf = Vec::new();
952 {
953 let mut writer = WriterBuilder::new()
954 .with_time_format("%H-%M-%S %f".to_string())
955 .build::<_, LineDelimited>(&mut buf);
956 writer.write_batches(&[&batch]).unwrap();
957 }
958
959 assert_json_eq(
960 &buf,
961 r#"{"time32sec":"00-02-00 000000000","time32msec":"00-00-00 120000000","time64usec":"00-00-00 000120000","time64nsec":"00-00-00 000000120","name":"a"}
962{"name":"b"}
963"#,
964 );
965 }
966
967 #[test]
968 fn write_durations() {
969 let arr_durationsec = DurationSecondArray::from(vec![Some(120), None]);
970 let arr_durationmsec = DurationMillisecondArray::from(vec![Some(120), None]);
971 let arr_durationusec = DurationMicrosecondArray::from(vec![Some(120), None]);
972 let arr_durationnsec = DurationNanosecondArray::from(vec![Some(120), None]);
973 let arr_names = StringArray::from(vec![Some("a"), Some("b")]);
974
975 let schema = Schema::new(vec![
976 Field::new("duration_sec", arr_durationsec.data_type().clone(), true),
977 Field::new("duration_msec", arr_durationmsec.data_type().clone(), true),
978 Field::new("duration_usec", arr_durationusec.data_type().clone(), true),
979 Field::new("duration_nsec", arr_durationnsec.data_type().clone(), true),
980 Field::new("name", arr_names.data_type().clone(), true),
981 ]);
982 let schema = Arc::new(schema);
983
984 let batch = RecordBatch::try_new(
985 schema,
986 vec![
987 Arc::new(arr_durationsec),
988 Arc::new(arr_durationmsec),
989 Arc::new(arr_durationusec),
990 Arc::new(arr_durationnsec),
991 Arc::new(arr_names),
992 ],
993 )
994 .unwrap();
995
996 let mut buf = Vec::new();
997 {
998 let mut writer = LineDelimitedWriter::new(&mut buf);
999 writer.write_batches(&[&batch]).unwrap();
1000 }
1001
1002 assert_json_eq(
1003 &buf,
1004 r#"{"duration_sec":"PT120S","duration_msec":"PT0.12S","duration_usec":"PT0.00012S","duration_nsec":"PT0.00000012S","name":"a"}
1005{"name":"b"}
1006"#,
1007 );
1008 }
1009
1010 #[test]
1011 fn write_nested_structs() {
1012 let schema = Schema::new(vec![
1013 Field::new(
1014 "c1",
1015 DataType::Struct(Fields::from(vec![
1016 Field::new("c11", DataType::Int32, true),
1017 Field::new(
1018 "c12",
1019 DataType::Struct(vec![Field::new("c121", DataType::Utf8, false)].into()),
1020 false,
1021 ),
1022 ])),
1023 false,
1024 ),
1025 Field::new("c2", DataType::Utf8, false),
1026 ]);
1027
1028 let c1 = StructArray::from(vec![
1029 (
1030 Arc::new(Field::new("c11", DataType::Int32, true)),
1031 Arc::new(Int32Array::from(vec![Some(1), None, Some(5)])) as ArrayRef,
1032 ),
1033 (
1034 Arc::new(Field::new(
1035 "c12",
1036 DataType::Struct(vec![Field::new("c121", DataType::Utf8, false)].into()),
1037 false,
1038 )),
1039 Arc::new(StructArray::from(vec![(
1040 Arc::new(Field::new("c121", DataType::Utf8, false)),
1041 Arc::new(StringArray::from(vec![Some("e"), Some("f"), Some("g")])) as ArrayRef,
1042 )])) as ArrayRef,
1043 ),
1044 ]);
1045 let c2 = StringArray::from(vec![Some("a"), Some("b"), Some("c")]);
1046
1047 let batch =
1048 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c1), Arc::new(c2)]).unwrap();
1049
1050 let mut buf = Vec::new();
1051 {
1052 let mut writer = LineDelimitedWriter::new(&mut buf);
1053 writer.write_batches(&[&batch]).unwrap();
1054 }
1055
1056 assert_json_eq(
1057 &buf,
1058 r#"{"c1":{"c11":1,"c12":{"c121":"e"}},"c2":"a"}
1059{"c1":{"c12":{"c121":"f"}},"c2":"b"}
1060{"c1":{"c11":5,"c12":{"c121":"g"}},"c2":"c"}
1061"#,
1062 );
1063 }
1064
1065 #[test]
1066 fn write_struct_with_list_field() {
1067 let field_c_list = Arc::new(Field::new("c_list", DataType::Utf8, false));
1068 let field_c1 = Field::new("c1", DataType::List(field_c_list.clone()), false);
1069 let field_c2 = Field::new("c2", DataType::Int32, false);
1070 let schema = Schema::new(vec![field_c1.clone(), field_c2]);
1071
1072 let a_values = StringArray::from(vec!["a", "a1", "b", "c", "d", "e"]);
1073 let a = ListArray::new(
1075 field_c_list,
1076 OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 2, 3, 4, 5, 6])),
1077 Arc::new(a_values),
1078 None,
1079 );
1080
1081 let b = Int32Array::from(vec![1, 2, 3, 4, 5]);
1082
1083 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]).unwrap();
1084
1085 let mut buf = Vec::new();
1086 {
1087 let mut writer = LineDelimitedWriter::new(&mut buf);
1088 writer.write_batches(&[&batch]).unwrap();
1089 }
1090
1091 assert_json_eq(
1092 &buf,
1093 r#"{"c1":["a","a1"],"c2":1}
1094{"c1":["b"],"c2":2}
1095{"c1":["c"],"c2":3}
1096{"c1":["d"],"c2":4}
1097{"c1":["e"],"c2":5}
1098"#,
1099 );
1100 }
1101
1102 #[test]
1103 fn write_nested_list() {
1104 let field_b = Arc::new(Field::new("b", DataType::Int32, false));
1105 let field_a = Arc::new(Field::new("a", DataType::List(field_b.clone()), false));
1106 let field_c1 = Field::new("c1", DataType::List(field_a.clone()), false);
1107 let field_c2 = Field::new("c2", DataType::Utf8, true);
1108 let schema = Schema::new(vec![field_c1.clone(), field_c2]);
1109
1110 let a_values = Int32Array::from(vec![1, 2, 3, 4, 5, 6]);
1112
1113 let a_list = ListArray::new(
1114 field_b,
1115 OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 2, 3, 6])),
1116 Arc::new(a_values),
1117 None,
1118 );
1119
1120 let c1 = ListArray::new(
1121 field_a,
1122 OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 2, 2, 3])),
1123 Arc::new(a_list),
1124 None,
1125 );
1126 let c2 = StringArray::from(vec![Some("foo"), Some("bar"), None]);
1127
1128 let batch =
1129 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c1), Arc::new(c2)]).unwrap();
1130
1131 let mut buf = Vec::new();
1132 {
1133 let mut writer = LineDelimitedWriter::new(&mut buf);
1134 writer.write_batches(&[&batch]).unwrap();
1135 }
1136
1137 assert_json_eq(
1138 &buf,
1139 r#"{"c1":[[1,2],[3]],"c2":"foo"}
1140{"c1":[],"c2":"bar"}
1141{"c1":[[4,5,6]]}
1142"#,
1143 );
1144 }
1145
1146 #[test]
1147 fn write_list_of_struct() {
1148 let field_c1 = Field::new(
1149 "c1",
1150 DataType::List(Arc::new(Field::new(
1151 "s",
1152 DataType::Struct(Fields::from(vec![
1153 Field::new("c11", DataType::Int32, true),
1154 Field::new(
1155 "c12",
1156 DataType::Struct(vec![Field::new("c121", DataType::Utf8, false)].into()),
1157 false,
1158 ),
1159 ])),
1160 false,
1161 ))),
1162 true,
1163 );
1164 let field_c2 = Field::new("c2", DataType::Int32, false);
1165 let schema = Schema::new(vec![field_c1.clone(), field_c2]);
1166
1167 let struct_values = StructArray::from(vec![
1168 (
1169 Arc::new(Field::new("c11", DataType::Int32, true)),
1170 Arc::new(Int32Array::from(vec![Some(1), None, Some(5)])) as ArrayRef,
1171 ),
1172 (
1173 Arc::new(Field::new(
1174 "c12",
1175 DataType::Struct(vec![Field::new("c121", DataType::Utf8, false)].into()),
1176 false,
1177 )),
1178 Arc::new(StructArray::from(vec![(
1179 Arc::new(Field::new("c121", DataType::Utf8, false)),
1180 Arc::new(StringArray::from(vec![Some("e"), Some("f"), Some("g")])) as ArrayRef,
1181 )])) as ArrayRef,
1182 ),
1183 ]);
1184
1185 let c1_inner = match field_c1.data_type() {
1190 DataType::List(f) => f.clone(),
1191 _ => unreachable!(),
1192 };
1193 let c1 = ListArray::new(
1194 c1_inner,
1195 OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 2, 2, 3])),
1196 Arc::new(struct_values),
1197 Some(NullBuffer::from(vec![true, false, true])),
1198 );
1199
1200 let c2 = Int32Array::from(vec![1, 2, 3]);
1201
1202 let batch =
1203 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c1), Arc::new(c2)]).unwrap();
1204
1205 let mut buf = Vec::new();
1206 {
1207 let mut writer = LineDelimitedWriter::new(&mut buf);
1208 writer.write_batches(&[&batch]).unwrap();
1209 }
1210
1211 assert_json_eq(
1212 &buf,
1213 r#"{"c1":[{"c11":1,"c12":{"c121":"e"}},{"c12":{"c121":"f"}}],"c2":1}
1214{"c2":2}
1215{"c1":[{"c11":5,"c12":{"c121":"g"}}],"c2":3}
1216"#,
1217 );
1218 }
1219
1220 fn assert_write_list_view<O: OffsetSizeTrait>() {
1221 let field = Arc::new(Field::new("item", DataType::Int32, true));
1222 let data_type = GenericListViewArray::<O>::DATA_TYPE_CONSTRUCTOR(field.clone());
1223 let schema = Schema::new(vec![Field::new("lv", data_type, true)]);
1224
1225 let values = Int32Array::from(vec![Some(1), Some(2), Some(3), Some(4), None, Some(6)]);
1227 let offsets = [0, 3, 0, 5]
1228 .iter()
1229 .map(|&v| O::from_usize(v).unwrap())
1230 .collect::<Vec<_>>();
1231 let sizes = [3, 2, 0, 1]
1232 .iter()
1233 .map(|&v| O::from_usize(v).unwrap())
1234 .collect::<Vec<_>>();
1235 let list_view = GenericListViewArray::<O>::try_new(
1236 field,
1237 ScalarBuffer::from(offsets),
1238 ScalarBuffer::from(sizes),
1239 Arc::new(values),
1240 Some(NullBuffer::from_iter([true, true, false, true])),
1241 )
1242 .unwrap();
1243
1244 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(list_view)]).unwrap();
1245
1246 let mut buf = Vec::new();
1247 {
1248 let mut writer = LineDelimitedWriter::new(&mut buf);
1249 writer.write_batches(&[&batch]).unwrap();
1250 }
1251
1252 assert_json_eq(
1253 &buf,
1254 r#"{"lv":[1,2,3]}
1255{"lv":[4,null]}
1256{}
1257{"lv":[6]}
1258"#,
1259 );
1260 }
1261
1262 #[test]
1263 fn write_list_view() {
1264 assert_write_list_view::<i32>();
1265 assert_write_list_view::<i64>();
1266 }
1267
1268 fn test_write_for_file(test_file: &str, remove_nulls: bool) {
1269 let file = File::open(test_file).unwrap();
1270 let mut reader = BufReader::new(file);
1271 let (schema, _) = infer_json_schema(&mut reader, None).unwrap();
1272 reader.rewind().unwrap();
1273
1274 let builder = ReaderBuilder::new(Arc::new(schema)).with_batch_size(1024);
1275 let mut reader = builder.build(reader).unwrap();
1276 let batch = reader.next().unwrap().unwrap();
1277
1278 let mut buf = Vec::new();
1279 {
1280 if remove_nulls {
1281 let mut writer = LineDelimitedWriter::new(&mut buf);
1282 writer.write_batches(&[&batch]).unwrap();
1283 } else {
1284 let mut writer = WriterBuilder::new()
1285 .with_explicit_nulls(true)
1286 .build::<_, LineDelimited>(&mut buf);
1287 writer.write_batches(&[&batch]).unwrap();
1288 }
1289 }
1290
1291 let result = str::from_utf8(&buf).unwrap();
1292 let expected = read_to_string(test_file).unwrap();
1293 for (r, e) in result.lines().zip(expected.lines()) {
1294 let mut expected_json = serde_json::from_str::<Value>(e).unwrap();
1295 if remove_nulls {
1296 if let Value::Object(obj) = expected_json {
1298 expected_json =
1299 Value::Object(obj.into_iter().filter(|(_, v)| *v != Value::Null).collect());
1300 }
1301 }
1302 assert_eq!(serde_json::from_str::<Value>(r).unwrap(), expected_json,);
1303 }
1304 }
1305
1306 #[test]
1307 fn write_basic_rows() {
1308 test_write_for_file("test/data/basic.json", true);
1309 }
1310
1311 #[test]
1312 fn write_arrays() {
1313 test_write_for_file("test/data/arrays.json", true);
1314 }
1315
1316 #[test]
1317 fn write_basic_nulls() {
1318 test_write_for_file("test/data/basic_nulls.json", true);
1319 }
1320
1321 #[test]
1322 fn write_nested_with_nulls() {
1323 test_write_for_file("test/data/nested_with_nulls.json", false);
1324 }
1325
1326 #[test]
1327 fn json_line_writer_empty() {
1328 let mut writer = LineDelimitedWriter::new(vec![] as Vec<u8>);
1329 writer.finish().unwrap();
1330 assert_eq!(str::from_utf8(&writer.into_inner()).unwrap(), "");
1331 }
1332
1333 #[test]
1334 fn json_array_writer_empty() {
1335 let mut writer = ArrayWriter::new(vec![] as Vec<u8>);
1336 writer.finish().unwrap();
1337 assert_eq!(str::from_utf8(&writer.into_inner()).unwrap(), "[]");
1338 }
1339
1340 #[test]
1341 fn json_line_writer_empty_batch() {
1342 let mut writer = LineDelimitedWriter::new(vec![] as Vec<u8>);
1343
1344 let array = Int32Array::from(Vec::<i32>::new());
1345 let schema = Schema::new(vec![Field::new("c", DataType::Int32, true)]);
1346 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap();
1347
1348 writer.write(&batch).unwrap();
1349 writer.finish().unwrap();
1350 assert_eq!(str::from_utf8(&writer.into_inner()).unwrap(), "");
1351 }
1352
1353 #[test]
1354 fn json_array_writer_empty_batch() {
1355 let mut writer = ArrayWriter::new(vec![] as Vec<u8>);
1356
1357 let array = Int32Array::from(Vec::<i32>::new());
1358 let schema = Schema::new(vec![Field::new("c", DataType::Int32, true)]);
1359 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap();
1360
1361 writer.write(&batch).unwrap();
1362 writer.finish().unwrap();
1363 assert_eq!(str::from_utf8(&writer.into_inner()).unwrap(), "[]");
1364 }
1365
1366 #[test]
1367 fn json_struct_array_nulls() {
1368 let inner = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
1369 Some(vec![Some(1), Some(2)]),
1370 Some(vec![None]),
1371 Some(vec![]),
1372 Some(vec![Some(3), None]), Some(vec![Some(4), Some(5)]),
1374 None, None,
1376 ]);
1377
1378 let field = Arc::new(Field::new("list", inner.data_type().clone(), true));
1379 let array = Arc::new(inner) as ArrayRef;
1380 let struct_array_a = StructArray::from((
1381 vec![(field.clone(), array.clone())],
1382 Buffer::from([0b01010111]),
1383 ));
1384 let struct_array_b = StructArray::from(vec![(field, array)]);
1385
1386 let schema = Schema::new(vec![
1387 Field::new_struct("a", struct_array_a.fields().clone(), true),
1388 Field::new_struct("b", struct_array_b.fields().clone(), true),
1389 ]);
1390
1391 let batch = RecordBatch::try_new(
1392 Arc::new(schema),
1393 vec![Arc::new(struct_array_a), Arc::new(struct_array_b)],
1394 )
1395 .unwrap();
1396
1397 let mut buf = Vec::new();
1398 {
1399 let mut writer = LineDelimitedWriter::new(&mut buf);
1400 writer.write_batches(&[&batch]).unwrap();
1401 }
1402
1403 assert_json_eq(
1404 &buf,
1405 r#"{"a":{"list":[1,2]},"b":{"list":[1,2]}}
1406{"a":{"list":[null]},"b":{"list":[null]}}
1407{"a":{"list":[]},"b":{"list":[]}}
1408{"b":{"list":[3,null]}}
1409{"a":{"list":[4,5]},"b":{"list":[4,5]}}
1410{"b":{}}
1411{"a":{},"b":{}}
1412"#,
1413 );
1414 }
1415
1416 fn run_json_writer_map_with_keys(keys_array: ArrayRef) {
1417 let values_array = super::Int64Array::from(vec![10, 20, 30, 40, 50]);
1418
1419 let keys_field = Arc::new(Field::new(
1420 Field::MAP_KEY_FIELD_DEFAULT_NAME,
1421 keys_array.data_type().clone(),
1422 false,
1423 ));
1424 let values_field = Arc::new(Field::new(
1425 Field::MAP_VALUE_FIELD_DEFAULT_NAME,
1426 DataType::Int64,
1427 false,
1428 ));
1429 let entry_struct = StructArray::from(vec![
1430 (keys_field, keys_array.clone()),
1431 (values_field, Arc::new(values_array) as ArrayRef),
1432 ]);
1433
1434 let entries_field = Arc::new(Field::new(
1435 Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
1436 entry_struct.data_type().clone(),
1437 false,
1438 ));
1439
1440 let map = MapArray::new(
1442 entries_field.clone(),
1443 OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 1, 1, 1, 4, 5, 5])),
1444 entry_struct,
1445 Some(NullBuffer::from(vec![true, false, true, true, true, true])),
1446 false,
1447 );
1448
1449 let map_field = Field::new("map", DataType::Map(entries_field, false), true);
1450 let schema = Arc::new(Schema::new(vec![map_field]));
1451
1452 let batch = RecordBatch::try_new(schema, vec![Arc::new(map)]).unwrap();
1453
1454 let mut buf = Vec::new();
1455 {
1456 let mut writer = LineDelimitedWriter::new(&mut buf);
1457 writer.write_batches(&[&batch]).unwrap();
1458 }
1459
1460 assert_json_eq(
1461 &buf,
1462 r#"{"map":{"foo":10}}
1463{}
1464{"map":{}}
1465{"map":{"bar":20,"baz":30,"qux":40}}
1466{"map":{"quux":50}}
1467{"map":{}}
1468"#,
1469 );
1470 }
1471
1472 #[test]
1473 fn json_writer_map() {
1474 let keys_utf8 = super::StringArray::from(vec!["foo", "bar", "baz", "qux", "quux"]);
1476 run_json_writer_map_with_keys(Arc::new(keys_utf8) as ArrayRef);
1477
1478 let keys_large = super::LargeStringArray::from(vec!["foo", "bar", "baz", "qux", "quux"]);
1480 run_json_writer_map_with_keys(Arc::new(keys_large) as ArrayRef);
1481
1482 let keys_view = super::StringViewArray::from(vec!["foo", "bar", "baz", "qux", "quux"]);
1484 run_json_writer_map_with_keys(Arc::new(keys_view) as ArrayRef);
1485 }
1486
1487 #[test]
1488 fn test_write_single_batch() {
1489 let test_file = "test/data/basic.json";
1490 let file = File::open(test_file).unwrap();
1491 let mut reader = BufReader::new(file);
1492 let (schema, _) = infer_json_schema(&mut reader, None).unwrap();
1493 reader.rewind().unwrap();
1494
1495 let builder = ReaderBuilder::new(Arc::new(schema)).with_batch_size(1024);
1496 let mut reader = builder.build(reader).unwrap();
1497 let batch = reader.next().unwrap().unwrap();
1498
1499 let mut buf = Vec::new();
1500 {
1501 let mut writer = LineDelimitedWriter::new(&mut buf);
1502 writer.write(&batch).unwrap();
1503 }
1504
1505 let result = str::from_utf8(&buf).unwrap();
1506 let expected = read_to_string(test_file).unwrap();
1507 for (r, e) in result.lines().zip(expected.lines()) {
1508 let mut expected_json = serde_json::from_str::<Value>(e).unwrap();
1509 if let Value::Object(obj) = expected_json {
1511 expected_json =
1512 Value::Object(obj.into_iter().filter(|(_, v)| *v != Value::Null).collect());
1513 }
1514 assert_eq!(serde_json::from_str::<Value>(r).unwrap(), expected_json,);
1515 }
1516 }
1517
1518 #[test]
1519 #[cfg_attr(miri, ignore)] fn test_write_multi_batches() {
1521 let test_file = "test/data/basic.json";
1522
1523 let schema = SchemaRef::new(Schema::new(vec![
1524 Field::new("a", DataType::Int64, true),
1525 Field::new("b", DataType::Float64, true),
1526 Field::new("c", DataType::Boolean, true),
1527 Field::new("d", DataType::Utf8, true),
1528 Field::new("e", DataType::Utf8, true),
1529 Field::new("f", DataType::Utf8, true),
1530 Field::new("g", DataType::Timestamp(TimeUnit::Millisecond, None), true),
1531 Field::new("h", DataType::Float16, true),
1532 ]));
1533
1534 let mut reader = ReaderBuilder::new(schema.clone())
1535 .build(BufReader::new(File::open(test_file).unwrap()))
1536 .unwrap();
1537 let batch = reader.next().unwrap().unwrap();
1538
1539 let batches = [&RecordBatch::new_empty(schema), &batch, &batch];
1541
1542 let mut buf = Vec::new();
1543 {
1544 let mut writer = LineDelimitedWriter::new(&mut buf);
1545 writer.write_batches(&batches).unwrap();
1546 }
1547
1548 let result = str::from_utf8(&buf).unwrap();
1549 let expected = read_to_string(test_file).unwrap();
1550 let expected = format!("{expected}\n{expected}");
1552 for (r, e) in result.lines().zip(expected.lines()) {
1553 let mut expected_json = serde_json::from_str::<Value>(e).unwrap();
1554 if let Value::Object(obj) = expected_json {
1556 expected_json =
1557 Value::Object(obj.into_iter().filter(|(_, v)| *v != Value::Null).collect());
1558 }
1559 assert_eq!(serde_json::from_str::<Value>(r).unwrap(), expected_json,);
1560 }
1561 }
1562
1563 #[test]
1564 fn test_writer_explicit_nulls() -> Result<(), ArrowError> {
1565 fn nested_list() -> (Arc<ListArray>, Arc<Field>) {
1566 let array = Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
1567 Some(vec![None, None, None]),
1568 Some(vec![Some(1), Some(2), Some(3)]),
1569 None,
1570 Some(vec![None, None, None]),
1571 ]));
1572 let field = Arc::new(Field::new("list", array.data_type().clone(), true));
1573 (array, field)
1575 }
1576
1577 fn nested_dict() -> (Arc<DictionaryArray<Int32Type>>, Arc<Field>) {
1578 let array = Arc::new(DictionaryArray::from_iter(vec![
1579 Some("cupcakes"),
1580 None,
1581 Some("bear"),
1582 Some("kuma"),
1583 ]));
1584 let field = Arc::new(Field::new("dict", array.data_type().clone(), true));
1585 (array, field)
1587 }
1588
1589 fn nested_map() -> (Arc<MapArray>, Arc<Field>) {
1590 let string_builder = StringBuilder::new();
1591 let int_builder = Int64Builder::new();
1592 let mut builder = MapBuilder::new(None, string_builder, int_builder);
1593
1594 builder.keys().append_value("foo");
1596 builder.values().append_value(10);
1597 builder.append(true).unwrap();
1598
1599 builder.append(false).unwrap();
1600
1601 builder.append(true).unwrap();
1602
1603 builder.keys().append_value("bar");
1604 builder.values().append_value(20);
1605 builder.keys().append_value("baz");
1606 builder.values().append_value(30);
1607 builder.keys().append_value("qux");
1608 builder.values().append_value(40);
1609 builder.append(true).unwrap();
1610
1611 let array = Arc::new(builder.finish());
1612 let field = Arc::new(Field::new("map", array.data_type().clone(), true));
1613 (array, field)
1614 }
1615
1616 fn root_list() -> (Arc<ListArray>, Field) {
1617 let struct_array = StructArray::from(vec![
1618 (
1619 Arc::new(Field::new("utf8", DataType::Utf8, true)),
1620 Arc::new(StringArray::from(vec![Some("a"), Some("b"), None, None])) as ArrayRef,
1621 ),
1622 (
1623 Arc::new(Field::new("int32", DataType::Int32, true)),
1624 Arc::new(Int32Array::from(vec![Some(1), None, Some(5), None])) as ArrayRef,
1625 ),
1626 ]);
1627
1628 let values_field =
1629 Arc::new(Field::new("struct", struct_array.data_type().clone(), true));
1630 let field = Field::new_list("list", values_field.as_ref().clone(), true);
1631
1632 let array = Arc::new(ListArray::new(
1634 values_field,
1635 OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 2, 2, 3, 3])),
1636 Arc::new(struct_array),
1637 Some(NullBuffer::from(vec![true, false, true, false])),
1638 ));
1639 (array, field)
1640 }
1641
1642 let (nested_list_array, nested_list_field) = nested_list();
1643 let (nested_dict_array, nested_dict_field) = nested_dict();
1644 let (nested_map_array, nested_map_field) = nested_map();
1645 let (root_list_array, root_list_field) = root_list();
1646
1647 let schema = Schema::new(vec![
1648 Field::new("date", DataType::Date32, true),
1649 Field::new("null", DataType::Null, true),
1650 Field::new_struct(
1651 "struct",
1652 vec![
1653 Arc::new(Field::new("utf8", DataType::Utf8, true)),
1654 nested_list_field.clone(),
1655 nested_dict_field.clone(),
1656 nested_map_field.clone(),
1657 ],
1658 true,
1659 ),
1660 root_list_field,
1661 ]);
1662
1663 let arr_date32 = Date32Array::from(vec![Some(0), None, Some(1), None]);
1664 let arr_null = NullArray::new(4);
1665 let arr_struct = StructArray::from(vec![
1666 (
1668 Arc::new(Field::new("utf8", DataType::Utf8, true)),
1669 Arc::new(StringArray::from(vec![Some("a"), None, None, Some("b")])) as ArrayRef,
1670 ),
1671 (nested_list_field, nested_list_array as ArrayRef),
1673 (nested_dict_field, nested_dict_array as ArrayRef),
1675 (nested_map_field, nested_map_array as ArrayRef),
1677 ]);
1678
1679 let batch = RecordBatch::try_new(
1680 Arc::new(schema),
1681 vec![
1682 Arc::new(arr_date32),
1684 Arc::new(arr_null),
1686 Arc::new(arr_struct),
1687 root_list_array,
1689 ],
1690 )?;
1691
1692 let mut buf = Vec::new();
1693 {
1694 let mut writer = WriterBuilder::new()
1695 .with_explicit_nulls(true)
1696 .build::<_, JsonArray>(&mut buf);
1697 writer.write_batches(&[&batch])?;
1698 writer.finish()?;
1699 }
1700
1701 let actual = serde_json::from_slice::<Vec<Value>>(&buf).unwrap();
1702 let expected = serde_json::from_value::<Vec<Value>>(json!([
1703 {
1704 "date": "1970-01-01",
1705 "list": [
1706 {
1707 "int32": 1,
1708 "utf8": "a"
1709 },
1710 {
1711 "int32": null,
1712 "utf8": "b"
1713 }
1714 ],
1715 "null": null,
1716 "struct": {
1717 "dict": "cupcakes",
1718 "list": [
1719 null,
1720 null,
1721 null
1722 ],
1723 "map": {
1724 "foo": 10
1725 },
1726 "utf8": "a"
1727 }
1728 },
1729 {
1730 "date": null,
1731 "list": null,
1732 "null": null,
1733 "struct": {
1734 "dict": null,
1735 "list": [
1736 1,
1737 2,
1738 3
1739 ],
1740 "map": null,
1741 "utf8": null
1742 }
1743 },
1744 {
1745 "date": "1970-01-02",
1746 "list": [
1747 {
1748 "int32": 5,
1749 "utf8": null
1750 }
1751 ],
1752 "null": null,
1753 "struct": {
1754 "dict": "bear",
1755 "list": null,
1756 "map": {},
1757 "utf8": null
1758 }
1759 },
1760 {
1761 "date": null,
1762 "list": null,
1763 "null": null,
1764 "struct": {
1765 "dict": "kuma",
1766 "list": [
1767 null,
1768 null,
1769 null
1770 ],
1771 "map": {
1772 "bar": 20,
1773 "baz": 30,
1774 "qux": 40
1775 },
1776 "utf8": "b"
1777 }
1778 }
1779 ]))
1780 .unwrap();
1781
1782 assert_eq!(actual, expected);
1783
1784 Ok(())
1785 }
1786
1787 fn build_array_binary<O: OffsetSizeTrait>(values: &[Option<&[u8]>]) -> RecordBatch {
1788 let schema = SchemaRef::new(Schema::new(vec![Field::new(
1789 "bytes",
1790 GenericBinaryType::<O>::DATA_TYPE,
1791 true,
1792 )]));
1793 let mut builder = GenericByteBuilder::<GenericBinaryType<O>>::new();
1794 for value in values {
1795 match value {
1796 Some(v) => builder.append_value(v),
1797 None => builder.append_null(),
1798 }
1799 }
1800 let array = Arc::new(builder.finish()) as ArrayRef;
1801 RecordBatch::try_new(schema, vec![array]).unwrap()
1802 }
1803
1804 fn build_array_binary_view(values: &[Option<&[u8]>]) -> RecordBatch {
1805 let schema = SchemaRef::new(Schema::new(vec![Field::new(
1806 "bytes",
1807 DataType::BinaryView,
1808 true,
1809 )]));
1810 let mut builder = BinaryViewBuilder::new();
1811 for value in values {
1812 match value {
1813 Some(v) => builder.append_value(v),
1814 None => builder.append_null(),
1815 }
1816 }
1817 let array = Arc::new(builder.finish()) as ArrayRef;
1818 RecordBatch::try_new(schema, vec![array]).unwrap()
1819 }
1820
1821 fn assert_binary_json(batch: &RecordBatch) {
1822 {
1824 let mut buf = Vec::new();
1825 let json_value: Value = {
1826 let mut writer = WriterBuilder::new()
1827 .with_explicit_nulls(true)
1828 .build::<_, JsonArray>(&mut buf);
1829 writer.write(batch).unwrap();
1830 writer.close().unwrap();
1831 serde_json::from_slice(&buf).unwrap()
1832 };
1833
1834 assert_eq!(
1835 json!([
1836 {
1837 "bytes": "426f622054686f6d70736f6e"
1838 },
1839 {
1840 "bytes": null },
1842 {
1843 "bytes": "54726f79204d63436c757265"
1844 }
1845 ]),
1846 json_value,
1847 );
1848 }
1849
1850 {
1852 let mut buf = Vec::new();
1853 let json_value: Value = {
1854 let mut writer = ArrayWriter::new(&mut buf);
1857 writer.write(batch).unwrap();
1858 writer.close().unwrap();
1859 serde_json::from_slice(&buf).unwrap()
1860 };
1861
1862 assert_eq!(
1863 json!([
1864 { "bytes": "426f622054686f6d70736f6e" },
1865 {},
1866 { "bytes": "54726f79204d63436c757265" }
1867 ]),
1868 json_value
1869 );
1870 }
1871 }
1872
1873 #[test]
1874 fn test_writer_binary() {
1875 let values: [Option<&[u8]>; 3] = [
1876 Some(b"Bob Thompson" as &[u8]),
1877 None,
1878 Some(b"Troy McClure" as &[u8]),
1879 ];
1880 {
1882 let batch = build_array_binary::<i32>(&values);
1883 assert_binary_json(&batch);
1884 }
1885 {
1887 let batch = build_array_binary::<i64>(&values);
1888 assert_binary_json(&batch);
1889 }
1890 {
1891 let batch = build_array_binary_view(&values);
1892 assert_binary_json(&batch);
1893 }
1894 }
1895
1896 #[test]
1897 fn test_writer_fixed_size_binary() {
1898 let size = 11;
1900 let schema = SchemaRef::new(Schema::new(vec![Field::new(
1901 "bytes",
1902 DataType::FixedSizeBinary(size),
1903 true,
1904 )]));
1905
1906 let mut builder = FixedSizeBinaryBuilder::new(size);
1908 let values = [Some(b"hello world"), None, Some(b"summer rain")];
1909 for value in values {
1910 match value {
1911 Some(v) => builder.append_value(v).unwrap(),
1912 None => builder.append_null(),
1913 }
1914 }
1915 let array = Arc::new(builder.finish()) as ArrayRef;
1916 let batch = RecordBatch::try_new(schema, vec![array]).unwrap();
1917
1918 {
1920 let mut buf = Vec::new();
1921 let json_value: Value = {
1922 let mut writer = WriterBuilder::new()
1923 .with_explicit_nulls(true)
1924 .build::<_, JsonArray>(&mut buf);
1925 writer.write(&batch).unwrap();
1926 writer.close().unwrap();
1927 serde_json::from_slice(&buf).unwrap()
1928 };
1929
1930 assert_eq!(
1931 json!([
1932 {
1933 "bytes": "68656c6c6f20776f726c64"
1934 },
1935 {
1936 "bytes": null },
1938 {
1939 "bytes": "73756d6d6572207261696e"
1940 }
1941 ]),
1942 json_value,
1943 );
1944 }
1945 {
1947 let mut buf = Vec::new();
1948 let json_value: Value = {
1949 let mut writer = ArrayWriter::new(&mut buf);
1952 writer.write(&batch).unwrap();
1953 writer.close().unwrap();
1954 serde_json::from_slice(&buf).unwrap()
1955 };
1956
1957 assert_eq!(
1958 json!([
1959 {
1960 "bytes": "68656c6c6f20776f726c64"
1961 },
1962 {}, {
1964 "bytes": "73756d6d6572207261696e"
1965 }
1966 ]),
1967 json_value,
1968 );
1969 }
1970 }
1971
1972 #[test]
1973 fn test_writer_fixed_size_list() {
1974 let size = 3;
1975 let field = FieldRef::new(Field::new_list_field(DataType::Int32, true));
1976 let schema = SchemaRef::new(Schema::new(vec![Field::new(
1977 "list",
1978 DataType::FixedSizeList(field, size),
1979 true,
1980 )]));
1981
1982 let values_builder = Int32Builder::new();
1983 let mut list_builder = FixedSizeListBuilder::new(values_builder, size);
1984 let lists = [
1985 Some([Some(1), Some(2), None]),
1986 Some([Some(3), None, Some(4)]),
1987 Some([None, Some(5), Some(6)]),
1988 None,
1989 ];
1990 for list in lists {
1991 match list {
1992 Some(l) => {
1993 for value in l {
1994 match value {
1995 Some(v) => list_builder.values().append_value(v),
1996 None => list_builder.values().append_null(),
1997 }
1998 }
1999 list_builder.append(true);
2000 }
2001 None => {
2002 for _ in 0..size {
2003 list_builder.values().append_null();
2004 }
2005 list_builder.append(false);
2006 }
2007 }
2008 }
2009 let array = Arc::new(list_builder.finish()) as ArrayRef;
2010 let batch = RecordBatch::try_new(schema, vec![array]).unwrap();
2011
2012 {
2014 let json_value: Value = {
2015 let mut buf = Vec::new();
2016 let mut writer = WriterBuilder::new()
2017 .with_explicit_nulls(true)
2018 .build::<_, JsonArray>(&mut buf);
2019 writer.write(&batch).unwrap();
2020 writer.close().unwrap();
2021 serde_json::from_slice(&buf).unwrap()
2022 };
2023 assert_eq!(
2024 json!([
2025 {"list": [1, 2, null]},
2026 {"list": [3, null, 4]},
2027 {"list": [null, 5, 6]},
2028 {"list": null},
2029 ]),
2030 json_value
2031 );
2032 }
2033 {
2035 let json_value: Value = {
2036 let mut buf = Vec::new();
2037 let mut writer = ArrayWriter::new(&mut buf);
2038 writer.write(&batch).unwrap();
2039 writer.close().unwrap();
2040 serde_json::from_slice(&buf).unwrap()
2041 };
2042 assert_eq!(
2043 json!([
2044 {"list": [1, 2, null]},
2045 {"list": [3, null, 4]},
2046 {"list": [null, 5, 6]},
2047 {}, ]),
2049 json_value
2050 );
2051 }
2052 }
2053
2054 #[test]
2055 fn test_writer_null_dict() {
2056 let keys = Int32Array::from_iter(vec![Some(0), None, Some(1)]);
2057 let values = Arc::new(StringArray::from_iter(vec![Some("a"), None]));
2058 let dict = DictionaryArray::new(keys, values);
2059
2060 let schema = SchemaRef::new(Schema::new(vec![Field::new(
2061 "my_dict",
2062 DataType::Dictionary(DataType::Int32.into(), DataType::Utf8.into()),
2063 true,
2064 )]));
2065
2066 let array = Arc::new(dict) as ArrayRef;
2067 let batch = RecordBatch::try_new(schema, vec![array]).unwrap();
2068
2069 let mut json = Vec::new();
2070 let write_builder = WriterBuilder::new().with_explicit_nulls(true);
2071 let mut writer = write_builder.build::<_, JsonArray>(&mut json);
2072 writer.write(&batch).unwrap();
2073 writer.close().unwrap();
2074
2075 let json_str = str::from_utf8(&json).unwrap();
2076 assert_eq!(
2077 json_str,
2078 r#"[{"my_dict":"a"},{"my_dict":null},{"my_dict":null}]"#
2079 )
2080 }
2081
2082 #[test]
2083 fn test_decimal32_encoder() {
2084 let array = Decimal32Array::from_iter_values([1234, 5678, 9012])
2085 .with_precision_and_scale(8, 2)
2086 .unwrap();
2087 let field = Arc::new(Field::new("decimal", array.data_type().clone(), true));
2088 let schema = Schema::new(vec![field]);
2089 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap();
2090
2091 let mut buf = Vec::new();
2092 {
2093 let mut writer = LineDelimitedWriter::new(&mut buf);
2094 writer.write_batches(&[&batch]).unwrap();
2095 }
2096
2097 assert_json_eq(
2098 &buf,
2099 r#"{"decimal":12.34}
2100{"decimal":56.78}
2101{"decimal":90.12}
2102"#,
2103 );
2104 }
2105
2106 #[test]
2107 fn test_decimal64_encoder() {
2108 let array = Decimal64Array::from_iter_values([1234, 5678, 9012])
2109 .with_precision_and_scale(10, 2)
2110 .unwrap();
2111 let field = Arc::new(Field::new("decimal", array.data_type().clone(), true));
2112 let schema = Schema::new(vec![field]);
2113 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap();
2114
2115 let mut buf = Vec::new();
2116 {
2117 let mut writer = LineDelimitedWriter::new(&mut buf);
2118 writer.write_batches(&[&batch]).unwrap();
2119 }
2120
2121 assert_json_eq(
2122 &buf,
2123 r#"{"decimal":12.34}
2124{"decimal":56.78}
2125{"decimal":90.12}
2126"#,
2127 );
2128 }
2129
2130 #[test]
2131 fn test_decimal128_encoder() {
2132 let array = Decimal128Array::from_iter_values([1234, 5678, 9012])
2133 .with_precision_and_scale(10, 2)
2134 .unwrap();
2135 let field = Arc::new(Field::new("decimal", array.data_type().clone(), true));
2136 let schema = Schema::new(vec![field]);
2137 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap();
2138
2139 let mut buf = Vec::new();
2140 {
2141 let mut writer = LineDelimitedWriter::new(&mut buf);
2142 writer.write_batches(&[&batch]).unwrap();
2143 }
2144
2145 assert_json_eq(
2146 &buf,
2147 r#"{"decimal":12.34}
2148{"decimal":56.78}
2149{"decimal":90.12}
2150"#,
2151 );
2152 }
2153
2154 #[test]
2155 fn test_decimal256_encoder() {
2156 let array = Decimal256Array::from_iter_values([
2157 i256::from(123400),
2158 i256::from(567800),
2159 i256::from(901200),
2160 ])
2161 .with_precision_and_scale(10, 4)
2162 .unwrap();
2163 let field = Arc::new(Field::new("decimal", array.data_type().clone(), true));
2164 let schema = Schema::new(vec![field]);
2165 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap();
2166
2167 let mut buf = Vec::new();
2168 {
2169 let mut writer = LineDelimitedWriter::new(&mut buf);
2170 writer.write_batches(&[&batch]).unwrap();
2171 }
2172
2173 assert_json_eq(
2174 &buf,
2175 r#"{"decimal":12.3400}
2176{"decimal":56.7800}
2177{"decimal":90.1200}
2178"#,
2179 );
2180 }
2181
2182 #[test]
2183 fn test_decimal_encoder_with_nulls() {
2184 let array = Decimal128Array::from_iter([Some(1234), None, Some(5678)])
2185 .with_precision_and_scale(10, 2)
2186 .unwrap();
2187 let field = Arc::new(Field::new("decimal", array.data_type().clone(), true));
2188 let schema = Schema::new(vec![field]);
2189 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap();
2190
2191 let mut buf = Vec::new();
2192 {
2193 let mut writer = LineDelimitedWriter::new(&mut buf);
2194 writer.write_batches(&[&batch]).unwrap();
2195 }
2196
2197 assert_json_eq(
2198 &buf,
2199 r#"{"decimal":12.34}
2200{}
2201{"decimal":56.78}
2202"#,
2203 );
2204 }
2205
2206 #[test]
2207 fn test_decimal_encoder_negative_scale() {
2208 let array = Decimal128Array::from_iter([Some(0), Some(12), Some(-12)])
2210 .with_precision_and_scale(10, -2)
2211 .unwrap();
2212 let field = Arc::new(Field::new("decimal", array.data_type().clone(), true));
2213 let schema = Schema::new(vec![field]);
2214 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap();
2215
2216 let mut buf = Vec::new();
2217 {
2218 let mut writer = LineDelimitedWriter::new(&mut buf);
2219 writer.write_batches(&[&batch]).unwrap();
2220 }
2221
2222 assert_json_eq(
2223 &buf,
2224 r#"{"decimal":0}
2225{"decimal":1200}
2226{"decimal":-1200}
2227"#,
2228 );
2229 }
2230
2231 #[test]
2232 fn write_structs_as_list() {
2233 let schema = Schema::new(vec![
2234 Field::new(
2235 "c1",
2236 DataType::Struct(Fields::from(vec![
2237 Field::new("c11", DataType::Int32, true),
2238 Field::new(
2239 "c12",
2240 DataType::Struct(vec![Field::new("c121", DataType::Utf8, false)].into()),
2241 false,
2242 ),
2243 ])),
2244 false,
2245 ),
2246 Field::new("c2", DataType::Utf8, false),
2247 ]);
2248
2249 let c1 = StructArray::from(vec![
2250 (
2251 Arc::new(Field::new("c11", DataType::Int32, true)),
2252 Arc::new(Int32Array::from(vec![Some(1), None, Some(5)])) as ArrayRef,
2253 ),
2254 (
2255 Arc::new(Field::new(
2256 "c12",
2257 DataType::Struct(vec![Field::new("c121", DataType::Utf8, false)].into()),
2258 false,
2259 )),
2260 Arc::new(StructArray::from(vec![(
2261 Arc::new(Field::new("c121", DataType::Utf8, false)),
2262 Arc::new(StringArray::from(vec![Some("e"), Some("f"), Some("g")])) as ArrayRef,
2263 )])) as ArrayRef,
2264 ),
2265 ]);
2266 let c2 = StringArray::from(vec![Some("a"), Some("b"), Some("c")]);
2267
2268 let batch =
2269 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(c1), Arc::new(c2)]).unwrap();
2270
2271 let expected = r#"[[1,["e"]],"a"]
2272[[null,["f"]],"b"]
2273[[5,["g"]],"c"]
2274"#;
2275
2276 let mut buf = Vec::new();
2277 {
2278 let builder = WriterBuilder::new()
2279 .with_explicit_nulls(true)
2280 .with_struct_mode(StructMode::ListOnly);
2281 let mut writer = builder.build::<_, LineDelimited>(&mut buf);
2282 writer.write_batches(&[&batch]).unwrap();
2283 }
2284 assert_json_eq(&buf, expected);
2285
2286 let mut buf = Vec::new();
2287 {
2288 let builder = WriterBuilder::new()
2289 .with_explicit_nulls(false)
2290 .with_struct_mode(StructMode::ListOnly);
2291 let mut writer = builder.build::<_, LineDelimited>(&mut buf);
2292 writer.write_batches(&[&batch]).unwrap();
2293 }
2294 assert_json_eq(&buf, expected);
2295 }
2296
2297 fn make_fallback_encoder_test_data() -> (RecordBatch, Arc<dyn EncoderFactory>) {
2298 #[derive(Debug)]
2301 enum UnionValue {
2302 Int32(i32),
2303 String(String),
2304 }
2305
2306 #[derive(Debug)]
2307 struct UnionEncoder {
2308 array: Vec<Option<UnionValue>>,
2309 }
2310
2311 impl Encoder for UnionEncoder {
2312 fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
2313 match &self.array[idx] {
2314 None => out.extend_from_slice(b"null"),
2315 Some(UnionValue::Int32(v)) => out.extend_from_slice(v.to_string().as_bytes()),
2316 Some(UnionValue::String(v)) => {
2317 out.extend_from_slice(format!("\"{v}\"").as_bytes())
2318 }
2319 }
2320 }
2321 }
2322
2323 #[derive(Debug)]
2324 struct UnionEncoderFactory;
2325
2326 impl EncoderFactory for UnionEncoderFactory {
2327 fn make_default_encoder<'a>(
2328 &self,
2329 _field: &'a FieldRef,
2330 array: &'a dyn Array,
2331 _options: &'a EncoderOptions,
2332 ) -> Result<Option<NullableEncoder<'a>>, ArrowError> {
2333 let data_type = array.data_type();
2334 let DataType::Union(fields, UnionMode::Sparse) = data_type else {
2335 return Ok(None);
2336 };
2337 let fields = fields.iter().map(|(_, f)| f).collect::<Vec<_>>();
2339 for f in &fields {
2340 match f.data_type() {
2341 DataType::Null => {}
2342 DataType::Int32 => {}
2343 DataType::Utf8 => {}
2344 _ => return Ok(None),
2345 }
2346 }
2347 let (_, type_ids, _, buffers) = array.as_union().clone().into_parts();
2348 let mut values = Vec::with_capacity(type_ids.len());
2349 for idx in 0..type_ids.len() {
2350 let type_id = type_ids[idx];
2351 let field = &fields[type_id as usize];
2352 let value = match field.data_type() {
2353 DataType::Null => None,
2354 DataType::Int32 => Some(UnionValue::Int32(
2355 buffers[type_id as usize]
2356 .as_primitive::<Int32Type>()
2357 .value(idx),
2358 )),
2359 DataType::Utf8 => Some(UnionValue::String(
2360 buffers[type_id as usize]
2361 .as_string::<i32>()
2362 .value(idx)
2363 .to_string(),
2364 )),
2365 _ => unreachable!(),
2366 };
2367 values.push(value);
2368 }
2369 let array_encoder =
2370 Box::new(UnionEncoder { array: values }) as Box<dyn Encoder + 'a>;
2371 let nulls = array.nulls().cloned();
2372 Ok(Some(NullableEncoder::new(array_encoder, nulls)))
2373 }
2374 }
2375
2376 let int_array = Int32Array::from(vec![Some(1), None, None]);
2377 let string_array = StringArray::from(vec![None, Some("a"), None]);
2378 let null_array = NullArray::new(3);
2379 let type_ids = [0_i8, 1, 2].into_iter().collect::<ScalarBuffer<i8>>();
2380
2381 let union_fields = [
2382 (0, Arc::new(Field::new("A", DataType::Int32, false))),
2383 (1, Arc::new(Field::new("B", DataType::Utf8, false))),
2384 (2, Arc::new(Field::new("C", DataType::Null, false))),
2385 ]
2386 .into_iter()
2387 .collect::<UnionFields>();
2388
2389 let children = vec![
2390 Arc::new(int_array) as Arc<dyn Array>,
2391 Arc::new(string_array),
2392 Arc::new(null_array),
2393 ];
2394
2395 let array = UnionArray::try_new(union_fields.clone(), type_ids, None, children).unwrap();
2396
2397 let float_array = Float64Array::from(vec![Some(1.0), None, Some(3.4)]);
2398
2399 let fields = vec![
2400 Field::new(
2401 "union",
2402 DataType::Union(union_fields, UnionMode::Sparse),
2403 true,
2404 ),
2405 Field::new("float", DataType::Float64, true),
2406 ];
2407
2408 let batch = RecordBatch::try_new(
2409 Arc::new(Schema::new(fields)),
2410 vec![
2411 Arc::new(array) as Arc<dyn Array>,
2412 Arc::new(float_array) as Arc<dyn Array>,
2413 ],
2414 )
2415 .unwrap();
2416
2417 (batch, Arc::new(UnionEncoderFactory))
2418 }
2419
2420 #[test]
2421 fn test_fallback_encoder_factory_line_delimited_implicit_nulls() {
2422 let (batch, encoder_factory) = make_fallback_encoder_test_data();
2423
2424 let mut buf = Vec::new();
2425 {
2426 let mut writer = WriterBuilder::new()
2427 .with_encoder_factory(encoder_factory)
2428 .with_explicit_nulls(false)
2429 .build::<_, LineDelimited>(&mut buf);
2430 writer.write_batches(&[&batch]).unwrap();
2431 writer.finish().unwrap();
2432 }
2433
2434 println!("{}", str::from_utf8(&buf).unwrap());
2435
2436 assert_json_eq(
2437 &buf,
2438 r#"{"union":1,"float":1.0}
2439{"union":"a"}
2440{"union":null,"float":3.4}
2441"#,
2442 );
2443 }
2444
2445 #[test]
2446 fn test_fallback_encoder_factory_line_delimited_explicit_nulls() {
2447 let (batch, encoder_factory) = make_fallback_encoder_test_data();
2448
2449 let mut buf = Vec::new();
2450 {
2451 let mut writer = WriterBuilder::new()
2452 .with_encoder_factory(encoder_factory)
2453 .with_explicit_nulls(true)
2454 .build::<_, LineDelimited>(&mut buf);
2455 writer.write_batches(&[&batch]).unwrap();
2456 writer.finish().unwrap();
2457 }
2458
2459 assert_json_eq(
2460 &buf,
2461 r#"{"union":1,"float":1.0}
2462{"union":"a","float":null}
2463{"union":null,"float":3.4}
2464"#,
2465 );
2466 }
2467
2468 #[test]
2469 fn test_fallback_encoder_factory_array_implicit_nulls() {
2470 let (batch, encoder_factory) = make_fallback_encoder_test_data();
2471
2472 let json_value: Value = {
2473 let mut buf = Vec::new();
2474 let mut writer = WriterBuilder::new()
2475 .with_encoder_factory(encoder_factory)
2476 .build::<_, JsonArray>(&mut buf);
2477 writer.write_batches(&[&batch]).unwrap();
2478 writer.finish().unwrap();
2479 serde_json::from_slice(&buf).unwrap()
2480 };
2481
2482 let expected = json!([
2483 {"union":1,"float":1.0},
2484 {"union":"a"},
2485 {"float":3.4,"union":null},
2486 ]);
2487
2488 assert_eq!(json_value, expected);
2489 }
2490
2491 #[test]
2492 fn test_fallback_encoder_factory_array_explicit_nulls() {
2493 let (batch, encoder_factory) = make_fallback_encoder_test_data();
2494
2495 let json_value: Value = {
2496 let mut buf = Vec::new();
2497 let mut writer = WriterBuilder::new()
2498 .with_encoder_factory(encoder_factory)
2499 .with_explicit_nulls(true)
2500 .build::<_, JsonArray>(&mut buf);
2501 writer.write_batches(&[&batch]).unwrap();
2502 writer.finish().unwrap();
2503 serde_json::from_slice(&buf).unwrap()
2504 };
2505
2506 let expected = json!([
2507 {"union":1,"float":1.0},
2508 {"union":"a", "float": null},
2509 {"union":null,"float":3.4},
2510 ]);
2511
2512 assert_eq!(json_value, expected);
2513 }
2514
2515 #[test]
2516 fn test_default_encoder_byte_array() {
2517 struct IntArrayBinaryEncoder<B> {
2518 array: B,
2519 }
2520
2521 impl<'a, B> Encoder for IntArrayBinaryEncoder<B>
2522 where
2523 B: ArrayAccessor<Item = &'a [u8]>,
2524 {
2525 fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
2526 out.push(b'[');
2527 let child = self.array.value(idx);
2528 for (idx, byte) in child.iter().enumerate() {
2529 write!(out, "{byte}").unwrap();
2530 if idx < child.len() - 1 {
2531 out.push(b',');
2532 }
2533 }
2534 out.push(b']');
2535 }
2536 }
2537
2538 #[derive(Debug)]
2539 struct IntArrayBinaryEncoderFactory;
2540
2541 impl EncoderFactory for IntArrayBinaryEncoderFactory {
2542 fn make_default_encoder<'a>(
2543 &self,
2544 _field: &'a FieldRef,
2545 array: &'a dyn Array,
2546 _options: &'a EncoderOptions,
2547 ) -> Result<Option<NullableEncoder<'a>>, ArrowError> {
2548 match array.data_type() {
2549 DataType::Binary => {
2550 let array = array.as_binary::<i32>();
2551 let encoder = IntArrayBinaryEncoder { array };
2552 let array_encoder = Box::new(encoder) as Box<dyn Encoder + 'a>;
2553 let nulls = array.nulls().cloned();
2554 Ok(Some(NullableEncoder::new(array_encoder, nulls)))
2555 }
2556 _ => Ok(None),
2557 }
2558 }
2559 }
2560
2561 let binary_array = BinaryArray::from_opt_vec(vec![Some(b"a"), None, Some(b"b")]);
2562 let float_array = Float64Array::from(vec![Some(1.0), Some(2.3), None]);
2563 let fields = vec![
2564 Field::new("bytes", DataType::Binary, true),
2565 Field::new("float", DataType::Float64, true),
2566 ];
2567 let batch = RecordBatch::try_new(
2568 Arc::new(Schema::new(fields)),
2569 vec![
2570 Arc::new(binary_array) as Arc<dyn Array>,
2571 Arc::new(float_array) as Arc<dyn Array>,
2572 ],
2573 )
2574 .unwrap();
2575
2576 let json_value: Value = {
2577 let mut buf = Vec::new();
2578 let mut writer = WriterBuilder::new()
2579 .with_encoder_factory(Arc::new(IntArrayBinaryEncoderFactory))
2580 .build::<_, JsonArray>(&mut buf);
2581 writer.write_batches(&[&batch]).unwrap();
2582 writer.finish().unwrap();
2583 serde_json::from_slice(&buf).unwrap()
2584 };
2585
2586 let expected = json!([
2587 {"bytes": [97], "float": 1.0},
2588 {"float": 2.3},
2589 {"bytes": [98]},
2590 ]);
2591
2592 assert_eq!(json_value, expected);
2593 }
2594
2595 #[test]
2596 fn test_encoder_factory_customize_dictionary() {
2597 struct PaddedInt32Encoder {
2602 array: Int32Array,
2603 }
2604
2605 impl Encoder for PaddedInt32Encoder {
2606 fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
2607 let value = self.array.value(idx);
2608 write!(out, "\"{value:0>8}\"").unwrap();
2609 }
2610 }
2611
2612 #[derive(Debug)]
2613 struct CustomEncoderFactory;
2614
2615 impl EncoderFactory for CustomEncoderFactory {
2616 fn make_default_encoder<'a>(
2617 &self,
2618 field: &'a FieldRef,
2619 array: &'a dyn Array,
2620 _options: &'a EncoderOptions,
2621 ) -> Result<Option<NullableEncoder<'a>>, ArrowError> {
2622 let padded = field.metadata().get("padded").is_some_and(|v| v == "true");
2627 match (array.data_type(), padded) {
2628 (DataType::Int32, true) => {
2629 let array = array.as_primitive::<Int32Type>();
2630 let nulls = array.nulls().cloned();
2631 let encoder = PaddedInt32Encoder {
2632 array: array.clone(),
2633 };
2634 let array_encoder = Box::new(encoder) as Box<dyn Encoder + 'a>;
2635 Ok(Some(NullableEncoder::new(array_encoder, nulls)))
2636 }
2637 _ => Ok(None),
2638 }
2639 }
2640 }
2641
2642 let to_json = |batch| {
2643 let mut buf = Vec::new();
2644 let mut writer = WriterBuilder::new()
2645 .with_encoder_factory(Arc::new(CustomEncoderFactory))
2646 .build::<_, JsonArray>(&mut buf);
2647 writer.write_batches(&[batch]).unwrap();
2648 writer.finish().unwrap();
2649 serde_json::from_slice::<Value>(&buf).unwrap()
2650 };
2651
2652 let array = Int32Array::from(vec![Some(1), None, Some(2)]);
2654 let field = Arc::new(Field::new("int", DataType::Int32, true).with_metadata(
2655 HashMap::from_iter(vec![("padded".to_string(), "true".to_string())]),
2656 ));
2657 let batch = RecordBatch::try_new(
2658 Arc::new(Schema::new(vec![field.clone()])),
2659 vec![Arc::new(array)],
2660 )
2661 .unwrap();
2662
2663 let json_value = to_json(&batch);
2664
2665 let expected = json!([
2666 {"int": "00000001"},
2667 {},
2668 {"int": "00000002"},
2669 ]);
2670
2671 assert_eq!(json_value, expected);
2672
2673 let mut array_builder = PrimitiveDictionaryBuilder::<UInt16Type, Int32Type>::new();
2675 array_builder.append_value(1);
2676 array_builder.append_null();
2677 array_builder.append_value(1);
2678 let array = array_builder.finish();
2679 let field = Field::new(
2680 "int",
2681 DataType::Dictionary(Box::new(DataType::UInt16), Box::new(DataType::Int32)),
2682 true,
2683 )
2684 .with_metadata(HashMap::from_iter(vec![(
2685 "padded".to_string(),
2686 "true".to_string(),
2687 )]));
2688 let batch = RecordBatch::try_new(Arc::new(Schema::new(vec![field])), vec![Arc::new(array)])
2689 .unwrap();
2690
2691 let json_value = to_json(&batch);
2692
2693 let expected = json!([
2694 {"int": "00000001"},
2695 {},
2696 {"int": "00000001"},
2697 ]);
2698
2699 assert_eq!(json_value, expected);
2700 }
2701
2702 #[test]
2703 fn test_write_run_end_encoded() {
2704 let run_ends = Int32Array::from(vec![2, 5, 6]);
2705 let values = StringArray::from(vec![Some("a"), Some("b"), None]);
2706 let ree = RunArray::<Int32Type>::try_new(&run_ends, &values).unwrap();
2707
2708 let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
2709 "c1",
2710 ree.data_type().clone(),
2711 true,
2712 )]));
2713
2714 let batch = RecordBatch::try_new(schema, vec![Arc::new(ree)]).unwrap();
2715
2716 let mut buf = Vec::new();
2717 {
2718 let mut writer = LineDelimitedWriter::new(&mut buf);
2719 writer.write_batches(&[&batch]).unwrap();
2720 }
2721
2722 assert_json_eq(
2723 &buf,
2724 r#"{"c1":"a"}
2725{"c1":"a"}
2726{"c1":"b"}
2727{"c1":"b"}
2728{"c1":"b"}
2729{}
2730"#,
2731 );
2732 }
2733
2734 #[test]
2735 fn test_write_run_end_encoded_int_values() {
2736 let run_ends = Int32Array::from(vec![3, 5]);
2737 let values = Int32Array::from(vec![10, 20]);
2738 let ree = RunArray::<Int32Type>::try_new(&run_ends, &values).unwrap();
2739
2740 let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
2741 "n",
2742 ree.data_type().clone(),
2743 true,
2744 )]));
2745
2746 let batch = RecordBatch::try_new(schema, vec![Arc::new(ree)]).unwrap();
2747
2748 let json_value: Value = {
2749 let mut buf = Vec::new();
2750 let mut writer = WriterBuilder::new().build::<_, JsonArray>(&mut buf);
2751 writer.write_batches(&[&batch]).unwrap();
2752 writer.finish().unwrap();
2753 serde_json::from_slice(&buf).unwrap()
2754 };
2755
2756 let expected = json!([
2757 {"n": 10},
2758 {"n": 10},
2759 {"n": 10},
2760 {"n": 20},
2761 {"n": 20},
2762 ]);
2763
2764 assert_eq!(json_value, expected);
2765 }
2766
2767 #[test]
2768 fn test_run_end_encoded_roundtrip() {
2769 let run_ends = Int32Array::from(vec![3, 5, 7]);
2770 let values = StringArray::from(vec![Some("a"), None, Some("b")]);
2771 let ree = RunArray::<Int32Type>::try_new(&run_ends, &values).unwrap();
2772
2773 let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
2774 "c",
2775 ree.data_type().clone(),
2776 true,
2777 )]));
2778 let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(ree)]).unwrap();
2779
2780 let mut buf = Vec::new();
2781 {
2782 let mut writer = super::LineDelimitedWriter::new(&mut buf);
2783 writer.write_batches(&[&batch]).unwrap();
2784 }
2785
2786 let batches: Vec<RecordBatch> = ReaderBuilder::new(schema)
2787 .with_batch_size(1024)
2788 .build(std::io::Cursor::new(&buf))
2789 .unwrap()
2790 .collect::<Result<Vec<_>, _>>()
2791 .unwrap();
2792 assert_eq!(batches.len(), 1);
2793
2794 let col = batches[0].column(0);
2795 let run_array = col.as_run::<Int32Type>();
2796
2797 assert_eq!(run_array.len(), 7);
2798 assert_eq!(run_array.run_ends().values(), &[3, 5, 7]);
2799
2800 let values = run_array.values().as_string::<i32>();
2801 assert_eq!(values.len(), 3);
2802 assert_eq!(values.value(0), "a");
2803 assert!(values.is_null(1));
2804 assert_eq!(values.value(2), "b");
2805 }
2806}