1use std::io::Write;
6use std::sync::Arc;
7
8use crate::StructMode;
9use arrow_array::cast::AsArray;
10use arrow_array::types::*;
11use arrow_array::*;
12use arrow_buffer::{ArrowNativeType, NullBuffer, OffsetBuffer, ScalarBuffer};
13use arrow_cast::display::{ArrayFormatter, FormatOptions};
14use arrow_schema::{ArrowError, DataType, FieldRef};
15use half::f16;
16use lexical_core::FormattedSize;
17use serde_core::Serializer;
18
19#[derive(Debug, Clone, Default)]
21pub struct EncoderOptions {
22 explicit_nulls: bool,
24 struct_mode: StructMode,
26 encoder_factory: Option<Arc<dyn EncoderFactory>>,
28 date_format: Option<String>,
30 datetime_format: Option<String>,
32 timestamp_format: Option<String>,
34 timestamp_tz_format: Option<String>,
36 time_format: Option<String>,
38}
39
40impl EncoderOptions {
41 pub fn with_explicit_nulls(mut self, explicit_nulls: bool) -> Self {
43 self.explicit_nulls = explicit_nulls;
44 self
45 }
46
47 pub fn with_struct_mode(mut self, struct_mode: StructMode) -> Self {
49 self.struct_mode = struct_mode;
50 self
51 }
52
53 pub fn with_encoder_factory(mut self, encoder_factory: Arc<dyn EncoderFactory>) -> Self {
55 self.encoder_factory = Some(encoder_factory);
56 self
57 }
58
59 pub fn explicit_nulls(&self) -> bool {
61 self.explicit_nulls
62 }
63
64 pub fn struct_mode(&self) -> StructMode {
66 self.struct_mode
67 }
68
69 pub fn encoder_factory(&self) -> Option<&Arc<dyn EncoderFactory>> {
71 self.encoder_factory.as_ref()
72 }
73
74 pub fn with_date_format(mut self, format: String) -> Self {
76 self.date_format = Some(format);
77 self
78 }
79
80 pub fn date_format(&self) -> Option<&str> {
82 self.date_format.as_deref()
83 }
84
85 pub fn with_datetime_format(mut self, format: String) -> Self {
87 self.datetime_format = Some(format);
88 self
89 }
90
91 pub fn datetime_format(&self) -> Option<&str> {
93 self.datetime_format.as_deref()
94 }
95
96 pub fn with_time_format(mut self, format: String) -> Self {
98 self.time_format = Some(format);
99 self
100 }
101
102 pub fn time_format(&self) -> Option<&str> {
104 self.time_format.as_deref()
105 }
106
107 pub fn with_timestamp_format(mut self, format: String) -> Self {
109 self.timestamp_format = Some(format);
110 self
111 }
112
113 pub fn timestamp_format(&self) -> Option<&str> {
115 self.timestamp_format.as_deref()
116 }
117
118 pub fn with_timestamp_tz_format(mut self, tz_format: String) -> Self {
120 self.timestamp_tz_format = Some(tz_format);
121 self
122 }
123
124 pub fn timestamp_tz_format(&self) -> Option<&str> {
126 self.timestamp_tz_format.as_deref()
127 }
128}
129
130pub trait EncoderFactory: std::fmt::Debug + Send + Sync {
247 fn make_default_encoder<'a>(
255 &self,
256 _field: &'a FieldRef,
257 _array: &'a dyn Array,
258 _options: &'a EncoderOptions,
259 ) -> Result<Option<NullableEncoder<'a>>, ArrowError> {
260 Ok(None)
261 }
262}
263
264pub struct NullableEncoder<'a> {
267 encoder: Box<dyn Encoder + 'a>,
268 nulls: Option<NullBuffer>,
269}
270
271impl<'a> NullableEncoder<'a> {
272 #[inline]
274 pub fn new(encoder: Box<dyn Encoder + 'a>, nulls: Option<NullBuffer>) -> Self {
275 Self { encoder, nulls }
276 }
277
278 #[inline]
280 pub fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
281 self.encoder.encode(idx, out)
282 }
283
284 #[inline]
286 pub fn is_null(&self, idx: usize) -> bool {
287 match self.nulls {
288 Some(ref nulls) => nulls.is_null(idx),
289 None => false,
290 }
291 }
292
293 #[inline]
295 pub fn has_nulls(&self) -> bool {
296 match self.nulls {
297 Some(ref nulls) => nulls.null_count() > 0,
298 None => false,
299 }
300 }
301}
302
303impl Encoder for NullableEncoder<'_> {
304 #[inline]
305 fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
306 self.encoder.encode(idx, out)
307 }
308}
309
310pub trait Encoder {
314 fn encode(&mut self, idx: usize, out: &mut Vec<u8>);
318}
319
320pub fn make_encoder<'a>(
324 field: &'a FieldRef,
325 array: &'a dyn Array,
326 options: &'a EncoderOptions,
327) -> Result<NullableEncoder<'a>, ArrowError> {
328 macro_rules! primitive_helper {
329 ($t:ty) => {{
330 let array = array.as_primitive::<$t>();
331 let nulls = array.nulls().cloned();
332 NullableEncoder::new(Box::new(PrimitiveEncoder::new(array)), nulls)
333 }};
334 }
335
336 if let Some(factory) = options.encoder_factory()
337 && let Some(encoder) = factory.make_default_encoder(field, array, options)?
338 {
339 return Ok(encoder);
340 }
341
342 let nulls = array.nulls().cloned();
343 let encoder = downcast_integer! {
344 array.data_type() => (primitive_helper),
345 DataType::Float16 => primitive_helper!(Float16Type),
346 DataType::Float32 => primitive_helper!(Float32Type),
347 DataType::Float64 => primitive_helper!(Float64Type),
348 DataType::Boolean => {
349 let array = array.as_boolean();
350 NullableEncoder::new(Box::new(BooleanEncoder(array)), array.nulls().cloned())
351 }
352 DataType::Null => NullableEncoder::new(Box::new(NullEncoder), array.logical_nulls()),
353 DataType::Utf8 => {
354 let array = array.as_string::<i32>();
355 NullableEncoder::new(Box::new(StringEncoder(array)), array.nulls().cloned())
356 }
357 DataType::LargeUtf8 => {
358 let array = array.as_string::<i64>();
359 NullableEncoder::new(Box::new(StringEncoder(array)), array.nulls().cloned())
360 }
361 DataType::Utf8View => {
362 let array = array.as_string_view();
363 NullableEncoder::new(Box::new(StringViewEncoder(array)), array.nulls().cloned())
364 }
365 DataType::BinaryView => {
366 let array = array.as_binary_view();
367 NullableEncoder::new(Box::new(BinaryViewEncoder(array)), array.nulls().cloned())
368 }
369 DataType::List(_) => {
370 let array = array.as_list::<i32>();
371 NullableEncoder::new(Box::new(ListLikeEncoder::try_new(field, array, options)?), array.nulls().cloned())
372 }
373 DataType::LargeList(_) => {
374 let array = array.as_list::<i64>();
375 NullableEncoder::new(Box::new(ListLikeEncoder::try_new(field, array, options)?), array.nulls().cloned())
376 }
377 DataType::ListView(_) => {
378 let array = array.as_list_view::<i32>();
379 NullableEncoder::new(Box::new(ListLikeEncoder::try_new(field, array, options)?), array.nulls().cloned())
380 }
381 DataType::LargeListView(_) => {
382 let array = array.as_list_view::<i64>();
383 NullableEncoder::new(Box::new(ListLikeEncoder::try_new(field, array, options)?), array.nulls().cloned())
384 }
385 DataType::FixedSizeList(_, _) => {
386 let array = array.as_fixed_size_list();
387 NullableEncoder::new(Box::new(ListLikeEncoder::try_new(field, array, options)?), array.nulls().cloned())
388 }
389
390 DataType::Dictionary(_, _) => downcast_dictionary_array! {
391 array => {
392 NullableEncoder::new(Box::new(DictionaryEncoder::try_new(field, array, options)?), array.logical_nulls())
393 },
394 _ => unreachable!()
395 }
396
397 DataType::RunEndEncoded(_, _) => downcast_run_array! {
398 array => {
399 NullableEncoder::new(
400 Box::new(RunEndEncodedEncoder::try_new(field, array, options)?),
401 array.logical_nulls(),
402 )
403 },
404 _ => unreachable!()
405 }
406
407 DataType::Map(_, _) => {
408 let array = array.as_map();
409 NullableEncoder::new(Box::new(MapEncoder::try_new(field, array, options)?), array.nulls().cloned())
410 }
411
412 DataType::FixedSizeBinary(_) => {
413 let array = array.as_fixed_size_binary();
414 NullableEncoder::new(Box::new(BinaryEncoder::new(array)) as _, array.nulls().cloned())
415 }
416
417 DataType::Binary => {
418 let array: &BinaryArray = array.as_binary();
419 NullableEncoder::new(Box::new(BinaryEncoder::new(array)), array.nulls().cloned())
420 }
421
422 DataType::LargeBinary => {
423 let array: &LargeBinaryArray = array.as_binary();
424 NullableEncoder::new(Box::new(BinaryEncoder::new(array)), array.nulls().cloned())
425 }
426
427 DataType::Struct(fields) => {
428 let array = array.as_struct();
429 let encoders = fields.iter().zip(array.columns()).map(|(field, array)| {
430 let encoder = make_encoder(field, array, options)?;
431
432 let mut field_name = Vec::with_capacity(field.name().len() + 3);
434 encode_string(field.name(), &mut field_name);
435 field_name.push(b':');
436
437 Ok(FieldEncoder {
438 field_name,
439 encoder,
440 })
441 }).collect::<Result<Vec<_>, ArrowError>>()?;
442
443 let encoder = StructArrayEncoder{
444 encoders,
445 explicit_nulls: options.explicit_nulls(),
446 struct_mode: options.struct_mode(),
447 };
448 let nulls = array.nulls().cloned();
449 NullableEncoder::new(Box::new(encoder) as Box<dyn Encoder + 'a>, nulls)
450 }
451 DataType::Decimal32(_, _) | DataType::Decimal64(_, _) | DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => {
452 let options = FormatOptions::new().with_display_error(true);
453 let formatter = JsonArrayFormatter::new(ArrayFormatter::try_new(array, &options)?);
454 NullableEncoder::new(Box::new(RawArrayFormatter(formatter)) as Box<dyn Encoder + 'a>, nulls)
455 }
456 d => match d.is_temporal() {
457 true => {
458 let fops = FormatOptions::new().with_display_error(true)
463 .with_date_format(options.date_format.as_deref())
464 .with_datetime_format(options.datetime_format.as_deref())
465 .with_timestamp_format(options.timestamp_format.as_deref())
466 .with_timestamp_tz_format(options.timestamp_tz_format.as_deref())
467 .with_time_format(options.time_format.as_deref());
468
469 let formatter = ArrayFormatter::try_new(array, &fops)?;
470 let formatter = JsonArrayFormatter::new(formatter);
471 NullableEncoder::new(Box::new(formatter) as Box<dyn Encoder + 'a>, nulls)
472 }
473 false => return Err(ArrowError::JsonError(format!(
474 "Unsupported data type for JSON encoding: {d:?}",
475 )))
476 }
477 };
478
479 Ok(encoder)
480}
481
482fn encode_string(s: &str, out: &mut Vec<u8>) {
483 let mut serializer = serde_json::Serializer::new(out);
484 serializer.serialize_str(s).unwrap();
485}
486
487fn encode_binary(bytes: &[u8], out: &mut Vec<u8>) {
488 out.push(b'"');
489 for byte in bytes {
490 write!(out, "{byte:02x}").unwrap();
491 }
492 out.push(b'"');
493}
494
495struct FieldEncoder<'a> {
496 field_name: Vec<u8>,
497 encoder: NullableEncoder<'a>,
498}
499
500impl FieldEncoder<'_> {
501 #[inline]
502 fn is_null(&self, idx: usize) -> bool {
503 self.encoder.is_null(idx)
504 }
505}
506
507struct StructArrayEncoder<'a> {
508 encoders: Vec<FieldEncoder<'a>>,
509 explicit_nulls: bool,
510 struct_mode: StructMode,
511}
512
513impl Encoder for StructArrayEncoder<'_> {
514 fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
515 match self.struct_mode {
516 StructMode::ObjectOnly => out.push(b'{'),
517 StructMode::ListOnly => out.push(b'['),
518 }
519 let mut is_first = true;
520 let drop_nulls = (self.struct_mode == StructMode::ObjectOnly) && !self.explicit_nulls;
522
523 for field_encoder in &mut self.encoders {
524 let is_null = field_encoder.is_null(idx);
525 if is_null && drop_nulls {
526 continue;
527 }
528
529 if !is_first {
530 out.push(b',');
531 }
532 is_first = false;
533
534 if self.struct_mode == StructMode::ObjectOnly {
535 out.extend_from_slice(&field_encoder.field_name);
536 }
537
538 if is_null {
539 out.extend_from_slice(b"null");
540 } else {
541 field_encoder.encoder.encode(idx, out);
542 }
543 }
544 match self.struct_mode {
545 StructMode::ObjectOnly => out.push(b'}'),
546 StructMode::ListOnly => out.push(b']'),
547 }
548 }
549}
550
551trait PrimitiveEncode: ArrowNativeType {
552 type Buffer;
553
554 fn init_buffer() -> Self::Buffer;
556
557 fn encode(self, buf: &mut Self::Buffer) -> &[u8];
561}
562
563macro_rules! integer_encode {
564 ($($t:ty),*) => {
565 $(
566 impl PrimitiveEncode for $t {
567 type Buffer = [u8; Self::FORMATTED_SIZE];
568
569 fn init_buffer() -> Self::Buffer {
570 [0; Self::FORMATTED_SIZE]
571 }
572
573 fn encode(self, buf: &mut Self::Buffer) -> &[u8] {
574 lexical_core::write(self, buf)
575 }
576 }
577 )*
578 };
579}
580integer_encode!(i8, i16, i32, i64, u8, u16, u32, u64);
581
582macro_rules! float_encode {
583 ($($t:ty),*) => {
584 $(
585 impl PrimitiveEncode for $t {
586 type Buffer = [u8; Self::FORMATTED_SIZE];
587
588 fn init_buffer() -> Self::Buffer {
589 [0; Self::FORMATTED_SIZE]
590 }
591
592 fn encode(self, buf: &mut Self::Buffer) -> &[u8] {
593 if self.is_infinite() || self.is_nan() {
594 b"null"
595 } else {
596 lexical_core::write(self, buf)
597 }
598 }
599 }
600 )*
601 };
602}
603float_encode!(f32, f64);
604
605impl PrimitiveEncode for f16 {
606 type Buffer = <f32 as PrimitiveEncode>::Buffer;
607
608 fn init_buffer() -> Self::Buffer {
609 f32::init_buffer()
610 }
611
612 fn encode(self, buf: &mut Self::Buffer) -> &[u8] {
613 self.to_f32().encode(buf)
614 }
615}
616
617struct PrimitiveEncoder<N: PrimitiveEncode> {
618 values: ScalarBuffer<N>,
619 buffer: N::Buffer,
620}
621
622impl<N: PrimitiveEncode> PrimitiveEncoder<N> {
623 fn new<P: ArrowPrimitiveType<Native = N>>(array: &PrimitiveArray<P>) -> Self {
624 Self {
625 values: array.values().clone(),
626 buffer: N::init_buffer(),
627 }
628 }
629}
630
631impl<N: PrimitiveEncode> Encoder for PrimitiveEncoder<N> {
632 fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
633 out.extend_from_slice(self.values[idx].encode(&mut self.buffer));
634 }
635}
636
637struct BooleanEncoder<'a>(&'a BooleanArray);
638
639impl Encoder for BooleanEncoder<'_> {
640 fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
641 match self.0.value(idx) {
642 true => out.extend_from_slice(b"true"),
643 false => out.extend_from_slice(b"false"),
644 }
645 }
646}
647
648struct StringEncoder<'a, O: OffsetSizeTrait>(&'a GenericStringArray<O>);
649
650impl<O: OffsetSizeTrait> Encoder for StringEncoder<'_, O> {
651 fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
652 encode_string(self.0.value(idx), out);
653 }
654}
655
656struct StringViewEncoder<'a>(&'a StringViewArray);
657
658impl Encoder for StringViewEncoder<'_> {
659 fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
660 encode_string(self.0.value(idx), out);
661 }
662}
663
664struct BinaryViewEncoder<'a>(&'a BinaryViewArray);
665
666impl Encoder for BinaryViewEncoder<'_> {
667 fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
668 encode_binary(self.0.value(idx), out);
669 }
670}
671
672struct ListLikeEncoder<'a, L: ListLikeArray> {
673 list_array: &'a L,
674 encoder: NullableEncoder<'a>,
675}
676
677impl<'a, L: ListLikeArray> ListLikeEncoder<'a, L> {
678 fn try_new(
679 field: &'a FieldRef,
680 array: &'a L,
681 options: &'a EncoderOptions,
682 ) -> Result<Self, ArrowError> {
683 let encoder = make_encoder(field, array.values().as_ref(), options)?;
684 Ok(Self {
685 list_array: array,
686 encoder,
687 })
688 }
689}
690
691impl<L: ListLikeArray> Encoder for ListLikeEncoder<'_, L> {
692 fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
693 let range = self.list_array.element_range(idx);
694 let start = range.start;
695 let end = range.end;
696 out.push(b'[');
697 if self.encoder.has_nulls() {
698 for idx in start..end {
699 if idx != start {
700 out.push(b',')
701 }
702 if self.encoder.is_null(idx) {
703 out.extend_from_slice(b"null");
704 } else {
705 self.encoder.encode(idx, out);
706 }
707 }
708 } else {
709 for idx in start..end {
710 if idx != start {
711 out.push(b',')
712 }
713 self.encoder.encode(idx, out);
714 }
715 }
716 out.push(b']');
717 }
718}
719
720struct DictionaryEncoder<'a, K: ArrowDictionaryKeyType> {
721 keys: ScalarBuffer<K::Native>,
722 encoder: NullableEncoder<'a>,
723}
724
725impl<'a, K: ArrowDictionaryKeyType> DictionaryEncoder<'a, K> {
726 fn try_new(
727 field: &'a FieldRef,
728 array: &'a DictionaryArray<K>,
729 options: &'a EncoderOptions,
730 ) -> Result<Self, ArrowError> {
731 let encoder = make_encoder(field, array.values().as_ref(), options)?;
732
733 Ok(Self {
734 keys: array.keys().values().clone(),
735 encoder,
736 })
737 }
738}
739
740impl<K: ArrowDictionaryKeyType> Encoder for DictionaryEncoder<'_, K> {
741 fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
742 self.encoder.encode(self.keys[idx].as_usize(), out)
743 }
744}
745
746struct RunEndEncodedEncoder<'a, R: RunEndIndexType> {
747 run_array: &'a RunArray<R>,
748 encoder: NullableEncoder<'a>,
749}
750
751impl<'a, R: RunEndIndexType> RunEndEncodedEncoder<'a, R> {
752 fn try_new(
753 field: &'a FieldRef,
754 array: &'a RunArray<R>,
755 options: &'a EncoderOptions,
756 ) -> Result<Self, ArrowError> {
757 let encoder = make_encoder(field, array.values().as_ref(), options)?;
758 Ok(Self {
759 run_array: array,
760 encoder,
761 })
762 }
763}
764
765impl<R: RunEndIndexType> Encoder for RunEndEncodedEncoder<'_, R> {
766 fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
767 let physical_idx = self.run_array.get_physical_index(idx);
768 self.encoder.encode(physical_idx, out)
769 }
770}
771
772struct JsonArrayFormatter<'a> {
774 formatter: ArrayFormatter<'a>,
775}
776
777impl<'a> JsonArrayFormatter<'a> {
778 fn new(formatter: ArrayFormatter<'a>) -> Self {
779 Self { formatter }
780 }
781}
782
783impl Encoder for JsonArrayFormatter<'_> {
784 fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
785 out.push(b'"');
786 let _ = write!(out, "{}", self.formatter.value(idx));
789 out.push(b'"')
790 }
791}
792
793struct RawArrayFormatter<'a>(JsonArrayFormatter<'a>);
795
796impl Encoder for RawArrayFormatter<'_> {
797 fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
798 let _ = write!(out, "{}", self.0.formatter.value(idx));
799 }
800}
801
802struct NullEncoder;
803
804impl Encoder for NullEncoder {
805 fn encode(&mut self, _idx: usize, _out: &mut Vec<u8>) {
806 unreachable!()
807 }
808}
809
810struct MapEncoder<'a> {
811 offsets: OffsetBuffer<i32>,
812 keys: NullableEncoder<'a>,
813 values: NullableEncoder<'a>,
814 explicit_nulls: bool,
815}
816
817impl<'a> MapEncoder<'a> {
818 fn try_new(
819 field: &'a FieldRef,
820 array: &'a MapArray,
821 options: &'a EncoderOptions,
822 ) -> Result<Self, ArrowError> {
823 let values = array.values();
824 let keys = array.keys();
825
826 if !matches!(
827 keys.data_type(),
828 DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
829 ) {
830 return Err(ArrowError::JsonError(format!(
831 "Only UTF8 keys supported by JSON MapArray Writer: got {:?}",
832 keys.data_type()
833 )));
834 }
835
836 let keys = make_encoder(field, keys, options)?;
837 let values = make_encoder(field, values, options)?;
838
839 if keys.has_nulls() {
841 return Err(ArrowError::InvalidArgumentError(
842 "Encountered nulls in MapArray keys".to_string(),
843 ));
844 }
845
846 if array.entries().nulls().is_some_and(|x| x.null_count() != 0) {
847 return Err(ArrowError::InvalidArgumentError(
848 "Encountered nulls in MapArray entries".to_string(),
849 ));
850 }
851
852 Ok(Self {
853 offsets: array.offsets().clone(),
854 keys,
855 values,
856 explicit_nulls: options.explicit_nulls(),
857 })
858 }
859}
860
861impl Encoder for MapEncoder<'_> {
862 fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
863 let end = self.offsets[idx + 1].as_usize();
864 let start = self.offsets[idx].as_usize();
865
866 let mut is_first = true;
867
868 out.push(b'{');
869
870 for idx in start..end {
871 let is_null = self.values.is_null(idx);
872 if is_null && !self.explicit_nulls {
873 continue;
874 }
875
876 if !is_first {
877 out.push(b',');
878 }
879 is_first = false;
880
881 self.keys.encode(idx, out);
882 out.push(b':');
883
884 if is_null {
885 out.extend_from_slice(b"null");
886 } else {
887 self.values.encode(idx, out);
888 }
889 }
890 out.push(b'}');
891 }
892}
893
894struct BinaryEncoder<B>(B);
897
898impl<'a, B> BinaryEncoder<B>
899where
900 B: ArrayAccessor<Item = &'a [u8]>,
901{
902 fn new(array: B) -> Self {
903 Self(array)
904 }
905}
906
907impl<'a, B> Encoder for BinaryEncoder<B>
908where
909 B: ArrayAccessor<Item = &'a [u8]>,
910{
911 fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
912 out.push(b'"');
913 for byte in self.0.value(idx) {
914 write!(out, "{byte:02x}").unwrap();
916 }
917 out.push(b'"');
918 }
919}