1use crate::codec::AvroFieldBuilder;
134use crate::compression::CompressionCodec;
135use crate::errors::AvroError;
136use crate::schema::{
137 AvroSchema, Fingerprint, FingerprintAlgorithm, FingerprintStrategy, SCHEMA_METADATA_KEY,
138};
139use crate::writer::encoder::{RecordEncoder, RecordEncoderBuilder, write_long};
140use crate::writer::format::{AvroFormat, AvroOcfFormat, AvroSoeFormat};
141use arrow_array::RecordBatch;
142use arrow_schema::{Schema, SchemaRef};
143use bytes::{Bytes, BytesMut};
144use std::io::Write;
145use std::sync::Arc;
146
147mod encoder;
149pub mod format;
151
152#[derive(Debug, Clone)]
163pub struct EncodedRows {
164 data: Bytes,
165 offsets: Vec<usize>,
166}
167
168impl EncodedRows {
169 pub fn new(data: Bytes, offsets: Vec<usize>) -> Self {
174 Self { data, offsets }
175 }
176
177 #[inline]
179 pub fn len(&self) -> usize {
180 self.offsets.len().saturating_sub(1)
181 }
182
183 #[inline]
185 pub fn is_empty(&self) -> bool {
186 self.len() == 0
187 }
188
189 #[inline]
198 pub fn bytes(&self) -> &Bytes {
199 &self.data
200 }
201
202 #[inline]
207 pub fn offsets(&self) -> &[usize] {
208 &self.offsets
209 }
210
211 pub fn row(&self, n: usize) -> Result<Bytes, AvroError> {
243 if n >= self.len() {
244 return Err(AvroError::General(format!(
245 "Row index {n} out of bounds for len {}",
246 self.len()
247 )));
248 }
249 let (start, end) = unsafe {
254 (
255 *self.offsets.get_unchecked(n),
256 *self.offsets.get_unchecked(n + 1),
257 )
258 };
259 if start > end || end > self.data.len() {
260 return Err(AvroError::General(format!(
261 "Invalid row offsets for row {n}: start={start}, end={end}, data_len={}",
262 self.data.len()
263 )));
264 }
265 Ok(self.data.slice(start..end))
266 }
267
268 #[inline]
298 pub fn iter(&self) -> impl ExactSizeIterator<Item = Bytes> + '_ {
299 self.offsets.windows(2).map(|w| self.data.slice(w[0]..w[1]))
300 }
301}
302
303#[derive(Debug, Clone)]
305pub struct WriterBuilder {
306 schema: Schema,
307 codec: Option<CompressionCodec>,
308 row_capacity: Option<usize>,
309 capacity: usize,
310 fingerprint_strategy: Option<FingerprintStrategy>,
311}
312
313impl WriterBuilder {
314 pub fn new(schema: Schema) -> Self {
321 Self {
322 schema,
323 codec: None,
324 row_capacity: None,
325 capacity: 1024,
326 fingerprint_strategy: None,
327 }
328 }
329
330 pub fn with_fingerprint_strategy(mut self, strategy: FingerprintStrategy) -> Self {
333 self.fingerprint_strategy = Some(strategy);
334 self
335 }
336
337 pub fn with_compression(mut self, codec: Option<CompressionCodec>) -> Self {
339 self.codec = codec;
340 self
341 }
342
343 pub fn with_capacity(mut self, capacity: usize) -> Self {
347 self.capacity = capacity;
348 self
349 }
350
351 pub fn with_row_capacity(mut self, capacity: usize) -> Self {
356 self.row_capacity = Some(capacity);
357 self
358 }
359
360 fn prepare_encoder<F: AvroFormat>(&self) -> Result<(Arc<Schema>, RecordEncoder), AvroError> {
361 let avro_schema = match self.schema.metadata.get(SCHEMA_METADATA_KEY) {
362 Some(json) => AvroSchema::new(json.clone()),
363 None => AvroSchema::try_from(&self.schema)?,
364 };
365 let maybe_fingerprint = if F::NEEDS_PREFIX {
366 match &self.fingerprint_strategy {
367 Some(FingerprintStrategy::Id(id)) => Some(Fingerprint::Id(*id)),
368 Some(FingerprintStrategy::Id64(id)) => Some(Fingerprint::Id64(*id)),
369 Some(strategy) => {
370 Some(avro_schema.fingerprint(FingerprintAlgorithm::from(*strategy))?)
371 }
372 None => Some(
373 avro_schema
374 .fingerprint(FingerprintAlgorithm::from(FingerprintStrategy::Rabin))?,
375 ),
376 }
377 } else {
378 None
379 };
380 let mut md = self.schema.metadata().clone();
381 md.insert(
382 SCHEMA_METADATA_KEY.to_string(),
383 avro_schema.clone().json_string,
384 );
385 let schema = Arc::new(Schema::new_with_metadata(self.schema.fields().clone(), md));
386 let avro_root = AvroFieldBuilder::new(&avro_schema.schema()?).build()?;
387 let encoder = RecordEncoderBuilder::new(&avro_root, schema.as_ref())
388 .with_fingerprint(maybe_fingerprint)
389 .build()?;
390 Ok((schema, encoder))
391 }
392
393 pub fn build_encoder<F: AvroFormat>(self) -> Result<Encoder, AvroError> {
398 if F::default().sync_marker().is_some() {
399 return Err(AvroError::InvalidArgument(
400 "Encoder only supports stream formats (no OCF header/sync marker)".to_string(),
401 ));
402 }
403 let (schema, encoder) = self.prepare_encoder::<F>()?;
404 Ok(Encoder {
405 schema,
406 encoder,
407 row_capacity: self.row_capacity,
408 buffer: BytesMut::with_capacity(self.capacity),
409 offsets: vec![0],
410 })
411 }
412
413 pub fn build<W, F>(self, mut writer: W) -> Result<Writer<W, F>, AvroError>
415 where
416 W: Write,
417 F: AvroFormat,
418 {
419 let mut format = F::default();
420 if format.sync_marker().is_none() && !F::NEEDS_PREFIX {
421 return Err(AvroError::InvalidArgument(
422 "AvroBinaryFormat is only supported with Encoder, use build_encoder instead"
423 .to_string(),
424 ));
425 }
426 let (schema, encoder) = self.prepare_encoder::<F>()?;
427 format.start_stream(&mut writer, &schema, self.codec)?;
428 Ok(Writer {
429 writer,
430 schema,
431 format,
432 compression: self.codec,
433 capacity: self.capacity,
434 encoder,
435 })
436 }
437}
438
439#[derive(Debug)]
485pub struct Encoder {
486 schema: SchemaRef,
487 encoder: RecordEncoder,
488 row_capacity: Option<usize>,
489 buffer: BytesMut,
490 offsets: Vec<usize>,
491}
492
493impl Encoder {
494 pub fn encode(&mut self, batch: &RecordBatch) -> Result<(), AvroError> {
496 if batch.schema().fields() != self.schema.fields() {
497 return Err(AvroError::SchemaError(
498 "Schema of RecordBatch differs from Writer schema".to_string(),
499 ));
500 }
501 self.encoder.encode_rows(
502 batch,
503 self.row_capacity.unwrap_or(0),
504 &mut self.buffer,
505 &mut self.offsets,
506 )?;
507 Ok(())
508 }
509
510 pub fn encode_batches(&mut self, batches: &[RecordBatch]) -> Result<(), AvroError> {
512 for b in batches {
513 self.encode(b)?;
514 }
515 Ok(())
516 }
517
518 pub fn flush(&mut self) -> EncodedRows {
522 let data = self.buffer.split().freeze();
523 let mut offsets = Vec::with_capacity(self.offsets.len());
524 offsets.append(&mut self.offsets);
525 self.offsets.push(0);
526 EncodedRows::new(data, offsets)
527 }
528
529 pub fn schema(&self) -> SchemaRef {
534 self.schema.clone()
535 }
536
537 pub fn buffered_len(&self) -> usize {
539 self.offsets.len().saturating_sub(1)
540 }
541}
542
543#[derive(Debug)]
551pub struct Writer<W: Write, F: AvroFormat> {
552 writer: W,
553 schema: SchemaRef,
554 format: F,
555 compression: Option<CompressionCodec>,
556 capacity: usize,
557 encoder: RecordEncoder,
558}
559
560pub type AvroWriter<W> = Writer<W, AvroOcfFormat>;
601
602pub type AvroStreamWriter<W> = Writer<W, AvroSoeFormat>;
634
635impl<W: Write> Writer<W, AvroOcfFormat> {
636 pub fn new(writer: W, schema: Schema) -> Result<Self, AvroError> {
662 WriterBuilder::new(schema).build::<W, AvroOcfFormat>(writer)
663 }
664
665 pub fn sync_marker(&self) -> Option<&[u8; 16]> {
667 self.format.sync_marker()
668 }
669}
670
671impl<W: Write> Writer<W, AvroSoeFormat> {
672 pub fn new(writer: W, schema: Schema) -> Result<Self, AvroError> {
700 WriterBuilder::new(schema).build::<W, AvroSoeFormat>(writer)
701 }
702}
703
704impl<W: Write, F: AvroFormat> Writer<W, F> {
705 pub fn write(&mut self, batch: &RecordBatch) -> Result<(), AvroError> {
707 if batch.schema().fields() != self.schema.fields() {
708 return Err(AvroError::SchemaError(
709 "Schema of RecordBatch differs from Writer schema".to_string(),
710 ));
711 }
712 match self.format.sync_marker() {
713 Some(&sync) => self.write_ocf_block(batch, &sync),
714 None => self.write_stream(batch),
715 }
716 }
717
718 pub fn write_batches(&mut self, batches: &[&RecordBatch]) -> Result<(), AvroError> {
722 for b in batches {
723 self.write(b)?;
724 }
725 Ok(())
726 }
727
728 pub fn finish(&mut self) -> Result<(), AvroError> {
730 self.writer
731 .flush()
732 .map_err(|e| AvroError::IoError(format!("Error flushing writer: {e}"), e))
733 }
734
735 pub fn into_inner(self) -> W {
737 self.writer
738 }
739
740 fn write_ocf_block(&mut self, batch: &RecordBatch, sync: &[u8; 16]) -> Result<(), AvroError> {
741 let mut buf = Vec::<u8>::with_capacity(self.capacity);
742 self.encoder.encode(&mut buf, batch)?;
743 let encoded = match self.compression {
744 Some(codec) => codec.compress(&buf)?,
745 None => buf,
746 };
747 write_long(&mut self.writer, batch.num_rows() as i64)?;
748 write_long(&mut self.writer, encoded.len() as i64)?;
749 self.writer
750 .write_all(&encoded)
751 .map_err(|e| AvroError::IoError(format!("Error writing Avro block: {e}"), e))?;
752 self.writer
753 .write_all(sync)
754 .map_err(|e| AvroError::IoError(format!("Error writing Avro sync: {e}"), e))?;
755 Ok(())
756 }
757
758 fn write_stream(&mut self, batch: &RecordBatch) -> Result<(), AvroError> {
759 self.encoder.encode(&mut self.writer, batch)?;
760 Ok(())
761 }
762}
763
764#[cfg(test)]
765mod tests {
766 use super::*;
767 use crate::compression::CompressionCodec;
768 use crate::reader::ReaderBuilder;
769 use crate::schema::AVRO_NAME_METADATA_KEY;
770 use crate::schema::{AvroSchema, SchemaStore};
771 use crate::test_util::arrow_test_data;
772 use arrow::datatypes::TimeUnit;
773 use arrow::util::pretty::pretty_format_batches;
774 #[cfg(not(feature = "avro_custom_types"))]
775 use arrow_array::Float32Array;
776 #[cfg(feature = "avro_custom_types")]
777 use arrow_array::RunArray;
778 use arrow_array::builder::{Int32Builder, ListBuilder};
779 use arrow_array::cast::AsArray;
780 #[cfg(feature = "avro_custom_types")]
781 use arrow_array::types::{Int16Type, Int64Type};
782 use arrow_array::types::{
783 Int32Type, Time32MillisecondType, Time64MicrosecondType, TimestampMicrosecondType,
784 TimestampMillisecondType, TimestampNanosecondType,
785 };
786 use arrow_array::{
787 Array, ArrayRef, BinaryArray, BooleanArray, Date32Array, Date64Array, Float16Array,
788 Int8Array, Int16Array, Int32Array, Int64Array, IntervalDayTimeArray,
789 IntervalMonthDayNanoArray, IntervalYearMonthArray, PrimitiveArray, RecordBatch,
790 StringArray, StructArray, Time32MillisecondArray, Time32SecondArray,
791 Time64MicrosecondArray, Time64NanosecondArray, TimestampMillisecondArray,
792 TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, UnionArray,
793 };
794 use arrow_buffer::{IntervalDayTime, IntervalMonthDayNano};
795 #[cfg(not(feature = "avro_custom_types"))]
796 use arrow_schema::{DataType, Field, Schema};
797 #[cfg(feature = "avro_custom_types")]
798 use arrow_schema::{DataType, Field, Schema};
799 use arrow_schema::{IntervalUnit, UnionMode};
800 use bytes::BytesMut;
801 use half::f16;
802 use serde_json::{Value, json};
803 use std::collections::HashMap;
804 use std::collections::HashSet;
805 use std::fs::File;
806 use std::io::{BufReader, Cursor};
807 use std::path::PathBuf;
808 use std::sync::Arc;
809 use tempfile::NamedTempFile;
810
811 fn files() -> impl Iterator<Item = &'static str> {
812 [
813 #[cfg(feature = "snappy")]
815 "avro/alltypes_plain.avro",
816 #[cfg(all(feature = "snappy", not(miri)))]
818 "avro/alltypes_plain.snappy.avro",
819 #[cfg(all(feature = "zstd", not(miri)))]
820 "avro/alltypes_plain.zstandard.avro",
821 #[cfg(all(feature = "bzip2", not(miri)))]
822 "avro/alltypes_plain.bzip2.avro",
823 #[cfg(all(feature = "xz", not(miri)))]
824 "avro/alltypes_plain.xz.avro",
825 ]
826 .into_iter()
827 }
828
829 fn make_schema() -> Schema {
830 Schema::new(vec![
831 Field::new("id", DataType::Int32, false),
832 Field::new("name", DataType::Binary, false),
833 ])
834 }
835
836 fn make_batch() -> RecordBatch {
837 let ids = Int32Array::from(vec![1, 2, 3]);
838 let names = BinaryArray::from_vec(vec![b"a".as_ref(), b"b".as_ref(), b"c".as_ref()]);
839 RecordBatch::try_new(
840 Arc::new(make_schema()),
841 vec![Arc::new(ids) as ArrayRef, Arc::new(names) as ArrayRef],
842 )
843 .expect("failed to build test RecordBatch")
844 }
845
846 #[test]
847 fn test_stream_writer_writes_prefix_per_row_rt() -> Result<(), AvroError> {
848 let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
849 let batch = RecordBatch::try_new(
850 Arc::new(schema.clone()),
851 vec![Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef],
852 )?;
853 let buf: Vec<u8> = Vec::new();
854 let mut writer = AvroStreamWriter::new(buf, schema.clone())?;
855 writer.write(&batch)?;
856 let encoded = writer.into_inner();
857 let mut store = SchemaStore::new(); let avro_schema = AvroSchema::try_from(&schema)?;
859 let _fp = store.register(avro_schema)?;
860 let mut decoder = ReaderBuilder::new()
861 .with_writer_schema_store(store)
862 .build_decoder()?;
863 let _consumed = decoder.decode(&encoded)?;
864 let decoded = decoder
865 .flush()?
866 .expect("expected at least one batch from decoder");
867 assert_eq!(decoded.num_columns(), 1);
868 assert_eq!(decoded.num_rows(), 2);
869 let col = decoded.column(0).as_primitive::<Int32Type>();
870 assert_eq!(col, &Int32Array::from(vec![10, 20]));
871 Ok(())
872 }
873
874 #[test]
875 fn test_nullable_struct_with_nonnullable_field_sliced_encoding() {
876 use arrow_array::{ArrayRef, Int32Array, StringArray, StructArray};
877 use arrow_buffer::NullBuffer;
878 use arrow_schema::{DataType, Field, Fields, Schema};
879 use std::sync::Arc;
880 let inner_fields = Fields::from(vec![
881 Field::new("id", DataType::Int32, false), Field::new("name", DataType::Utf8, true), ]);
884 let inner_struct_type = DataType::Struct(inner_fields.clone());
885 let schema = Schema::new(vec![
886 Field::new("before", inner_struct_type.clone(), true), Field::new("after", inner_struct_type.clone(), true), Field::new("op", DataType::Utf8, false), ]);
890 let before_ids = Int32Array::from(vec![None, None]);
891 let before_names = StringArray::from(vec![None::<&str>, None]);
892 let before_struct = StructArray::new(
893 inner_fields.clone(),
894 vec![
895 Arc::new(before_ids) as ArrayRef,
896 Arc::new(before_names) as ArrayRef,
897 ],
898 Some(NullBuffer::from(vec![false, false])),
899 );
900 let after_ids = Int32Array::from(vec![1, 2]); let after_names = StringArray::from(vec![Some("Alice"), Some("Bob")]);
902 let after_struct = StructArray::new(
903 inner_fields.clone(),
904 vec![
905 Arc::new(after_ids) as ArrayRef,
906 Arc::new(after_names) as ArrayRef,
907 ],
908 Some(NullBuffer::from(vec![true, true])),
909 );
910 let op_col = StringArray::from(vec!["r", "r"]);
911 let batch = RecordBatch::try_new(
912 Arc::new(schema.clone()),
913 vec![
914 Arc::new(before_struct) as ArrayRef,
915 Arc::new(after_struct) as ArrayRef,
916 Arc::new(op_col) as ArrayRef,
917 ],
918 )
919 .expect("failed to create test batch");
920 let mut sink = Vec::new();
921 let mut writer = WriterBuilder::new(schema)
922 .with_fingerprint_strategy(FingerprintStrategy::Id(1))
923 .build::<_, AvroSoeFormat>(&mut sink)
924 .expect("failed to create writer");
925 for row_idx in 0..batch.num_rows() {
926 let single_row = batch.slice(row_idx, 1);
927 let after_col = single_row.column(1);
928 assert_eq!(
929 after_col.null_count(),
930 0,
931 "after column should have no nulls in sliced row"
932 );
933 writer
934 .write(&single_row)
935 .unwrap_or_else(|e| panic!("Failed to encode row {row_idx}: {e}"));
936 }
937 writer.finish().expect("failed to finish writer");
938 assert!(!sink.is_empty(), "encoded output should not be empty");
939 }
940
941 #[test]
942 fn test_nullable_struct_with_decimal_and_timestamp_sliced() {
943 use arrow_array::{
944 ArrayRef, Decimal128Array, Int32Array, StringArray, StructArray,
945 TimestampMicrosecondArray,
946 };
947 use arrow_buffer::NullBuffer;
948 use arrow_schema::{DataType, Field, Fields, Schema};
949 use std::sync::Arc;
950 let row_fields = Fields::from(vec![
951 Field::new("id", DataType::Int32, false),
952 Field::new("name", DataType::Utf8, true),
953 Field::new("category", DataType::Utf8, true),
954 Field::new("price", DataType::Decimal128(10, 2), true),
955 Field::new("stock_quantity", DataType::Int32, true),
956 Field::new(
957 "created_at",
958 DataType::Timestamp(TimeUnit::Microsecond, None),
959 true,
960 ),
961 ]);
962 let row_struct_type = DataType::Struct(row_fields.clone());
963 let schema = Schema::new(vec![
964 Field::new("before", row_struct_type.clone(), true),
965 Field::new("after", row_struct_type.clone(), true),
966 Field::new("op", DataType::Utf8, false),
967 ]);
968 let before_struct = StructArray::new_null(row_fields.clone(), 2);
969 let ids = Int32Array::from(vec![1, 2]);
970 let names = StringArray::from(vec![Some("Widget"), Some("Gadget")]);
971 let categories = StringArray::from(vec![Some("Electronics"), Some("Electronics")]);
972 let prices = Decimal128Array::from(vec![Some(1999), Some(2999)])
973 .with_precision_and_scale(10, 2)
974 .unwrap();
975 let quantities = Int32Array::from(vec![Some(100), Some(50)]);
976 let timestamps = TimestampMicrosecondArray::from(vec![
977 Some(1700000000000000i64),
978 Some(1700000001000000i64),
979 ]);
980 let after_struct = StructArray::new(
981 row_fields.clone(),
982 vec![
983 Arc::new(ids) as ArrayRef,
984 Arc::new(names) as ArrayRef,
985 Arc::new(categories) as ArrayRef,
986 Arc::new(prices) as ArrayRef,
987 Arc::new(quantities) as ArrayRef,
988 Arc::new(timestamps) as ArrayRef,
989 ],
990 Some(NullBuffer::from(vec![true, true])),
991 );
992 let op_col = StringArray::from(vec!["r", "r"]);
993 let batch = RecordBatch::try_new(
994 Arc::new(schema.clone()),
995 vec![
996 Arc::new(before_struct) as ArrayRef,
997 Arc::new(after_struct) as ArrayRef,
998 Arc::new(op_col) as ArrayRef,
999 ],
1000 )
1001 .expect("failed to create products batch");
1002 let mut sink = Vec::new();
1003 let mut writer = WriterBuilder::new(schema)
1004 .with_fingerprint_strategy(FingerprintStrategy::Id(1))
1005 .build::<_, AvroSoeFormat>(&mut sink)
1006 .expect("failed to create writer");
1007 for row_idx in 0..batch.num_rows() {
1009 let single_row = batch.slice(row_idx, 1);
1010 writer
1011 .write(&single_row)
1012 .unwrap_or_else(|e| panic!("Failed to encode product row {row_idx}: {e}"));
1013 }
1014 writer.finish().expect("failed to finish writer");
1015 assert!(!sink.is_empty());
1016 }
1017
1018 #[test]
1019 fn non_nullable_child_in_nullable_struct_should_encode_per_row() {
1020 use arrow_array::{
1021 ArrayRef, Int32Array, Int64Array, RecordBatch, StringArray, StructArray,
1022 };
1023 use arrow_schema::{DataType, Field, Fields, Schema};
1024 use std::sync::Arc;
1025 let row_fields = Fields::from(vec![
1026 Field::new("id", DataType::Int32, false),
1027 Field::new("name", DataType::Utf8, true),
1028 ]);
1029 let row_struct_dt = DataType::Struct(row_fields.clone());
1030 let before: ArrayRef = Arc::new(StructArray::new_null(row_fields.clone(), 1));
1031 let id_col: ArrayRef = Arc::new(Int32Array::from(vec![1]));
1032 let name_col: ArrayRef = Arc::new(StringArray::from(vec![None::<&str>]));
1033 let after: ArrayRef = Arc::new(StructArray::new(
1034 row_fields.clone(),
1035 vec![id_col, name_col],
1036 None,
1037 ));
1038 let schema = Arc::new(Schema::new(vec![
1039 Field::new("before", row_struct_dt.clone(), true),
1040 Field::new("after", row_struct_dt, true),
1041 Field::new("op", DataType::Utf8, false),
1042 Field::new("ts_ms", DataType::Int64, false),
1043 ]));
1044 let op = Arc::new(StringArray::from(vec!["r"])) as ArrayRef;
1045 let ts_ms = Arc::new(Int64Array::from(vec![1732900000000_i64])) as ArrayRef;
1046 let batch = RecordBatch::try_new(schema.clone(), vec![before, after, op, ts_ms]).unwrap();
1047 let mut buf = Vec::new();
1048 let mut writer = WriterBuilder::new(schema.as_ref().clone())
1049 .build::<_, AvroSoeFormat>(&mut buf)
1050 .unwrap();
1051 let single = batch.slice(0, 1);
1052 let res = writer.write(&single);
1053 assert!(
1054 res.is_ok(),
1055 "expected to encode successfully, got: {:?}",
1056 res.err()
1057 );
1058 }
1059
1060 #[test]
1061 fn test_union_nonzero_type_ids() -> Result<(), AvroError> {
1062 use arrow_array::UnionArray;
1063 use arrow_buffer::Buffer;
1064 use arrow_schema::UnionFields;
1065 let union_fields = UnionFields::try_new(
1066 vec![2, 5],
1067 vec![
1068 Field::new("v_str", DataType::Utf8, true),
1069 Field::new("v_int", DataType::Int32, true),
1070 ],
1071 )
1072 .unwrap();
1073 let strings = StringArray::from(vec!["hello", "world"]);
1074 let ints = Int32Array::from(vec![10, 20, 30]);
1075 let type_ids = Buffer::from_slice_ref([2_i8, 5, 5, 2, 5]);
1076 let offsets = Buffer::from_slice_ref([0_i32, 0, 1, 1, 2]);
1077 let union_array = UnionArray::try_new(
1078 union_fields.clone(),
1079 type_ids.into(),
1080 Some(offsets.into()),
1081 vec![Arc::new(strings) as ArrayRef, Arc::new(ints) as ArrayRef],
1082 )?;
1083 let schema = Schema::new(vec![Field::new(
1084 "union_col",
1085 DataType::Union(union_fields, UnionMode::Dense),
1086 false,
1087 )]);
1088 let batch = RecordBatch::try_new(
1089 Arc::new(schema.clone()),
1090 vec![Arc::new(union_array) as ArrayRef],
1091 )?;
1092 let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
1093 assert!(
1094 writer.write(&batch).is_ok(),
1095 "Expected no error from writing"
1096 );
1097 writer.finish()?;
1098 assert!(
1099 writer.finish().is_ok(),
1100 "Expected no error from finishing writer"
1101 );
1102 Ok(())
1103 }
1104
1105 #[test]
1106 fn test_stream_writer_with_id_fingerprint_rt() -> Result<(), AvroError> {
1107 let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
1108 let batch = RecordBatch::try_new(
1109 Arc::new(schema.clone()),
1110 vec![Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef],
1111 )?;
1112 let schema_id: u32 = 42;
1113 let mut writer = WriterBuilder::new(schema.clone())
1114 .with_fingerprint_strategy(FingerprintStrategy::Id(schema_id))
1115 .build::<_, AvroSoeFormat>(Vec::new())?;
1116 writer.write(&batch)?;
1117 let encoded = writer.into_inner();
1118 let mut store = SchemaStore::new_with_type(FingerprintAlgorithm::Id);
1119 let avro_schema = AvroSchema::try_from(&schema)?;
1120 let _ = store.set(Fingerprint::Id(schema_id), avro_schema)?;
1121 let mut decoder = ReaderBuilder::new()
1122 .with_writer_schema_store(store)
1123 .build_decoder()?;
1124 let _ = decoder.decode(&encoded)?;
1125 let decoded = decoder
1126 .flush()?
1127 .expect("expected at least one batch from decoder");
1128 assert_eq!(decoded.num_columns(), 1);
1129 assert_eq!(decoded.num_rows(), 3);
1130 let col = decoded.column(0).as_primitive::<Int32Type>();
1131 assert_eq!(col, &Int32Array::from(vec![1, 2, 3]));
1132 Ok(())
1133 }
1134
1135 #[test]
1136 fn test_stream_writer_with_id64_fingerprint_rt() -> Result<(), AvroError> {
1137 let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
1138 let batch = RecordBatch::try_new(
1139 Arc::new(schema.clone()),
1140 vec![Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef],
1141 )?;
1142 let schema_id: u64 = 42;
1143 let mut writer = WriterBuilder::new(schema.clone())
1144 .with_fingerprint_strategy(FingerprintStrategy::Id64(schema_id))
1145 .build::<_, AvroSoeFormat>(Vec::new())?;
1146 writer.write(&batch)?;
1147 let encoded = writer.into_inner();
1148 let mut store = SchemaStore::new_with_type(FingerprintAlgorithm::Id64);
1149 let avro_schema = AvroSchema::try_from(&schema)?;
1150 let _ = store.set(Fingerprint::Id64(schema_id), avro_schema)?;
1151 let mut decoder = ReaderBuilder::new()
1152 .with_writer_schema_store(store)
1153 .build_decoder()?;
1154 let _ = decoder.decode(&encoded)?;
1155 let decoded = decoder
1156 .flush()?
1157 .expect("expected at least one batch from decoder");
1158 assert_eq!(decoded.num_columns(), 1);
1159 assert_eq!(decoded.num_rows(), 3);
1160 let col = decoded.column(0).as_primitive::<Int32Type>();
1161 assert_eq!(col, &Int32Array::from(vec![1, 2, 3]));
1162 Ok(())
1163 }
1164
1165 #[test]
1166 fn test_ocf_writer_generates_header_and_sync() -> Result<(), AvroError> {
1167 let batch = make_batch();
1168 let buffer: Vec<u8> = Vec::new();
1169 let mut writer = AvroWriter::new(buffer, make_schema())?;
1170 writer.write(&batch)?;
1171 writer.finish()?;
1172 let out = writer.into_inner();
1173 assert_eq!(&out[..4], b"Obj\x01", "OCF magic bytes missing/incorrect");
1174 let trailer = &out[out.len() - 16..];
1175 assert_eq!(trailer.len(), 16, "expected 16‑byte sync marker");
1176 Ok(())
1177 }
1178
1179 #[test]
1180 fn test_schema_mismatch_yields_error() {
1181 let batch = make_batch();
1182 let alt_schema = Schema::new(vec![Field::new("x", DataType::Int32, false)]);
1183 let buffer = Vec::<u8>::new();
1184 let mut writer = AvroWriter::new(buffer, alt_schema).unwrap();
1185 let err = writer.write(&batch).unwrap_err();
1186 assert!(matches!(err, AvroError::SchemaError(_)));
1187 }
1188
1189 #[test]
1190 fn test_write_batches_accumulates_multiple() -> Result<(), AvroError> {
1191 let batch1 = make_batch();
1192 let batch2 = make_batch();
1193 let buffer = Vec::<u8>::new();
1194 let mut writer = AvroWriter::new(buffer, make_schema())?;
1195 writer.write_batches(&[&batch1, &batch2])?;
1196 writer.finish()?;
1197 let out = writer.into_inner();
1198 assert!(out.len() > 4, "combined batches produced tiny file");
1199 Ok(())
1200 }
1201
1202 #[test]
1203 fn test_finish_without_write_adds_header() -> Result<(), AvroError> {
1204 let buffer = Vec::<u8>::new();
1205 let mut writer = AvroWriter::new(buffer, make_schema())?;
1206 writer.finish()?;
1207 let out = writer.into_inner();
1208 assert_eq!(&out[..4], b"Obj\x01", "finish() should emit OCF header");
1209 Ok(())
1210 }
1211
1212 #[test]
1213 fn test_write_long_encodes_zigzag_varint() -> Result<(), AvroError> {
1214 let mut buf = Vec::new();
1215 write_long(&mut buf, 0)?;
1216 write_long(&mut buf, -1)?;
1217 write_long(&mut buf, 1)?;
1218 write_long(&mut buf, -2)?;
1219 write_long(&mut buf, 2147483647)?;
1220 assert!(
1221 buf.starts_with(&[0x00, 0x01, 0x02, 0x03]),
1222 "zig‑zag varint encodings incorrect: {buf:?}"
1223 );
1224 Ok(())
1225 }
1226
1227 #[test]
1228 fn test_roundtrip_alltypes_roundtrip_writer() -> Result<(), AvroError> {
1229 for rel in files() {
1230 let path = arrow_test_data(rel);
1231 let rdr_file = File::open(&path).expect("open input avro");
1232 let reader = ReaderBuilder::new()
1233 .build(BufReader::new(rdr_file))
1234 .expect("build reader");
1235 let schema = reader.schema();
1236 let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
1237 let original =
1238 arrow::compute::concat_batches(&schema, &input_batches).expect("concat input");
1239 let tmp = NamedTempFile::new().expect("create temp file");
1240 let out_path = tmp.into_temp_path();
1241 let out_file = File::create(&out_path).expect("create temp avro");
1242 let codec = if rel.contains(".snappy.") {
1243 Some(CompressionCodec::Snappy)
1244 } else if rel.contains(".zstandard.") {
1245 Some(CompressionCodec::ZStandard)
1246 } else if rel.contains(".bzip2.") {
1247 Some(CompressionCodec::Bzip2)
1248 } else if rel.contains(".xz.") {
1249 Some(CompressionCodec::Xz)
1250 } else {
1251 None
1252 };
1253 let mut writer = WriterBuilder::new(original.schema().as_ref().clone())
1254 .with_compression(codec)
1255 .build::<_, AvroOcfFormat>(out_file)?;
1256 writer.write(&original)?;
1257 writer.finish()?;
1258 drop(writer);
1259 let rt_file = File::open(&out_path).expect("open roundtrip avro");
1260 let rt_reader = ReaderBuilder::new()
1261 .build(BufReader::new(rt_file))
1262 .expect("build roundtrip reader");
1263 let rt_schema = rt_reader.schema();
1264 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1265 let roundtrip =
1266 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat roundtrip");
1267 assert_eq!(
1268 roundtrip, original,
1269 "Round-trip batch mismatch for file: {rel}"
1270 );
1271 }
1272 Ok(())
1273 }
1274
1275 #[test]
1276 fn test_roundtrip_nested_records_writer() -> Result<(), AvroError> {
1277 let path = arrow_test_data("avro/nested_records.avro");
1278 let rdr_file = File::open(&path).expect("open nested_records.avro");
1279 let reader = ReaderBuilder::new()
1280 .build(BufReader::new(rdr_file))
1281 .expect("build reader for nested_records.avro");
1282 let schema = reader.schema();
1283 let batches = reader.collect::<Result<Vec<_>, _>>()?;
1284 let original = arrow::compute::concat_batches(&schema, &batches).expect("concat original");
1285 let tmp = NamedTempFile::new().expect("create temp file");
1286 let out_path = tmp.into_temp_path();
1287 {
1288 let out_file = File::create(&out_path).expect("create output avro");
1289 let mut writer = AvroWriter::new(out_file, original.schema().as_ref().clone())?;
1290 writer.write(&original)?;
1291 writer.finish()?;
1292 }
1293 let rt_file = File::open(&out_path).expect("open round_trip avro");
1294 let rt_reader = ReaderBuilder::new()
1295 .build(BufReader::new(rt_file))
1296 .expect("build round_trip reader");
1297 let rt_schema = rt_reader.schema();
1298 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1299 let round_trip =
1300 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
1301 assert_eq!(
1302 round_trip, original,
1303 "Round-trip batch mismatch for nested_records.avro"
1304 );
1305 Ok(())
1306 }
1307
1308 #[test]
1309 #[cfg(feature = "snappy")]
1310 fn test_roundtrip_nested_lists_writer() -> Result<(), AvroError> {
1311 let path = arrow_test_data("avro/nested_lists.snappy.avro");
1312 let rdr_file = File::open(&path).expect("open nested_lists.snappy.avro");
1313 let reader = ReaderBuilder::new()
1314 .build(BufReader::new(rdr_file))
1315 .expect("build reader for nested_lists.snappy.avro");
1316 let schema = reader.schema();
1317 let batches = reader.collect::<Result<Vec<_>, _>>()?;
1318 let original = arrow::compute::concat_batches(&schema, &batches).expect("concat original");
1319 let tmp = NamedTempFile::new().expect("create temp file");
1320 let out_path = tmp.into_temp_path();
1321 {
1322 let out_file = File::create(&out_path).expect("create output avro");
1323 let mut writer = WriterBuilder::new(original.schema().as_ref().clone())
1324 .with_compression(Some(CompressionCodec::Snappy))
1325 .build::<_, AvroOcfFormat>(out_file)?;
1326 writer.write(&original)?;
1327 writer.finish()?;
1328 }
1329 let rt_file = File::open(&out_path).expect("open round_trip avro");
1330 let rt_reader = ReaderBuilder::new()
1331 .build(BufReader::new(rt_file))
1332 .expect("build round_trip reader");
1333 let rt_schema = rt_reader.schema();
1334 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1335 let round_trip =
1336 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
1337 assert_eq!(
1338 round_trip, original,
1339 "Round-trip batch mismatch for nested_lists.snappy.avro"
1340 );
1341 Ok(())
1342 }
1343
1344 #[test]
1345 fn test_round_trip_simple_fixed_ocf() -> Result<(), AvroError> {
1346 let path = arrow_test_data("avro/simple_fixed.avro");
1347 let rdr_file = File::open(&path).expect("open avro/simple_fixed.avro");
1348 let reader = ReaderBuilder::new()
1349 .build(BufReader::new(rdr_file))
1350 .expect("build avro reader");
1351 let schema = reader.schema();
1352 let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
1353 let original =
1354 arrow::compute::concat_batches(&schema, &input_batches).expect("concat input");
1355 let tmp = NamedTempFile::new().expect("create temp file");
1356 let out_file = File::create(tmp.path()).expect("create temp avro");
1357 let mut writer = AvroWriter::new(out_file, original.schema().as_ref().clone())?;
1358 writer.write(&original)?;
1359 writer.finish()?;
1360 drop(writer);
1361 let rt_file = File::open(tmp.path()).expect("open round_trip avro");
1362 let rt_reader = ReaderBuilder::new()
1363 .build(BufReader::new(rt_file))
1364 .expect("build round_trip reader");
1365 let rt_schema = rt_reader.schema();
1366 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1367 let round_trip =
1368 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
1369 assert_eq!(round_trip, original);
1370 Ok(())
1371 }
1372
1373 #[test]
1375 #[cfg(feature = "canonical_extension_types")]
1376 fn test_round_trip_duration_and_uuid_ocf() -> Result<(), AvroError> {
1377 use arrow_schema::{DataType, IntervalUnit};
1378 let in_file =
1379 File::open("test/data/duration_uuid.avro").expect("open test/data/duration_uuid.avro");
1380 let reader = ReaderBuilder::new()
1381 .build(BufReader::new(in_file))
1382 .expect("build reader for duration_uuid.avro");
1383 let in_schema = reader.schema();
1384 let has_mdn = in_schema.fields().iter().any(|f| {
1385 matches!(
1386 f.data_type(),
1387 DataType::Interval(IntervalUnit::MonthDayNano)
1388 )
1389 });
1390 assert!(
1391 has_mdn,
1392 "expected at least one Interval(MonthDayNano) field in duration_uuid.avro"
1393 );
1394 let has_uuid_fixed = in_schema
1395 .fields()
1396 .iter()
1397 .any(|f| matches!(f.data_type(), DataType::FixedSizeBinary(16)));
1398 assert!(
1399 has_uuid_fixed,
1400 "expected at least one FixedSizeBinary(16) (uuid) field in duration_uuid.avro"
1401 );
1402 let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
1403 let input =
1404 arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
1405 let mut writer = AvroWriter::new(Vec::<u8>::new(), in_schema.as_ref().clone())?;
1407 writer.write(&input)?;
1408 writer.finish()?;
1409 let bytes = writer.into_inner();
1410 let rt_reader = ReaderBuilder::new()
1411 .build(Cursor::new(bytes))
1412 .expect("build round_trip reader");
1413 let rt_schema = rt_reader.schema();
1414 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1415 let round_trip =
1416 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
1417 assert_eq!(round_trip, input);
1418 Ok(())
1419 }
1420
1421 #[test]
1423 #[cfg(not(feature = "canonical_extension_types"))]
1424 fn test_duration_and_uuid_ocf_without_extensions_round_trips_values() -> Result<(), AvroError> {
1425 use arrow::datatypes::{DataType, IntervalUnit};
1426 use std::io::BufReader;
1427
1428 let in_file =
1430 File::open("test/data/duration_uuid.avro").expect("open test/data/duration_uuid.avro");
1431 let reader = ReaderBuilder::new()
1432 .build(BufReader::new(in_file))
1433 .expect("build reader for duration_uuid.avro");
1434 let in_schema = reader.schema();
1435
1436 assert!(
1438 in_schema.fields().iter().any(|f| {
1439 matches!(
1440 f.data_type(),
1441 DataType::Interval(IntervalUnit::MonthDayNano)
1442 )
1443 }),
1444 "expected at least one Interval(MonthDayNano) field"
1445 );
1446 assert!(
1447 in_schema
1448 .fields()
1449 .iter()
1450 .any(|f| matches!(f.data_type(), DataType::FixedSizeBinary(16))),
1451 "expected a FixedSizeBinary(16) field (uuid)"
1452 );
1453
1454 let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
1455 let input =
1456 arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
1457
1458 let mut writer = AvroWriter::new(Vec::<u8>::new(), in_schema.as_ref().clone())?;
1460 writer.write(&input)?;
1461 writer.finish()?;
1462 let bytes = writer.into_inner();
1463 let rt_reader = ReaderBuilder::new()
1464 .build(Cursor::new(bytes))
1465 .expect("build round_trip reader");
1466 let rt_schema = rt_reader.schema();
1467 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1468 let round_trip =
1469 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
1470
1471 assert_eq!(
1473 round_trip.column(0),
1474 input.column(0),
1475 "duration column values differ"
1476 );
1477 assert_eq!(round_trip.column(1), input.column(1), "uuid bytes differ");
1478
1479 let uuid_rt = rt_schema.field_with_name("uuid_field")?;
1482 assert_eq!(uuid_rt.data_type(), &DataType::FixedSizeBinary(16));
1483 assert_eq!(
1484 uuid_rt.metadata().get("logicalType").map(|s| s.as_str()),
1485 Some("uuid"),
1486 "expected `logicalType = \"uuid\"` on round-tripped field metadata"
1487 );
1488
1489 let dur_rt = rt_schema.field_with_name("duration_field")?;
1491 assert!(matches!(
1492 dur_rt.data_type(),
1493 DataType::Interval(IntervalUnit::MonthDayNano)
1494 ));
1495
1496 Ok(())
1497 }
1498
1499 #[test]
1503 #[cfg(feature = "snappy")]
1505 fn test_nonnullable_impala_roundtrip_writer() -> Result<(), AvroError> {
1506 let path = arrow_test_data("avro/nonnullable.impala.avro");
1508 let rdr_file = File::open(&path).expect("open avro/nonnullable.impala.avro");
1509 let reader = ReaderBuilder::new()
1510 .build(BufReader::new(rdr_file))
1511 .expect("build reader for nonnullable.impala.avro");
1512 let in_schema = reader.schema();
1514 let has_map = in_schema
1516 .fields()
1517 .iter()
1518 .any(|f| matches!(f.data_type(), DataType::Map(_, _)));
1519 assert!(
1520 has_map,
1521 "expected at least one Map field in avro/nonnullable.impala.avro"
1522 );
1523
1524 let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
1525 let original =
1526 arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
1527 let buffer = Vec::<u8>::new();
1529 let mut writer = AvroWriter::new(buffer, in_schema.as_ref().clone())?;
1530 writer.write(&original)?;
1531 writer.finish()?;
1532 let out_bytes = writer.into_inner();
1533 let rt_reader = ReaderBuilder::new()
1535 .build(Cursor::new(out_bytes))
1536 .expect("build reader for round-tripped in-memory OCF");
1537 let rt_schema = rt_reader.schema();
1538 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1539 let roundtrip =
1540 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat roundtrip");
1541 assert_eq!(
1543 roundtrip, original,
1544 "Round-trip Avro map data mismatch for nonnullable.impala.avro"
1545 );
1546 Ok(())
1547 }
1548
1549 #[test]
1550 #[cfg(feature = "snappy")]
1552 fn test_roundtrip_decimals_via_writer() -> Result<(), AvroError> {
1553 let files: [(&str, bool); 8] = [
1555 ("avro/fixed_length_decimal.avro", true), ("avro/fixed_length_decimal_legacy.avro", true), ("avro/int32_decimal.avro", true), ("avro/int64_decimal.avro", true), ("test/data/int256_decimal.avro", false), ("test/data/fixed256_decimal.avro", false), ("test/data/fixed_length_decimal_legacy_32.avro", false), ("test/data/int128_decimal.avro", false), ];
1564 for (rel, in_test_data_dir) in files {
1565 let path: String = if in_test_data_dir {
1567 arrow_test_data(rel)
1568 } else {
1569 PathBuf::from(env!("CARGO_MANIFEST_DIR"))
1570 .join(rel)
1571 .to_string_lossy()
1572 .into_owned()
1573 };
1574 let f_in = File::open(&path).expect("open input avro");
1576 let rdr = ReaderBuilder::new().build(BufReader::new(f_in))?;
1577 let in_schema = rdr.schema();
1578 let in_batches = rdr.collect::<Result<Vec<_>, _>>()?;
1579 let original =
1580 arrow::compute::concat_batches(&in_schema, &in_batches).expect("concat input");
1581 let tmp = NamedTempFile::new().expect("create temp file");
1583 let out_path = tmp.into_temp_path();
1584 let out_file = File::create(&out_path).expect("create temp avro");
1585 let mut writer = AvroWriter::new(out_file, original.schema().as_ref().clone())?;
1586 writer.write(&original)?;
1587 writer.finish()?;
1588 let f_rt = File::open(&out_path).expect("open roundtrip avro");
1590 let rt_rdr = ReaderBuilder::new().build(BufReader::new(f_rt))?;
1591 let rt_schema = rt_rdr.schema();
1592 let rt_batches = rt_rdr.collect::<Result<Vec<_>, _>>()?;
1593 let roundtrip =
1594 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat rt");
1595 assert_eq!(roundtrip, original, "decimal round-trip mismatch for {rel}");
1596 }
1597 Ok(())
1598 }
1599
1600 #[test]
1601 fn test_named_types_complex_roundtrip() -> Result<(), AvroError> {
1602 let path =
1604 PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("test/data/named_types_complex.avro");
1605 let rdr_file = File::open(&path).expect("open avro/named_types_complex.avro");
1606
1607 let reader = ReaderBuilder::new()
1608 .build(BufReader::new(rdr_file))
1609 .expect("build reader for named_types_complex.avro");
1610
1611 let in_schema = reader.schema();
1613 let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
1614 let original =
1615 arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
1616
1617 {
1619 let arrow_schema = original.schema();
1620
1621 let author_field = arrow_schema.field_with_name("author")?;
1623 let author_type = author_field.data_type();
1624 let editors_field = arrow_schema.field_with_name("editors")?;
1625 let editors_item_type = match editors_field.data_type() {
1626 DataType::List(item_field) => item_field.data_type(),
1627 other => panic!("Editors field should be a List, but was {other:?}"),
1628 };
1629 assert_eq!(
1630 author_type, editors_item_type,
1631 "The DataType for the 'author' struct and the 'editors' list items must be identical"
1632 );
1633
1634 let status_field = arrow_schema.field_with_name("status")?;
1636 let status_type = status_field.data_type();
1637 assert!(
1638 matches!(status_type, DataType::Dictionary(_, _)),
1639 "Status field should be a Dictionary (Enum)"
1640 );
1641
1642 let prev_status_field = arrow_schema.field_with_name("previous_status")?;
1643 let prev_status_type = prev_status_field.data_type();
1644 assert_eq!(
1645 status_type, prev_status_type,
1646 "The DataType for 'status' and 'previous_status' enums must be identical"
1647 );
1648
1649 let content_hash_field = arrow_schema.field_with_name("content_hash")?;
1651 let content_hash_type = content_hash_field.data_type();
1652 assert!(
1653 matches!(content_hash_type, DataType::FixedSizeBinary(16)),
1654 "Content hash should be FixedSizeBinary(16)"
1655 );
1656
1657 let thumb_hash_field = arrow_schema.field_with_name("thumbnail_hash")?;
1658 let thumb_hash_type = thumb_hash_field.data_type();
1659 assert_eq!(
1660 content_hash_type, thumb_hash_type,
1661 "The DataType for 'content_hash' and 'thumbnail_hash' fixed types must be identical"
1662 );
1663 }
1664
1665 let buffer: Vec<u8> = Vec::new();
1667 let mut writer = AvroWriter::new(buffer, original.schema().as_ref().clone())?;
1668 writer.write(&original)?;
1669 writer.finish()?;
1670 let bytes = writer.into_inner();
1671
1672 let rt_reader = ReaderBuilder::new()
1674 .build(Cursor::new(bytes))
1675 .expect("build reader for round-trip");
1676 let rt_schema = rt_reader.schema();
1677 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1678 let roundtrip =
1679 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat roundtrip");
1680
1681 assert_eq!(
1682 roundtrip, original,
1683 "Avro complex named types round-trip mismatch"
1684 );
1685
1686 Ok(())
1687 }
1688
1689 fn assert_schema_is_semantically_equivalent(expected: &Schema, actual: &Schema) {
1696 assert_metadata_is_superset(expected.metadata(), actual.metadata(), "Schema");
1698
1699 assert_eq!(
1701 expected.fields().len(),
1702 actual.fields().len(),
1703 "Schema must have the same number of fields"
1704 );
1705
1706 for (expected_field, actual_field) in expected.fields().iter().zip(actual.fields().iter()) {
1707 assert_field_is_semantically_equivalent(expected_field, actual_field);
1708 }
1709 }
1710
1711 fn assert_field_is_semantically_equivalent(expected: &Field, actual: &Field) {
1712 let context = format!("Field '{}'", expected.name());
1713
1714 assert_eq!(
1715 expected.name(),
1716 actual.name(),
1717 "{context}: names must match"
1718 );
1719 assert_eq!(
1720 expected.is_nullable(),
1721 actual.is_nullable(),
1722 "{context}: nullability must match"
1723 );
1724
1725 assert_datatype_is_semantically_equivalent(
1727 expected.data_type(),
1728 actual.data_type(),
1729 &context,
1730 );
1731
1732 assert_metadata_is_superset(expected.metadata(), actual.metadata(), &context);
1734 }
1735
1736 fn assert_datatype_is_semantically_equivalent(
1737 expected: &DataType,
1738 actual: &DataType,
1739 context: &str,
1740 ) {
1741 match (expected, actual) {
1742 (DataType::List(expected_field), DataType::List(actual_field))
1743 | (DataType::LargeList(expected_field), DataType::LargeList(actual_field))
1744 | (DataType::Map(expected_field, _), DataType::Map(actual_field, _)) => {
1745 assert_field_is_semantically_equivalent(expected_field, actual_field);
1746 }
1747 (DataType::Struct(expected_fields), DataType::Struct(actual_fields)) => {
1748 assert_eq!(
1749 expected_fields.len(),
1750 actual_fields.len(),
1751 "{context}: struct must have same number of fields"
1752 );
1753 for (ef, af) in expected_fields.iter().zip(actual_fields.iter()) {
1754 assert_field_is_semantically_equivalent(ef, af);
1755 }
1756 }
1757 (
1758 DataType::Union(expected_fields, expected_mode),
1759 DataType::Union(actual_fields, actual_mode),
1760 ) => {
1761 assert_eq!(
1762 expected_mode, actual_mode,
1763 "{context}: union mode must match"
1764 );
1765 assert_eq!(
1766 expected_fields.len(),
1767 actual_fields.len(),
1768 "{context}: union must have same number of variants"
1769 );
1770 for ((exp_id, exp_field), (act_id, act_field)) in
1771 expected_fields.iter().zip(actual_fields.iter())
1772 {
1773 assert_eq!(exp_id, act_id, "{context}: union type ids must match");
1774 assert_field_is_semantically_equivalent(exp_field, act_field);
1775 }
1776 }
1777 _ => {
1778 assert_eq!(expected, actual, "{context}: data types must be identical");
1779 }
1780 }
1781 }
1782
1783 fn assert_batch_data_is_identical(expected: &RecordBatch, actual: &RecordBatch) {
1784 assert_eq!(
1785 expected.num_columns(),
1786 actual.num_columns(),
1787 "RecordBatches must have the same number of columns"
1788 );
1789 assert_eq!(
1790 expected.num_rows(),
1791 actual.num_rows(),
1792 "RecordBatches must have the same number of rows"
1793 );
1794
1795 for i in 0..expected.num_columns() {
1796 let context = format!("Column {i}");
1797 let expected_col = expected.column(i);
1798 let actual_col = actual.column(i);
1799 assert_array_data_is_identical(expected_col, actual_col, &context);
1800 }
1801 }
1802
1803 fn assert_array_data_is_identical(expected: &dyn Array, actual: &dyn Array, context: &str) {
1805 assert_eq!(
1806 expected.nulls(),
1807 actual.nulls(),
1808 "{context}: null buffers must match"
1809 );
1810 assert_eq!(
1811 expected.len(),
1812 actual.len(),
1813 "{context}: array lengths must match"
1814 );
1815
1816 match (expected.data_type(), actual.data_type()) {
1817 (DataType::Union(expected_fields, _), DataType::Union(..)) => {
1818 let expected_union = expected.as_any().downcast_ref::<UnionArray>().unwrap();
1819 let actual_union = actual.as_any().downcast_ref::<UnionArray>().unwrap();
1820
1821 assert_eq!(
1823 &expected.to_data().buffers()[0],
1824 &actual.to_data().buffers()[0],
1825 "{context}: union type_ids buffer mismatch"
1826 );
1827
1828 if expected.to_data().buffers().len() > 1 {
1830 assert_eq!(
1831 &expected.to_data().buffers()[1],
1832 &actual.to_data().buffers()[1],
1833 "{context}: union value_offsets buffer mismatch"
1834 );
1835 }
1836
1837 for (type_id, _) in expected_fields.iter() {
1839 let child_context = format!("{context} -> child variant {type_id}");
1840 assert_array_data_is_identical(
1841 expected_union.child(type_id),
1842 actual_union.child(type_id),
1843 &child_context,
1844 );
1845 }
1846 }
1847 (DataType::Struct(_), DataType::Struct(_)) => {
1848 let expected_struct = expected.as_any().downcast_ref::<StructArray>().unwrap();
1849 let actual_struct = actual.as_any().downcast_ref::<StructArray>().unwrap();
1850 for i in 0..expected_struct.num_columns() {
1851 let child_context = format!("{context} -> struct child {i}");
1852 assert_array_data_is_identical(
1853 expected_struct.column(i),
1854 actual_struct.column(i),
1855 &child_context,
1856 );
1857 }
1858 }
1859 _ => {
1861 assert_eq!(
1862 expected.to_data().buffers(),
1863 actual.to_data().buffers(),
1864 "{context}: data buffers must match"
1865 );
1866 }
1867 }
1868 }
1869
1870 fn assert_metadata_is_superset(
1873 expected_meta: &arrow_schema::Metadata,
1874 actual_meta: &arrow_schema::Metadata,
1875 context: &str,
1876 ) {
1877 let allowed_additions: HashSet<&str> =
1878 vec!["arrowUnionMode", "arrowUnionTypeIds", "avro.name"]
1879 .into_iter()
1880 .collect();
1881 for (key, expected_value) in expected_meta {
1882 match actual_meta.get(key) {
1883 Some(actual_value) => assert_eq!(
1884 expected_value, actual_value,
1885 "{context}: preserved metadata for key '{key}' must have the same value"
1886 ),
1887 None => panic!("{context}: metadata key '{key}' was lost during roundtrip"),
1888 }
1889 }
1890 for key in actual_meta.keys() {
1891 if !expected_meta.contains_key(key) && !allowed_additions.contains(key.as_str()) {
1892 panic!("{context}: unexpected metadata key '{key}' was added during roundtrip");
1893 }
1894 }
1895 }
1896
1897 #[test]
1898 fn test_union_roundtrip() -> Result<(), AvroError> {
1899 let file_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
1900 .join("test/data/union_fields.avro")
1901 .to_string_lossy()
1902 .into_owned();
1903 let rdr_file = File::open(&file_path).expect("open avro/union_fields.avro");
1904 let reader = ReaderBuilder::new()
1905 .build(BufReader::new(rdr_file))
1906 .expect("build reader for union_fields.avro");
1907 let schema = reader.schema();
1908 let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
1909 let original =
1910 arrow::compute::concat_batches(&schema, &input_batches).expect("concat input");
1911 let mut writer = AvroWriter::new(Vec::<u8>::new(), original.schema().as_ref().clone())?;
1912 writer.write(&original)?;
1913 writer.finish()?;
1914 let bytes = writer.into_inner();
1915 let rt_reader = ReaderBuilder::new()
1916 .build(Cursor::new(bytes))
1917 .expect("build round_trip reader");
1918 let rt_schema = rt_reader.schema();
1919 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1920 let round_trip =
1921 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
1922
1923 assert_schema_is_semantically_equivalent(&original.schema(), &round_trip.schema());
1926
1927 assert_batch_data_is_identical(&original, &round_trip);
1928 Ok(())
1929 }
1930
1931 #[test]
1932 fn test_enum_roundtrip_uses_reader_fixture() -> Result<(), AvroError> {
1933 let path = arrow_test_data("avro/simple_enum.avro");
1935 let rdr_file = File::open(&path).expect("open avro/simple_enum.avro");
1936 let reader = ReaderBuilder::new()
1937 .build(BufReader::new(rdr_file))
1938 .expect("build reader for simple_enum.avro");
1939 let in_schema = reader.schema();
1941 let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
1942 let original =
1943 arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
1944 let has_enum_dict = in_schema.fields().iter().any(|f| {
1946 matches!(
1947 f.data_type(),
1948 DataType::Dictionary(k, v) if **k == DataType::Int32 && **v == DataType::Utf8
1949 )
1950 });
1951 assert!(
1952 has_enum_dict,
1953 "Expected at least one enum-mapped Dictionary<Int32, Utf8> field"
1954 );
1955 let buffer: Vec<u8> = Vec::new();
1958 let mut writer = AvroWriter::new(buffer, in_schema.as_ref().clone())?;
1959 writer.write(&original)?;
1960 writer.finish()?;
1961 let bytes = writer.into_inner();
1962 let rt_reader = ReaderBuilder::new()
1964 .build(Cursor::new(bytes))
1965 .expect("reader for round-trip");
1966 let rt_schema = rt_reader.schema();
1967 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1968 let roundtrip =
1969 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat roundtrip");
1970 assert_eq!(roundtrip, original, "Avro enum round-trip mismatch");
1971 Ok(())
1972 }
1973
1974 #[test]
1975 fn test_builder_propagates_capacity_to_writer() -> Result<(), AvroError> {
1976 let cap = 64 * 1024;
1977 let buffer = Vec::<u8>::new();
1978 let mut writer = WriterBuilder::new(make_schema())
1979 .with_capacity(cap)
1980 .build::<_, AvroOcfFormat>(buffer)?;
1981 assert_eq!(writer.capacity, cap, "builder capacity not propagated");
1982 let batch = make_batch();
1983 writer.write(&batch)?;
1984 writer.finish()?;
1985 let out = writer.into_inner();
1986 assert_eq!(&out[..4], b"Obj\x01", "OCF magic missing/incorrect");
1987 Ok(())
1988 }
1989
1990 #[test]
1991 fn test_stream_writer_stores_capacity_direct_writes() -> Result<(), AvroError> {
1992 use arrow_array::{ArrayRef, Int32Array};
1993 use arrow_schema::{DataType, Field, Schema};
1994 let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
1995 let batch = RecordBatch::try_new(
1996 Arc::new(schema.clone()),
1997 vec![Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef],
1998 )?;
1999 let cap = 8192;
2000 let mut writer = WriterBuilder::new(schema)
2001 .with_capacity(cap)
2002 .build::<_, AvroSoeFormat>(Vec::new())?;
2003 assert_eq!(writer.capacity, cap);
2004 writer.write(&batch)?;
2005 let _bytes = writer.into_inner();
2006 Ok(())
2007 }
2008
2009 #[cfg(feature = "avro_custom_types")]
2010 #[test]
2011 fn test_roundtrip_duration_logical_types_ocf() -> Result<(), AvroError> {
2012 let file_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
2013 .join("test/data/duration_logical_types.avro")
2014 .to_string_lossy()
2015 .into_owned();
2016
2017 let in_file = File::open(&file_path)
2018 .unwrap_or_else(|_| panic!("Failed to open test file: {file_path}"));
2019
2020 let reader = ReaderBuilder::new()
2021 .build(BufReader::new(in_file))
2022 .expect("build reader for duration_logical_types.avro");
2023 let in_schema = reader.schema();
2024
2025 let expected_units: HashSet<TimeUnit> = [
2026 TimeUnit::Nanosecond,
2027 TimeUnit::Microsecond,
2028 TimeUnit::Millisecond,
2029 TimeUnit::Second,
2030 ]
2031 .into_iter()
2032 .collect();
2033
2034 let found_units: HashSet<TimeUnit> = in_schema
2035 .fields()
2036 .iter()
2037 .filter_map(|f| match f.data_type() {
2038 DataType::Duration(unit) => Some(*unit),
2039 _ => None,
2040 })
2041 .collect();
2042
2043 assert_eq!(
2044 found_units, expected_units,
2045 "Expected to find all four Duration TimeUnits in the schema from the initial read"
2046 );
2047
2048 let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
2049 let input =
2050 arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
2051
2052 let tmp = NamedTempFile::new().expect("create temp file");
2053 {
2054 let out_file = File::create(tmp.path()).expect("create temp avro");
2055 let mut writer = AvroWriter::new(out_file, in_schema.as_ref().clone())?;
2056 writer.write(&input)?;
2057 writer.finish()?;
2058 }
2059
2060 let rt_file = File::open(tmp.path()).expect("open round_trip avro");
2061 let rt_reader = ReaderBuilder::new()
2062 .build(BufReader::new(rt_file))
2063 .expect("build round_trip reader");
2064 let rt_schema = rt_reader.schema();
2065 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2066 let round_trip =
2067 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
2068
2069 assert_eq!(round_trip, input);
2070 Ok(())
2071 }
2072
2073 #[cfg(feature = "avro_custom_types")]
2074 #[test]
2075 fn test_run_end_encoded_roundtrip_writer() -> Result<(), AvroError> {
2076 let run_ends = Int32Array::from(vec![3, 5, 7, 8]);
2077 let run_values = Int32Array::from(vec![Some(1), Some(2), None, Some(3)]);
2078 let ree = RunArray::<Int32Type>::try_new(&run_ends, &run_values)?;
2079 let field = Field::new("x", ree.data_type().clone(), true);
2080 let schema = Schema::new(vec![field]);
2081 let batch = RecordBatch::try_new(
2082 Arc::new(schema.clone()),
2083 vec![Arc::new(ree.clone()) as ArrayRef],
2084 )?;
2085 let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
2086 writer.write(&batch)?;
2087 writer.finish()?;
2088 let bytes = writer.into_inner();
2089 let reader = ReaderBuilder::new().build(Cursor::new(bytes))?;
2090 let out_schema = reader.schema();
2091 let batches = reader.collect::<Result<Vec<_>, _>>()?;
2092 let out = arrow::compute::concat_batches(&out_schema, &batches).expect("concat output");
2093 assert_eq!(out.num_columns(), 1);
2094 assert_eq!(out.num_rows(), 8);
2095 match out.schema().field(0).data_type() {
2096 DataType::RunEndEncoded(run_ends_field, values_field) => {
2097 assert_eq!(
2098 run_ends_field.name(),
2099 Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME
2100 );
2101 assert_eq!(run_ends_field.data_type(), &DataType::Int32);
2102 assert_eq!(values_field.name(), Field::REE_VALUES_FIELD_DEFAULT_NAME);
2103 assert_eq!(values_field.data_type(), &DataType::Int32);
2104 assert!(values_field.is_nullable());
2105 let got_ree = out
2106 .column(0)
2107 .as_any()
2108 .downcast_ref::<RunArray<Int32Type>>()
2109 .expect("RunArray<Int32Type>");
2110 assert_eq!(got_ree, &ree);
2111 }
2112 other => {
2113 panic!("Unexpected DataType for round-tripped RunEndEncoded column: {other:?}")
2114 }
2115 }
2116 Ok(())
2117 }
2118
2119 #[cfg(feature = "avro_custom_types")]
2120 #[test]
2121 fn test_run_end_encoded_string_values_int16_run_ends_roundtrip_writer() -> Result<(), AvroError>
2122 {
2123 let run_ends = Int16Array::from(vec![2, 5, 7]); let run_values = StringArray::from(vec![Some("a"), None, Some("c")]);
2125 let ree = RunArray::<Int16Type>::try_new(&run_ends, &run_values)?;
2126 let field = Field::new("s", ree.data_type().clone(), true);
2127 let schema = Schema::new(vec![field]);
2128 let batch = RecordBatch::try_new(
2129 Arc::new(schema.clone()),
2130 vec![Arc::new(ree.clone()) as ArrayRef],
2131 )?;
2132 let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
2133 writer.write(&batch)?;
2134 writer.finish()?;
2135 let bytes = writer.into_inner();
2136 let reader = ReaderBuilder::new().build(Cursor::new(bytes))?;
2137 let out_schema = reader.schema();
2138 let batches = reader.collect::<Result<Vec<_>, _>>()?;
2139 let out = arrow::compute::concat_batches(&out_schema, &batches).expect("concat output");
2140 assert_eq!(out.num_columns(), 1);
2141 assert_eq!(out.num_rows(), 7);
2142 match out.schema().field(0).data_type() {
2143 DataType::RunEndEncoded(run_ends_field, values_field) => {
2144 assert_eq!(run_ends_field.data_type(), &DataType::Int16);
2145 assert_eq!(values_field.data_type(), &DataType::Utf8);
2146 assert!(
2147 values_field.is_nullable(),
2148 "REE 'values' child should be nullable"
2149 );
2150 let got = out
2151 .column(0)
2152 .as_any()
2153 .downcast_ref::<RunArray<Int16Type>>()
2154 .expect("RunArray<Int16Type>");
2155 assert_eq!(got, &ree);
2156 }
2157 other => panic!("Unexpected DataType: {other:?}"),
2158 }
2159 Ok(())
2160 }
2161
2162 #[cfg(feature = "avro_custom_types")]
2163 #[test]
2164 fn test_run_end_encoded_int64_run_ends_numeric_values_roundtrip_writer() -> Result<(), AvroError>
2165 {
2166 let run_ends = Int64Array::from(vec![4_i64, 8_i64]);
2167 let run_values = Int32Array::from(vec![Some(999), Some(-5)]);
2168 let ree = RunArray::<Int64Type>::try_new(&run_ends, &run_values)?;
2169 let field = Field::new("y", ree.data_type().clone(), true);
2170 let schema = Schema::new(vec![field]);
2171 let batch = RecordBatch::try_new(
2172 Arc::new(schema.clone()),
2173 vec![Arc::new(ree.clone()) as ArrayRef],
2174 )?;
2175 let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
2176 writer.write(&batch)?;
2177 writer.finish()?;
2178 let bytes = writer.into_inner();
2179 let reader = ReaderBuilder::new().build(Cursor::new(bytes))?;
2180 let out_schema = reader.schema();
2181 let batches = reader.collect::<Result<Vec<_>, _>>()?;
2182 let out = arrow::compute::concat_batches(&out_schema, &batches).expect("concat output");
2183 assert_eq!(out.num_columns(), 1);
2184 assert_eq!(out.num_rows(), 8);
2185 match out.schema().field(0).data_type() {
2186 DataType::RunEndEncoded(run_ends_field, values_field) => {
2187 assert_eq!(run_ends_field.data_type(), &DataType::Int64);
2188 assert_eq!(values_field.data_type(), &DataType::Int32);
2189 assert!(values_field.is_nullable());
2190 let got = out
2191 .column(0)
2192 .as_any()
2193 .downcast_ref::<RunArray<Int64Type>>()
2194 .expect("RunArray<Int64Type>");
2195 assert_eq!(got, &ree);
2196 }
2197 other => panic!("Unexpected DataType for REE column: {other:?}"),
2198 }
2199 Ok(())
2200 }
2201
2202 #[cfg(feature = "avro_custom_types")]
2203 #[test]
2204 fn test_run_end_encoded_sliced_roundtrip_writer() -> Result<(), AvroError> {
2205 let run_ends = Int32Array::from(vec![3, 5, 7, 8]);
2206 let run_values = Int32Array::from(vec![Some(1), Some(2), None, Some(3)]);
2207 let base = RunArray::<Int32Type>::try_new(&run_ends, &run_values)?;
2208 let offset = 1usize;
2209 let length = 6usize;
2210 let base_values = base.values().as_primitive::<Int32Type>();
2211 let mut logical_window: Vec<Option<i32>> = Vec::with_capacity(length);
2212 for i in offset..offset + length {
2213 let phys = base.get_physical_index(i);
2214 let v = if base_values.is_null(phys) {
2215 None
2216 } else {
2217 Some(base_values.value(phys))
2218 };
2219 logical_window.push(v);
2220 }
2221
2222 fn compress_run_ends_i32(vals: &[Option<i32>]) -> (Int32Array, Int32Array) {
2223 if vals.is_empty() {
2224 return (Int32Array::new_null(0), Int32Array::new_null(0));
2225 }
2226 let mut run_ends_out: Vec<i32> = Vec::new();
2227 let mut run_vals_out: Vec<Option<i32>> = Vec::new();
2228 let mut cur = vals[0];
2229 let mut len = 1i32;
2230 for v in &vals[1..] {
2231 if *v == cur {
2232 len += 1;
2233 } else {
2234 let last_end = run_ends_out.last().copied().unwrap_or(0);
2235 run_ends_out.push(last_end + len);
2236 run_vals_out.push(cur);
2237 cur = *v;
2238 len = 1;
2239 }
2240 }
2241 let last_end = run_ends_out.last().copied().unwrap_or(0);
2242 run_ends_out.push(last_end + len);
2243 run_vals_out.push(cur);
2244 (
2245 Int32Array::from(run_ends_out),
2246 Int32Array::from(run_vals_out),
2247 )
2248 }
2249 let (owned_run_ends, owned_run_values) = compress_run_ends_i32(&logical_window);
2250 let owned_slice = RunArray::<Int32Type>::try_new(&owned_run_ends, &owned_run_values)?;
2251 let field = Field::new("x", owned_slice.data_type().clone(), true);
2252 let schema = Schema::new(vec![field]);
2253 let batch = RecordBatch::try_new(
2254 Arc::new(schema.clone()),
2255 vec![Arc::new(owned_slice.clone()) as ArrayRef],
2256 )?;
2257 let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
2258 writer.write(&batch)?;
2259 writer.finish()?;
2260 let bytes = writer.into_inner();
2261 let reader = ReaderBuilder::new().build(Cursor::new(bytes))?;
2262 let out_schema = reader.schema();
2263 let batches = reader.collect::<Result<Vec<_>, _>>()?;
2264 let out = arrow::compute::concat_batches(&out_schema, &batches).expect("concat output");
2265 assert_eq!(out.num_columns(), 1);
2266 assert_eq!(out.num_rows(), length);
2267 match out.schema().field(0).data_type() {
2268 DataType::RunEndEncoded(run_ends_field, values_field) => {
2269 assert_eq!(run_ends_field.data_type(), &DataType::Int32);
2270 assert_eq!(values_field.data_type(), &DataType::Int32);
2271 assert!(values_field.is_nullable());
2272 let got = out
2273 .column(0)
2274 .as_any()
2275 .downcast_ref::<RunArray<Int32Type>>()
2276 .expect("RunArray<Int32Type>");
2277 fn expand_ree_to_int32(a: &RunArray<Int32Type>) -> Int32Array {
2278 let vals = a.values().as_primitive::<Int32Type>();
2279 let mut out: Vec<Option<i32>> = Vec::with_capacity(a.len());
2280 for i in 0..a.len() {
2281 let phys = a.get_physical_index(i);
2282 out.push(if vals.is_null(phys) {
2283 None
2284 } else {
2285 Some(vals.value(phys))
2286 });
2287 }
2288 Int32Array::from(out)
2289 }
2290 let got_logical = expand_ree_to_int32(got);
2291 let expected_logical = Int32Array::from(logical_window);
2292 assert_eq!(
2293 got_logical, expected_logical,
2294 "Logical values differ after REE slice round-trip"
2295 );
2296 }
2297 other => panic!("Unexpected DataType for REE column: {other:?}"),
2298 }
2299 Ok(())
2300 }
2301
2302 #[cfg(not(feature = "avro_custom_types"))]
2303 #[test]
2304 fn test_run_end_encoded_roundtrip_writer_feature_off() -> Result<(), AvroError> {
2305 use arrow_schema::{DataType, Field, Schema};
2306 let run_ends = arrow_array::Int32Array::from(vec![3, 5, 7, 8]);
2307 let run_values = arrow_array::Int32Array::from(vec![Some(1), Some(2), None, Some(3)]);
2308 let ree = arrow_array::RunArray::<arrow_array::types::Int32Type>::try_new(
2309 &run_ends,
2310 &run_values,
2311 )?;
2312 let field = Field::new("x", ree.data_type().clone(), true);
2313 let schema = Schema::new(vec![field]);
2314 let batch =
2315 RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(ree) as ArrayRef])?;
2316 let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
2317 writer.write(&batch)?;
2318 writer.finish()?;
2319 let bytes = writer.into_inner();
2320 let reader = ReaderBuilder::new().build(Cursor::new(bytes))?;
2321 let out_schema = reader.schema();
2322 let batches = reader.collect::<Result<Vec<_>, _>>()?;
2323 let out = arrow::compute::concat_batches(&out_schema, &batches).expect("concat output");
2324 assert_eq!(out.num_columns(), 1);
2325 assert_eq!(out.num_rows(), 8);
2326 assert_eq!(out.schema().field(0).data_type(), &DataType::Int32);
2327 let got = out.column(0).as_primitive::<Int32Type>();
2328 let expected = Int32Array::from(vec![
2329 Some(1),
2330 Some(1),
2331 Some(1),
2332 Some(2),
2333 Some(2),
2334 None,
2335 None,
2336 Some(3),
2337 ]);
2338 assert_eq!(got, &expected);
2339 Ok(())
2340 }
2341
2342 #[cfg(not(feature = "avro_custom_types"))]
2343 #[test]
2344 fn test_run_end_encoded_string_values_int16_run_ends_roundtrip_writer_feature_off()
2345 -> Result<(), AvroError> {
2346 use arrow_schema::{DataType, Field, Schema};
2347 let run_ends = arrow_array::Int16Array::from(vec![2, 5, 7]);
2348 let run_values = arrow_array::StringArray::from(vec![Some("a"), None, Some("c")]);
2349 let ree = arrow_array::RunArray::<arrow_array::types::Int16Type>::try_new(
2350 &run_ends,
2351 &run_values,
2352 )?;
2353 let field = Field::new("s", ree.data_type().clone(), true);
2354 let schema = Schema::new(vec![field]);
2355 let batch =
2356 RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(ree) as ArrayRef])?;
2357 let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
2358 writer.write(&batch)?;
2359 writer.finish()?;
2360 let bytes = writer.into_inner();
2361 let reader = ReaderBuilder::new().build(Cursor::new(bytes))?;
2362 let out_schema = reader.schema();
2363 let batches = reader.collect::<Result<Vec<_>, _>>()?;
2364 let out = arrow::compute::concat_batches(&out_schema, &batches).expect("concat output");
2365 assert_eq!(out.num_columns(), 1);
2366 assert_eq!(out.num_rows(), 7);
2367 assert_eq!(out.schema().field(0).data_type(), &DataType::Utf8);
2368 let got = out
2369 .column(0)
2370 .as_any()
2371 .downcast_ref::<arrow_array::StringArray>()
2372 .expect("StringArray");
2373 let expected = arrow_array::StringArray::from(vec![
2374 Some("a"),
2375 Some("a"),
2376 None,
2377 None,
2378 None,
2379 Some("c"),
2380 Some("c"),
2381 ]);
2382 assert_eq!(got, &expected);
2383 Ok(())
2384 }
2385
2386 #[cfg(not(feature = "avro_custom_types"))]
2387 #[test]
2388 fn test_run_end_encoded_int64_run_ends_numeric_values_roundtrip_writer_feature_off()
2389 -> Result<(), AvroError> {
2390 use arrow_schema::{DataType, Field, Schema};
2391 let run_ends = arrow_array::Int64Array::from(vec![4_i64, 8_i64]);
2392 let run_values = Int32Array::from(vec![Some(999), Some(-5)]);
2393 let ree = arrow_array::RunArray::<arrow_array::types::Int64Type>::try_new(
2394 &run_ends,
2395 &run_values,
2396 )?;
2397 let field = Field::new("y", ree.data_type().clone(), true);
2398 let schema = Schema::new(vec![field]);
2399 let batch =
2400 RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(ree) as ArrayRef])?;
2401 let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
2402 writer.write(&batch)?;
2403 writer.finish()?;
2404 let bytes = writer.into_inner();
2405 let reader = ReaderBuilder::new().build(Cursor::new(bytes))?;
2406 let out_schema = reader.schema();
2407 let batches = reader.collect::<Result<Vec<_>, _>>()?;
2408 let out = arrow::compute::concat_batches(&out_schema, &batches).expect("concat output");
2409 assert_eq!(out.num_columns(), 1);
2410 assert_eq!(out.num_rows(), 8);
2411 assert_eq!(out.schema().field(0).data_type(), &DataType::Int32);
2412 let got = out.column(0).as_primitive::<Int32Type>();
2413 let expected = Int32Array::from(vec![
2414 Some(999),
2415 Some(999),
2416 Some(999),
2417 Some(999),
2418 Some(-5),
2419 Some(-5),
2420 Some(-5),
2421 Some(-5),
2422 ]);
2423 assert_eq!(got, &expected);
2424 Ok(())
2425 }
2426
2427 #[cfg(not(feature = "avro_custom_types"))]
2428 #[test]
2429 fn test_run_end_encoded_sliced_roundtrip_writer_feature_off() -> Result<(), AvroError> {
2430 use arrow_schema::{DataType, Field, Schema};
2431 let run_ends = Int32Array::from(vec![2, 4, 6]);
2432 let run_values = Int32Array::from(vec![Some(1), Some(2), None]);
2433 let ree = arrow_array::RunArray::<arrow_array::types::Int32Type>::try_new(
2434 &run_ends,
2435 &run_values,
2436 )?;
2437 let field = Field::new("x", ree.data_type().clone(), true);
2438 let schema = Schema::new(vec![field]);
2439 let batch =
2440 RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(ree) as ArrayRef])?;
2441 let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
2442 writer.write(&batch)?;
2443 writer.finish()?;
2444 let bytes = writer.into_inner();
2445 let reader = ReaderBuilder::new().build(Cursor::new(bytes))?;
2446 let out_schema = reader.schema();
2447 let batches = reader.collect::<Result<Vec<_>, _>>()?;
2448 let out = arrow::compute::concat_batches(&out_schema, &batches).expect("concat output");
2449 assert_eq!(out.num_columns(), 1);
2450 assert_eq!(out.num_rows(), 6);
2451 assert_eq!(out.schema().field(0).data_type(), &DataType::Int32);
2452 let got = out.column(0).as_primitive::<Int32Type>();
2453 let expected = Int32Array::from(vec![Some(1), Some(1), Some(2), Some(2), None, None]);
2454 assert_eq!(got, &expected);
2455 Ok(())
2456 }
2457
2458 #[test]
2459 #[cfg(feature = "snappy")]
2461 fn test_nullable_impala_roundtrip() -> Result<(), AvroError> {
2462 let path = arrow_test_data("avro/nullable.impala.avro");
2463 let rdr_file = File::open(&path).expect("open avro/nullable.impala.avro");
2464 let reader = ReaderBuilder::new()
2465 .build(BufReader::new(rdr_file))
2466 .expect("build reader for nullable.impala.avro");
2467 let in_schema = reader.schema();
2468 assert!(
2469 in_schema.fields().iter().any(|f| f.is_nullable()),
2470 "expected at least one nullable field in avro/nullable.impala.avro"
2471 );
2472 let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
2473 let original =
2474 arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
2475 let buffer: Vec<u8> = Vec::new();
2476 let mut writer = AvroWriter::new(buffer, in_schema.as_ref().clone())?;
2477 writer.write(&original)?;
2478 writer.finish()?;
2479 let out_bytes = writer.into_inner();
2480 let rt_reader = ReaderBuilder::new()
2481 .build(Cursor::new(out_bytes))
2482 .expect("build reader for round-tripped in-memory OCF");
2483 let rt_schema = rt_reader.schema();
2484 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2485 let roundtrip =
2486 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat roundtrip");
2487 assert_eq!(
2488 roundtrip, original,
2489 "Round-trip Avro data mismatch for nullable.impala.avro"
2490 );
2491 Ok(())
2492 }
2493
2494 #[test]
2495 #[cfg(feature = "snappy")]
2496 fn test_datapage_v2_roundtrip() -> Result<(), AvroError> {
2497 let path = arrow_test_data("avro/datapage_v2.snappy.avro");
2498 let rdr_file = File::open(&path).expect("open avro/datapage_v2.snappy.avro");
2499 let reader = ReaderBuilder::new()
2500 .build(BufReader::new(rdr_file))
2501 .expect("build reader for datapage_v2.snappy.avro");
2502 let in_schema = reader.schema();
2503 let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
2504 let original =
2505 arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
2506 let mut writer = AvroWriter::new(Vec::<u8>::new(), in_schema.as_ref().clone())?;
2507 writer.write(&original)?;
2508 writer.finish()?;
2509 let bytes = writer.into_inner();
2510 let rt_reader = ReaderBuilder::new()
2511 .build(Cursor::new(bytes))
2512 .expect("build round-trip reader");
2513 let rt_schema = rt_reader.schema();
2514 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2515 let round_trip =
2516 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
2517 assert_eq!(
2518 round_trip, original,
2519 "Round-trip batch mismatch for datapage_v2.snappy.avro"
2520 );
2521 Ok(())
2522 }
2523
2524 #[test]
2525 #[cfg(feature = "snappy")]
2526 fn test_single_nan_roundtrip() -> Result<(), AvroError> {
2527 let path = arrow_test_data("avro/single_nan.avro");
2528 let in_file = File::open(&path).expect("open avro/single_nan.avro");
2529 let reader = ReaderBuilder::new()
2530 .build(BufReader::new(in_file))
2531 .expect("build reader for single_nan.avro");
2532 let in_schema = reader.schema();
2533 let in_batches = reader.collect::<Result<Vec<_>, _>>()?;
2534 let original =
2535 arrow::compute::concat_batches(&in_schema, &in_batches).expect("concat input");
2536 let mut writer = AvroWriter::new(Vec::<u8>::new(), original.schema().as_ref().clone())?;
2537 writer.write(&original)?;
2538 writer.finish()?;
2539 let bytes = writer.into_inner();
2540 let rt_reader = ReaderBuilder::new()
2541 .build(Cursor::new(bytes))
2542 .expect("build round_trip reader");
2543 let rt_schema = rt_reader.schema();
2544 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2545 let round_trip =
2546 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
2547 assert_eq!(
2548 round_trip, original,
2549 "Round-trip batch mismatch for avro/single_nan.avro"
2550 );
2551 Ok(())
2552 }
2553 #[test]
2554 #[cfg(feature = "snappy")]
2556 fn test_dict_pages_offset_zero_roundtrip() -> Result<(), AvroError> {
2557 let path = arrow_test_data("avro/dict-page-offset-zero.avro");
2558 let rdr_file = File::open(&path).expect("open avro/dict-page-offset-zero.avro");
2559 let reader = ReaderBuilder::new()
2560 .build(BufReader::new(rdr_file))
2561 .expect("build reader for dict-page-offset-zero.avro");
2562 let in_schema = reader.schema();
2563 let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
2564 let original =
2565 arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
2566 let buffer: Vec<u8> = Vec::new();
2567 let mut writer = AvroWriter::new(buffer, original.schema().as_ref().clone())?;
2568 writer.write(&original)?;
2569 writer.finish()?;
2570 let bytes = writer.into_inner();
2571 let rt_reader = ReaderBuilder::new()
2572 .build(Cursor::new(bytes))
2573 .expect("build reader for round-trip");
2574 let rt_schema = rt_reader.schema();
2575 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2576 let roundtrip =
2577 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat roundtrip");
2578 assert_eq!(
2579 roundtrip, original,
2580 "Round-trip batch mismatch for avro/dict-page-offset-zero.avro"
2581 );
2582 Ok(())
2583 }
2584
2585 #[test]
2586 #[cfg(feature = "snappy")]
2587 fn test_repeated_no_annotation_roundtrip() -> Result<(), AvroError> {
2588 let path = arrow_test_data("avro/repeated_no_annotation.avro");
2589 let in_file = File::open(&path).expect("open avro/repeated_no_annotation.avro");
2590 let reader = ReaderBuilder::new()
2591 .build(BufReader::new(in_file))
2592 .expect("build reader for repeated_no_annotation.avro");
2593 let in_schema = reader.schema();
2594 let in_batches = reader.collect::<Result<Vec<_>, _>>()?;
2595 let original =
2596 arrow::compute::concat_batches(&in_schema, &in_batches).expect("concat input");
2597 let mut writer = AvroWriter::new(Vec::<u8>::new(), original.schema().as_ref().clone())?;
2598 writer.write(&original)?;
2599 writer.finish()?;
2600 let bytes = writer.into_inner();
2601 let rt_reader = ReaderBuilder::new()
2602 .build(Cursor::new(bytes))
2603 .expect("build reader for round-trip buffer");
2604 let rt_schema = rt_reader.schema();
2605 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2606 let round_trip =
2607 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round-trip");
2608 assert_eq!(
2609 round_trip, original,
2610 "Round-trip batch mismatch for avro/repeated_no_annotation.avro"
2611 );
2612 Ok(())
2613 }
2614
2615 #[test]
2616 fn test_nested_record_type_reuse_roundtrip() -> Result<(), AvroError> {
2617 let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
2618 .join("test/data/nested_record_reuse.avro")
2619 .to_string_lossy()
2620 .into_owned();
2621 let in_file = File::open(&path).expect("open avro/nested_record_reuse.avro");
2622 let reader = ReaderBuilder::new()
2623 .build(BufReader::new(in_file))
2624 .expect("build reader for nested_record_reuse.avro");
2625 let in_schema = reader.schema();
2626 let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
2627 let input =
2628 arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
2629 let mut writer = AvroWriter::new(Vec::<u8>::new(), in_schema.as_ref().clone())?;
2630 writer.write(&input)?;
2631 writer.finish()?;
2632 let bytes = writer.into_inner();
2633 let rt_reader = ReaderBuilder::new()
2634 .build(Cursor::new(bytes))
2635 .expect("build round_trip reader");
2636 let rt_schema = rt_reader.schema();
2637 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2638 let round_trip =
2639 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
2640 assert_eq!(
2641 round_trip, input,
2642 "Round-trip batch mismatch for nested_record_reuse.avro"
2643 );
2644 Ok(())
2645 }
2646
2647 #[test]
2648 fn test_enum_type_reuse_roundtrip() -> Result<(), AvroError> {
2649 let path =
2650 std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("test/data/enum_reuse.avro");
2651 let rdr_file = std::fs::File::open(&path).expect("open test/data/enum_reuse.avro");
2652 let reader = ReaderBuilder::new()
2653 .build(std::io::BufReader::new(rdr_file))
2654 .expect("build reader for enum_reuse.avro");
2655 let in_schema = reader.schema();
2656 let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
2657 let original =
2658 arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
2659 let mut writer = AvroWriter::new(Vec::<u8>::new(), original.schema().as_ref().clone())?;
2660 writer.write(&original)?;
2661 writer.finish()?;
2662 let bytes = writer.into_inner();
2663 let rt_reader = ReaderBuilder::new()
2664 .build(std::io::Cursor::new(bytes))
2665 .expect("build round_trip reader");
2666 let rt_schema = rt_reader.schema();
2667 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2668 let round_trip =
2669 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
2670 assert_eq!(
2671 round_trip, original,
2672 "Avro enum type reuse round-trip mismatch"
2673 );
2674 Ok(())
2675 }
2676
2677 #[test]
2678 #[cfg_attr(miri, ignore)] fn comprehensive_e2e_test_roundtrip() -> Result<(), AvroError> {
2680 let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
2681 .join("test/data/comprehensive_e2e.avro");
2682 let rdr_file = File::open(&path).expect("open test/data/comprehensive_e2e.avro");
2683 let reader = ReaderBuilder::new()
2684 .build(BufReader::new(rdr_file))
2685 .expect("build reader for comprehensive_e2e.avro");
2686 let in_schema = reader.schema();
2687 let in_batches = reader.collect::<Result<Vec<_>, _>>()?;
2688 let original =
2689 arrow::compute::concat_batches(&in_schema, &in_batches).expect("concat input");
2690 let sink: Vec<u8> = Vec::new();
2691 let mut writer = AvroWriter::new(sink, original.schema().as_ref().clone())?;
2692 writer.write(&original)?;
2693 writer.finish()?;
2694 let bytes = writer.into_inner();
2695 let rt_reader = ReaderBuilder::new()
2696 .build(Cursor::new(bytes))
2697 .expect("build round-trip reader");
2698 let rt_schema = rt_reader.schema();
2699 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2700 let roundtrip =
2701 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat roundtrip");
2702 assert_eq!(
2703 roundtrip, original,
2704 "Round-trip batch mismatch for comprehensive_e2e.avro"
2705 );
2706 Ok(())
2707 }
2708
2709 #[test]
2710 fn test_roundtrip_new_time_encoders_writer() -> Result<(), AvroError> {
2711 let schema = Schema::new(vec![
2712 Field::new("d32", DataType::Date32, false),
2713 Field::new("t32_ms", DataType::Time32(TimeUnit::Millisecond), false),
2714 Field::new("t64_us", DataType::Time64(TimeUnit::Microsecond), false),
2715 Field::new(
2716 "ts_ms",
2717 DataType::Timestamp(TimeUnit::Millisecond, None),
2718 false,
2719 ),
2720 Field::new(
2721 "ts_us",
2722 DataType::Timestamp(TimeUnit::Microsecond, None),
2723 false,
2724 ),
2725 Field::new(
2726 "ts_ns",
2727 DataType::Timestamp(TimeUnit::Nanosecond, None),
2728 false,
2729 ),
2730 ]);
2731 let d32 = Date32Array::from(vec![0, 1, -1]);
2732 let t32_ms: PrimitiveArray<Time32MillisecondType> =
2733 vec![0_i32, 12_345_i32, 86_399_999_i32].into();
2734 let t64_us: PrimitiveArray<Time64MicrosecondType> =
2735 vec![0_i64, 1_234_567_i64, 86_399_999_999_i64].into();
2736 let ts_ms: PrimitiveArray<TimestampMillisecondType> =
2737 vec![0_i64, -1_i64, 1_700_000_000_000_i64].into();
2738 let ts_us: PrimitiveArray<TimestampMicrosecondType> = vec![0_i64, 1_i64, -1_i64].into();
2739 let ts_ns: PrimitiveArray<TimestampNanosecondType> = vec![0_i64, 1_i64, -1_i64].into();
2740 let batch = RecordBatch::try_new(
2741 Arc::new(schema.clone()),
2742 vec![
2743 Arc::new(d32) as ArrayRef,
2744 Arc::new(t32_ms) as ArrayRef,
2745 Arc::new(t64_us) as ArrayRef,
2746 Arc::new(ts_ms) as ArrayRef,
2747 Arc::new(ts_us) as ArrayRef,
2748 Arc::new(ts_ns) as ArrayRef,
2749 ],
2750 )?;
2751 let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
2752 writer.write(&batch)?;
2753 writer.finish()?;
2754 let bytes = writer.into_inner();
2755 let rt_reader = ReaderBuilder::new()
2756 .build(std::io::Cursor::new(bytes))
2757 .expect("build reader for round-trip of new time encoders");
2758 let rt_schema = rt_reader.schema();
2759 let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2760 let roundtrip =
2761 arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat roundtrip");
2762 assert_eq!(roundtrip, batch);
2763 Ok(())
2764 }
2765
2766 fn make_encoder_schema() -> Schema {
2767 Schema::new(vec![
2768 Field::new("a", DataType::Int32, false),
2769 Field::new("b", DataType::Int32, false),
2770 ])
2771 }
2772
2773 fn make_encoder_batch(schema: &Schema) -> RecordBatch {
2774 let a = Int32Array::from(vec![1, 2, 3]);
2775 let b = Int32Array::from(vec![10, 20, 30]);
2776 RecordBatch::try_new(
2777 Arc::new(schema.clone()),
2778 vec![Arc::new(a) as ArrayRef, Arc::new(b) as ArrayRef],
2779 )
2780 .expect("failed to build test RecordBatch")
2781 }
2782
2783 fn make_real_avro_schema_and_batch() -> Result<(Schema, RecordBatch, AvroSchema), AvroError> {
2784 let avro_json = r#"
2785 {
2786 "type": "record",
2787 "name": "User",
2788 "fields": [
2789 { "name": "id", "type": "long" },
2790 { "name": "name", "type": "string" },
2791 { "name": "active", "type": "boolean" },
2792 { "name": "tags", "type": { "type": "array", "items": "int" } },
2793 { "name": "opt", "type": ["null", "string"], "default": null }
2794 ]
2795 }"#;
2796 let avro_schema = AvroSchema::new(avro_json.to_string());
2797 let mut md = HashMap::new();
2798 md.insert(
2799 SCHEMA_METADATA_KEY.to_string(),
2800 avro_schema.json_string.clone(),
2801 );
2802 let item_field = Arc::new(Field::new(
2803 Field::LIST_FIELD_DEFAULT_NAME,
2804 DataType::Int32,
2805 false,
2806 ));
2807 let schema = Schema::new_with_metadata(
2808 vec![
2809 Field::new("id", DataType::Int64, false),
2810 Field::new("name", DataType::Utf8, false),
2811 Field::new("active", DataType::Boolean, false),
2812 Field::new("tags", DataType::List(item_field.clone()), false),
2813 Field::new("opt", DataType::Utf8, true),
2814 ],
2815 md,
2816 );
2817 let id = Int64Array::from(vec![1, 2, 3]);
2818 let name = StringArray::from(vec!["alice", "bob", "carol"]);
2819 let active = BooleanArray::from(vec![true, false, true]);
2820 let mut tags_builder = ListBuilder::new(Int32Builder::new()).with_field(item_field);
2821 tags_builder.values().append_value(1);
2822 tags_builder.values().append_value(2);
2823 tags_builder.append(true);
2824 tags_builder.append(true);
2825 tags_builder.values().append_value(3);
2826 tags_builder.append(true);
2827 let tags = tags_builder.finish();
2828 let opt = StringArray::from(vec![Some("x"), None, Some("z")]);
2829 let batch = RecordBatch::try_new(
2830 Arc::new(schema.clone()),
2831 vec![
2832 Arc::new(id) as ArrayRef,
2833 Arc::new(name) as ArrayRef,
2834 Arc::new(active) as ArrayRef,
2835 Arc::new(tags) as ArrayRef,
2836 Arc::new(opt) as ArrayRef,
2837 ],
2838 )?;
2839 Ok((schema, batch, avro_schema))
2840 }
2841
2842 #[test]
2843 fn test_row_writer_matches_stream_writer_soe() -> Result<(), AvroError> {
2844 let schema = make_encoder_schema();
2845 let batch = make_encoder_batch(&schema);
2846 let mut stream = AvroStreamWriter::new(Vec::<u8>::new(), schema.clone())?;
2847 stream.write(&batch)?;
2848 stream.finish()?;
2849 let stream_bytes = stream.into_inner();
2850 let mut row_writer = WriterBuilder::new(schema).build_encoder::<AvroSoeFormat>()?;
2851 row_writer.encode(&batch)?;
2852 let rows = row_writer.flush();
2853 let row_bytes: Vec<u8> = rows.bytes().to_vec();
2854 assert_eq!(stream_bytes, row_bytes);
2855 Ok(())
2856 }
2857
2858 #[test]
2859 fn test_row_writer_flush_clears_buffer() -> Result<(), AvroError> {
2860 let schema = make_encoder_schema();
2861 let batch = make_encoder_batch(&schema);
2862 let mut row_writer = WriterBuilder::new(schema).build_encoder::<AvroSoeFormat>()?;
2863 row_writer.encode(&batch)?;
2864 assert_eq!(row_writer.buffered_len(), batch.num_rows());
2865 let out1 = row_writer.flush();
2866 assert_eq!(out1.len(), batch.num_rows());
2867 assert_eq!(row_writer.buffered_len(), 0);
2868 let out2 = row_writer.flush();
2869 assert_eq!(out2.len(), 0);
2870 Ok(())
2871 }
2872
2873 #[test]
2874 fn test_row_writer_roundtrip_decoder_soe_real_avro_data() -> Result<(), AvroError> {
2875 let (schema, batch, avro_schema) = make_real_avro_schema_and_batch()?;
2876 let mut store = SchemaStore::new();
2877 store.register(avro_schema.clone())?;
2878 let mut row_writer = WriterBuilder::new(schema).build_encoder::<AvroSoeFormat>()?;
2879 row_writer.encode(&batch)?;
2880 let rows = row_writer.flush();
2881 let mut decoder = ReaderBuilder::new()
2882 .with_writer_schema_store(store)
2883 .with_batch_size(1024)
2884 .build_decoder()?;
2885 for row in rows.iter() {
2886 let consumed = decoder.decode(row.as_ref())?;
2887 assert_eq!(
2888 consumed,
2889 row.len(),
2890 "decoder should consume the full row frame"
2891 );
2892 }
2893 let out = decoder.flush()?.expect("decoded batch");
2894 let expected = pretty_format_batches(std::slice::from_ref(&batch))?.to_string();
2895 let actual = pretty_format_batches(&[out])?.to_string();
2896 assert_eq!(expected, actual);
2897 Ok(())
2898 }
2899
2900 #[test]
2901 fn test_row_writer_roundtrip_decoder_soe_streaming_chunks() -> Result<(), AvroError> {
2902 let (schema, batch, avro_schema) = make_real_avro_schema_and_batch()?;
2903 let mut store = SchemaStore::new();
2904 store.register(avro_schema.clone())?;
2905 let mut row_writer = WriterBuilder::new(schema).build_encoder::<AvroSoeFormat>()?;
2906 row_writer.encode(&batch)?;
2907 let rows = row_writer.flush();
2908 let mut stream: Vec<u8> = Vec::new();
2910 let mut boundaries: Vec<usize> = Vec::with_capacity(rows.len() + 1);
2911 boundaries.push(0usize);
2912 for row in rows.iter() {
2913 stream.extend_from_slice(row.as_ref());
2914 boundaries.push(stream.len());
2915 }
2916 let mut decoder = ReaderBuilder::new()
2917 .with_writer_schema_store(store)
2918 .with_batch_size(1024)
2919 .build_decoder()?;
2920 let mut buffered = BytesMut::new();
2921 let chunk_rows = [1usize, 2, 3, 1, 4, 2];
2922 let mut row_idx = 0usize;
2923 let mut i = 0usize;
2924 let n_rows = rows.len();
2925 while row_idx < n_rows {
2926 let take = chunk_rows[i % chunk_rows.len()];
2927 i += 1;
2928 let end_row = (row_idx + take).min(n_rows);
2929 let byte_start = boundaries[row_idx];
2930 let byte_end = boundaries[end_row];
2931 buffered.extend_from_slice(&stream[byte_start..byte_end]);
2932 loop {
2933 let consumed = decoder.decode(&buffered)?;
2934 if consumed == 0 {
2935 break;
2936 }
2937 let _ = buffered.split_to(consumed);
2938 }
2939 assert!(
2940 buffered.is_empty(),
2941 "expected decoder to consume the entire frame-aligned chunk"
2942 );
2943 row_idx = end_row;
2944 }
2945 let out = decoder.flush()?.expect("decoded batch");
2946 let expected = pretty_format_batches(std::slice::from_ref(&batch))?.to_string();
2947 let actual = pretty_format_batches(&[out])?.to_string();
2948 assert_eq!(expected, actual);
2949 Ok(())
2950 }
2951
2952 #[test]
2953 fn test_row_writer_roundtrip_decoder_confluent_wire_format_id() -> Result<(), AvroError> {
2954 let (schema, batch, avro_schema) = make_real_avro_schema_and_batch()?;
2955 let schema_id: u32 = 42;
2956 let mut store = SchemaStore::new_with_type(FingerprintAlgorithm::Id);
2957 store.set(Fingerprint::Id(schema_id), avro_schema.clone())?;
2958 let mut row_writer = WriterBuilder::new(schema)
2959 .with_fingerprint_strategy(FingerprintStrategy::Id(schema_id))
2960 .build_encoder::<AvroSoeFormat>()?;
2961 row_writer.encode(&batch)?;
2962 let rows = row_writer.flush();
2963 let mut decoder = ReaderBuilder::new()
2964 .with_writer_schema_store(store)
2965 .with_batch_size(1024)
2966 .build_decoder()?;
2967 for row in rows.iter() {
2968 let consumed = decoder.decode(row.as_ref())?;
2969 assert_eq!(consumed, row.len());
2970 }
2971 let out = decoder.flush()?.expect("decoded batch");
2972 let expected = pretty_format_batches(std::slice::from_ref(&batch))?.to_string();
2973 let actual = pretty_format_batches(&[out])?.to_string();
2974 assert_eq!(expected, actual);
2975 Ok(())
2976 }
2977 #[test]
2978 fn test_encoder_encode_batches_flush_and_encoded_rows_methods_with_avro_binary_format()
2979 -> Result<(), AvroError> {
2980 use crate::writer::format::AvroBinaryFormat;
2981 use arrow_array::{ArrayRef, Int32Array, RecordBatch};
2982 use arrow_schema::{DataType, Field, Schema};
2983 use std::sync::Arc;
2984 let schema = Schema::new(vec![
2985 Field::new("a", DataType::Int32, false),
2986 Field::new("b", DataType::Int32, false),
2987 ]);
2988 let schema_ref = Arc::new(schema.clone());
2989 let batch1 = RecordBatch::try_new(
2990 schema_ref.clone(),
2991 vec![
2992 Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef,
2993 Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef,
2994 ],
2995 )?;
2996 let batch2 = RecordBatch::try_new(
2997 schema_ref,
2998 vec![
2999 Arc::new(Int32Array::from(vec![4, 5])) as ArrayRef,
3000 Arc::new(Int32Array::from(vec![40, 50])) as ArrayRef,
3001 ],
3002 )?;
3003 let mut encoder = WriterBuilder::new(schema).build_encoder::<AvroBinaryFormat>()?;
3004 let empty = Encoder::flush(&mut encoder);
3005 assert_eq!(EncodedRows::len(&empty), 0);
3006 assert!(EncodedRows::is_empty(&empty));
3007 assert_eq!(EncodedRows::bytes(&empty).as_ref(), &[] as &[u8]);
3008 assert_eq!(EncodedRows::offsets(&empty), &[0usize]);
3009 assert_eq!(EncodedRows::iter(&empty).count(), 0);
3010 let empty_vecs: Vec<Vec<u8>> = empty.iter().map(|b| b.to_vec()).collect();
3011 assert!(empty_vecs.is_empty());
3012 let batches = vec![batch1, batch2];
3013 Encoder::encode_batches(&mut encoder, &batches)?;
3014 assert_eq!(encoder.buffered_len(), 5);
3015 let rows = Encoder::flush(&mut encoder);
3016 assert_eq!(
3017 encoder.buffered_len(),
3018 0,
3019 "Encoder::flush should reset the internal offsets"
3020 );
3021 assert_eq!(EncodedRows::len(&rows), 5);
3022 assert!(!EncodedRows::is_empty(&rows));
3023 let expected_offsets: &[usize] = &[0, 2, 4, 6, 8, 10];
3024 assert_eq!(EncodedRows::offsets(&rows), expected_offsets);
3025 let expected_rows: Vec<Vec<u8>> = vec![
3026 vec![2, 20],
3027 vec![4, 40],
3028 vec![6, 60],
3029 vec![8, 80],
3030 vec![10, 100],
3031 ];
3032 let expected_stream: Vec<u8> = expected_rows.concat();
3033 assert_eq!(
3034 EncodedRows::bytes(&rows).as_ref(),
3035 expected_stream.as_slice()
3036 );
3037 for (i, expected) in expected_rows.iter().enumerate() {
3038 assert_eq!(EncodedRows::row(&rows, i)?.as_ref(), expected.as_slice());
3039 }
3040 let iter_rows: Vec<Vec<u8>> = EncodedRows::iter(&rows).map(|b| b.to_vec()).collect();
3041 assert_eq!(iter_rows, expected_rows);
3042 let recreated = EncodedRows::new(
3043 EncodedRows::bytes(&rows).clone(),
3044 EncodedRows::offsets(&rows).to_vec(),
3045 );
3046 assert_eq!(EncodedRows::len(&recreated), EncodedRows::len(&rows));
3047 assert_eq!(EncodedRows::bytes(&recreated), EncodedRows::bytes(&rows));
3048 assert_eq!(
3049 EncodedRows::offsets(&recreated),
3050 EncodedRows::offsets(&rows)
3051 );
3052 let rec_vecs: Vec<Vec<u8>> = recreated.iter().map(|b| b.to_vec()).collect();
3053 assert_eq!(rec_vecs, iter_rows);
3054 let empty_again = Encoder::flush(&mut encoder);
3055 assert!(EncodedRows::is_empty(&empty_again));
3056 Ok(())
3057 }
3058
3059 #[test]
3060 fn test_writer_builder_build_rejects_avro_binary_format() {
3061 use crate::writer::format::AvroBinaryFormat;
3062 use arrow_schema::{DataType, Field, Schema};
3063 let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
3064 let err = WriterBuilder::new(schema)
3065 .build::<_, AvroBinaryFormat>(Vec::<u8>::new())
3066 .unwrap_err();
3067 match err {
3068 AvroError::InvalidArgument(msg) => assert_eq!(
3069 msg,
3070 "AvroBinaryFormat is only supported with Encoder, use build_encoder instead"
3071 ),
3072 other => panic!("expected InvalidArgumentError, got {other:?}"),
3073 }
3074 }
3075 #[test]
3076 fn test_row_encoder_avro_binary_format_roundtrip_decoder_with_soe_framing()
3077 -> Result<(), AvroError> {
3078 use crate::writer::format::AvroBinaryFormat;
3079 let (schema, batch, avro_schema) = make_real_avro_schema_and_batch()?;
3080 let batches: Vec<RecordBatch> = vec![batch.clone(), batch.slice(1, 2)];
3081 let expected = arrow::compute::concat_batches(&batch.schema(), &batches)?;
3082 let mut binary_encoder =
3083 WriterBuilder::new(schema.clone()).build_encoder::<AvroBinaryFormat>()?;
3084 binary_encoder.encode_batches(&batches)?;
3085 let binary_rows = binary_encoder.flush();
3086 assert_eq!(
3087 binary_rows.len(),
3088 expected.num_rows(),
3089 "binary encoder row count mismatch"
3090 );
3091 let mut soe_encoder = WriterBuilder::new(schema).build_encoder::<AvroSoeFormat>()?;
3092 soe_encoder.encode_batches(&batches)?;
3093 let soe_rows = soe_encoder.flush();
3094 assert_eq!(
3095 soe_rows.len(),
3096 binary_rows.len(),
3097 "SOE vs binary row count mismatch"
3098 );
3099 let mut store = SchemaStore::new(); let fp = store.register(avro_schema)?;
3101 let fp_le_bytes = match fp {
3102 Fingerprint::Rabin(v) => v.to_le_bytes(),
3103 other => panic!("expected Rabin fingerprint from SchemaStore::new(), got {other:?}"),
3104 };
3105 const SOE_MAGIC: [u8; 2] = [0xC3, 0x01];
3106 const SOE_PREFIX_LEN: usize = 2 + 8;
3107 for i in 0..binary_rows.len() {
3108 let body = binary_rows.row(i)?;
3109 let soe = soe_rows.row(i)?;
3110 assert!(
3111 soe.len() >= SOE_PREFIX_LEN,
3112 "expected SOE row to include prefix"
3113 );
3114 assert_eq!(&soe.as_ref()[..2], &SOE_MAGIC);
3115 assert_eq!(&soe.as_ref()[2..SOE_PREFIX_LEN], &fp_le_bytes);
3116 assert_eq!(
3117 &soe.as_ref()[SOE_PREFIX_LEN..],
3118 body.as_ref(),
3119 "SOE body bytes differ from AvroBinaryFormat body bytes (row {i})"
3120 );
3121 }
3122 let mut decoder = ReaderBuilder::new()
3123 .with_writer_schema_store(store)
3124 .with_batch_size(1024)
3125 .build_decoder()?;
3126 for body in binary_rows.iter() {
3127 let mut framed = Vec::with_capacity(SOE_PREFIX_LEN + body.len());
3128 framed.extend_from_slice(&SOE_MAGIC);
3129 framed.extend_from_slice(&fp_le_bytes);
3130 framed.extend_from_slice(body.as_ref());
3131 let consumed = decoder.decode(&framed)?;
3132 assert_eq!(
3133 consumed,
3134 framed.len(),
3135 "decoder should consume the full SOE-framed message"
3136 );
3137 }
3138 let out = decoder.flush()?.expect("expected a decoded RecordBatch");
3139 let expected_str = pretty_format_batches(&[expected])?.to_string();
3140 let actual_str = pretty_format_batches(&[out])?.to_string();
3141 assert_eq!(expected_str, actual_str);
3142 Ok(())
3143 }
3144
3145 #[test]
3146 fn test_row_encoder_avro_binary_format_roundtrip_decoder_streaming_chunks()
3147 -> Result<(), AvroError> {
3148 use crate::writer::format::AvroBinaryFormat;
3149 let (schema, batch, avro_schema) = make_real_avro_schema_and_batch()?;
3150 let mut encoder = WriterBuilder::new(schema).build_encoder::<AvroBinaryFormat>()?;
3151 encoder.encode(&batch)?;
3152 let rows = encoder.flush();
3153 let mut store = SchemaStore::new();
3154 let fp = store.register(avro_schema)?;
3155 let fp_le_bytes = match fp {
3156 Fingerprint::Rabin(v) => v.to_le_bytes(),
3157 other => panic!("expected Rabin fingerprint from SchemaStore::new(), got {other:?}"),
3158 };
3159 const SOE_MAGIC: [u8; 2] = [0xC3, 0x01];
3160 const SOE_PREFIX_LEN: usize = 2 + 8;
3161 let mut stream: Vec<u8> = Vec::new();
3162 for body in rows.iter() {
3163 let msg_len: u32 = (SOE_PREFIX_LEN + body.len())
3164 .try_into()
3165 .expect("message length must fit in u32");
3166 stream.extend_from_slice(&msg_len.to_le_bytes());
3167 stream.extend_from_slice(&SOE_MAGIC);
3168 stream.extend_from_slice(&fp_le_bytes);
3169 stream.extend_from_slice(body.as_ref());
3170 }
3171 let mut decoder = ReaderBuilder::new()
3172 .with_writer_schema_store(store)
3173 .with_batch_size(1024)
3174 .build_decoder()?;
3175 let chunk_sizes = [1usize, 2, 3, 5, 8, 13, 21, 34];
3176 let mut pos = 0usize;
3177 let mut i = 0usize;
3178 let mut buffered = BytesMut::new();
3179 let mut decoded_frames = 0usize;
3180 while pos < stream.len() {
3181 let take = chunk_sizes[i % chunk_sizes.len()];
3182 i += 1;
3183 let end = (pos + take).min(stream.len());
3184 buffered.extend_from_slice(&stream[pos..end]);
3185 pos = end;
3186 loop {
3187 if buffered.len() < 4 {
3188 break;
3189 }
3190 let msg_len =
3191 u32::from_le_bytes([buffered[0], buffered[1], buffered[2], buffered[3]])
3192 as usize;
3193 if buffered.len() < 4 + msg_len {
3194 break;
3195 }
3196 let frame = buffered.split_to(4 + msg_len);
3197 let payload = &frame[4..];
3198 let consumed = decoder.decode(payload)?;
3199 assert_eq!(
3200 consumed,
3201 payload.len(),
3202 "decoder should consume the full SOE-framed message"
3203 );
3204
3205 decoded_frames += 1;
3206 }
3207 }
3208 assert!(
3209 buffered.is_empty(),
3210 "expected transport framer to consume all bytes; leftover = {}",
3211 buffered.len()
3212 );
3213 assert_eq!(
3214 decoded_frames,
3215 rows.len(),
3216 "expected to decode exactly one frame per encoded row"
3217 );
3218 let out = decoder.flush()?.expect("expected decoded RecordBatch");
3219 let expected_str = pretty_format_batches(std::slice::from_ref(&batch))?.to_string();
3220 let actual_str = pretty_format_batches(&[out])?.to_string();
3221 assert_eq!(expected_str, actual_str);
3222 Ok(())
3223 }
3224
3225 fn roundtrip_ocf(batch: &RecordBatch) -> Result<RecordBatch, AvroError> {
3227 let schema = batch.schema();
3228 let mut buffer = Vec::<u8>::new();
3229 let mut writer = AvroWriter::new(&mut buffer, schema.as_ref().clone())?;
3230 writer.write(batch)?;
3231 writer.finish()?;
3232 drop(writer);
3233 let reader = ReaderBuilder::new()
3234 .build(Cursor::new(buffer))
3235 .expect("build reader for roundtrip OCF");
3236 let avro_schema_json = reader
3238 .avro_header()
3239 .get(SCHEMA_METADATA_KEY)
3240 .map(|raw| std::str::from_utf8(raw).expect("valid UTF-8").to_string());
3241 let arrow_schema = reader.schema();
3243 let rt_schema = if let Some(json) = avro_schema_json {
3244 let mut metadata = arrow_schema.metadata().clone();
3245 metadata.insert(SCHEMA_METADATA_KEY.to_string(), json);
3246 Arc::new(Schema::new_with_metadata(
3247 arrow_schema.fields().clone(),
3248 metadata,
3249 ))
3250 } else {
3251 arrow_schema
3252 };
3253 let rt_batches: Vec<RecordBatch> = reader.collect::<Result<Vec<_>, _>>()?;
3254 Ok(arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat roundtrip"))
3255 }
3256
3257 #[cfg(feature = "avro_custom_types")]
3259 fn assert_round_trip(array: ArrayRef) {
3260 assert_round_trip_widened(array.clone(), array);
3261 }
3262
3263 fn assert_round_trip_widened(input: ArrayRef, expected: ArrayRef) {
3265 let schema = Schema::new(vec![Field::new("val", input.data_type().clone(), true)]);
3266 let batch =
3267 RecordBatch::try_new(Arc::new(schema), vec![input]).expect("failed to create batch");
3268 let roundtrip = roundtrip_ocf(&batch).expect("roundtrip failed");
3269 assert_eq!(
3270 roundtrip.column(0).data_type(),
3271 expected.data_type(),
3272 "output data type mismatch"
3273 );
3274 assert_eq!(
3275 roundtrip.column(0).to_data(),
3276 expected.to_data(),
3277 "output data mismatch"
3278 );
3279 }
3280
3281 #[cfg(feature = "avro_custom_types")]
3282 #[test]
3283 fn test_roundtrip_int8_custom_types() {
3284 assert_round_trip(Arc::new(Int8Array::from(vec![
3285 Some(i8::MIN),
3286 Some(-1),
3287 Some(0),
3288 None,
3289 Some(1),
3290 Some(i8::MAX),
3291 ])));
3292 }
3293
3294 #[cfg(not(feature = "avro_custom_types"))]
3295 #[test]
3296 fn test_roundtrip_int8_no_custom_widens_to_int32() {
3297 assert_round_trip_widened(
3298 Arc::new(Int8Array::from(vec![
3299 Some(i8::MIN),
3300 Some(-1),
3301 Some(0),
3302 None,
3303 Some(1),
3304 Some(i8::MAX),
3305 ])),
3306 Arc::new(Int32Array::from(vec![
3307 Some(i8::MIN as i32),
3308 Some(-1),
3309 Some(0),
3310 None,
3311 Some(1),
3312 Some(i8::MAX as i32),
3313 ])),
3314 );
3315 }
3316
3317 #[cfg(feature = "avro_custom_types")]
3318 #[test]
3319 fn test_roundtrip_int16_custom_types() {
3320 assert_round_trip(Arc::new(Int16Array::from(vec![
3321 Some(i16::MIN),
3322 Some(-1),
3323 Some(0),
3324 None,
3325 Some(1),
3326 Some(i16::MAX),
3327 ])));
3328 }
3329
3330 #[cfg(not(feature = "avro_custom_types"))]
3331 #[test]
3332 fn test_roundtrip_int16_no_custom_widens_to_int32() {
3333 assert_round_trip_widened(
3334 Arc::new(Int16Array::from(vec![
3335 Some(i16::MIN),
3336 Some(-1),
3337 Some(0),
3338 None,
3339 Some(1),
3340 Some(i16::MAX),
3341 ])),
3342 Arc::new(Int32Array::from(vec![
3343 Some(i16::MIN as i32),
3344 Some(-1),
3345 Some(0),
3346 None,
3347 Some(1),
3348 Some(i16::MAX as i32),
3349 ])),
3350 );
3351 }
3352
3353 #[cfg(feature = "avro_custom_types")]
3354 #[test]
3355 fn test_roundtrip_uint8_custom_types() {
3356 assert_round_trip(Arc::new(UInt8Array::from(vec![
3357 Some(0u8),
3358 Some(1),
3359 None,
3360 Some(127),
3361 Some(u8::MAX),
3362 ])));
3363 }
3364
3365 #[cfg(not(feature = "avro_custom_types"))]
3366 #[test]
3367 fn test_roundtrip_uint8_no_custom_widens_to_int32() {
3368 assert_round_trip_widened(
3369 Arc::new(UInt8Array::from(vec![
3370 Some(0u8),
3371 Some(1),
3372 None,
3373 Some(127),
3374 Some(u8::MAX),
3375 ])),
3376 Arc::new(Int32Array::from(vec![
3377 Some(0i32),
3378 Some(1),
3379 None,
3380 Some(127),
3381 Some(u8::MAX as i32),
3382 ])),
3383 );
3384 }
3385
3386 #[cfg(feature = "avro_custom_types")]
3387 #[test]
3388 fn test_roundtrip_uint16_custom_types() {
3389 assert_round_trip(Arc::new(UInt16Array::from(vec![
3390 Some(0u16),
3391 Some(1),
3392 None,
3393 Some(32767),
3394 Some(u16::MAX),
3395 ])));
3396 }
3397
3398 #[cfg(not(feature = "avro_custom_types"))]
3399 #[test]
3400 fn test_roundtrip_uint16_no_custom_widens_to_int32() {
3401 assert_round_trip_widened(
3402 Arc::new(UInt16Array::from(vec![
3403 Some(0u16),
3404 Some(1),
3405 None,
3406 Some(32767),
3407 Some(u16::MAX),
3408 ])),
3409 Arc::new(Int32Array::from(vec![
3410 Some(0i32),
3411 Some(1),
3412 None,
3413 Some(32767),
3414 Some(u16::MAX as i32),
3415 ])),
3416 );
3417 }
3418
3419 #[cfg(feature = "avro_custom_types")]
3420 #[test]
3421 fn test_roundtrip_uint32_custom_types() {
3422 assert_round_trip(Arc::new(UInt32Array::from(vec![
3423 Some(0u32),
3424 Some(1),
3425 None,
3426 Some(i32::MAX as u32),
3427 Some(u32::MAX),
3428 ])));
3429 }
3430
3431 #[cfg(not(feature = "avro_custom_types"))]
3432 #[test]
3433 fn test_roundtrip_uint32_no_custom_widens_to_int64() {
3434 assert_round_trip_widened(
3435 Arc::new(UInt32Array::from(vec![
3436 Some(0u32),
3437 Some(1),
3438 None,
3439 Some(i32::MAX as u32),
3440 Some(u32::MAX),
3441 ])),
3442 Arc::new(Int64Array::from(vec![
3443 Some(0i64),
3444 Some(1),
3445 None,
3446 Some(i32::MAX as i64),
3447 Some(u32::MAX as i64),
3448 ])),
3449 );
3450 }
3451
3452 #[cfg(feature = "avro_custom_types")]
3453 #[test]
3454 fn test_roundtrip_uint64_custom_types() {
3455 assert_round_trip(Arc::new(UInt64Array::from(vec![
3456 Some(0u64),
3457 Some(1),
3458 None,
3459 Some(i64::MAX as u64),
3460 Some(u64::MAX),
3461 ])));
3462 }
3463
3464 #[cfg(not(feature = "avro_custom_types"))]
3465 #[test]
3466 fn test_roundtrip_uint64_no_custom_widens_to_int64() {
3467 assert_round_trip_widened(
3468 Arc::new(UInt64Array::from(vec![
3469 Some(0u64),
3470 Some(1),
3471 None,
3472 Some(i64::MAX as u64),
3473 ])),
3474 Arc::new(Int64Array::from(vec![
3475 Some(0i64),
3476 Some(1),
3477 None,
3478 Some(i64::MAX),
3479 ])),
3480 );
3481 }
3482
3483 #[cfg(not(feature = "avro_custom_types"))]
3484 #[test]
3485 fn test_roundtrip_uint64_overflow_errors_without_custom() {
3486 use arrow_array::UInt64Array;
3487 let schema = Schema::new(vec![Field::new("val", DataType::UInt64, false)]);
3488 let values: Vec<u64> = vec![u64::MAX];
3489 let array = UInt64Array::from(values);
3490 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array) as ArrayRef])
3491 .expect("create batch");
3492 let result = roundtrip_ocf(&batch);
3493 assert!(
3494 result.is_err(),
3495 "Expected error when encoding UInt64 > i64::MAX without avro_custom_types"
3496 );
3497 }
3498
3499 #[cfg(feature = "avro_custom_types")]
3500 #[test]
3501 fn test_roundtrip_float16_custom_types() {
3502 assert_round_trip(Arc::new(Float16Array::from(vec![
3503 Some(f16::ZERO),
3504 Some(f16::ONE),
3505 None,
3506 Some(f16::NEG_ONE),
3507 Some(f16::MAX),
3508 Some(f16::MIN),
3509 ])));
3510 }
3511
3512 #[cfg(not(feature = "avro_custom_types"))]
3513 #[test]
3514 #[cfg_attr(miri, ignore)] fn test_roundtrip_float16_no_custom_widens_to_float32() {
3516 assert_round_trip_widened(
3517 Arc::new(Float16Array::from(vec![
3518 Some(f16::ZERO),
3519 Some(f16::ONE),
3520 None,
3521 Some(f16::NEG_ONE),
3522 ])),
3523 Arc::new(Float32Array::from(vec![
3524 Some(0.0f32),
3525 Some(1.0),
3526 None,
3527 Some(-1.0),
3528 ])),
3529 );
3530 }
3531
3532 #[cfg(feature = "avro_custom_types")]
3533 #[test]
3534 fn test_roundtrip_date64_custom_types() {
3535 assert_round_trip(Arc::new(Date64Array::from(vec![
3536 Some(0i64),
3537 Some(86_400_000),
3538 None,
3539 Some(1_609_459_200_000),
3540 ])));
3541 }
3542
3543 #[cfg(not(feature = "avro_custom_types"))]
3544 #[test]
3545 fn test_roundtrip_date64_no_custom_as_timestamp_millis() {
3546 assert_round_trip_widened(
3547 Arc::new(Date64Array::from(vec![
3548 Some(0i64),
3549 Some(86_400_000),
3550 None,
3551 Some(1_609_459_200_000),
3552 ])),
3553 Arc::new(TimestampMillisecondArray::from(vec![
3554 Some(0i64),
3555 Some(86_400_000),
3556 None,
3557 Some(1_609_459_200_000),
3558 ])),
3559 );
3560 }
3561
3562 #[cfg(feature = "avro_custom_types")]
3563 #[test]
3564 fn test_roundtrip_time64_nanosecond_custom_types() {
3565 assert_round_trip(Arc::new(Time64NanosecondArray::from(vec![
3566 Some(0i64),
3567 Some(1_000_000_000),
3568 None,
3569 Some(86_399_999_999_999),
3570 ])));
3571 }
3572
3573 #[cfg(not(feature = "avro_custom_types"))]
3574 #[test]
3575 fn test_roundtrip_time64_nanos_no_custom_truncates_to_micros() {
3576 assert_round_trip_widened(
3578 Arc::new(Time64NanosecondArray::from(vec![
3579 Some(0i64),
3580 Some(1_000_000_000),
3581 None,
3582 Some(86_399_999_000_000),
3583 ])),
3584 Arc::new(Time64MicrosecondArray::from(vec![
3585 Some(0i64),
3586 Some(1_000_000),
3587 None,
3588 Some(86_399_999_000),
3589 ])),
3590 );
3591 }
3592
3593 #[cfg(feature = "avro_custom_types")]
3594 #[test]
3595 fn test_roundtrip_time32_second_custom_types() {
3596 assert_round_trip(Arc::new(Time32SecondArray::from(vec![
3597 Some(0i32),
3598 Some(3600),
3599 None,
3600 Some(86399),
3601 ])));
3602 }
3603
3604 #[cfg(not(feature = "avro_custom_types"))]
3605 #[test]
3606 fn test_roundtrip_time32_second_no_custom_scales_to_millis() {
3607 assert_round_trip_widened(
3608 Arc::new(Time32SecondArray::from(vec![
3609 Some(0i32),
3610 Some(3600),
3611 None,
3612 Some(86399),
3613 ])),
3614 Arc::new(Time32MillisecondArray::from(vec![
3615 Some(0i32),
3616 Some(3_600_000),
3617 None,
3618 Some(86_399_000),
3619 ])),
3620 );
3621 }
3622
3623 #[cfg(feature = "avro_custom_types")]
3624 #[test]
3625 fn test_roundtrip_timestamp_second_custom_types() {
3626 assert_round_trip(Arc::new(
3627 TimestampSecondArray::from(vec![Some(0i64), Some(1609459200), None, Some(1735689600)])
3628 .with_timezone("+00:00"),
3629 ));
3630 }
3631
3632 #[cfg(not(feature = "avro_custom_types"))]
3633 #[test]
3634 fn test_roundtrip_timestamp_second_no_custom_scales_to_millis() {
3635 assert_round_trip_widened(
3636 Arc::new(
3637 TimestampSecondArray::from(vec![
3638 Some(0i64),
3639 Some(1609459200),
3640 None,
3641 Some(1735689600),
3642 ])
3643 .with_timezone("+00:00"),
3644 ),
3645 Arc::new(
3646 TimestampMillisecondArray::from(vec![
3647 Some(0i64),
3648 Some(1_609_459_200_000),
3649 None,
3650 Some(1_735_689_600_000),
3651 ])
3652 .with_timezone("+00:00"),
3653 ),
3654 );
3655 }
3656
3657 #[cfg(feature = "avro_custom_types")]
3658 #[test]
3659 fn test_roundtrip_interval_year_month_custom_types() {
3660 assert_round_trip(Arc::new(IntervalYearMonthArray::from(vec![
3661 Some(0i32),
3662 Some(12),
3663 None,
3664 Some(-6),
3665 Some(25),
3666 ])));
3667 }
3668
3669 #[cfg(not(feature = "avro_custom_types"))]
3670 #[test]
3671 fn test_roundtrip_interval_year_month_no_custom() {
3672 assert_round_trip_widened(
3674 Arc::new(IntervalYearMonthArray::from(vec![
3675 Some(0i32),
3676 Some(12),
3677 None,
3678 Some(25),
3679 ])),
3680 Arc::new(IntervalMonthDayNanoArray::from(vec![
3681 Some(IntervalMonthDayNano::new(0, 0, 0)),
3682 Some(IntervalMonthDayNano::new(12, 0, 0)),
3683 None,
3684 Some(IntervalMonthDayNano::new(25, 0, 0)),
3685 ])),
3686 );
3687 }
3688
3689 #[cfg(feature = "avro_custom_types")]
3690 #[test]
3691 fn test_roundtrip_interval_day_time_custom_types() {
3692 assert_round_trip(Arc::new(IntervalDayTimeArray::from(vec![
3693 Some(IntervalDayTime::new(0, 0)),
3694 Some(IntervalDayTime::new(1, 1000)),
3695 None,
3696 Some(IntervalDayTime::new(30, 3600000)),
3697 ])));
3698 }
3699
3700 #[cfg(not(feature = "avro_custom_types"))]
3701 #[test]
3702 fn test_roundtrip_interval_day_time_no_custom() {
3703 assert_round_trip_widened(
3704 Arc::new(IntervalDayTimeArray::from(vec![
3705 Some(IntervalDayTime::new(0, 0)),
3706 Some(IntervalDayTime::new(1, 1000)),
3707 None,
3708 Some(IntervalDayTime::new(30, 3600000)),
3709 ])),
3710 Arc::new(IntervalMonthDayNanoArray::from(vec![
3711 Some(IntervalMonthDayNano::new(0, 0, 0)),
3712 Some(IntervalMonthDayNano::new(0, 1, 1_000_000_000)),
3713 None,
3714 Some(IntervalMonthDayNano::new(0, 30, 3_600_000_000_000)),
3715 ])),
3716 );
3717 }
3718
3719 #[cfg(feature = "avro_custom_types")]
3720 #[test]
3721 fn test_roundtrip_interval_month_day_nano_custom_types() {
3722 assert_round_trip(Arc::new(IntervalMonthDayNanoArray::from(vec![
3723 Some(IntervalMonthDayNano::new(0, 0, 0)),
3724 Some(IntervalMonthDayNano::new(1, 2, 3)),
3725 None,
3726 Some(IntervalMonthDayNano::new(-4, -5, -6)),
3727 ])));
3728 }
3729
3730 #[cfg(not(feature = "avro_custom_types"))]
3731 #[test]
3732 fn test_roundtrip_interval_month_day_nano_no_custom() {
3733 assert_round_trip_widened(
3735 Arc::new(IntervalMonthDayNanoArray::from(vec![
3736 Some(IntervalMonthDayNano::new(0, 0, 0)),
3737 Some(IntervalMonthDayNano::new(1, 2, 3_000_000)),
3738 None,
3739 Some(IntervalMonthDayNano::new(4, 5, 6_000_000)),
3740 ])),
3741 Arc::new(IntervalMonthDayNanoArray::from(vec![
3742 Some(IntervalMonthDayNano::new(0, 0, 0)),
3743 Some(IntervalMonthDayNano::new(1, 2, 3_000_000)),
3744 None,
3745 Some(IntervalMonthDayNano::new(4, 5, 6_000_000)),
3746 ])),
3747 );
3748 }
3749
3750 fn schemas_equal_ignoring_metadata(left: &Schema, right: &Schema) -> bool {
3751 if left.fields().len() != right.fields().len() {
3752 return false;
3753 }
3754 for (l, r) in left.fields().iter().zip(right.fields().iter()) {
3755 if l.name() != r.name()
3756 || l.data_type() != r.data_type()
3757 || l.is_nullable() != r.is_nullable()
3758 {
3759 return false;
3760 }
3761 }
3762 true
3763 }
3764
3765 fn avro_field_type<'a>(avro_schema: &'a Value, name: &str) -> &'a Value {
3766 let fields = avro_schema
3767 .get("fields")
3768 .and_then(|v| v.as_array())
3769 .expect("avro schema has 'fields' array");
3770 fields
3771 .iter()
3772 .find(|f| f.get("name").and_then(|n| n.as_str()) == Some(name))
3773 .unwrap_or_else(|| panic!("avro schema missing field '{name}'"))
3774 .get("type")
3775 .expect("field has 'type'")
3776 }
3777
3778 #[test]
3779 #[cfg_attr(miri, ignore)] fn e2e_types_and_schema_alignment() -> Result<(), AvroError> {
3781 let i8_values: Vec<Option<i8>> = vec![Some(i8::MIN), Some(-1), Some(i8::MAX)];
3786 let i16_values: Vec<Option<i16>> = vec![Some(i16::MIN), Some(-1), Some(i16::MAX)];
3787 let u8_values: Vec<Option<u8>> = vec![Some(0), Some(1), Some(u8::MAX)];
3788 let u16_values: Vec<Option<u16>> = vec![Some(0), Some(1), Some(u16::MAX)];
3789 let u32_values: Vec<Option<u32>> = vec![Some(0), Some(1), Some(u32::MAX)];
3790 let u64_values: Vec<Option<u64>> = if cfg!(feature = "avro_custom_types") {
3791 vec![Some(0), Some(i64::MAX as u64), Some((i64::MAX as u64) + 1)]
3792 } else {
3793 vec![Some(0), Some((i64::MAX as u64) - 1), Some(i64::MAX as u64)]
3795 };
3796 let f16_values: Vec<Option<f16>> = vec![
3797 Some(f16::from_f32(1.5)),
3798 Some(f16::from_f32(-2.0)),
3799 Some(f16::from_f32(0.0)),
3800 ];
3801 let date64_values: Vec<Option<i64>> = vec![Some(-86_400_000), Some(0), Some(86_400_000)];
3802 let time32s_values: Vec<Option<i32>> = vec![Some(0), Some(1), Some(86_399)];
3803 let time64ns_values: Vec<Option<i64>> = vec![
3804 Some(0),
3805 Some(1_234_567_890), Some(86_399_000_000_123_i64), ];
3808 let ts_s_local_values: Vec<Option<i64>> = vec![Some(-1), Some(0), Some(1)];
3809 let ts_s_utc_values: Vec<Option<i64>> = vec![Some(1), Some(2), Some(3)];
3810 let iv_ym_values: Vec<Option<i32>> = if cfg!(feature = "avro_custom_types") {
3811 vec![Some(0), Some(-6), Some(25)]
3812 } else {
3813 vec![Some(0), Some(12), Some(25)]
3815 };
3816 let iv_dt_values: Vec<Option<IntervalDayTime>> = if cfg!(feature = "avro_custom_types") {
3817 vec![
3818 Some(IntervalDayTime::new(0, 0)),
3819 Some(IntervalDayTime::new(1, 1000)),
3820 Some(IntervalDayTime::new(-1, -1000)),
3821 ]
3822 } else {
3823 vec![
3825 Some(IntervalDayTime::new(0, 0)),
3826 Some(IntervalDayTime::new(1, 1000)),
3827 Some(IntervalDayTime::new(30, 3_600_000)),
3828 ]
3829 };
3830 let iv_mdn_values: Vec<Option<IntervalMonthDayNano>> =
3831 if cfg!(feature = "avro_custom_types") {
3832 vec![
3833 Some(IntervalMonthDayNano::new(0, 0, 0)),
3834 Some(IntervalMonthDayNano::new(1, 2, 3)), Some(IntervalMonthDayNano::new(-1, -2, -3)), ]
3837 } else {
3838 vec![
3840 Some(IntervalMonthDayNano::new(0, 0, 0)),
3841 Some(IntervalMonthDayNano::new(1, 2, 3_000_000)), Some(IntervalMonthDayNano::new(10, 20, 30_000_000_000)), ]
3844 };
3845 let schema = Schema::new(vec![
3847 Field::new("i8", DataType::Int8, false),
3848 Field::new("i16", DataType::Int16, false),
3849 Field::new("u8", DataType::UInt8, false),
3850 Field::new("u16", DataType::UInt16, false),
3851 Field::new("u32", DataType::UInt32, false),
3852 Field::new("u64", DataType::UInt64, false),
3853 Field::new("f16", DataType::Float16, false),
3854 Field::new("date64", DataType::Date64, false),
3855 Field::new("time32s", DataType::Time32(TimeUnit::Second), false),
3856 Field::new("time64ns", DataType::Time64(TimeUnit::Nanosecond), false),
3857 Field::new(
3858 "ts_s_local",
3859 DataType::Timestamp(TimeUnit::Second, None),
3860 false,
3861 ),
3862 Field::new(
3863 "ts_s_utc",
3864 DataType::Timestamp(TimeUnit::Second, Some("+00:00".into())),
3865 false,
3866 ),
3867 Field::new("iv_ym", DataType::Interval(IntervalUnit::YearMonth), false),
3868 Field::new("iv_dt", DataType::Interval(IntervalUnit::DayTime), false),
3869 Field::new(
3870 "iv_mdn",
3871 DataType::Interval(IntervalUnit::MonthDayNano),
3872 false,
3873 ),
3874 ]);
3875 let batch = RecordBatch::try_new(
3876 Arc::new(schema.clone()),
3877 vec![
3878 Arc::new(Int8Array::from(i8_values.clone())) as ArrayRef,
3879 Arc::new(Int16Array::from(i16_values.clone())) as ArrayRef,
3880 Arc::new(UInt8Array::from(u8_values.clone())) as ArrayRef,
3881 Arc::new(UInt16Array::from(u16_values.clone())) as ArrayRef,
3882 Arc::new(UInt32Array::from(u32_values.clone())) as ArrayRef,
3883 Arc::new(UInt64Array::from(u64_values.clone())) as ArrayRef,
3884 Arc::new(Float16Array::from(f16_values.clone())) as ArrayRef,
3885 Arc::new(Date64Array::from(date64_values.clone())) as ArrayRef,
3886 Arc::new(Time32SecondArray::from(time32s_values.clone())) as ArrayRef,
3887 Arc::new(Time64NanosecondArray::from(time64ns_values.clone())) as ArrayRef,
3888 Arc::new(TimestampSecondArray::from(ts_s_local_values.clone())) as ArrayRef,
3889 Arc::new(
3890 TimestampSecondArray::from(ts_s_utc_values.clone()).with_timezone("+00:00"),
3891 ) as ArrayRef,
3892 Arc::new(IntervalYearMonthArray::from(iv_ym_values.clone())) as ArrayRef,
3893 Arc::new(IntervalDayTimeArray::from(iv_dt_values.clone())) as ArrayRef,
3894 Arc::new(IntervalMonthDayNanoArray::from(iv_mdn_values.clone())) as ArrayRef,
3895 ],
3896 )?;
3897 let rt = roundtrip_ocf(&batch)?;
3898 let rt_schema = rt.schema();
3899 let avro_schema_json = rt_schema
3900 .metadata()
3901 .get(SCHEMA_METADATA_KEY)
3902 .expect("avro.schema missing in round-tripped batch metadata");
3903 let avro_schema: Value =
3904 serde_json::from_str(avro_schema_json).expect("valid avro schema json");
3905 let rt_arrow_schema = rt.schema();
3906 if cfg!(feature = "avro_custom_types") {
3907 assert!(
3908 schemas_equal_ignoring_metadata(rt_arrow_schema.as_ref(), &schema),
3909 "Schema fields mismatch.\nExpected: {schema:?}\nGot: {rt_arrow_schema:?}"
3910 );
3911 for field_name in ["u64", "f16", "iv_ym", "iv_dt", "iv_mdn"] {
3912 let field = rt_arrow_schema
3913 .field_with_name(field_name)
3914 .expect("field exists");
3915 assert!(
3916 field.metadata().get(AVRO_NAME_METADATA_KEY).is_some(),
3917 "Field '{field_name}' should have avro.name metadata"
3918 );
3919 }
3920 } else {
3921 let exp_schema = Schema::new(vec![
3925 Field::new("i8", DataType::Int32, false),
3926 Field::new("i16", DataType::Int32, false),
3927 Field::new("u8", DataType::Int32, false),
3928 Field::new("u16", DataType::Int32, false),
3929 Field::new("u32", DataType::Int64, false),
3930 Field::new("u64", DataType::Int64, false),
3931 Field::new("f16", DataType::Float32, false),
3932 Field::new(
3933 "date64",
3934 DataType::Timestamp(TimeUnit::Millisecond, None),
3935 false,
3936 ),
3937 Field::new("time32s", DataType::Time32(TimeUnit::Millisecond), false),
3938 Field::new("time64ns", DataType::Time64(TimeUnit::Microsecond), false),
3939 Field::new(
3940 "ts_s_local",
3941 DataType::Timestamp(TimeUnit::Millisecond, None),
3942 false,
3943 ),
3944 Field::new(
3945 "ts_s_utc",
3946 DataType::Timestamp(TimeUnit::Millisecond, Some("+00:00".into())),
3947 false,
3948 ),
3949 Field::new(
3950 "iv_ym",
3951 DataType::Interval(IntervalUnit::MonthDayNano),
3952 false,
3953 ),
3954 Field::new(
3955 "iv_dt",
3956 DataType::Interval(IntervalUnit::MonthDayNano),
3957 false,
3958 ),
3959 Field::new(
3960 "iv_mdn",
3961 DataType::Interval(IntervalUnit::MonthDayNano),
3962 false,
3963 ),
3964 ]);
3965 assert!(
3966 schemas_equal_ignoring_metadata(rt_arrow_schema.as_ref(), &exp_schema),
3967 "Schema fields mismatch.\nExpected: {exp_schema:?}\nGot: {rt_arrow_schema:?}"
3968 );
3969 for field_name in ["iv_ym", "iv_dt", "iv_mdn"] {
3970 let field = rt_arrow_schema
3971 .field_with_name(field_name)
3972 .expect("field exists");
3973 assert!(
3974 field.metadata().get(AVRO_NAME_METADATA_KEY).is_some(),
3975 "Field '{field_name}' should have avro.name metadata"
3976 );
3977 }
3978 }
3979 if cfg!(feature = "avro_custom_types") {
3980 assert_eq!(
3981 avro_field_type(&avro_schema, "i8"),
3982 &json!({"type":"int","logicalType":"arrow.int8"})
3983 );
3984 assert_eq!(
3985 avro_field_type(&avro_schema, "i16"),
3986 &json!({"type":"int","logicalType":"arrow.int16"})
3987 );
3988 assert_eq!(
3989 avro_field_type(&avro_schema, "u8"),
3990 &json!({"type":"int","logicalType":"arrow.uint8"})
3991 );
3992 assert_eq!(
3993 avro_field_type(&avro_schema, "u16"),
3994 &json!({"type":"int","logicalType":"arrow.uint16"})
3995 );
3996 assert_eq!(
3997 avro_field_type(&avro_schema, "u32"),
3998 &json!({"type":"long","logicalType":"arrow.uint32"})
3999 );
4000 assert_eq!(
4001 avro_field_type(&avro_schema, "u64"),
4002 &json!({"type":"fixed","name":"u64","size":8,"logicalType":"arrow.uint64"})
4003 );
4004 assert_eq!(
4005 avro_field_type(&avro_schema, "f16"),
4006 &json!({"type":"fixed","name":"f16","size":2,"logicalType":"arrow.float16"})
4007 );
4008 assert_eq!(
4009 avro_field_type(&avro_schema, "date64"),
4010 &json!({"type":"long","logicalType":"arrow.date64"})
4011 );
4012 assert_eq!(
4013 avro_field_type(&avro_schema, "time32s"),
4014 &json!({"type":"int","logicalType":"arrow.time32-second"})
4015 );
4016 assert_eq!(
4017 avro_field_type(&avro_schema, "time64ns"),
4018 &json!({"type":"long","logicalType":"arrow.time64-nanosecond"})
4019 );
4020 assert_eq!(
4021 avro_field_type(&avro_schema, "ts_s_local"),
4022 &json!({"type":"long","logicalType":"arrow.local-timestamp-second"})
4023 );
4024 assert_eq!(
4025 avro_field_type(&avro_schema, "ts_s_utc"),
4026 &json!({"type":"long","logicalType":"arrow.timestamp-second"})
4027 );
4028 assert_eq!(
4029 avro_field_type(&avro_schema, "iv_ym"),
4030 &json!({"type":"fixed","name":"iv_ym","size":4,"logicalType":"arrow.interval-year-month"})
4031 );
4032 assert_eq!(
4033 avro_field_type(&avro_schema, "iv_dt"),
4034 &json!({"type":"fixed","name":"iv_dt","size":8,"logicalType":"arrow.interval-day-time"})
4035 );
4036 assert_eq!(
4037 avro_field_type(&avro_schema, "iv_mdn"),
4038 &json!({"type":"fixed","name":"iv_mdn","size":16,"logicalType":"arrow.interval-month-day-nano"})
4039 );
4040 } else {
4041 assert_eq!(avro_field_type(&avro_schema, "i8"), &json!("int"));
4051 assert_eq!(avro_field_type(&avro_schema, "i16"), &json!("int"));
4052 assert_eq!(avro_field_type(&avro_schema, "u8"), &json!("int"));
4053 assert_eq!(avro_field_type(&avro_schema, "u16"), &json!("int"));
4054 assert_eq!(avro_field_type(&avro_schema, "u32"), &json!("long"));
4055 assert_eq!(avro_field_type(&avro_schema, "u64"), &json!("long"));
4056 assert_eq!(avro_field_type(&avro_schema, "f16"), &json!("float"));
4057 assert_eq!(
4058 avro_field_type(&avro_schema, "date64"),
4059 &json!({"type":"long","logicalType":"local-timestamp-millis"})
4060 );
4061 assert_eq!(
4062 avro_field_type(&avro_schema, "time32s"),
4063 &json!({"type":"int","logicalType":"time-millis"})
4064 );
4065 assert_eq!(
4066 avro_field_type(&avro_schema, "time64ns"),
4067 &json!({"type":"long","logicalType":"time-micros"})
4068 );
4069 assert_eq!(
4070 avro_field_type(&avro_schema, "ts_s_local"),
4071 &json!({"type":"long","logicalType":"local-timestamp-millis"})
4072 );
4073 assert_eq!(
4074 avro_field_type(&avro_schema, "ts_s_utc"),
4075 &json!({"type":"long","logicalType":"timestamp-millis"})
4076 );
4077 assert_eq!(
4078 avro_field_type(&avro_schema, "iv_ym"),
4079 &json!({"type":"fixed","name":"iv_ym","size":12,"logicalType":"duration"})
4080 );
4081 assert_eq!(
4082 avro_field_type(&avro_schema, "iv_dt"),
4083 &json!({"type":"fixed","name":"iv_dt","size":12,"logicalType":"duration"})
4084 );
4085 assert_eq!(
4086 avro_field_type(&avro_schema, "iv_mdn"),
4087 &json!({"type":"fixed","name":"iv_mdn","size":12,"logicalType":"duration"})
4088 );
4089 }
4090 if cfg!(feature = "avro_custom_types") {
4091 assert_eq!(
4092 rt.column(0).as_ref(),
4093 &Int8Array::from(i8_values) as &dyn Array
4094 );
4095 assert_eq!(
4096 rt.column(1).as_ref(),
4097 &Int16Array::from(i16_values) as &dyn Array
4098 );
4099 assert_eq!(
4100 rt.column(2).as_ref(),
4101 &UInt8Array::from(u8_values) as &dyn Array
4102 );
4103 assert_eq!(
4104 rt.column(3).as_ref(),
4105 &UInt16Array::from(u16_values) as &dyn Array
4106 );
4107 assert_eq!(
4108 rt.column(4).as_ref(),
4109 &UInt32Array::from(u32_values) as &dyn Array
4110 );
4111 assert_eq!(
4112 rt.column(5).as_ref(),
4113 &UInt64Array::from(u64_values) as &dyn Array
4114 );
4115 assert_eq!(
4116 rt.column(6).as_ref(),
4117 &Float16Array::from(f16_values) as &dyn Array
4118 );
4119 assert_eq!(
4120 rt.column(7).as_ref(),
4121 &Date64Array::from(date64_values) as &dyn Array
4122 );
4123 assert_eq!(
4124 rt.column(8).as_ref(),
4125 &Time32SecondArray::from(time32s_values) as &dyn Array
4126 );
4127 assert_eq!(
4128 rt.column(9).as_ref(),
4129 &Time64NanosecondArray::from(time64ns_values) as &dyn Array
4130 );
4131 assert_eq!(
4132 rt.column(10).as_ref(),
4133 &TimestampSecondArray::from(ts_s_local_values) as &dyn Array
4134 );
4135 assert_eq!(
4136 rt.column(11).as_ref(),
4137 &TimestampSecondArray::from(ts_s_utc_values).with_timezone("+00:00") as &dyn Array
4138 );
4139 assert_eq!(
4140 rt.column(12).as_ref(),
4141 &IntervalYearMonthArray::from(iv_ym_values) as &dyn Array
4142 );
4143 assert_eq!(
4144 rt.column(13).as_ref(),
4145 &IntervalDayTimeArray::from(iv_dt_values) as &dyn Array
4146 );
4147 assert_eq!(
4148 rt.column(14).as_ref(),
4149 &IntervalMonthDayNanoArray::from(iv_mdn_values) as &dyn Array
4150 );
4151 } else {
4152 let exp_i8: Vec<Option<i32>> = i8_values.iter().map(|v| v.map(|x| x as i32)).collect();
4153 let exp_i16: Vec<Option<i32>> =
4154 i16_values.iter().map(|v| v.map(|x| x as i32)).collect();
4155 let exp_u8: Vec<Option<i32>> = u8_values.iter().map(|v| v.map(|x| x as i32)).collect();
4156 let exp_u16: Vec<Option<i32>> =
4157 u16_values.iter().map(|v| v.map(|x| x as i32)).collect();
4158 let exp_u32: Vec<Option<i64>> =
4159 u32_values.iter().map(|v| v.map(|x| x as i64)).collect();
4160 let exp_u64: Vec<Option<i64>> =
4161 u64_values.iter().map(|v| v.map(|x| x as i64)).collect();
4162 let exp_f16: Vec<Option<f32>> =
4163 f16_values.iter().map(|v| v.map(|x| x.to_f32())).collect();
4164 let exp_time32_ms: Vec<Option<i32>> = time32s_values
4165 .iter()
4166 .map(|v| v.map(|x| x.saturating_mul(1000)))
4167 .collect();
4168 let exp_time64_us: Vec<Option<i64>> = time64ns_values
4169 .iter()
4170 .map(|v| v.map(|x| x / 1000))
4171 .collect();
4172 let exp_ts_local_ms: Vec<Option<i64>> = ts_s_local_values
4173 .iter()
4174 .map(|v| v.map(|x| x * 1000))
4175 .collect();
4176 let exp_ts_utc_ms: Vec<Option<i64>> = ts_s_utc_values
4177 .iter()
4178 .map(|v| v.map(|x| x * 1000))
4179 .collect();
4180 let exp_iv_ym: Vec<Option<IntervalMonthDayNano>> = iv_ym_values
4182 .iter()
4183 .map(|v| v.map(|months| IntervalMonthDayNano::new(months, 0, 0)))
4184 .collect();
4185 let exp_iv_dt: Vec<Option<IntervalMonthDayNano>> = iv_dt_values
4186 .iter()
4187 .map(|v| {
4188 v.map(|dt| {
4189 IntervalMonthDayNano::new(0, dt.days, (dt.milliseconds as i64) * 1_000_000)
4190 })
4191 })
4192 .collect();
4193 assert_eq!(
4194 rt.column(0).as_ref(),
4195 &Int32Array::from(exp_i8) as &dyn Array
4196 );
4197 assert_eq!(
4198 rt.column(1).as_ref(),
4199 &Int32Array::from(exp_i16) as &dyn Array
4200 );
4201 assert_eq!(
4202 rt.column(2).as_ref(),
4203 &Int32Array::from(exp_u8) as &dyn Array
4204 );
4205 assert_eq!(
4206 rt.column(3).as_ref(),
4207 &Int32Array::from(exp_u16) as &dyn Array
4208 );
4209 assert_eq!(
4210 rt.column(4).as_ref(),
4211 &arrow_array::Int64Array::from(exp_u32) as &dyn Array
4212 );
4213 assert_eq!(
4214 rt.column(5).as_ref(),
4215 &arrow_array::Int64Array::from(exp_u64) as &dyn Array
4216 );
4217 assert_eq!(
4218 rt.column(6).as_ref(),
4219 &arrow_array::Float32Array::from(exp_f16) as &dyn Array
4220 );
4221 assert_eq!(
4222 rt.column(7).as_ref(),
4223 &TimestampMillisecondArray::from(date64_values) as &dyn Array
4224 );
4225 assert_eq!(
4226 rt.column(8).as_ref(),
4227 &Time32MillisecondArray::from(exp_time32_ms) as &dyn Array
4228 );
4229 assert_eq!(
4230 rt.column(9).as_ref(),
4231 &Time64MicrosecondArray::from(exp_time64_us) as &dyn Array
4232 );
4233 assert_eq!(
4234 rt.column(10).as_ref(),
4235 &TimestampMillisecondArray::from(exp_ts_local_ms) as &dyn Array
4236 );
4237 assert_eq!(
4238 rt.column(11).as_ref(),
4239 &TimestampMillisecondArray::from(exp_ts_utc_ms).with_timezone("+00:00")
4240 as &dyn Array
4241 );
4242 assert_eq!(
4243 rt.column(12).as_ref(),
4244 &IntervalMonthDayNanoArray::from(exp_iv_ym) as &dyn Array
4245 );
4246 assert_eq!(
4247 rt.column(13).as_ref(),
4248 &IntervalMonthDayNanoArray::from(exp_iv_dt) as &dyn Array
4249 );
4250 assert_eq!(
4251 rt.column(14).as_ref(),
4252 &IntervalMonthDayNanoArray::from(iv_mdn_values) as &dyn Array
4253 );
4254 }
4255 Ok(())
4256 }
4257
4258 #[cfg(not(feature = "avro_custom_types"))]
4259 #[test]
4260 fn non_custom_uint64_overflow_errors() -> Result<(), AvroError> {
4261 let schema = Schema::new(vec![Field::new("u64", DataType::UInt64, false)]);
4262 let values: Vec<Option<u64>> = vec![Some((i64::MAX as u64) + 1)];
4263 let batch = RecordBatch::try_new(
4264 Arc::new(schema.clone()),
4265 vec![Arc::new(UInt64Array::from(values)) as ArrayRef],
4266 )?;
4267 let mut w = AvroWriter::new(Vec::<u8>::new(), schema)?;
4268 let err = w
4269 .write(&batch)
4270 .expect_err("expected UInt64 overflow error when avro_custom_types is disabled");
4271 match err {
4272 AvroError::InvalidArgument(msg) => {
4273 assert_eq!(
4274 msg,
4275 "UInt64 value 9223372036854775808 exceeds i64::MAX; enable avro_custom_types feature for full UInt64 support"
4276 );
4277 }
4278 other => panic!("expected InvalidArgument, got {other:?}"),
4279 }
4280 Ok(())
4281 }
4282
4283 #[cfg(not(feature = "avro_custom_types"))]
4284 #[test]
4285 fn non_custom_interval_year_month_negative_errors() -> Result<(), AvroError> {
4286 let schema = Schema::new(vec![Field::new(
4287 "iv_ym",
4288 DataType::Interval(IntervalUnit::YearMonth),
4289 false,
4290 )]);
4291 let values: Vec<Option<i32>> = vec![Some(-1)];
4292 let batch = RecordBatch::try_new(
4293 Arc::new(schema.clone()),
4294 vec![Arc::new(IntervalYearMonthArray::from(values)) as ArrayRef],
4295 )?;
4296
4297 let mut w = AvroWriter::new(Vec::<u8>::new(), schema)?;
4298 let err = w
4299 .write(&batch)
4300 .expect_err("expected negative Interval(YearMonth) error");
4301 match err {
4302 AvroError::InvalidArgument(msg) => {
4303 assert_eq!(
4304 msg,
4305 "Avro 'duration' cannot encode negative months; enable `avro_custom_types` to round-trip signed Arrow Interval(YearMonth)"
4306 );
4307 }
4308 other => panic!("expected InvalidArgument, got {other:?}"),
4309 }
4310 Ok(())
4311 }
4312
4313 #[cfg(not(feature = "avro_custom_types"))]
4314 #[test]
4315 fn non_custom_interval_day_time_negative_errors() -> Result<(), AvroError> {
4316 let schema = Schema::new(vec![Field::new(
4317 "iv_dt",
4318 DataType::Interval(IntervalUnit::DayTime),
4319 false,
4320 )]);
4321 let values: Vec<Option<IntervalDayTime>> = vec![Some(IntervalDayTime::new(-1, 0))];
4322 let batch = RecordBatch::try_new(
4323 Arc::new(schema.clone()),
4324 vec![Arc::new(IntervalDayTimeArray::from(values)) as ArrayRef],
4325 )?;
4326 let mut w = AvroWriter::new(Vec::<u8>::new(), schema)?;
4327 let err = w
4328 .write(&batch)
4329 .expect_err("expected negative Interval(DayTime) error");
4330 match err {
4331 AvroError::InvalidArgument(msg) => {
4332 assert_eq!(
4333 msg,
4334 "Avro 'duration' cannot encode negative days or milliseconds; enable `avro_custom_types` to round-trip signed Arrow Interval(DayTime)"
4335 );
4336 }
4337 other => panic!("expected InvalidArgument, got {other:?}"),
4338 }
4339 Ok(())
4340 }
4341
4342 #[cfg(not(feature = "avro_custom_types"))]
4343 #[test]
4344 fn non_custom_interval_month_day_nano_negative_errors() -> Result<(), AvroError> {
4345 let schema = Schema::new(vec![Field::new(
4346 "iv_mdn",
4347 DataType::Interval(IntervalUnit::MonthDayNano),
4348 false,
4349 )]);
4350 let values: Vec<Option<IntervalMonthDayNano>> =
4351 vec![Some(IntervalMonthDayNano::new(-1, 0, 0))];
4352 let batch = RecordBatch::try_new(
4353 Arc::new(schema.clone()),
4354 vec![Arc::new(IntervalMonthDayNanoArray::from(values)) as ArrayRef],
4355 )?;
4356 let mut w = AvroWriter::new(Vec::<u8>::new(), schema)?;
4357 let err = w
4358 .write(&batch)
4359 .expect_err("expected negative Interval(MonthDayNano) error");
4360 match err {
4361 AvroError::InvalidArgument(msg) => {
4362 assert_eq!(
4363 msg,
4364 "Avro 'duration' cannot encode negative months/days/nanoseconds; enable `avro_custom_types` to round-trip signed Arrow intervals"
4365 );
4366 }
4367 other => panic!("expected InvalidArgument, got {other:?}"),
4368 }
4369 Ok(())
4370 }
4371
4372 #[cfg(not(feature = "avro_custom_types"))]
4373 #[test]
4374 fn non_custom_interval_month_day_nano_sub_millis_errors() -> Result<(), AvroError> {
4375 let schema = Schema::new(vec![Field::new(
4376 "iv_mdn",
4377 DataType::Interval(IntervalUnit::MonthDayNano),
4378 false,
4379 )]);
4380 let values: Vec<Option<IntervalMonthDayNano>> =
4381 vec![Some(IntervalMonthDayNano::new(0, 0, 1))];
4382 let batch = RecordBatch::try_new(
4383 Arc::new(schema.clone()),
4384 vec![Arc::new(IntervalMonthDayNanoArray::from(values)) as ArrayRef],
4385 )?;
4386 let mut w = AvroWriter::new(Vec::<u8>::new(), schema)?;
4387 let err = w
4388 .write(&batch)
4389 .expect_err("expected sub-millisecond Interval(MonthDayNano) error");
4390 match err {
4391 AvroError::InvalidArgument(msg) => {
4392 assert_eq!(
4393 msg,
4394 "Avro 'duration' requires whole milliseconds; nanoseconds must be divisible by 1_000_000 (enable `avro_custom_types` to preserve nanosecond intervals)"
4395 );
4396 }
4397 other => panic!("expected InvalidArgument, got {other:?}"),
4398 }
4399 Ok(())
4400 }
4401
4402 #[cfg(not(feature = "avro_custom_types"))]
4403 #[test]
4404 fn non_custom_time32_second_scaling_overflow_errors() -> Result<(), AvroError> {
4405 let schema = Schema::new(vec![Field::new(
4406 "time32s",
4407 DataType::Time32(TimeUnit::Second),
4408 false,
4409 )]);
4410 let values: Vec<Option<i32>> = vec![Some((i32::MAX / 1000) + 1)];
4411 let batch = RecordBatch::try_new(
4412 Arc::new(schema.clone()),
4413 vec![Arc::new(Time32SecondArray::from(values)) as ArrayRef],
4414 )?;
4415 let mut w = AvroWriter::new(Vec::<u8>::new(), schema)?;
4416 let err = w
4417 .write(&batch)
4418 .expect_err("expected time32 seconds->millis overflow error");
4419 match err {
4420 AvroError::InvalidArgument(msg) => {
4421 assert_eq!(msg, "time32(secs) * 1000 overflowed");
4422 }
4423 other => panic!("expected InvalidArgument, got {other:?}"),
4424 }
4425 Ok(())
4426 }
4427
4428 #[cfg(not(feature = "avro_custom_types"))]
4429 #[test]
4430 fn non_custom_timestamp_second_scaling_overflow_errors() -> Result<(), AvroError> {
4431 let schema = Schema::new(vec![Field::new(
4432 "ts_s_local",
4433 DataType::Timestamp(TimeUnit::Second, None),
4434 false,
4435 )]);
4436 let values: Vec<Option<i64>> = vec![Some((i64::MAX / 1000) + 1)];
4438 let batch = RecordBatch::try_new(
4439 Arc::new(schema.clone()),
4440 vec![Arc::new(TimestampSecondArray::from(values)) as ArrayRef],
4441 )?;
4442 let mut w = AvroWriter::new(Vec::<u8>::new(), schema)?;
4443 let err = w
4444 .write(&batch)
4445 .expect_err("expected timestamp seconds->millis overflow error");
4446 match err {
4447 AvroError::InvalidArgument(msg) => {
4448 assert_eq!(msg, "timestamp(secs) * 1000 overflowed");
4449 }
4450 other => panic!("expected InvalidArgument, got {other:?}"),
4451 }
4452 Ok(())
4453 }
4454}