1use std::cmp::min;
15use std::collections::HashMap;
16use std::io::{BufWriter, Write};
17use std::mem::size_of;
18use std::sync::Arc;
19
20use flatbuffers::FlatBufferBuilder;
21
22use arrow_array::cast::*;
23use arrow_array::types::{Int16Type, Int32Type, Int64Type, RunEndIndexType};
24use arrow_array::*;
25use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer, ToByteSlice};
26use arrow_data::{ArrayData, ArrayDataBuilder, BufferSpec, layout};
27use arrow_schema::*;
28
29use crate::CONTINUATION_MARKER;
30use crate::compression::CompressionCodec;
31#[expect(deprecated)]
32pub use crate::compression::{CompressionContext, IpcWriteContext};
33use crate::convert::IpcSchemaEncoder;
34
35#[derive(Debug, Clone)]
37pub struct IpcWriteOptions {
38 alignment: u8,
41 write_legacy_ipc_format: bool,
43 metadata_version: crate::MetadataVersion,
52 batch_compression_type: Option<crate::CompressionType>,
55 batch_compression_level: Option<i32>,
57 dictionary_handling: DictionaryHandling,
59}
60
61enum EncodedBuffer {
67 Raw(Buffer),
69 Compressed(Vec<u8>),
71}
72
73impl EncodedBuffer {
74 fn as_slice(&self) -> &[u8] {
75 match self {
76 EncodedBuffer::Raw(b) => b.as_slice(),
77 EncodedBuffer::Compressed(v) => v.as_slice(),
78 }
79 }
80
81 fn len(&self) -> usize {
82 match self {
83 EncodedBuffer::Raw(b) => b.len(),
84 EncodedBuffer::Compressed(v) => v.len(),
85 }
86 }
87}
88#[derive(Default)]
93struct IpcMetadataBuilder {
94 nodes: Vec<crate::FieldNode>,
95 buffers: Vec<crate::Buffer>,
96}
97
98enum IpcBodySink<'a> {
103 Write(&'a mut Vec<u8>),
105 Collect(&'a mut Vec<EncodedBuffer>),
107}
108impl IpcBodySink<'_> {
109 pub fn write(&mut self, pad_len: usize, buffer: EncodedBuffer) {
111 match self {
112 IpcBodySink::Write(vec) => {
113 vec.extend_from_slice(buffer.as_slice());
114 vec.extend_from_slice(&PADDING[..pad_len]);
115 }
116 IpcBodySink::Collect(vec) => {
117 vec.push(buffer);
118 }
119 }
120 }
121}
122
123struct MetadataLayout {
124 padded_header_len: usize,
125 padded_metadata_len: usize,
126 metadata_padding: usize,
127}
128
129impl MetadataLayout {
130 fn new(metadata_len: usize, write_options: &IpcWriteOptions) -> MetadataLayout {
131 let prefix_size = if write_options.write_legacy_ipc_format {
132 4
133 } else {
134 8
135 };
136 let alignment_mask = usize::from(write_options.alignment - 1);
137 let padded_header_len = (metadata_len + prefix_size + alignment_mask) & !alignment_mask;
138 let padded_metadata_len = padded_header_len - prefix_size;
139 let metadata_padding = padded_metadata_len - metadata_len;
140
141 MetadataLayout {
142 padded_header_len,
143 padded_metadata_len,
144 metadata_padding,
145 }
146 }
147}
148
149trait IpcMessageSink {
155 fn write_slice(&mut self, bytes: &[u8]) -> Result<(), ArrowError>;
156
157 fn write_vec(&mut self, bytes: Vec<u8>) -> Result<(), ArrowError> {
158 self.write_slice(&bytes)
159 }
160
161 fn write_encoded_buffer(&mut self, buffer: EncodedBuffer) -> Result<(), ArrowError> {
162 self.write_slice(buffer.as_slice())
163 }
164}
165
166trait IpcMessageSinkExt: IpcMessageSink {
168 fn write_padding(&mut self, len: usize) -> Result<(), ArrowError> {
169 self.write_slice(&PADDING[..len])
170 }
171
172 fn write_continuation(
174 &mut self,
175 write_options: &IpcWriteOptions,
176 metadata_len: i32,
177 ) -> Result<(), ArrowError> {
178 let mut buffer = [0; 8];
179 let len = match write_options.metadata_version {
180 crate::MetadataVersion::V1
181 | crate::MetadataVersion::V2
182 | crate::MetadataVersion::V3 => {
183 unreachable!("Options with the metadata version cannot be created")
184 }
185 crate::MetadataVersion::V4 => {
186 let metadata_len_bytes = metadata_len.to_le_bytes();
187 if !write_options.write_legacy_ipc_format {
188 buffer[..4].copy_from_slice(&CONTINUATION_MARKER);
190 buffer[4..].copy_from_slice(&metadata_len_bytes);
191 8
192 } else {
193 buffer[..4].copy_from_slice(&metadata_len_bytes);
194 4
195 }
196 }
197 crate::MetadataVersion::V5 => {
198 buffer[..4].copy_from_slice(&CONTINUATION_MARKER);
199 buffer[4..].copy_from_slice(&metadata_len.to_le_bytes());
200 8
201 }
202 z => panic!("Unsupported crate::MetadataVersion {z:?}"),
203 };
204 self.write_slice(&buffer[..len])
205 }
206
207 fn write_body_data(&mut self, data: Vec<u8>, alignment: u8) -> Result<usize, ArrowError> {
208 let len = data.len();
209 let pad_len = pad_to_alignment(alignment, len);
210 self.write_vec(data)?;
211 self.write_padding(pad_len)?;
212 Ok(len + pad_len)
213 }
214
215 fn write_encoded_data(
220 &mut self,
221 encoded: EncodedData,
222 write_options: &IpcWriteOptions,
223 ) -> Result<(usize, usize), ArrowError> {
224 let arrow_data_len = encoded.arrow_data.len();
225 if !arrow_data_len.is_multiple_of(usize::from(write_options.alignment)) {
226 return Err(ArrowError::MemoryError(
227 "Arrow data not aligned".to_string(),
228 ));
229 }
230
231 let metadata = encoded.ipc_message;
232 let metadata_len = metadata.len();
233 let layout = MetadataLayout::new(metadata_len, write_options);
234
235 self.write_continuation(write_options, layout.padded_metadata_len as i32)?;
236 self.write_vec(metadata)?;
237 self.write_padding(layout.metadata_padding)?;
238
239 let body_len = if arrow_data_len > 0 {
240 self.write_body_data(encoded.arrow_data, write_options.alignment)?
241 } else {
242 0
243 };
244
245 Ok((layout.padded_header_len, body_len))
246 }
247
248 fn write_eos(&mut self, write_options: &IpcWriteOptions) -> Result<(), ArrowError> {
250 self.write_continuation(write_options, 0)?;
251 Ok(())
252 }
253}
254
255impl<T: IpcMessageSink + ?Sized> IpcMessageSinkExt for T {}
256
257trait IpcRecordBatchSink: IpcMessageSinkExt {
259 fn write_record_batch(
273 &mut self,
274 metadata: &[u8],
275 encoded_buffers: &mut Vec<EncodedBuffer>,
276 body_len: usize,
277 write_options: &IpcWriteOptions,
278 ) -> Result<(usize, usize), ArrowError> {
279 let alignment = write_options.alignment;
280 let layout = MetadataLayout::new(metadata.len(), write_options);
281
282 self.write_continuation(write_options, layout.padded_metadata_len as i32)?;
283 self.write_slice(metadata)?;
284 self.write_padding(layout.metadata_padding)?;
285 for enc in encoded_buffers.drain(..) {
286 let len = enc.len();
287 self.write_encoded_buffer(enc)?;
288 self.write_padding(pad_to_alignment(alignment, len))?;
289 }
290
291 Ok((layout.padded_header_len, body_len))
292 }
293}
294
295impl<W> IpcMessageSink for W
296where
297 W: Write,
298{
299 fn write_slice(&mut self, bytes: &[u8]) -> Result<(), ArrowError> {
300 if !bytes.is_empty() {
301 self.write_all(bytes)?;
302 }
303 Ok(())
304 }
305}
306
307impl<W> IpcRecordBatchSink for W
308where
309 W: Write,
310{
311 fn write_record_batch(
312 &mut self,
313 metadata: &[u8],
314 encoded_buffers: &mut Vec<EncodedBuffer>,
315 body_len: usize,
316 write_options: &IpcWriteOptions,
317 ) -> Result<(usize, usize), ArrowError> {
318 let alignment = write_options.alignment;
319 let layout = MetadataLayout::new(metadata.len(), write_options);
320
321 self.write_continuation(write_options, layout.padded_metadata_len as i32)?;
322 self.write_all(metadata)?;
323 self.write_all(&PADDING[..layout.metadata_padding])?;
324 for enc in encoded_buffers.iter_mut() {
325 self.write_all(enc.as_slice())?;
326 self.write_all(&PADDING[..pad_to_alignment(alignment, enc.len())])?;
327 }
328 encoded_buffers.clear();
334
335 Ok((layout.padded_header_len, body_len))
336 }
337}
338
339struct Buffers<'a> {
341 out: &'a mut Vec<Buffer>,
342}
343
344impl IpcMessageSink for Buffers<'_> {
345 fn write_slice(&mut self, bytes: &[u8]) -> Result<(), ArrowError> {
346 if !bytes.is_empty() {
347 self.out.push(Buffer::from(bytes));
348 }
349 Ok(())
350 }
351
352 fn write_vec(&mut self, bytes: Vec<u8>) -> Result<(), ArrowError> {
353 if !bytes.is_empty() {
354 self.out.push(Buffer::from(bytes));
355 }
356 Ok(())
357 }
358
359 fn write_encoded_buffer(&mut self, buffer: EncodedBuffer) -> Result<(), ArrowError> {
360 match buffer {
361 EncodedBuffer::Raw(buffer) => self.out.push(buffer),
362 EncodedBuffer::Compressed(bytes) => self.out.push(Buffer::from(bytes)),
363 }
364 Ok(())
365 }
366}
367
368impl IpcRecordBatchSink for Buffers<'_> {}
369
370struct IpcWriteMetadata {
375 dictionary_block_sizes: Vec<(usize, usize)>,
378 padded_header_len: usize,
380 body_len: usize,
382}
383
384impl IpcWriteOptions {
385 pub fn try_with_compression(
390 mut self,
391 batch_compression_type: Option<crate::CompressionType>,
392 ) -> Result<Self, ArrowError> {
393 self.batch_compression_type = batch_compression_type;
394
395 if self.batch_compression_type.is_some()
396 && self.metadata_version < crate::MetadataVersion::V5
397 {
398 return Err(ArrowError::InvalidArgumentError(
399 "Compression only supported in metadata v5 and above".to_string(),
400 ));
401 }
402 Ok(self)
403 }
404
405 pub fn try_with_compression_level(
410 mut self,
411 batch_compression_level: Option<i32>,
412 ) -> Result<Self, ArrowError> {
413 self.batch_compression_level = batch_compression_level;
414
415 if self.batch_compression_level.is_some()
416 && self.metadata_version < crate::MetadataVersion::V5
417 {
418 return Err(ArrowError::InvalidArgumentError(
419 "Compression only supported in metadata v5 and above".to_string(),
420 ));
421 }
422
423 match (self.batch_compression_type, self.batch_compression_level) {
424 (Some(crate::CompressionType::ZSTD), Some(level)) => {
425 return self.check_zstd_level(level);
426 }
427 (Some(crate::CompressionType::LZ4_FRAME), Some(_)) => {
428 return Err(ArrowError::InvalidArgumentError(
429 "LZ4 Frame compression does not support configurable compression levels"
430 .to_string(),
431 ));
432 }
433 _ => {}
434 }
435
436 Ok(self)
437 }
438
439 #[cfg(not(feature = "zstd"))]
440 fn check_zstd_level(self, _level: i32) -> Result<Self, ArrowError> {
441 Err(ArrowError::InvalidArgumentError(
442 "zstd IPC compression requires the zstd feature".to_string(),
443 ))
444 }
445
446 #[cfg(feature = "zstd")]
447 fn check_zstd_level(self, level: i32) -> Result<Self, ArrowError> {
448 let range = zstd::compression_level_range();
449 if !range.contains(&level) {
450 return Err(ArrowError::InvalidArgumentError(format!(
451 "ZSTD compression level must be between {} and {}, got {}",
452 range.start(),
453 range.end(),
454 level,
455 )));
456 }
457
458 Ok(self)
459 }
460
461 pub fn try_new(
463 alignment: usize,
464 write_legacy_ipc_format: bool,
465 metadata_version: crate::MetadataVersion,
466 ) -> Result<Self, ArrowError> {
467 let alignment: u8 = match alignment {
468 8 => 8,
469 16 => 16,
470 32 => 32,
471 64 => 64,
472 _ => {
473 return Err(ArrowError::InvalidArgumentError(
474 "Alignment should be 8, 16, 32, or 64.".to_string(),
475 ));
476 }
477 };
478 match metadata_version {
479 crate::MetadataVersion::V1
480 | crate::MetadataVersion::V2
481 | crate::MetadataVersion::V3 => Err(ArrowError::InvalidArgumentError(
482 "Writing IPC metadata version 3 and lower not supported".to_string(),
483 )),
484 crate::MetadataVersion::V4 => Ok(Self {
485 alignment,
486 write_legacy_ipc_format,
487 metadata_version,
488 batch_compression_type: None,
489 batch_compression_level: None,
490 dictionary_handling: DictionaryHandling::default(),
491 }),
492 crate::MetadataVersion::V5 => {
493 if write_legacy_ipc_format {
494 Err(ArrowError::InvalidArgumentError(
495 "Legacy IPC format only supported on metadata version 4".to_string(),
496 ))
497 } else {
498 Ok(Self {
499 alignment,
500 write_legacy_ipc_format,
501 metadata_version,
502 batch_compression_type: None,
503 batch_compression_level: None,
504 dictionary_handling: DictionaryHandling::default(),
505 })
506 }
507 }
508 z => Err(ArrowError::InvalidArgumentError(format!(
509 "Unsupported crate::MetadataVersion {z:?}"
510 ))),
511 }
512 }
513
514 pub fn with_dictionary_handling(mut self, dictionary_handling: DictionaryHandling) -> Self {
516 self.dictionary_handling = dictionary_handling;
517 self
518 }
519}
520
521impl Default for IpcWriteOptions {
522 fn default() -> Self {
523 Self {
524 alignment: 64,
525 write_legacy_ipc_format: false,
526 metadata_version: crate::MetadataVersion::V5,
527 batch_compression_type: None,
528 batch_compression_level: None,
529 dictionary_handling: DictionaryHandling::default(),
530 }
531 }
532}
533
534#[derive(Debug, Default)]
535pub struct IpcDataGenerator {}
569
570struct DictionaryToEncode {
572 id: i64,
573 data: ArrayData,
574 is_delta: bool,
575}
576
577struct EncodedRecordBatchMeta {
584 fb_offset: flatbuffers::WIPOffset<crate::RecordBatch<'static>>,
586 body_len: usize,
589}
590
591impl IpcDataGenerator {
592 pub fn schema_to_bytes_with_dictionary_tracker(
595 &self,
596 schema: &Schema,
597 dictionary_tracker: &mut DictionaryTracker,
598 write_options: &IpcWriteOptions,
599 ) -> EncodedData {
600 let mut fbb = FlatBufferBuilder::new();
601 let schema = {
602 let fb = IpcSchemaEncoder::new()
603 .with_dictionary_tracker(dictionary_tracker)
604 .schema_to_fb_offset(&mut fbb, schema);
605 fb.as_union_value()
606 };
607
608 let mut message = crate::MessageBuilder::new(&mut fbb);
609 message.add_version(write_options.metadata_version);
610 message.add_header_type(crate::MessageHeader::Schema);
611 message.add_bodyLength(0);
612 message.add_header(schema);
613 let root = message.finish();
615 fbb.finish(root, None);
616
617 let metadata = fbb.finished_data();
618 EncodedData {
619 ipc_message: metadata.to_vec(),
620 arrow_data: vec![],
621 }
622 }
623
624 fn _collect_dict_updates<I: Iterator<Item = i64>>(
625 &self,
626 column: &ArrayRef,
627 dictionaries_to_encode: &mut Vec<DictionaryToEncode>,
628 dictionary_tracker: &mut DictionaryTracker,
629 write_options: &IpcWriteOptions,
630 dict_id: &mut I,
631 ) -> Result<(), ArrowError> {
632 match column.data_type() {
633 DataType::Struct(fields) => {
634 let s = as_struct_array(column);
635 for (field, column) in fields.iter().zip(s.columns()) {
636 self.collect_dict_updates(
637 field,
638 column,
639 dictionaries_to_encode,
640 dictionary_tracker,
641 write_options,
642 dict_id,
643 )?;
644 }
645 }
646 DataType::RunEndEncoded(_, values) => {
647 let data = column.to_data();
648 if data.child_data().len() != 2 {
649 return Err(ArrowError::InvalidArgumentError(format!(
650 "The run encoded array should have exactly two child arrays. Found {}",
651 data.child_data().len()
652 )));
653 }
654 let values_array = make_array(data.child_data()[1].clone());
657 self.collect_dict_updates(
658 values,
659 &values_array,
660 dictionaries_to_encode,
661 dictionary_tracker,
662 write_options,
663 dict_id,
664 )?;
665 }
666 DataType::List(field) => {
667 let list = as_list_array(column);
668 self.collect_dict_updates(
669 field,
670 list.values(),
671 dictionaries_to_encode,
672 dictionary_tracker,
673 write_options,
674 dict_id,
675 )?;
676 }
677 DataType::LargeList(field) => {
678 let list = as_large_list_array(column);
679 self.collect_dict_updates(
680 field,
681 list.values(),
682 dictionaries_to_encode,
683 dictionary_tracker,
684 write_options,
685 dict_id,
686 )?;
687 }
688 DataType::ListView(field) => {
689 let list = column.as_list_view::<i32>();
690 self.collect_dict_updates(
691 field,
692 list.values(),
693 dictionaries_to_encode,
694 dictionary_tracker,
695 write_options,
696 dict_id,
697 )?;
698 }
699 DataType::LargeListView(field) => {
700 let list = column.as_list_view::<i64>();
701 self.collect_dict_updates(
702 field,
703 list.values(),
704 dictionaries_to_encode,
705 dictionary_tracker,
706 write_options,
707 dict_id,
708 )?;
709 }
710 DataType::FixedSizeList(field, _) => {
711 let list = column
712 .as_any()
713 .downcast_ref::<FixedSizeListArray>()
714 .expect("Unable to downcast to fixed size list array");
715 self.collect_dict_updates(
716 field,
717 list.values(),
718 dictionaries_to_encode,
719 dictionary_tracker,
720 write_options,
721 dict_id,
722 )?;
723 }
724 DataType::Map(field, _) => {
725 let map_array = as_map_array(column);
726
727 let (keys, values) = match field.data_type() {
728 DataType::Struct(fields) if fields.len() == 2 => (&fields[0], &fields[1]),
729 _ => panic!("Incorrect field data type {:?}", field.data_type()),
730 };
731
732 self.collect_dict_updates(
734 keys,
735 map_array.keys(),
736 dictionaries_to_encode,
737 dictionary_tracker,
738 write_options,
739 dict_id,
740 )?;
741
742 self.collect_dict_updates(
744 values,
745 map_array.values(),
746 dictionaries_to_encode,
747 dictionary_tracker,
748 write_options,
749 dict_id,
750 )?;
751 }
752 DataType::Union(fields, _) => {
753 let union = as_union_array(column);
754 for (type_id, field) in fields.iter() {
755 let column = union.child(type_id);
756 self.collect_dict_updates(
757 field,
758 column,
759 dictionaries_to_encode,
760 dictionary_tracker,
761 write_options,
762 dict_id,
763 )?;
764 }
765 }
766 _ => (),
767 }
768
769 Ok(())
770 }
771
772 fn collect_dict_updates<I: Iterator<Item = i64>>(
776 &self,
777 field: &Field,
778 column: &ArrayRef,
779 dictionaries_to_encode: &mut Vec<DictionaryToEncode>,
780 dictionary_tracker: &mut DictionaryTracker,
781 write_options: &IpcWriteOptions,
782 dict_id_seq: &mut I,
783 ) -> Result<(), ArrowError> {
784 match column.data_type() {
785 DataType::Dictionary(_key_type, value_type) => {
786 if matches!(value_type.as_ref(), DataType::Dictionary(_, _)) {
787 return Err(ArrowError::InvalidArgumentError(format!(
788 "Arrow IPC field metadata cannot encode direct dictionary-of-dictionary values for field {:?}",
789 field.name()
790 )));
791 }
792
793 let dict_data = column.to_data();
794 let dict_values = &dict_data.child_data()[0];
795
796 let values = make_array(dict_data.child_data()[0].clone());
797
798 self._collect_dict_updates(
799 &values,
800 dictionaries_to_encode,
801 dictionary_tracker,
802 write_options,
803 dict_id_seq,
804 )?;
805
806 let dict_id = dict_id_seq.next().ok_or_else(|| {
810 ArrowError::IpcError(format!(
811 "no dict id for field {:?}: field.data_type={:?}, column.data_type={:?}",
812 field.name(),
813 field.data_type(),
814 column.data_type()
815 ))
816 })?;
817
818 match dictionary_tracker.insert_column(
819 dict_id,
820 column,
821 write_options.dictionary_handling,
822 )? {
823 DictionaryUpdate::None => {}
824 DictionaryUpdate::New | DictionaryUpdate::Replaced => {
825 dictionaries_to_encode.push(DictionaryToEncode {
826 id: dict_id,
827 data: dict_values.clone(),
828 is_delta: false,
829 });
830 }
831 DictionaryUpdate::Delta(data) => {
832 dictionaries_to_encode.push(DictionaryToEncode {
833 id: dict_id,
834 data,
835 is_delta: true,
836 });
837 }
838 }
839 }
840 _ => self._collect_dict_updates(
841 column,
842 dictionaries_to_encode,
843 dictionary_tracker,
844 write_options,
845 dict_id_seq,
846 )?,
847 }
848
849 Ok(())
850 }
851
852 pub fn encode(
856 &self,
857 batch: &RecordBatch,
858 dictionary_tracker: &mut DictionaryTracker,
859 write_options: &IpcWriteOptions,
860 ipc_write_context: &mut IpcWriteContext,
861 ) -> Result<(Vec<EncodedData>, EncodedData), ArrowError> {
862 let dictionaries = self.collect_all_dicts(batch, dictionary_tracker, write_options)?;
863 let mut encoded_dictionaries = Vec::with_capacity(dictionaries.len());
864 for dict in dictionaries {
865 encoded_dictionaries.push(self.dictionary_batch_to_bytes(
866 dict,
867 write_options,
868 ipc_write_context,
869 )?);
870 }
871 let mut arrow_data = ipc_write_context.scratch();
872 self.record_batch_to_bytes(
873 batch,
874 write_options,
875 ipc_write_context,
876 &mut IpcBodySink::Write(&mut arrow_data),
877 )?;
878 ipc_write_context.reserve_scratch_with_capacity(arrow_data.capacity());
879 Ok((
880 encoded_dictionaries,
881 EncodedData {
882 ipc_message: ipc_write_context.mut_fbb().finished_data().to_vec(),
883 arrow_data,
884 },
885 ))
886 }
887
888 fn collect_all_dicts(
890 &self,
891 batch: &RecordBatch,
892 dictionary_tracker: &mut DictionaryTracker,
893 write_options: &IpcWriteOptions,
894 ) -> Result<Vec<DictionaryToEncode>, ArrowError> {
895 let schema = batch.schema();
896 let mut dictionaries = Vec::with_capacity(schema.flattened_fields().len());
897 let mut dict_id = dictionary_tracker.dict_ids.clone().into_iter();
898 for (i, field) in schema.fields().iter().enumerate() {
899 self.collect_dict_updates(
900 field,
901 batch.column(i),
902 &mut dictionaries,
903 dictionary_tracker,
904 write_options,
905 &mut dict_id,
906 )?;
907 }
908 Ok(dictionaries)
909 }
910
911 fn write<W: Write>(
915 &self,
916 batch: &RecordBatch,
917 dictionary_tracker: &mut DictionaryTracker,
918 write_options: &IpcWriteOptions,
919 ipc_write_context: &mut IpcWriteContext,
920 writer: &mut W,
921 ) -> Result<IpcWriteMetadata, ArrowError> {
922 self.write_to_sink(
923 batch,
924 dictionary_tracker,
925 write_options,
926 ipc_write_context,
927 writer,
928 )
929 }
930
931 fn encode_to_buffers(
934 &self,
935 batch: &RecordBatch,
936 dictionary_tracker: &mut DictionaryTracker,
937 write_options: &IpcWriteOptions,
938 ipc_write_context: &mut IpcWriteContext,
939 out: &mut Vec<Buffer>,
940 ) -> Result<IpcWriteMetadata, ArrowError> {
941 let mut sink = Buffers { out };
942 self.write_to_sink(
943 batch,
944 dictionary_tracker,
945 write_options,
946 ipc_write_context,
947 &mut sink,
948 )
949 }
950
951 fn write_to_sink<S: IpcRecordBatchSink>(
952 &self,
953 batch: &RecordBatch,
954 dictionary_tracker: &mut DictionaryTracker,
955 write_options: &IpcWriteOptions,
956 ipc_write_context: &mut IpcWriteContext,
957 sink: &mut S,
958 ) -> Result<IpcWriteMetadata, ArrowError> {
959 let dictionaries = self.collect_all_dicts(batch, dictionary_tracker, write_options)?;
960 let mut dictionary_block_sizes = Vec::with_capacity(dictionaries.len());
961
962 let capacity = batch
963 .columns()
964 .iter()
965 .map(|a| estimate_encoded_buffer_count(a.data_type()))
966 .sum();
967 let mut encoded_buffers: Vec<EncodedBuffer> = Vec::with_capacity(capacity);
968
969 for dict in &dictionaries {
970 encoded_buffers.clear();
971
972 let body_len = self.dictionary_batch_to_sink(
973 dict,
974 write_options,
975 ipc_write_context,
976 &mut IpcBodySink::Collect(&mut encoded_buffers),
977 )?;
978
979 dictionary_block_sizes.push(sink.write_record_batch(
980 ipc_write_context.mut_fbb().finished_data(),
981 &mut encoded_buffers,
982 body_len,
983 write_options,
984 )?);
985 }
986
987 encoded_buffers.clear();
988 let body_len = self.record_batch_to_bytes(
989 batch,
990 write_options,
991 ipc_write_context,
992 &mut IpcBodySink::Collect(&mut encoded_buffers),
993 )?;
994
995 let (padded_header_len, body_len) = sink.write_record_batch(
996 ipc_write_context.mut_fbb().finished_data(),
997 &mut encoded_buffers,
998 body_len,
999 write_options,
1000 )?;
1001
1002 Ok(IpcWriteMetadata {
1003 dictionary_block_sizes,
1004 padded_header_len,
1005 body_len,
1006 })
1007 }
1008
1009 #[deprecated(since = "57.0.0", note = "Use `encode` instead")]
1013 pub fn encoded_batch(
1014 &self,
1015 batch: &RecordBatch,
1016 dictionary_tracker: &mut DictionaryTracker,
1017 write_options: &IpcWriteOptions,
1018 ) -> Result<(Vec<EncodedData>, EncodedData), ArrowError> {
1019 self.encode(
1020 batch,
1021 dictionary_tracker,
1022 write_options,
1023 &mut Default::default(),
1024 )
1025 }
1026
1027 fn record_batch_to_bytes(
1038 &self,
1039 batch: &RecordBatch,
1040 write_options: &IpcWriteOptions,
1041 ipc_write_context: &mut IpcWriteContext,
1042 sink: &mut IpcBodySink<'_>,
1043 ) -> Result<usize, ArrowError> {
1044 ipc_write_context.mut_fbb().reset();
1046
1047 let EncodedRecordBatchMeta {
1048 fb_offset: record_batch,
1049 body_len,
1050 } = self.encode_record_batch_data(
1051 batch.columns().iter().map(|array| array.to_data()),
1052 batch.num_rows() as i64,
1053 write_options,
1054 ipc_write_context,
1055 sink,
1056 )?;
1057
1058 let fbb = ipc_write_context.mut_fbb();
1059 let mut message = crate::MessageBuilder::new(fbb);
1060 message.add_version(write_options.metadata_version);
1061 message.add_header_type(crate::MessageHeader::RecordBatch);
1062 message.add_bodyLength(body_len as i64);
1063 message.add_header(record_batch.as_union_value());
1064 let root = message.finish();
1065 fbb.finish(root, None);
1066
1067 Ok(body_len)
1068 }
1069
1070 fn dictionary_batch_to_bytes(
1073 &self,
1074 dict: DictionaryToEncode,
1075 write_options: &IpcWriteOptions,
1076 ipc_write_context: &mut IpcWriteContext,
1077 ) -> Result<EncodedData, ArrowError> {
1078 let mut arrow_data: Vec<u8> = vec![];
1079 self.dictionary_batch_to_sink(
1080 &dict,
1081 write_options,
1082 ipc_write_context,
1083 &mut IpcBodySink::Write(&mut arrow_data),
1084 )?;
1085
1086 Ok(EncodedData {
1087 ipc_message: ipc_write_context.mut_fbb().finished_data().to_vec(),
1088 arrow_data,
1089 })
1090 }
1091
1092 fn dictionary_batch_to_sink(
1099 &self,
1100 dict: &DictionaryToEncode,
1101 write_options: &IpcWriteOptions,
1102 ipc_write_context: &mut IpcWriteContext,
1103 sink: &mut IpcBodySink<'_>,
1104 ) -> Result<usize, ArrowError> {
1105 ipc_write_context.mut_fbb().reset();
1107
1108 let EncodedRecordBatchMeta {
1112 fb_offset: record_batch,
1113 body_len,
1114 } = self.encode_record_batch_data(
1115 std::iter::once(dict.data.clone()),
1116 dict.data.len() as i64,
1117 write_options,
1118 ipc_write_context,
1119 sink,
1120 )?;
1121
1122 let fbb = ipc_write_context.mut_fbb();
1123 let root = {
1124 let mut batch_builder = crate::DictionaryBatchBuilder::new(fbb);
1125 batch_builder.add_id(dict.id);
1126 batch_builder.add_data(record_batch);
1127 batch_builder.add_isDelta(dict.is_delta);
1128 batch_builder.finish().as_union_value()
1129 };
1130
1131 let root = {
1132 let mut message_builder = crate::MessageBuilder::new(fbb);
1133 message_builder.add_version(write_options.metadata_version);
1134 message_builder.add_header_type(crate::MessageHeader::DictionaryBatch);
1135 message_builder.add_bodyLength(body_len as i64);
1136 message_builder.add_header(root);
1137 message_builder.finish()
1138 };
1139
1140 fbb.finish(root, None);
1141
1142 Ok(body_len)
1143 }
1144
1145 #[inline]
1158 fn encode_record_batch_data(
1159 &self,
1160 columns: impl IntoIterator<Item = ArrayData>,
1161 row_count: i64,
1162 write_options: &IpcWriteOptions,
1163 ipc_write_context: &mut IpcWriteContext,
1164 sink: &mut IpcBodySink<'_>,
1165 ) -> Result<EncodedRecordBatchMeta, ArrowError> {
1166 let batch_compression_type = write_options.batch_compression_type;
1167
1168 let compression = batch_compression_type.map(|batch_compression_type| {
1169 let fbb = ipc_write_context.mut_fbb();
1170 let mut c = crate::BodyCompressionBuilder::new(fbb);
1171 c.add_method(crate::BodyCompressionMethod::BUFFER);
1172 c.add_codec(batch_compression_type);
1173 c.finish()
1174 });
1175
1176 let batch_compression_level = write_options.batch_compression_level;
1177 let compression_codec: Option<CompressionCodec> = batch_compression_type
1178 .map(|compression_type| match batch_compression_level {
1179 Some(level) => {
1180 CompressionCodec::try_new_with_compression_level(compression_type, level)
1181 }
1182 None => compression_type.try_into(),
1183 })
1184 .transpose()?;
1185
1186 let mut meta = IpcMetadataBuilder::default();
1187 let mut variadic_buffer_counts: Vec<i64> = vec![];
1188 let mut offset = 0i64;
1189
1190 for array_data in columns {
1191 offset = write_array_data(
1192 &array_data,
1193 &mut meta,
1194 sink,
1195 offset,
1196 compression_codec,
1197 ipc_write_context,
1198 write_options,
1199 )?;
1200 append_variadic_buffer_counts(&mut variadic_buffer_counts, &array_data);
1201 }
1202
1203 debug_assert_eq!(offset % write_options.alignment as i64, 0);
1206 let body_len = offset as usize;
1207
1208 let fbb = ipc_write_context.mut_fbb();
1209 let buffers = fbb.create_vector(&meta.buffers);
1210 let nodes = fbb.create_vector(&meta.nodes);
1211 let variadic_buffer = if variadic_buffer_counts.is_empty() {
1212 None
1213 } else {
1214 Some(fbb.create_vector(&variadic_buffer_counts))
1215 };
1216
1217 let mut batch_builder = crate::RecordBatchBuilder::new(fbb);
1218 batch_builder.add_length(row_count);
1219 batch_builder.add_nodes(nodes);
1220 batch_builder.add_buffers(buffers);
1221 if let Some(c) = compression {
1222 batch_builder.add_compression(c);
1223 }
1224 if let Some(v) = variadic_buffer {
1225 batch_builder.add_variadicBufferCounts(v);
1226 }
1227 let record_batch = batch_builder.finish();
1228
1229 Ok(EncodedRecordBatchMeta {
1230 fb_offset: record_batch,
1231 body_len,
1232 })
1233 }
1234}
1235
1236fn ensure_supported_ipc_schema(schema: &Schema) -> Result<(), ArrowError> {
1237 schema
1238 .fields()
1239 .iter()
1240 .try_for_each(|field| ensure_supported_ipc_data_type(field.name(), field.data_type()))
1241}
1242
1243fn ensure_supported_ipc_data_type(
1244 field_name: &str,
1245 data_type: &DataType,
1246) -> Result<(), ArrowError> {
1247 match data_type {
1248 DataType::Dictionary(_, value_type)
1249 if matches!(value_type.as_ref(), DataType::Dictionary(_, _)) =>
1250 {
1251 Err(ArrowError::InvalidArgumentError(format!(
1252 "Arrow IPC field metadata cannot encode direct dictionary-of-dictionary values for field {field_name:?}"
1253 )))
1254 }
1255 DataType::Dictionary(_, value_type) => {
1256 ensure_supported_ipc_data_type(field_name, value_type)
1257 }
1258 DataType::Struct(fields) => fields
1259 .iter()
1260 .try_for_each(|field| ensure_supported_ipc_data_type(field.name(), field.data_type())),
1261 DataType::RunEndEncoded(_, field)
1262 | DataType::List(field)
1263 | DataType::LargeList(field)
1264 | DataType::ListView(field)
1265 | DataType::LargeListView(field)
1266 | DataType::FixedSizeList(field, _)
1267 | DataType::Map(field, _) => {
1268 ensure_supported_ipc_data_type(field.name(), field.data_type())
1269 }
1270 DataType::Union(fields, _) => fields.iter().try_for_each(|(_, field)| {
1271 ensure_supported_ipc_data_type(field.name(), field.data_type())
1272 }),
1273 _ => Ok(()),
1274 }
1275}
1276
1277fn append_variadic_buffer_counts(counts: &mut Vec<i64>, array: &ArrayData) {
1278 match array.data_type() {
1279 DataType::BinaryView | DataType::Utf8View => {
1280 counts.push(array.buffers().len() as i64 - 1);
1283 }
1284 DataType::Dictionary(_, _) => {
1285 }
1288 _ => {
1289 for child in array.child_data() {
1290 append_variadic_buffer_counts(counts, child)
1291 }
1292 }
1293 }
1294}
1295
1296pub(crate) fn unslice_run_array(arr: ArrayData) -> Result<ArrayData, ArrowError> {
1297 match arr.data_type() {
1298 DataType::RunEndEncoded(k, _) => match k.data_type() {
1299 DataType::Int16 => {
1300 Ok(into_zero_offset_run_array(RunArray::<Int16Type>::from(arr))?.into_data())
1301 }
1302 DataType::Int32 => {
1303 Ok(into_zero_offset_run_array(RunArray::<Int32Type>::from(arr))?.into_data())
1304 }
1305 DataType::Int64 => {
1306 Ok(into_zero_offset_run_array(RunArray::<Int64Type>::from(arr))?.into_data())
1307 }
1308 d => unreachable!("Unexpected data type {d}"),
1309 },
1310 d => Err(ArrowError::InvalidArgumentError(format!(
1311 "The given array is not a run array. Data type of given array: {d}"
1312 ))),
1313 }
1314}
1315
1316fn into_zero_offset_run_array<R: RunEndIndexType>(
1319 run_array: RunArray<R>,
1320) -> Result<RunArray<R>, ArrowError> {
1321 if run_array.is_empty() {
1323 return Ok(ArrayData::new_empty(run_array.data_type()).into());
1324 }
1325
1326 let run_ends = run_array.run_ends();
1327 if run_ends.offset() == 0 && run_ends.max_value() == run_ends.len() {
1328 return Ok(run_array);
1329 }
1330
1331 let start_physical_index = run_ends.get_start_physical_index();
1333
1334 let end_physical_index = run_ends.get_end_physical_index();
1336
1337 let physical_length = end_physical_index - start_physical_index + 1;
1338
1339 let offset = R::Native::usize_as(run_ends.offset());
1341 let mut run_ends_values = Vec::<R::Native>::with_capacity(physical_length);
1342 for run_end_value in &run_ends.values()[start_physical_index..end_physical_index] {
1343 run_ends_values.push(run_end_value.sub_wrapping(offset));
1344 }
1345 run_ends_values.push(R::Native::from_usize(run_array.len()).unwrap());
1346 let offset_buffer = Buffer::from_vec(run_ends_values);
1347 let new_run_ends = unsafe {
1348 ArrayDataBuilder::new(R::DATA_TYPE)
1351 .len(physical_length)
1352 .add_buffer(offset_buffer)
1353 .build_unchecked()
1354 };
1355
1356 let new_values = run_array
1358 .values()
1359 .slice(start_physical_index, physical_length)
1360 .into_data();
1361
1362 let builder = ArrayDataBuilder::new(run_array.data_type().clone())
1363 .len(run_array.len())
1364 .add_child_data(new_run_ends)
1365 .add_child_data(new_values);
1366 let array_data = unsafe {
1367 builder.build_unchecked()
1370 };
1371 Ok(array_data.into())
1372}
1373
1374#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
1376pub enum DictionaryHandling {
1377 #[default]
1379 Resend,
1380 Delta,
1386}
1387
1388#[derive(Debug, Clone)]
1390pub enum DictionaryUpdate {
1391 None,
1394 New,
1396 Replaced,
1398 Delta(ArrayData),
1400}
1401
1402#[derive(Debug)]
1408pub struct DictionaryTracker {
1409 written: HashMap<i64, ArrayData>,
1411 dict_ids: Vec<i64>,
1412 error_on_replacement: bool,
1413}
1414
1415impl DictionaryTracker {
1416 pub fn new(error_on_replacement: bool) -> Self {
1422 Self {
1423 written: HashMap::new(),
1424 dict_ids: Vec::new(),
1425 error_on_replacement,
1426 }
1427 }
1428
1429 pub fn next_dict_id(&mut self) -> i64 {
1431 let next = self
1432 .dict_ids
1433 .last()
1434 .copied()
1435 .map(|i| i + 1)
1436 .unwrap_or_default();
1437
1438 self.dict_ids.push(next);
1439 next
1440 }
1441
1442 pub fn dict_id(&mut self) -> &[i64] {
1445 &self.dict_ids
1446 }
1447
1448 #[deprecated(since = "56.1.0", note = "Use `insert_column` instead")]
1458 pub fn insert(&mut self, dict_id: i64, column: &ArrayRef) -> Result<bool, ArrowError> {
1459 let dict_data = column.to_data();
1460 let dict_values = &dict_data.child_data()[0];
1461
1462 if let Some(last) = self.written.get(&dict_id) {
1464 if ArrayData::ptr_eq(&last.child_data()[0], dict_values) {
1465 return Ok(false);
1467 }
1468 if self.error_on_replacement {
1469 if last.child_data()[0] == *dict_values {
1471 return Ok(false);
1473 }
1474 return Err(ArrowError::InvalidArgumentError(
1475 "Dictionary replacement detected when writing IPC file format. \
1476 Arrow IPC files only support a single dictionary for a given field \
1477 across all batches."
1478 .to_string(),
1479 ));
1480 }
1481 }
1482
1483 self.written.insert(dict_id, dict_data);
1484 Ok(true)
1485 }
1486
1487 pub fn insert_column(
1503 &mut self,
1504 dict_id: i64,
1505 column: &ArrayRef,
1506 dict_handling: DictionaryHandling,
1507 ) -> Result<DictionaryUpdate, ArrowError> {
1508 let new_data = column.to_data();
1509 let new_values = &new_data.child_data()[0];
1510
1511 let Some(old) = self.written.get(&dict_id) else {
1513 self.written.insert(dict_id, new_data);
1514 return Ok(DictionaryUpdate::New);
1515 };
1516
1517 let old_values = &old.child_data()[0];
1520 if ArrayData::ptr_eq(old_values, new_values) {
1521 return Ok(DictionaryUpdate::None);
1522 }
1523
1524 let comparison = compare_dictionaries(old_values, new_values);
1526 if matches!(comparison, DictionaryComparison::Equal) {
1527 return Ok(DictionaryUpdate::None);
1528 }
1529
1530 const REPLACEMENT_ERROR: &str = "Dictionary replacement detected when writing IPC file format. \
1531 Arrow IPC files only support a single dictionary for a given field \
1532 across all batches.";
1533
1534 match comparison {
1535 DictionaryComparison::NotEqual => {
1536 if self.error_on_replacement {
1537 return Err(ArrowError::InvalidArgumentError(
1538 REPLACEMENT_ERROR.to_string(),
1539 ));
1540 }
1541
1542 self.written.insert(dict_id, new_data);
1543 Ok(DictionaryUpdate::Replaced)
1544 }
1545 DictionaryComparison::Delta => match dict_handling {
1546 DictionaryHandling::Resend => {
1547 if self.error_on_replacement {
1548 return Err(ArrowError::InvalidArgumentError(
1549 REPLACEMENT_ERROR.to_string(),
1550 ));
1551 }
1552
1553 self.written.insert(dict_id, new_data);
1554 Ok(DictionaryUpdate::Replaced)
1555 }
1556 DictionaryHandling::Delta => {
1557 let delta =
1558 new_values.slice(old_values.len(), new_values.len() - old_values.len());
1559 self.written.insert(dict_id, new_data);
1560 Ok(DictionaryUpdate::Delta(delta))
1561 }
1562 },
1563 DictionaryComparison::Equal => unreachable!("Already checked equal case"),
1564 }
1565 }
1566
1567 pub fn clear(&mut self) {
1573 self.dict_ids.clear();
1574 self.written.clear();
1575 }
1576}
1577
1578#[derive(Debug, Clone)]
1580enum DictionaryComparison {
1581 NotEqual,
1583 Equal,
1585 Delta,
1588}
1589
1590fn compare_dictionaries(old: &ArrayData, new: &ArrayData) -> DictionaryComparison {
1592 let existing_len = old.len();
1594 let new_len = new.len();
1595 if existing_len == new_len {
1596 return if *old == *new {
1597 DictionaryComparison::Equal
1598 } else {
1599 DictionaryComparison::NotEqual
1600 };
1601 }
1602
1603 if new_len < existing_len {
1605 return DictionaryComparison::NotEqual;
1606 }
1607
1608 if new.slice(0, existing_len) == *old {
1610 return DictionaryComparison::Delta;
1611 }
1612
1613 DictionaryComparison::NotEqual
1614}
1615
1616pub struct FileWriter<W> {
1639 writer: W,
1641 write_options: IpcWriteOptions,
1643 schema: SchemaRef,
1645 block_offsets: usize,
1647 dictionary_blocks: Vec<crate::Block>,
1649 record_blocks: Vec<crate::Block>,
1651 finished: bool,
1653 dictionary_tracker: DictionaryTracker,
1655 custom_metadata: Metadata,
1657
1658 data_gen: IpcDataGenerator,
1659
1660 ipc_write_context: IpcWriteContext,
1664}
1665
1666impl<W: Write> FileWriter<BufWriter<W>> {
1667 pub fn try_new_buffered(writer: W, schema: &Schema) -> Result<Self, ArrowError> {
1671 Self::try_new(BufWriter::new(writer), schema)
1672 }
1673}
1674
1675impl<W: Write> FileWriter<W> {
1676 pub fn try_new(writer: W, schema: &Schema) -> Result<Self, ArrowError> {
1684 let write_options = IpcWriteOptions::default();
1685 Self::try_new_with_options(writer, schema, write_options)
1686 }
1687
1688 pub fn try_new_with_options(
1696 mut writer: W,
1697 schema: &Schema,
1698 write_options: IpcWriteOptions,
1699 ) -> Result<Self, ArrowError> {
1700 ensure_supported_ipc_schema(schema)?;
1701
1702 let data_gen = IpcDataGenerator::default();
1703 let pad_len = pad_to_alignment(write_options.alignment, super::ARROW_MAGIC.len());
1705 let header_size = super::ARROW_MAGIC.len() + pad_len;
1706 writer.write_all(&super::ARROW_MAGIC)?;
1707 writer.write_all(&PADDING[..pad_len])?;
1708 let mut dictionary_tracker = DictionaryTracker::new(true);
1710 let encoded_message = data_gen.schema_to_bytes_with_dictionary_tracker(
1711 schema,
1712 &mut dictionary_tracker,
1713 &write_options,
1714 );
1715 let (meta, data) = write_message(&mut writer, encoded_message, &write_options)?;
1716 Ok(Self {
1717 writer,
1718 write_options,
1719 schema: Arc::new(schema.clone()),
1720 block_offsets: meta + data + header_size,
1721 dictionary_blocks: vec![],
1722 record_blocks: vec![],
1723 finished: false,
1724 dictionary_tracker,
1725 custom_metadata: Default::default(),
1726 data_gen,
1727 ipc_write_context: IpcWriteContext::default(),
1728 })
1729 }
1730
1731 pub fn write_metadata(&mut self, key: impl Into<String>, value: impl Into<String>) {
1733 self.custom_metadata.insert(key.into(), value.into());
1734 }
1735
1736 pub fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
1738 if self.finished {
1739 return Err(ArrowError::IpcError(
1740 "Cannot write record batch to file writer as it is closed".to_string(),
1741 ));
1742 }
1743
1744 let meta = self.data_gen.write(
1745 batch,
1746 &mut self.dictionary_tracker,
1747 &self.write_options,
1748 &mut self.ipc_write_context,
1749 &mut self.writer,
1750 )?;
1751
1752 for (header_len, body_len) in meta.dictionary_block_sizes {
1753 let block = crate::Block::new(
1754 self.block_offsets as i64,
1755 header_len as i32,
1756 body_len as i64,
1757 );
1758 self.dictionary_blocks.push(block);
1759 self.block_offsets += header_len + body_len;
1760 }
1761
1762 let block = crate::Block::new(
1764 self.block_offsets as i64,
1765 meta.padded_header_len as i32,
1766 meta.body_len as i64,
1767 );
1768 self.record_blocks.push(block);
1769 self.block_offsets += meta.padded_header_len + meta.body_len;
1770 Ok(())
1771 }
1772
1773 pub fn finish(&mut self) -> Result<(), ArrowError> {
1775 if self.finished {
1776 return Err(ArrowError::IpcError(
1777 "Cannot write footer to file writer as it is closed".to_string(),
1778 ));
1779 }
1780
1781 self.writer.write_eos(&self.write_options)?;
1783
1784 let mut fbb = FlatBufferBuilder::new();
1785 let dictionaries = fbb.create_vector(&self.dictionary_blocks);
1786 let record_batches = fbb.create_vector(&self.record_blocks);
1787
1788 self.dictionary_tracker.clear();
1790 let schema = IpcSchemaEncoder::new()
1791 .with_dictionary_tracker(&mut self.dictionary_tracker)
1792 .schema_to_fb_offset(&mut fbb, &self.schema);
1793 let fb_custom_metadata = (!self.custom_metadata.is_empty())
1794 .then(|| crate::convert::metadata_to_fb(&mut fbb, &self.custom_metadata));
1795
1796 let root = {
1797 let mut footer_builder = crate::FooterBuilder::new(&mut fbb);
1798 footer_builder.add_version(self.write_options.metadata_version);
1799 footer_builder.add_schema(schema);
1800 footer_builder.add_dictionaries(dictionaries);
1801 footer_builder.add_recordBatches(record_batches);
1802 if let Some(fb_custom_metadata) = fb_custom_metadata {
1803 footer_builder.add_custom_metadata(fb_custom_metadata);
1804 }
1805 footer_builder.finish()
1806 };
1807 fbb.finish(root, None);
1808 let footer_data = fbb.finished_data();
1809 self.writer.write_all(footer_data)?;
1810 self.writer
1811 .write_all(&(footer_data.len() as i32).to_le_bytes())?;
1812 self.writer.write_all(&super::ARROW_MAGIC)?;
1813 self.writer.flush()?;
1814 self.finished = true;
1815
1816 Ok(())
1817 }
1818
1819 pub fn schema(&self) -> &SchemaRef {
1821 &self.schema
1822 }
1823
1824 pub fn get_ref(&self) -> &W {
1826 &self.writer
1827 }
1828
1829 pub fn get_mut(&mut self) -> &mut W {
1833 &mut self.writer
1834 }
1835
1836 pub fn flush(&mut self) -> Result<(), ArrowError> {
1840 self.writer.flush()?;
1841 Ok(())
1842 }
1843
1844 pub fn into_inner(mut self) -> Result<W, ArrowError> {
1853 if !self.finished {
1854 self.finish()?;
1856 }
1857 Ok(self.writer)
1858 }
1859}
1860
1861impl<W: Write> RecordBatchWriter for FileWriter<W> {
1862 fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
1863 self.write(batch)
1864 }
1865
1866 fn close(mut self) -> Result<(), ArrowError> {
1867 self.finish()
1868 }
1869}
1870
1871pub struct StreamEncoder {
1900 schema: Schema,
1901 write_options: IpcWriteOptions,
1903 schema_encoded: bool,
1905 dictionary_tracker: DictionaryTracker,
1907 data_gen: IpcDataGenerator,
1908 ipc_write_context: IpcWriteContext,
1909}
1910
1911impl StreamEncoder {
1912 pub fn try_new(schema: &Schema) -> Result<Self, ArrowError> {
1914 let write_options = IpcWriteOptions::default();
1915 Self::try_new_with_options(schema, write_options)
1916 }
1917
1918 pub fn try_new_with_options(
1920 schema: &Schema,
1921 write_options: IpcWriteOptions,
1922 ) -> Result<Self, ArrowError> {
1923 ensure_supported_ipc_schema(schema)?;
1924
1925 Ok(Self {
1926 schema: schema.clone(),
1927 write_options,
1928 schema_encoded: false,
1929 dictionary_tracker: DictionaryTracker::new(false),
1930 data_gen: IpcDataGenerator::default(),
1931 ipc_write_context: IpcWriteContext::default(),
1932 })
1933 }
1934
1935 pub fn encode(&mut self, batch: &RecordBatch) -> Result<Vec<Buffer>, ArrowError> {
1945 let mut out = vec![];
1946 self.encode_schema(&mut out)?;
1947 self.data_gen.encode_to_buffers(
1948 batch,
1949 &mut self.dictionary_tracker,
1950 &self.write_options,
1951 &mut self.ipc_write_context,
1952 &mut out,
1953 )?;
1954 Ok(out)
1955 }
1956
1957 pub fn finish(mut self) -> Result<Vec<Buffer>, ArrowError> {
1966 let mut out = vec![];
1967 self.encode_schema(&mut out)?;
1968 let mut sink = Buffers { out: &mut out };
1969 sink.write_eos(&self.write_options)?;
1970 Ok(out)
1971 }
1972
1973 fn encode_schema(&mut self, out: &mut Vec<Buffer>) -> Result<(), ArrowError> {
1974 if !self.schema_encoded {
1975 let encoded_message = self.data_gen.schema_to_bytes_with_dictionary_tracker(
1976 &self.schema,
1977 &mut self.dictionary_tracker,
1978 &self.write_options,
1979 );
1980 let mut sink = Buffers { out };
1981 sink.write_encoded_data(encoded_message, &self.write_options)?;
1982 self.schema_encoded = true;
1983 }
1984 Ok(())
1985 }
1986}
1987
1988pub struct StreamWriter<W> {
2062 writer: W,
2064 write_options: IpcWriteOptions,
2066 finished: bool,
2068 dictionary_tracker: DictionaryTracker,
2070
2071 data_gen: IpcDataGenerator,
2072
2073 ipc_write_context: IpcWriteContext,
2075}
2076
2077impl<W: Write> StreamWriter<BufWriter<W>> {
2078 pub fn try_new_buffered(writer: W, schema: &Schema) -> Result<Self, ArrowError> {
2082 Self::try_new(BufWriter::new(writer), schema)
2083 }
2084}
2085
2086impl<W: Write> StreamWriter<W> {
2087 pub fn try_new(writer: W, schema: &Schema) -> Result<Self, ArrowError> {
2095 let write_options = IpcWriteOptions::default();
2096 Self::try_new_with_options(writer, schema, write_options)
2097 }
2098
2099 pub fn try_new_with_options(
2105 mut writer: W,
2106 schema: &Schema,
2107 write_options: IpcWriteOptions,
2108 ) -> Result<Self, ArrowError> {
2109 ensure_supported_ipc_schema(schema)?;
2110
2111 let data_gen = IpcDataGenerator::default();
2112 let mut dictionary_tracker = DictionaryTracker::new(false);
2113
2114 let encoded_message = data_gen.schema_to_bytes_with_dictionary_tracker(
2116 schema,
2117 &mut dictionary_tracker,
2118 &write_options,
2119 );
2120 write_message(&mut writer, encoded_message, &write_options)?;
2121 Ok(Self {
2122 writer,
2123 write_options,
2124 finished: false,
2125 dictionary_tracker,
2126 data_gen,
2127 ipc_write_context: IpcWriteContext::default(),
2128 })
2129 }
2130
2131 pub fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
2133 if self.finished {
2134 return Err(ArrowError::IpcError(
2135 "Cannot write record batch to stream writer as it is closed".to_string(),
2136 ));
2137 }
2138
2139 self.data_gen.write(
2140 batch,
2141 &mut self.dictionary_tracker,
2142 &self.write_options,
2143 &mut self.ipc_write_context,
2144 &mut self.writer,
2145 )?;
2146 Ok(())
2147 }
2148
2149 pub fn finish(&mut self) -> Result<(), ArrowError> {
2151 if self.finished {
2152 return Err(ArrowError::IpcError(
2153 "Cannot write footer to stream writer as it is closed".to_string(),
2154 ));
2155 }
2156
2157 self.writer.write_eos(&self.write_options)?;
2158 self.writer.flush()?;
2159
2160 self.finished = true;
2161
2162 Ok(())
2163 }
2164
2165 pub fn get_ref(&self) -> &W {
2167 &self.writer
2168 }
2169
2170 pub fn get_mut(&mut self) -> &mut W {
2174 &mut self.writer
2175 }
2176
2177 pub fn flush(&mut self) -> Result<(), ArrowError> {
2181 self.writer.flush()?;
2182 Ok(())
2183 }
2184
2185 pub fn into_inner(mut self) -> Result<W, ArrowError> {
2223 if !self.finished {
2224 self.finish()?;
2226 }
2227 Ok(self.writer)
2228 }
2229}
2230
2231impl<W: Write> RecordBatchWriter for StreamWriter<W> {
2232 fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
2233 self.write(batch)
2234 }
2235
2236 fn close(mut self) -> Result<(), ArrowError> {
2237 self.finish()
2238 }
2239}
2240
2241pub struct EncodedData {
2243 pub ipc_message: Vec<u8>,
2245 pub arrow_data: Vec<u8>,
2247}
2248
2249pub fn write_message<W: Write>(
2251 mut writer: W,
2252 encoded: EncodedData,
2253 write_options: &IpcWriteOptions,
2254) -> Result<(usize, usize), ArrowError> {
2255 writer.write_encoded_data(encoded, write_options)
2256}
2257
2258fn has_validity_bitmap(data_type: &DataType, write_options: &IpcWriteOptions) -> bool {
2261 if write_options.metadata_version < crate::MetadataVersion::V5 {
2262 !matches!(data_type, DataType::Null | DataType::RunEndEncoded(_, _))
2263 } else {
2264 !matches!(
2265 data_type,
2266 DataType::Null | DataType::Union(_, _) | DataType::RunEndEncoded(_, _)
2267 )
2268 }
2269}
2270
2271#[inline]
2273fn buffer_need_truncate(
2274 array_offset: usize,
2275 buffer: &Buffer,
2276 spec: &BufferSpec,
2277 min_length: usize,
2278) -> bool {
2279 spec != &BufferSpec::AlwaysNull && (array_offset != 0 || min_length < buffer.len())
2280}
2281
2282#[inline]
2284fn get_buffer_element_width(spec: &BufferSpec) -> usize {
2285 match spec {
2286 BufferSpec::FixedWidth { byte_width, .. } => *byte_width,
2287 _ => 0,
2288 }
2289}
2290
2291fn reencode_offsets<O: OffsetSizeTrait>(
2294 offsets: &Buffer,
2295 data: &ArrayData,
2296) -> (Buffer, usize, usize) {
2297 let offsets_slice: &[O] = offsets.typed_data::<O>();
2298 let offset_slice = &offsets_slice[data.offset()..data.offset() + data.len() + 1];
2299
2300 let start_offset = offset_slice.first().unwrap();
2301 let end_offset = offset_slice.last().unwrap();
2302
2303 let offsets = match start_offset.as_usize() {
2304 0 => {
2305 let size = size_of::<O>();
2306 offsets.slice_with_length(data.offset() * size, (data.len() + 1) * size)
2307 }
2308 _ => offset_slice.iter().map(|x| *x - *start_offset).collect(),
2309 };
2310
2311 let start_offset = start_offset.as_usize();
2312 let end_offset = end_offset.as_usize();
2313
2314 (offsets, start_offset, end_offset - start_offset)
2315}
2316
2317fn get_byte_array_buffers<O: OffsetSizeTrait>(data: &ArrayData) -> [Buffer; 2] {
2324 if data.is_empty() {
2325 let mut offsets = MutableBuffer::new(size_of::<O>());
2328 offsets.extend_from_slice(O::usize_as(0).to_byte_slice());
2329 return [offsets.into(), MutableBuffer::new(0).into()];
2330 }
2331
2332 let (offsets, original_start_offset, len) = reencode_offsets::<O>(&data.buffers()[0], data);
2333 let values = data.buffers()[1].slice_with_length(original_start_offset, len);
2334 [offsets, values]
2335}
2336
2337fn get_list_array_buffers<O: OffsetSizeTrait>(data: &ArrayData) -> (Buffer, ArrayData) {
2340 if data.is_empty() {
2341 let mut offsets = MutableBuffer::new(size_of::<O>());
2344 offsets.extend_from_slice(O::usize_as(0).to_byte_slice());
2345 return (offsets.into(), data.child_data()[0].slice(0, 0));
2346 }
2347
2348 let (offsets, original_start_offset, len) = reencode_offsets::<O>(&data.buffers()[0], data);
2349 let child_data = data.child_data()[0].slice(original_start_offset, len);
2350 (offsets, child_data)
2351}
2352
2353fn get_list_view_array_buffers<O: OffsetSizeTrait>(
2359 data: &ArrayData,
2360) -> (Buffer, Buffer, ArrayData) {
2361 if data.is_empty() {
2362 return (
2363 MutableBuffer::new(0).into(),
2364 MutableBuffer::new(0).into(),
2365 data.child_data()[0].slice(0, 0),
2366 );
2367 }
2368
2369 let offsets = &data.buffers()[0];
2370 let sizes = &data.buffers()[1];
2371
2372 let element_size = std::mem::size_of::<O>();
2373 let offsets_slice =
2374 offsets.slice_with_length(data.offset() * element_size, data.len() * element_size);
2375 let sizes_slice =
2376 sizes.slice_with_length(data.offset() * element_size, data.len() * element_size);
2377
2378 let child_data = data.child_data()[0].clone();
2379
2380 (offsets_slice, sizes_slice, child_data)
2381}
2382
2383fn get_or_truncate_buffer(array_data: &ArrayData) -> Buffer {
2390 let buffer = &array_data.buffers()[0];
2391 let layout = layout(array_data.data_type());
2392 let spec = &layout.buffers[0];
2393
2394 let byte_width = get_buffer_element_width(spec);
2395 let min_length = array_data.len() * byte_width;
2396 if buffer_need_truncate(array_data.offset(), buffer, spec, min_length) {
2397 let byte_offset = array_data.offset() * byte_width;
2398 let buffer_length = min(min_length, buffer.len() - byte_offset);
2399 buffer.slice_with_length(byte_offset, buffer_length)
2400 } else {
2401 buffer.clone()
2402 }
2403}
2404
2405fn write_array_data(
2411 array_data: &ArrayData,
2412 meta: &mut IpcMetadataBuilder,
2413 sink: &mut IpcBodySink<'_>,
2414 offset: i64,
2415 compression_codec: Option<CompressionCodec>,
2416 ipc_write_context: &mut IpcWriteContext,
2417 write_options: &IpcWriteOptions,
2418) -> Result<i64, ArrowError> {
2419 let mut offset = offset;
2420 let num_rows = array_data.len();
2421 let null_count = array_data.null_count();
2422 if !matches!(array_data.data_type(), DataType::Null) {
2423 meta.nodes
2424 .push(crate::FieldNode::new(num_rows as i64, null_count as i64));
2425 } else {
2426 meta.nodes
2428 .push(crate::FieldNode::new(num_rows as i64, num_rows as i64));
2429 }
2430 if has_validity_bitmap(array_data.data_type(), write_options) {
2431 let null_buffer = match array_data.nulls() {
2434 Some(buffer) if null_count > 0 => buffer.inner().sliced(),
2435 _ => MutableBuffer::new(0).into(),
2436 };
2437
2438 offset = encode_sink_buffer(
2439 null_buffer,
2440 meta,
2441 sink,
2442 offset,
2443 compression_codec,
2444 ipc_write_context,
2445 write_options.alignment,
2446 )?;
2447 }
2448
2449 let data_type = array_data.data_type();
2450 if matches!(data_type, DataType::Binary | DataType::Utf8) {
2451 for buffer in get_byte_array_buffers::<i32>(array_data) {
2452 offset = encode_sink_buffer(
2453 buffer,
2454 meta,
2455 sink,
2456 offset,
2457 compression_codec,
2458 ipc_write_context,
2459 write_options.alignment,
2460 )?;
2461 }
2462 } else if matches!(data_type, DataType::BinaryView | DataType::Utf8View) {
2463 let views = get_or_truncate_buffer(array_data);
2470 offset = encode_sink_buffer(
2471 views,
2472 meta,
2473 sink,
2474 offset,
2475 compression_codec,
2476 ipc_write_context,
2477 write_options.alignment,
2478 )?;
2479
2480 for buffer in array_data.buffers().iter().skip(1) {
2481 offset = encode_sink_buffer(
2482 buffer.clone(),
2483 meta,
2484 sink,
2485 offset,
2486 compression_codec,
2487 ipc_write_context,
2488 write_options.alignment,
2489 )?;
2490 }
2491 } else if matches!(data_type, DataType::LargeBinary | DataType::LargeUtf8) {
2492 for buffer in get_byte_array_buffers::<i64>(array_data) {
2493 offset = encode_sink_buffer(
2494 buffer,
2495 meta,
2496 sink,
2497 offset,
2498 compression_codec,
2499 ipc_write_context,
2500 write_options.alignment,
2501 )?;
2502 }
2503 } else if DataType::is_numeric(data_type)
2504 || DataType::is_temporal(data_type)
2505 || matches!(
2506 array_data.data_type(),
2507 DataType::FixedSizeBinary(_) | DataType::Dictionary(_, _)
2508 )
2509 {
2510 assert_eq!(array_data.buffers().len(), 1);
2512
2513 let buffer = get_or_truncate_buffer(array_data);
2514 offset = encode_sink_buffer(
2515 buffer,
2516 meta,
2517 sink,
2518 offset,
2519 compression_codec,
2520 ipc_write_context,
2521 write_options.alignment,
2522 )?;
2523 } else if matches!(data_type, DataType::Boolean) {
2524 assert_eq!(array_data.buffers().len(), 1);
2527
2528 let buffer = &array_data.buffers()[0];
2529 let buffer = buffer.bit_slice(array_data.offset(), array_data.len());
2530 offset = encode_sink_buffer(
2531 buffer,
2532 meta,
2533 sink,
2534 offset,
2535 compression_codec,
2536 ipc_write_context,
2537 write_options.alignment,
2538 )?;
2539 } else if matches!(
2540 data_type,
2541 DataType::List(_) | DataType::LargeList(_) | DataType::Map(_, _)
2542 ) {
2543 assert_eq!(array_data.buffers().len(), 1);
2544 assert_eq!(array_data.child_data().len(), 1);
2545
2546 let (offsets, sliced_child_data) = match data_type {
2548 DataType::List(_) => get_list_array_buffers::<i32>(array_data),
2549 DataType::Map(_, _) => get_list_array_buffers::<i32>(array_data),
2550 DataType::LargeList(_) => get_list_array_buffers::<i64>(array_data),
2551 _ => unreachable!(),
2552 };
2553 offset = encode_sink_buffer(
2554 offsets,
2555 meta,
2556 sink,
2557 offset,
2558 compression_codec,
2559 ipc_write_context,
2560 write_options.alignment,
2561 )?;
2562 offset = write_array_data(
2563 &sliced_child_data,
2564 meta,
2565 sink,
2566 offset,
2567 compression_codec,
2568 ipc_write_context,
2569 write_options,
2570 )?;
2571 return Ok(offset);
2572 } else if matches!(
2573 data_type,
2574 DataType::ListView(_) | DataType::LargeListView(_)
2575 ) {
2576 assert_eq!(array_data.buffers().len(), 2); assert_eq!(array_data.child_data().len(), 1);
2578
2579 let (offsets, sizes, child_data) = match data_type {
2580 DataType::ListView(_) => get_list_view_array_buffers::<i32>(array_data),
2581 DataType::LargeListView(_) => get_list_view_array_buffers::<i64>(array_data),
2582 _ => unreachable!(),
2583 };
2584
2585 offset = encode_sink_buffer(
2586 offsets,
2587 meta,
2588 sink,
2589 offset,
2590 compression_codec,
2591 ipc_write_context,
2592 write_options.alignment,
2593 )?;
2594 offset = encode_sink_buffer(
2595 sizes,
2596 meta,
2597 sink,
2598 offset,
2599 compression_codec,
2600 ipc_write_context,
2601 write_options.alignment,
2602 )?;
2603
2604 offset = write_array_data(
2605 &child_data,
2606 meta,
2607 sink,
2608 offset,
2609 compression_codec,
2610 ipc_write_context,
2611 write_options,
2612 )?;
2613 return Ok(offset);
2614 } else if let DataType::FixedSizeList(_, fixed_size) = data_type {
2615 assert_eq!(array_data.child_data().len(), 1);
2616 let fixed_size = *fixed_size as usize;
2617
2618 let child_offset = array_data.offset() * fixed_size;
2619 let child_length = array_data.len() * fixed_size;
2620 let child_data = array_data.child_data()[0].slice(child_offset, child_length);
2621
2622 offset = write_array_data(
2623 &child_data,
2624 meta,
2625 sink,
2626 offset,
2627 compression_codec,
2628 ipc_write_context,
2629 write_options,
2630 )?;
2631 return Ok(offset);
2632 } else if matches!(data_type, DataType::Union(_, _)) {
2633 for (buffer, byte_width) in array_data
2634 .buffers()
2635 .iter()
2636 .zip([size_of::<i8>(), size_of::<i32>()])
2637 {
2638 let buffer =
2639 buffer.slice_with_length(array_data.offset() * byte_width, num_rows * byte_width);
2640 offset = encode_sink_buffer(
2641 buffer,
2642 meta,
2643 sink,
2644 offset,
2645 compression_codec,
2646 ipc_write_context,
2647 write_options.alignment,
2648 )?;
2649 }
2650 } else {
2651 for buffer in array_data.buffers() {
2652 offset = encode_sink_buffer(
2653 buffer.clone(),
2654 meta,
2655 sink,
2656 offset,
2657 compression_codec,
2658 ipc_write_context,
2659 write_options.alignment,
2660 )?;
2661 }
2662 }
2663
2664 match array_data.data_type() {
2665 DataType::Dictionary(_, _) => {}
2666 DataType::RunEndEncoded(_, _) => {
2667 let arr = unslice_run_array(array_data.clone())?;
2669 for data_ref in arr.child_data() {
2671 offset = write_array_data(
2673 data_ref,
2674 meta,
2675 sink,
2676 offset,
2677 compression_codec,
2678 ipc_write_context,
2679 write_options,
2680 )?;
2681 }
2682 }
2683 DataType::Union(_, UnionMode::Sparse) => {
2684 for data_ref in array_data.child_data() {
2685 let child_data = data_ref.slice(array_data.offset(), num_rows);
2686 offset = write_array_data(
2687 &child_data,
2688 meta,
2689 sink,
2690 offset,
2691 compression_codec,
2692 ipc_write_context,
2693 write_options,
2694 )?;
2695 }
2696 }
2697 _ => {
2698 for data_ref in array_data.child_data() {
2700 offset = write_array_data(
2702 data_ref,
2703 meta,
2704 sink,
2705 offset,
2706 compression_codec,
2707 ipc_write_context,
2708 write_options,
2709 )?;
2710 }
2711 }
2712 }
2713 Ok(offset)
2714}
2715
2716fn encode_sink_buffer(
2730 buffer: Buffer,
2731 ipc_meta_data: &mut IpcMetadataBuilder,
2732 sink: &mut IpcBodySink<'_>,
2733 offset: i64,
2734 compression_codec: Option<CompressionCodec>,
2735 ipc_write_context: &mut IpcWriteContext,
2736 alignment: u8,
2737) -> Result<i64, ArrowError> {
2738 let (encoded, len) = match compression_codec {
2739 None => {
2740 let len = buffer.len() as i64;
2741 (EncodedBuffer::Raw(buffer), len)
2742 }
2743 Some(codec) => {
2744 let mut scratch = Vec::new();
2745 let written =
2746 codec.compress_to_vec(buffer.as_slice(), &mut scratch, ipc_write_context)?;
2747 let len = i64::try_from(written)
2748 .map_err(|e| ArrowError::InvalidArgumentError(format!("{e}")))?;
2749 (EncodedBuffer::Compressed(scratch), len)
2750 }
2751 };
2752
2753 let pad_len = pad_to_alignment(alignment, len as usize);
2754 sink.write(pad_len, encoded);
2755 ipc_meta_data.buffers.push(crate::Buffer::new(offset, len));
2756 Ok(offset + len + pad_len as i64)
2757}
2758
2759const PADDING: [u8; 64] = [0; 64];
2760
2761#[inline]
2767fn estimate_encoded_buffer_count(dt: &DataType) -> usize {
2768 match dt {
2769 DataType::Null => 0,
2770
2771 DataType::Binary | DataType::Utf8 | DataType::LargeBinary | DataType::LargeUtf8 => 3,
2772
2773 DataType::BinaryView | DataType::Utf8View => 3,
2774
2775 DataType::List(f) | DataType::LargeList(f) | DataType::Map(f, _) => {
2776 2 + estimate_encoded_buffer_count(f.data_type())
2777 }
2778
2779 DataType::ListView(f) | DataType::LargeListView(f) => {
2780 3 + estimate_encoded_buffer_count(f.data_type())
2781 }
2782
2783 DataType::FixedSizeList(f, _) => 1 + estimate_encoded_buffer_count(f.data_type()),
2784
2785 DataType::Struct(fields) => {
2786 1 + fields
2787 .iter()
2788 .map(|f| estimate_encoded_buffer_count(f.data_type()))
2789 .sum::<usize>()
2790 }
2791
2792 DataType::Dictionary(_, _) => 2,
2794
2795 DataType::Union(fields, UnionMode::Sparse) => {
2796 1 + fields
2797 .iter()
2798 .map(|(_, f)| estimate_encoded_buffer_count(f.data_type()))
2799 .sum::<usize>()
2800 }
2801 DataType::Union(fields, UnionMode::Dense) => {
2802 2 + fields
2803 .iter()
2804 .map(|(_, f)| estimate_encoded_buffer_count(f.data_type()))
2805 .sum::<usize>()
2806 }
2807
2808 DataType::RunEndEncoded(run_ends, values) => {
2809 estimate_encoded_buffer_count(run_ends.data_type())
2810 + estimate_encoded_buffer_count(values.data_type())
2811 }
2812 _ => 2,
2814 }
2815}
2816
2817#[inline]
2819fn pad_to_alignment(alignment: u8, len: usize) -> usize {
2820 let a = usize::from(alignment - 1);
2821 ((len + a) & !a) - len
2822}
2823
2824#[cfg(test)]
2825mod tests {
2826 use std::hash::Hasher;
2827 use std::io::Cursor;
2828 use std::io::Seek;
2829
2830 use arrow_array::builder::FixedSizeListBuilder;
2831 use arrow_array::builder::Float32Builder;
2832 use arrow_array::builder::Int64Builder;
2833 use arrow_array::builder::MapBuilder;
2834 use arrow_array::builder::StringViewBuilder;
2835 use arrow_array::builder::UnionBuilder;
2836 use arrow_array::builder::{
2837 GenericListBuilder, GenericListViewBuilder, ListBuilder, StringBuilder,
2838 };
2839 use arrow_array::builder::{PrimitiveRunBuilder, UInt32Builder};
2840 use arrow_array::types::*;
2841 use arrow_buffer::ScalarBuffer;
2842
2843 use crate::MetadataVersion;
2844 use crate::convert::try_fb_to_schema;
2845 use crate::reader::*;
2846 use crate::root_as_footer;
2847
2848 use super::*;
2849
2850 #[test]
2851 fn test_roundtrip_sliced_list_of_union() {
2852 fn decoded(array: &ArrayRef) -> Vec<Vec<Option<i32>>> {
2853 (0..array.len())
2854 .map(|row| {
2855 let values = array.as_list::<i32>().value(row);
2856 let union = values.as_union();
2857 (0..union.len())
2858 .map(|index| {
2859 union
2860 .value(index)
2861 .as_primitive::<Int32Type>()
2862 .iter()
2863 .next()
2864 .unwrap()
2865 })
2866 .collect()
2867 })
2868 .collect()
2869 }
2870
2871 for mode in [UnionMode::Dense, UnionMode::Sparse] {
2872 let fields = UnionFields::try_new(
2873 [3, 7],
2874 [
2875 Field::new("a", DataType::Int32, true),
2876 Field::new("b", DataType::Int32, true),
2877 ],
2878 )
2879 .unwrap();
2880 let offsets = (mode == UnionMode::Dense).then(|| vec![2, 0, 1].into());
2881 let union: ArrayRef = Arc::new(
2882 UnionArray::try_new(
2883 fields,
2884 vec![3_i8, 7, 3].into(),
2885 offsets,
2886 vec![
2887 Arc::new(Int32Array::from(vec![Some(11), None, Some(33)])),
2888 Arc::new(Int32Array::from(vec![Some(101), Some(202), None])),
2889 ],
2890 )
2891 .unwrap(),
2892 );
2893 let item = Arc::new(Field::new("item", union.data_type().clone(), true));
2894 let list: ArrayRef = Arc::new(ListArray::new(
2895 item,
2896 arrow_buffer::OffsetBuffer::from_lengths([1, 2]),
2897 union,
2898 None,
2899 ));
2900 for (offset, len) in [(0, 2), (1, 1), (2, 0)] {
2901 let array = list.slice(offset, len);
2902 array.to_data().validate_full().unwrap();
2903 let expected = decoded(&array);
2904 let schema = Arc::new(Schema::new(vec![Field::new(
2905 "v",
2906 array.data_type().clone(),
2907 true,
2908 )]));
2909 let batch = RecordBatch::try_new(schema, vec![array]).unwrap();
2910 for output in [
2911 deserialize_file(serialize_file(&batch)),
2912 deserialize_stream(serialize_stream(&batch)),
2913 ] {
2914 output.column(0).to_data().validate_full().unwrap();
2915 assert_eq!(decoded(output.column(0)), expected);
2916 }
2917 }
2918 }
2919 }
2920
2921 fn serialize_file(rb: &RecordBatch) -> Vec<u8> {
2922 let mut writer = FileWriter::try_new(vec![], rb.schema_ref()).unwrap();
2923 writer.write(rb).unwrap();
2924 writer.finish().unwrap();
2925 writer.into_inner().unwrap()
2926 }
2927
2928 fn deserialize_file(bytes: Vec<u8>) -> RecordBatch {
2929 let mut reader = FileReader::try_new(Cursor::new(bytes), None).unwrap();
2930 reader.next().unwrap().unwrap()
2931 }
2932
2933 fn serialize_stream(record: &RecordBatch) -> Vec<u8> {
2934 const IPC_ALIGNMENT: usize = 8;
2938
2939 let mut stream_writer = StreamWriter::try_new_with_options(
2940 vec![],
2941 record.schema_ref(),
2942 IpcWriteOptions::try_new(IPC_ALIGNMENT, false, MetadataVersion::V5).unwrap(),
2943 )
2944 .unwrap();
2945 stream_writer.write(record).unwrap();
2946 stream_writer.finish().unwrap();
2947 stream_writer.into_inner().unwrap()
2948 }
2949
2950 fn deserialize_stream(bytes: Vec<u8>) -> RecordBatch {
2951 let mut stream_reader = StreamReader::try_new(Cursor::new(bytes), None).unwrap();
2952 stream_reader.next().unwrap().unwrap()
2953 }
2954
2955 fn encode_stream(
2960 schema: &Schema,
2961 batches: &[RecordBatch],
2962 options: IpcWriteOptions,
2963 ) -> Vec<u8> {
2964 let mut encoder = StreamEncoder::try_new_with_options(schema, options).unwrap();
2965 let mut bytes = Vec::new();
2966 for batch in batches {
2967 for buffer in encoder.encode(batch).unwrap() {
2968 bytes.write_all(buffer.as_slice()).unwrap();
2969 }
2970 }
2971 for buffer in encoder.finish().unwrap() {
2972 bytes.write_all(buffer.as_slice()).unwrap();
2973 }
2974 bytes
2975 }
2976
2977 fn write_stream(schema: &Schema, batches: &[RecordBatch], options: IpcWriteOptions) -> Vec<u8> {
2978 let mut bytes = Vec::new();
2979 let mut writer = StreamWriter::try_new_with_options(&mut bytes, schema, options).unwrap();
2980 for batch in batches {
2981 writer.write(batch).unwrap();
2982 }
2983 writer.finish().unwrap();
2984 bytes
2985 }
2986
2987 #[tokio::test]
2990 async fn test_stream_encoder_async_writer_matches_stream_writer() {
2991 use tokio::io::{AsyncReadExt, AsyncWriteExt};
2992
2993 let batch = record_batch!(("a", Int32, [1, 2, 3]), ("b", Utf8, ["x", "y", "z"])).unwrap();
2994 let options = IpcWriteOptions::default();
2995 let expected = write_stream(
2996 batch.schema_ref(),
2997 std::slice::from_ref(&batch),
2998 options.clone(),
2999 );
3000
3001 let (mut sink, mut source) = tokio::io::duplex(64);
3002 let read = tokio::spawn(async move {
3003 let mut bytes = Vec::new();
3004 source.read_to_end(&mut bytes).await.unwrap();
3005 bytes
3006 });
3007
3008 let mut encoder = StreamEncoder::try_new_with_options(batch.schema_ref(), options).unwrap();
3009 for buffer in encoder.encode(&batch).unwrap() {
3010 sink.write_all(buffer.as_slice()).await.unwrap();
3011 }
3012 for buffer in encoder.finish().unwrap() {
3013 sink.write_all(buffer.as_slice()).await.unwrap();
3014 }
3015 sink.shutdown().await.unwrap();
3016
3017 let encoded = read.await.unwrap();
3018 assert_eq!(encoded, expected);
3019
3020 let mut reader = StreamReader::try_new(Cursor::new(encoded), None).unwrap();
3021 assert_eq!(reader.next().unwrap().unwrap(), batch);
3022 assert!(reader.next().is_none());
3023 }
3024
3025 #[test]
3026 fn test_stream_encoder_empty_stream_matches_stream_writer() {
3027 let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);
3028 let options = IpcWriteOptions::default();
3029 let encoded = encode_stream(&schema, &[], options.clone());
3030 let written = write_stream(&schema, &[], options);
3031
3032 assert_eq!(encoded, written);
3033
3034 let mut reader = StreamReader::try_new(Cursor::new(encoded), None).unwrap();
3035 assert!(reader.next().is_none());
3036 }
3037
3038 #[test]
3039 fn test_stream_encoder_dictionary_batches_match_stream_writer() {
3040 let schema = Arc::new(Schema::new(vec![Field::new(
3041 "a",
3042 DataType::Dictionary(Box::new(DataType::UInt8), Box::new(DataType::Utf8)),
3043 false,
3044 )]));
3045 let batch = RecordBatch::try_new(
3046 schema.clone(),
3047 vec![Arc::new(DictionaryArray::new(
3048 UInt8Array::from_iter_values([0, 1, 0]),
3049 Arc::new(StringArray::from_iter_values(["a", "b"])),
3050 ))],
3051 )
3052 .unwrap();
3053 let options = IpcWriteOptions::default();
3054 let encoded = encode_stream(&schema, std::slice::from_ref(&batch), options.clone());
3055 let written = write_stream(&schema, std::slice::from_ref(&batch), options);
3056
3057 assert_eq!(encoded, written);
3058
3059 let mut reader = StreamReader::try_new(Cursor::new(encoded), None).unwrap();
3060 assert_eq!(reader.next().unwrap().unwrap(), batch);
3061 assert!(reader.next().is_none());
3062 }
3063
3064 #[test]
3065 #[cfg(feature = "lz4")]
3066 fn test_write_empty_record_batch_lz4_compression() {
3067 let schema = Schema::new(vec![Field::new("field1", DataType::Int32, true)]);
3068 let values: Vec<Option<i32>> = vec![];
3069 let array = Int32Array::from(values);
3070 let record_batch =
3071 RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(array)]).unwrap();
3072
3073 let mut file = tempfile::tempfile().unwrap();
3074
3075 {
3076 let write_option = IpcWriteOptions::try_new(8, false, crate::MetadataVersion::V5)
3077 .unwrap()
3078 .try_with_compression(Some(crate::CompressionType::LZ4_FRAME))
3079 .unwrap();
3080
3081 let mut writer =
3082 FileWriter::try_new_with_options(&mut file, &schema, write_option).unwrap();
3083 writer.write(&record_batch).unwrap();
3084 writer.finish().unwrap();
3085 }
3086 file.rewind().unwrap();
3087 {
3088 let reader = FileReader::try_new(file, None).unwrap();
3090 for read_batch in reader {
3091 read_batch
3092 .unwrap()
3093 .columns()
3094 .iter()
3095 .zip(record_batch.columns())
3096 .for_each(|(a, b)| {
3097 assert_eq!(a.data_type(), b.data_type());
3098 assert_eq!(a.len(), b.len());
3099 assert_eq!(a.null_count(), b.null_count());
3100 });
3101 }
3102 }
3103 }
3104
3105 #[test]
3106 #[cfg(feature = "lz4")]
3107 fn test_write_file_with_lz4_compression() {
3108 let schema = Schema::new(vec![Field::new("field1", DataType::Int32, true)]);
3109 let values: Vec<Option<i32>> = vec![Some(12), Some(1)];
3110 let array = Int32Array::from(values);
3111 let record_batch =
3112 RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(array)]).unwrap();
3113
3114 let mut file = tempfile::tempfile().unwrap();
3115 {
3116 let write_option = IpcWriteOptions::try_new(8, false, crate::MetadataVersion::V5)
3117 .unwrap()
3118 .try_with_compression(Some(crate::CompressionType::LZ4_FRAME))
3119 .unwrap();
3120
3121 let mut writer =
3122 FileWriter::try_new_with_options(&mut file, &schema, write_option).unwrap();
3123 writer.write(&record_batch).unwrap();
3124 writer.finish().unwrap();
3125 }
3126 file.rewind().unwrap();
3127 {
3128 let reader = FileReader::try_new(file, None).unwrap();
3130 for read_batch in reader {
3131 read_batch
3132 .unwrap()
3133 .columns()
3134 .iter()
3135 .zip(record_batch.columns())
3136 .for_each(|(a, b)| {
3137 assert_eq!(a.data_type(), b.data_type());
3138 assert_eq!(a.len(), b.len());
3139 assert_eq!(a.null_count(), b.null_count());
3140 });
3141 }
3142 }
3143 }
3144
3145 #[test]
3146 #[cfg(feature = "zstd")]
3147 fn test_write_file_with_zstd_compression() {
3148 let schema = Schema::new(vec![Field::new("field1", DataType::Int32, true)]);
3149 let values: Vec<Option<i32>> = vec![Some(12), Some(1)];
3150 let array = Int32Array::from(values);
3151 let record_batch =
3152 RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(array)]).unwrap();
3153 let mut file = tempfile::tempfile().unwrap();
3154 {
3155 let write_option = IpcWriteOptions::try_new(8, false, crate::MetadataVersion::V5)
3156 .unwrap()
3157 .try_with_compression(Some(crate::CompressionType::ZSTD))
3158 .unwrap()
3159 .try_with_compression_level(Some(1))
3160 .unwrap();
3161
3162 let mut writer =
3163 FileWriter::try_new_with_options(&mut file, &schema, write_option).unwrap();
3164 writer.write(&record_batch).unwrap();
3165 writer.finish().unwrap();
3166 }
3167 file.rewind().unwrap();
3168 {
3169 let reader = FileReader::try_new(file, None).unwrap();
3171 for read_batch in reader {
3172 read_batch
3173 .unwrap()
3174 .columns()
3175 .iter()
3176 .zip(record_batch.columns())
3177 .for_each(|(a, b)| {
3178 assert_eq!(a.data_type(), b.data_type());
3179 assert_eq!(a.len(), b.len());
3180 assert_eq!(a.null_count(), b.null_count());
3181 });
3182 }
3183 }
3184 }
3185
3186 #[test]
3187 fn test_write_file() {
3188 let schema = Schema::new(vec![Field::new("field1", DataType::UInt32, true)]);
3189 let values: Vec<Option<u32>> = vec![
3190 Some(999),
3191 None,
3192 Some(235),
3193 Some(123),
3194 None,
3195 None,
3196 None,
3197 None,
3198 None,
3199 ];
3200 let array1 = UInt32Array::from(values);
3201 let batch =
3202 RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(array1) as ArrayRef])
3203 .unwrap();
3204 let mut file = tempfile::tempfile().unwrap();
3205 {
3206 let mut writer = FileWriter::try_new(&mut file, &schema).unwrap();
3207
3208 writer.write(&batch).unwrap();
3209 writer.finish().unwrap();
3210 }
3211 file.rewind().unwrap();
3212
3213 {
3214 let mut reader = FileReader::try_new(file, None).unwrap();
3215 while let Some(Ok(read_batch)) = reader.next() {
3216 read_batch
3217 .columns()
3218 .iter()
3219 .zip(batch.columns())
3220 .for_each(|(a, b)| {
3221 assert_eq!(a.data_type(), b.data_type());
3222 assert_eq!(a.len(), b.len());
3223 assert_eq!(a.null_count(), b.null_count());
3224 });
3225 }
3226 }
3227 }
3228
3229 #[test]
3230 fn test_empty_utf8_ipc_writes_nonempty_offsets_buffer() {
3231 let name = StringArray::from(Vec::<String>::new());
3232 let [offsets, values] = get_byte_array_buffers::<i32>(&name.to_data());
3233
3234 assert_eq!(name.len(), 0);
3235 assert_eq!(
3236 offsets.len(),
3237 std::mem::size_of::<i32>(),
3238 "offsets buffer should contain one zero i32 offset"
3239 );
3240 assert_eq!(values.len(), 0, "values buffer should remain empty");
3241 }
3242
3243 #[test]
3244 fn test_empty_large_utf8_ipc_writes_nonempty_offsets_buffer() {
3245 let name = LargeStringArray::from(Vec::<String>::new());
3246 let [offsets, values] = get_byte_array_buffers::<i64>(&name.to_data());
3247
3248 assert_eq!(name.len(), 0);
3249 assert_eq!(
3250 offsets.len(),
3251 std::mem::size_of::<i64>(),
3252 "offsets buffer should contain one zero i64 offset"
3253 );
3254 assert_eq!(values.len(), 0, "values buffer should remain empty");
3255 }
3256
3257 #[test]
3258 fn test_empty_list_ipc_writes_nonempty_offsets_buffer() {
3259 let list = GenericListBuilder::<i32, _>::new(UInt32Builder::new()).finish();
3260 let (offsets, child_data) = get_list_array_buffers::<i32>(&list.to_data());
3261
3262 assert_eq!(list.len(), 0);
3263 assert_eq!(
3264 offsets.len(),
3265 std::mem::size_of::<i32>(),
3266 "offsets buffer should contain one zero i32 offset"
3267 );
3268 assert_eq!(child_data.len(), 0, "child data should remain empty");
3269 }
3270
3271 #[test]
3272 fn test_empty_large_list_ipc_writes_nonempty_offsets_buffer() {
3273 let list = GenericListBuilder::<i64, _>::new(UInt32Builder::new()).finish();
3274 let (offsets, child_data) = get_list_array_buffers::<i64>(&list.to_data());
3275
3276 assert_eq!(list.len(), 0);
3277 assert_eq!(
3278 offsets.len(),
3279 std::mem::size_of::<i64>(),
3280 "offsets buffer should contain one zero i64 offset"
3281 );
3282 assert_eq!(child_data.len(), 0, "child data should remain empty");
3283 }
3284
3285 fn write_null_file(options: IpcWriteOptions) {
3286 let schema = Schema::new(vec![
3287 Field::new("nulls", DataType::Null, true),
3288 Field::new("int32s", DataType::Int32, false),
3289 Field::new("nulls2", DataType::Null, true),
3290 Field::new("f64s", DataType::Float64, false),
3291 ]);
3292 let array1 = NullArray::new(32);
3293 let array2 = Int32Array::from(vec![1; 32]);
3294 let array3 = NullArray::new(32);
3295 let array4 = Float64Array::from(vec![f64::NAN; 32]);
3296 let batch = RecordBatch::try_new(
3297 Arc::new(schema.clone()),
3298 vec![
3299 Arc::new(array1) as ArrayRef,
3300 Arc::new(array2) as ArrayRef,
3301 Arc::new(array3) as ArrayRef,
3302 Arc::new(array4) as ArrayRef,
3303 ],
3304 )
3305 .unwrap();
3306 let mut file = tempfile::tempfile().unwrap();
3307 {
3308 let mut writer = FileWriter::try_new_with_options(&mut file, &schema, options).unwrap();
3309
3310 writer.write(&batch).unwrap();
3311 writer.finish().unwrap();
3312 }
3313
3314 file.rewind().unwrap();
3315
3316 {
3317 let reader = FileReader::try_new(file, None).unwrap();
3318 reader.for_each(|maybe_batch| {
3319 maybe_batch
3320 .unwrap()
3321 .columns()
3322 .iter()
3323 .zip(batch.columns())
3324 .for_each(|(a, b)| {
3325 assert_eq!(a.data_type(), b.data_type());
3326 assert_eq!(a.len(), b.len());
3327 assert_eq!(a.null_count(), b.null_count());
3328 });
3329 });
3330 }
3331 }
3332 #[test]
3333 fn test_write_null_file_v4() {
3334 write_null_file(IpcWriteOptions::try_new(8, false, MetadataVersion::V4).unwrap());
3335 write_null_file(IpcWriteOptions::try_new(8, true, MetadataVersion::V4).unwrap());
3336 write_null_file(IpcWriteOptions::try_new(64, false, MetadataVersion::V4).unwrap());
3337 write_null_file(IpcWriteOptions::try_new(64, true, MetadataVersion::V4).unwrap());
3338 }
3339
3340 #[test]
3341 fn test_write_null_file_v5() {
3342 write_null_file(IpcWriteOptions::try_new(8, false, MetadataVersion::V5).unwrap());
3343 write_null_file(IpcWriteOptions::try_new(64, false, MetadataVersion::V5).unwrap());
3344 }
3345
3346 #[test]
3347 fn track_union_nested_dict() {
3348 let inner: DictionaryArray<Int32Type> = vec!["a", "b", "a"].into_iter().collect();
3349
3350 let array = Arc::new(inner) as ArrayRef;
3351
3352 #[expect(deprecated)]
3354 let dctfield = Field::new_dict("dict", array.data_type().clone(), false, 0, false);
3355 let union_fields = std::iter::once((0, Arc::new(dctfield))).collect();
3356
3357 let types = [0, 0, 0].into_iter().collect::<ScalarBuffer<i8>>();
3358 let offsets = [0, 1, 2].into_iter().collect::<ScalarBuffer<i32>>();
3359
3360 let union = UnionArray::try_new(union_fields, types, Some(offsets), vec![array]).unwrap();
3361
3362 let schema = Arc::new(Schema::new(vec![Field::new(
3363 "union",
3364 union.data_type().clone(),
3365 false,
3366 )]));
3367
3368 let r#gen = IpcDataGenerator::default();
3369 let mut dict_tracker = DictionaryTracker::new(false);
3370 r#gen.schema_to_bytes_with_dictionary_tracker(
3371 &schema,
3372 &mut dict_tracker,
3373 &IpcWriteOptions::default(),
3374 );
3375
3376 let batch = RecordBatch::try_new(schema, vec![Arc::new(union)]).unwrap();
3377
3378 r#gen
3379 .encode(
3380 &batch,
3381 &mut dict_tracker,
3382 &Default::default(),
3383 &mut Default::default(),
3384 )
3385 .unwrap();
3386
3387 assert!(dict_tracker.written.contains_key(&0));
3390 }
3391
3392 #[test]
3393 fn track_struct_nested_dict() {
3394 let inner: DictionaryArray<Int32Type> = vec!["a", "b", "a"].into_iter().collect();
3395
3396 let array = Arc::new(inner) as ArrayRef;
3397
3398 #[expect(deprecated)]
3400 let dctfield = Arc::new(Field::new_dict(
3401 "dict",
3402 array.data_type().clone(),
3403 false,
3404 2,
3405 false,
3406 ));
3407
3408 let s = StructArray::from(vec![(dctfield, array)]);
3409 let struct_array = Arc::new(s) as ArrayRef;
3410
3411 let schema = Arc::new(Schema::new(vec![Field::new(
3412 "struct",
3413 struct_array.data_type().clone(),
3414 false,
3415 )]));
3416
3417 let r#gen = IpcDataGenerator::default();
3418 let mut dict_tracker = DictionaryTracker::new(false);
3419 r#gen.schema_to_bytes_with_dictionary_tracker(
3420 &schema,
3421 &mut dict_tracker,
3422 &IpcWriteOptions::default(),
3423 );
3424
3425 let batch = RecordBatch::try_new(schema, vec![struct_array]).unwrap();
3426
3427 r#gen
3428 .encode(
3429 &batch,
3430 &mut dict_tracker,
3431 &Default::default(),
3432 &mut Default::default(),
3433 )
3434 .unwrap();
3435
3436 assert!(dict_tracker.written.contains_key(&0));
3437 }
3438
3439 fn write_union_file(options: IpcWriteOptions) {
3440 let schema = Schema::new(vec![Field::new_union(
3441 "union",
3442 vec![0, 1],
3443 vec![
3444 Field::new("a", DataType::Int32, false),
3445 Field::new("c", DataType::Float64, false),
3446 ],
3447 UnionMode::Sparse,
3448 )]);
3449 let mut builder = UnionBuilder::with_capacity_sparse(5);
3450 builder.append::<Int32Type>("a", 1).unwrap();
3451 builder.append_null::<Int32Type>("a").unwrap();
3452 builder.append::<Float64Type>("c", 3.0).unwrap();
3453 builder.append_null::<Float64Type>("c").unwrap();
3454 builder.append::<Int32Type>("a", 4).unwrap();
3455 let union = builder.build().unwrap();
3456
3457 let batch =
3458 RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(union) as ArrayRef])
3459 .unwrap();
3460
3461 let mut file = tempfile::tempfile().unwrap();
3462 {
3463 let mut writer = FileWriter::try_new_with_options(&mut file, &schema, options).unwrap();
3464
3465 writer.write(&batch).unwrap();
3466 writer.finish().unwrap();
3467 }
3468 file.rewind().unwrap();
3469
3470 {
3471 let reader = FileReader::try_new(file, None).unwrap();
3472 reader.for_each(|maybe_batch| {
3473 maybe_batch
3474 .unwrap()
3475 .columns()
3476 .iter()
3477 .zip(batch.columns())
3478 .for_each(|(a, b)| {
3479 assert_eq!(a.data_type(), b.data_type());
3480 assert_eq!(a.len(), b.len());
3481 assert_eq!(a.null_count(), b.null_count());
3482 });
3483 });
3484 }
3485 }
3486
3487 #[test]
3488 fn test_write_union_file_v4_v5() {
3489 write_union_file(IpcWriteOptions::try_new(8, false, MetadataVersion::V4).unwrap());
3490 write_union_file(IpcWriteOptions::try_new(8, false, MetadataVersion::V5).unwrap());
3491 }
3492
3493 #[test]
3494 fn test_write_view_types() {
3495 const LONG_TEST_STRING: &str =
3496 "This is a long string to make sure binary view array handles it";
3497 let schema = Schema::new(vec![
3498 Field::new("field1", DataType::BinaryView, true),
3499 Field::new("field2", DataType::Utf8View, true),
3500 ]);
3501 let values: Vec<Option<&[u8]>> = vec![
3502 Some(b"foo"),
3503 Some(b"bar"),
3504 Some(LONG_TEST_STRING.as_bytes()),
3505 ];
3506 let binary_array = BinaryViewArray::from_iter(values);
3507 let utf8_array =
3508 StringViewArray::from_iter(vec![Some("foo"), Some("bar"), Some(LONG_TEST_STRING)]);
3509 let record_batch = RecordBatch::try_new(
3510 Arc::new(schema.clone()),
3511 vec![Arc::new(binary_array), Arc::new(utf8_array)],
3512 )
3513 .unwrap();
3514
3515 let mut file = tempfile::tempfile().unwrap();
3516 {
3517 let mut writer = FileWriter::try_new(&mut file, &schema).unwrap();
3518 writer.write(&record_batch).unwrap();
3519 writer.finish().unwrap();
3520 }
3521 file.rewind().unwrap();
3522 {
3523 let mut reader = FileReader::try_new(&file, None).unwrap();
3524 let read_batch = reader.next().unwrap().unwrap();
3525 read_batch
3526 .columns()
3527 .iter()
3528 .zip(record_batch.columns())
3529 .for_each(|(a, b)| {
3530 assert_eq!(a, b);
3531 });
3532 }
3533 file.rewind().unwrap();
3534 {
3535 let mut reader = FileReader::try_new(&file, Some(vec![0])).unwrap();
3536 let read_batch = reader.next().unwrap().unwrap();
3537 assert_eq!(read_batch.num_columns(), 1);
3538 let read_array = read_batch.column(0);
3539 let write_array = record_batch.column(0);
3540 assert_eq!(read_array, write_array);
3541 }
3542 }
3543
3544 #[test]
3545 #[cfg_attr(miri, ignore)] fn truncate_ipc_record_batch() {
3547 fn create_batch(rows: usize) -> RecordBatch {
3548 let schema = Schema::new(vec![
3549 Field::new("a", DataType::Int32, false),
3550 Field::new("b", DataType::Utf8, false),
3551 ]);
3552
3553 let a = Int32Array::from_iter_values(0..rows as i32);
3554 let b = StringArray::from_iter_values((0..rows).map(|i| i.to_string()));
3555
3556 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]).unwrap()
3557 }
3558
3559 let big_record_batch = create_batch(65536);
3560
3561 let length = 5;
3562 let small_record_batch = create_batch(length);
3563
3564 let offset = 2;
3565 let record_batch_slice = big_record_batch.slice(offset, length);
3566 assert!(
3567 serialize_stream(&big_record_batch).len() > serialize_stream(&small_record_batch).len()
3568 );
3569 assert_eq!(
3570 serialize_stream(&small_record_batch).len(),
3571 serialize_stream(&record_batch_slice).len()
3572 );
3573
3574 assert_eq!(
3575 deserialize_stream(serialize_stream(&record_batch_slice)),
3576 record_batch_slice
3577 );
3578 }
3579
3580 #[test]
3581 fn truncate_ipc_record_batch_with_nulls() {
3582 fn create_batch() -> RecordBatch {
3583 let schema = Schema::new(vec![
3584 Field::new("a", DataType::Int32, true),
3585 Field::new("b", DataType::Utf8, true),
3586 ]);
3587
3588 let a = Int32Array::from(vec![Some(1), None, Some(1), None, Some(1)]);
3589 let b = StringArray::from(vec![None, Some("a"), Some("a"), None, Some("a")]);
3590
3591 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]).unwrap()
3592 }
3593
3594 let record_batch = create_batch();
3595 let record_batch_slice = record_batch.slice(1, 2);
3596 let deserialized_batch = deserialize_stream(serialize_stream(&record_batch_slice));
3597
3598 assert!(
3599 serialize_stream(&record_batch).len() > serialize_stream(&record_batch_slice).len()
3600 );
3601
3602 assert!(deserialized_batch.column(0).is_null(0));
3603 assert!(deserialized_batch.column(0).is_valid(1));
3604 assert!(deserialized_batch.column(1).is_valid(0));
3605 assert!(deserialized_batch.column(1).is_valid(1));
3606
3607 assert_eq!(record_batch_slice, deserialized_batch);
3608 }
3609
3610 #[test]
3611 fn test_write_empty_validity_buffer_without_nulls() {
3612 let batch = RecordBatch::try_from_iter([
3613 ("a", Arc::new(Int32Array::from(vec![1, 2])) as _),
3614 (
3616 "b",
3617 Arc::new(Int32Array::from(vec![None, Some(2), Some(3)]).slice(1, 2)) as _,
3618 ),
3619 ("c", Arc::new(Int32Array::from(vec![Some(1), None])) as _),
3620 ])
3621 .unwrap();
3622 assert_eq!(batch.column(1).nulls().map(|n| n.null_count()), Some(0));
3623
3624 let (_, encoded) = IpcDataGenerator::default()
3625 .encode(
3626 &batch,
3627 &mut DictionaryTracker::new(false),
3628 &Default::default(),
3629 &mut Default::default(),
3630 )
3631 .unwrap();
3632 let message = crate::root_as_message(&encoded.ipc_message).unwrap();
3633 let buffers = message.header_as_record_batch().unwrap().buffers().unwrap();
3634 let validity_lengths: Vec<_> = buffers.iter().step_by(2).map(|b| b.length()).collect();
3636 assert_eq!(validity_lengths, [0, 0, 1]);
3637
3638 assert_eq!(deserialize_stream(serialize_stream(&batch)), batch);
3639 }
3640
3641 #[test]
3642 fn truncate_ipc_dictionary_array() {
3643 fn create_batch() -> RecordBatch {
3644 let values: StringArray = [Some("foo"), Some("bar"), Some("baz")]
3645 .into_iter()
3646 .collect();
3647 let keys: Int32Array = [Some(0), Some(2), None, Some(1)].into_iter().collect();
3648
3649 let array = DictionaryArray::new(keys, Arc::new(values));
3650
3651 let schema = Schema::new(vec![Field::new("dict", array.data_type().clone(), true)]);
3652
3653 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap()
3654 }
3655
3656 let record_batch = create_batch();
3657 let record_batch_slice = record_batch.slice(1, 2);
3658 let deserialized_batch = deserialize_stream(serialize_stream(&record_batch_slice));
3659
3660 assert!(
3661 serialize_stream(&record_batch).len() > serialize_stream(&record_batch_slice).len()
3662 );
3663
3664 assert!(deserialized_batch.column(0).is_valid(0));
3665 assert!(deserialized_batch.column(0).is_null(1));
3666
3667 assert_eq!(record_batch_slice, deserialized_batch);
3668 }
3669
3670 #[test]
3671 fn truncate_ipc_struct_array() {
3672 fn create_batch() -> RecordBatch {
3673 let strings: StringArray = [Some("foo"), None, Some("bar"), Some("baz")]
3674 .into_iter()
3675 .collect();
3676 let ints: Int32Array = [Some(0), Some(2), None, Some(1)].into_iter().collect();
3677
3678 let struct_array = StructArray::from(vec![
3679 (
3680 Arc::new(Field::new("s", DataType::Utf8, true)),
3681 Arc::new(strings) as ArrayRef,
3682 ),
3683 (
3684 Arc::new(Field::new("c", DataType::Int32, true)),
3685 Arc::new(ints) as ArrayRef,
3686 ),
3687 ]);
3688
3689 let schema = Schema::new(vec![Field::new(
3690 "struct_array",
3691 struct_array.data_type().clone(),
3692 true,
3693 )]);
3694
3695 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(struct_array)]).unwrap()
3696 }
3697
3698 let record_batch = create_batch();
3699 let record_batch_slice = record_batch.slice(1, 2);
3700 let deserialized_batch = deserialize_stream(serialize_stream(&record_batch_slice));
3701
3702 assert!(
3703 serialize_stream(&record_batch).len() > serialize_stream(&record_batch_slice).len()
3704 );
3705
3706 let structs = deserialized_batch
3707 .column(0)
3708 .as_any()
3709 .downcast_ref::<StructArray>()
3710 .unwrap();
3711
3712 assert!(structs.column(0).is_null(0));
3713 assert!(structs.column(0).is_valid(1));
3714 assert!(structs.column(1).is_valid(0));
3715 assert!(structs.column(1).is_null(1));
3716 assert_eq!(record_batch_slice, deserialized_batch);
3717 }
3718
3719 #[test]
3720 fn truncate_ipc_string_array_with_all_empty_string() {
3721 fn create_batch() -> RecordBatch {
3722 let schema = Schema::new(vec![Field::new("a", DataType::Utf8, true)]);
3723 let a = StringArray::from(vec![Some(""), Some(""), Some(""), Some(""), Some("")]);
3724 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a)]).unwrap()
3725 }
3726
3727 let record_batch = create_batch();
3728 let record_batch_slice = record_batch.slice(0, 1);
3729 let deserialized_batch = deserialize_stream(serialize_stream(&record_batch_slice));
3730
3731 assert!(
3732 serialize_stream(&record_batch).len() > serialize_stream(&record_batch_slice).len()
3733 );
3734 assert_eq!(record_batch_slice, deserialized_batch);
3735 }
3736
3737 #[test]
3738 fn test_stream_writer_writes_array_slice() {
3739 let array = UInt32Array::from(vec![Some(1), Some(2), Some(3)]);
3740 assert_eq!(
3741 vec![Some(1), Some(2), Some(3)],
3742 array.iter().collect::<Vec<_>>()
3743 );
3744
3745 let sliced = array.slice(1, 2);
3746 assert_eq!(vec![Some(2), Some(3)], sliced.iter().collect::<Vec<_>>());
3747
3748 let batch = RecordBatch::try_new(
3749 Arc::new(Schema::new(vec![Field::new("a", DataType::UInt32, true)])),
3750 vec![Arc::new(sliced)],
3751 )
3752 .expect("new batch");
3753
3754 let mut writer = StreamWriter::try_new(vec![], batch.schema_ref()).expect("new writer");
3755 writer.write(&batch).expect("write");
3756 let outbuf = writer.into_inner().expect("inner");
3757
3758 let mut reader = StreamReader::try_new(&outbuf[..], None).expect("new reader");
3759 let read_batch = reader.next().unwrap().expect("read batch");
3760
3761 let read_array: &UInt32Array = read_batch.column(0).as_primitive();
3762 assert_eq!(
3763 vec![Some(2), Some(3)],
3764 read_array.iter().collect::<Vec<_>>()
3765 );
3766 }
3767
3768 #[test]
3769 fn test_large_slice_uint32() {
3770 ensure_roundtrip(Arc::new(UInt32Array::from_iter(
3771 (0..8000).map(|i| if i % 2 == 0 { Some(i) } else { None }),
3772 )));
3773 }
3774
3775 #[test]
3776 #[cfg_attr(miri, ignore)] fn test_large_slice_string() {
3778 let strings: Vec<_> = (0..8000)
3779 .map(|i| {
3780 if i % 2 == 0 {
3781 Some(format!("value{i}"))
3782 } else {
3783 None
3784 }
3785 })
3786 .collect();
3787
3788 ensure_roundtrip(Arc::new(StringArray::from(strings)));
3789 }
3790
3791 #[test]
3792 #[cfg_attr(miri, ignore)] fn test_large_slice_string_list() {
3794 let mut ls = ListBuilder::new(StringBuilder::new());
3795
3796 let mut s = String::new();
3797 for row_number in 0..8000 {
3798 if row_number % 2 == 0 {
3799 for list_element in 0..1000 {
3800 s.clear();
3801 use std::fmt::Write;
3802 write!(&mut s, "value{row_number}-{list_element}").unwrap();
3803 ls.values().append_value(&s);
3804 }
3805 ls.append(true)
3806 } else {
3807 ls.append(false); }
3809 }
3810
3811 ensure_roundtrip(Arc::new(ls.finish()));
3812 }
3813
3814 #[test]
3815 #[cfg_attr(miri, ignore)] fn test_large_slice_string_list_of_lists() {
3817 let mut ls = ListBuilder::new(ListBuilder::new(StringBuilder::new()));
3821
3822 for _ in 0..4000 {
3823 ls.values().append(true);
3824 ls.append(true)
3825 }
3826
3827 let mut s = String::new();
3828 for row_number in 0..4000 {
3829 if row_number % 2 == 0 {
3830 for list_element in 0..1000 {
3831 s.clear();
3832 use std::fmt::Write;
3833 write!(&mut s, "value{row_number}-{list_element}").unwrap();
3834 ls.values().values().append_value(&s);
3835 }
3836 ls.values().append(true);
3837 ls.append(true)
3838 } else {
3839 ls.append(false); }
3841 }
3842
3843 ensure_roundtrip(Arc::new(ls.finish()));
3844 }
3845
3846 fn ensure_roundtrip(array: ArrayRef) {
3848 let num_rows = array.len();
3849 let orig_batch = RecordBatch::try_from_iter(vec![("a", array)]).unwrap();
3850 let sliced_batch = orig_batch.slice(1, num_rows - 1);
3852
3853 let schema = orig_batch.schema();
3854 let stream_data = {
3855 let mut writer = StreamWriter::try_new(vec![], &schema).unwrap();
3856 writer.write(&sliced_batch).unwrap();
3857 writer.into_inner().unwrap()
3858 };
3859 let read_batch = {
3860 let projection = None;
3861 let mut reader = StreamReader::try_new(Cursor::new(stream_data), projection).unwrap();
3862 reader
3863 .next()
3864 .expect("expect no errors reading batch")
3865 .expect("expect batch")
3866 };
3867 assert_eq!(sliced_batch, read_batch);
3868
3869 let file_data = {
3870 let mut writer = FileWriter::try_new_buffered(vec![], &schema).unwrap();
3871 writer.write(&sliced_batch).unwrap();
3872 writer.into_inner().unwrap().into_inner().unwrap()
3873 };
3874 let read_batch = {
3875 let projection = None;
3876 let mut reader = FileReader::try_new(Cursor::new(file_data), projection).unwrap();
3877 reader
3878 .next()
3879 .expect("expect no errors reading batch")
3880 .expect("expect batch")
3881 };
3882 assert_eq!(sliced_batch, read_batch);
3883
3884 }
3886
3887 #[test]
3888 fn encode_bools_slice() {
3889 assert_bool_roundtrip([true, false], 1, 1);
3891
3892 assert_bool_roundtrip(
3894 [
3895 true, false, true, true, false, false, true, true, true, false, false, false, true,
3896 true, true, true, false, false, false, false, true, true, true, true, true, false,
3897 false, false, false, false,
3898 ],
3899 13,
3900 17,
3901 );
3902
3903 assert_bool_roundtrip(
3905 [
3906 true, false, true, true, false, false, true, true, true, false, false, false,
3907 ],
3908 8,
3909 2,
3910 );
3911
3912 assert_bool_roundtrip(
3914 [
3915 true, false, true, true, false, false, true, true, true, false, false, false, true,
3916 true, true, true, true, false, false, false, false, false,
3917 ],
3918 8,
3919 8,
3920 );
3921 }
3922
3923 fn assert_bool_roundtrip<const N: usize>(bools: [bool; N], offset: usize, length: usize) {
3924 let val_bool_field = Field::new("val", DataType::Boolean, false);
3925
3926 let schema = Arc::new(Schema::new(vec![val_bool_field]));
3927
3928 let bools = BooleanArray::from(bools.to_vec());
3929
3930 let batch = RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(bools)]).unwrap();
3931 let batch = batch.slice(offset, length);
3932
3933 let data = serialize_stream(&batch);
3934 let batch2 = deserialize_stream(data);
3935 assert_eq!(batch, batch2);
3936 }
3937
3938 fn assert_empty_run_array_roundtrip<R: RunEndIndexType>() {
3939 let run_ends = PrimitiveArray::<R>::from_iter_values(
3940 [2, 5]
3941 .into_iter()
3942 .map(|v| R::Native::from_usize(v).unwrap()),
3943 );
3944 let values = Int32Array::from(vec![10, 20]);
3945 let array = RunArray::<R>::try_new(&run_ends, &values).unwrap();
3946 let empty = RunArray::<R>::from(ArrayData::new_empty(array.data_type()));
3947
3948 for source in [&array, &empty] {
3949 for offset in 0..=source.len() {
3950 let sliced = source.slice(offset, 0);
3951 let batch = RecordBatch::try_from_iter(vec![("run", Arc::new(sliced) as ArrayRef)])
3952 .unwrap();
3953 for decoded in [
3954 deserialize_stream(serialize_stream(&batch)),
3955 deserialize_file(serialize_file(&batch)),
3956 ] {
3957 assert_eq!(decoded, batch);
3958 let data = decoded.column(0).to_data();
3959 data.validate_full().unwrap();
3960 assert_eq!(data.offset(), 0);
3961 assert!(data.child_data().iter().all(ArrayData::is_empty));
3962 }
3963
3964 let normalized = into_zero_offset_run_array(source.slice(offset, 0)).unwrap();
3965 normalized.to_data().validate_full().unwrap();
3966 assert_eq!(normalized.offset(), 0);
3967 assert!(normalized.run_ends().values().is_empty());
3968 assert!(normalized.values().is_empty());
3969 }
3970 }
3971 }
3972
3973 #[test]
3974 fn test_empty_run_array_roundtrip_i16() {
3975 assert_empty_run_array_roundtrip::<Int16Type>();
3976 }
3977
3978 #[test]
3979 fn test_empty_run_array_roundtrip_i32() {
3980 assert_empty_run_array_roundtrip::<Int32Type>();
3981 }
3982
3983 #[test]
3984 fn test_empty_run_array_roundtrip_i64() {
3985 assert_empty_run_array_roundtrip::<Int64Type>();
3986 }
3987
3988 #[test]
3989 fn test_run_array_unslice() {
3990 let total_len = 80;
3991 let vals: Vec<Option<i32>> = vec![Some(1), None, Some(2), Some(3), Some(4), None, Some(5)];
3992 let repeats: Vec<usize> = vec![3, 4, 1, 2];
3993 let mut input_array: Vec<Option<i32>> = Vec::with_capacity(total_len);
3994 for ix in 0_usize..32 {
3995 let repeat: usize = repeats[ix % repeats.len()];
3996 let val: Option<i32> = vals[ix % vals.len()];
3997 input_array.resize(input_array.len() + repeat, val);
3998 }
3999
4000 let mut builder =
4002 PrimitiveRunBuilder::<Int16Type, Int32Type>::with_capacity(input_array.len());
4003 builder.extend(input_array.iter().copied());
4004 let run_array = builder.finish();
4005
4006 for slice_len in 1..=total_len {
4008 let sliced_run_array: RunArray<Int16Type> =
4010 run_array.slice(0, slice_len).into_data().into();
4011
4012 let unsliced_run_array = into_zero_offset_run_array(sliced_run_array).unwrap();
4014 let typed = unsliced_run_array
4015 .downcast::<PrimitiveArray<Int32Type>>()
4016 .unwrap();
4017 let expected: Vec<Option<i32>> = input_array.iter().take(slice_len).copied().collect();
4018 let actual: Vec<Option<i32>> = typed.into_iter().collect();
4019 assert_eq!(expected, actual);
4020
4021 let sliced_run_array: RunArray<Int16Type> = run_array
4023 .slice(total_len - slice_len, slice_len)
4024 .into_data()
4025 .into();
4026
4027 let unsliced_run_array = into_zero_offset_run_array(sliced_run_array).unwrap();
4029 let typed = unsliced_run_array
4030 .downcast::<PrimitiveArray<Int32Type>>()
4031 .unwrap();
4032 let expected: Vec<Option<i32>> = input_array
4033 .iter()
4034 .skip(total_len - slice_len)
4035 .copied()
4036 .collect();
4037 let actual: Vec<Option<i32>> = typed.into_iter().collect();
4038 assert_eq!(expected, actual);
4039 }
4040 }
4041
4042 fn generate_list_data<O: OffsetSizeTrait>() -> GenericListArray<O> {
4043 let mut ls = GenericListBuilder::<O, _>::new(UInt32Builder::new());
4044
4045 for i in 0..100_000 {
4046 for value in [i, i, i] {
4047 ls.values().append_value(value);
4048 }
4049 ls.append(true)
4050 }
4051
4052 ls.finish()
4053 }
4054
4055 fn generate_utf8view_list_data<O: OffsetSizeTrait>() -> GenericListArray<O> {
4056 let mut ls = GenericListBuilder::<O, _>::new(StringViewBuilder::new());
4057
4058 for i in 0..100_000 {
4059 for value in [
4060 format!("value{i}"),
4061 format!("value{i}"),
4062 format!("value{i}"),
4063 ] {
4064 ls.values().append_value(&value);
4065 }
4066 ls.append(true)
4067 }
4068
4069 ls.finish()
4070 }
4071
4072 fn generate_string_list_data<O: OffsetSizeTrait>() -> GenericListArray<O> {
4073 let mut ls = GenericListBuilder::<O, _>::new(StringBuilder::new());
4074
4075 for i in 0..100_000 {
4076 for value in [
4077 format!("value{i}"),
4078 format!("value{i}"),
4079 format!("value{i}"),
4080 ] {
4081 ls.values().append_value(&value);
4082 }
4083 ls.append(true)
4084 }
4085
4086 ls.finish()
4087 }
4088
4089 fn generate_nested_list_data<O: OffsetSizeTrait>() -> GenericListArray<O> {
4090 let mut ls =
4091 GenericListBuilder::<O, _>::new(GenericListBuilder::<O, _>::new(UInt32Builder::new()));
4092
4093 for _i in 0..10_000 {
4094 for j in 0..10 {
4095 for value in [j, j, j, j] {
4096 ls.values().values().append_value(value);
4097 }
4098 ls.values().append(true)
4099 }
4100 ls.append(true);
4101 }
4102
4103 ls.finish()
4104 }
4105
4106 fn generate_nested_list_data_starting_at_zero<O: OffsetSizeTrait>() -> GenericListArray<O> {
4107 let mut ls =
4108 GenericListBuilder::<O, _>::new(GenericListBuilder::<O, _>::new(UInt32Builder::new()));
4109
4110 for _i in 0..999 {
4111 ls.values().append(true);
4112 ls.append(true);
4113 }
4114
4115 for j in 0..10 {
4116 for value in [j, j, j, j] {
4117 ls.values().values().append_value(value);
4118 }
4119 ls.values().append(true)
4120 }
4121 ls.append(true);
4122
4123 for i in 0..9_000 {
4124 for j in 0..10 {
4125 for value in [i + j, i + j, i + j, i + j] {
4126 ls.values().values().append_value(value);
4127 }
4128 ls.values().append(true)
4129 }
4130 ls.append(true);
4131 }
4132
4133 ls.finish()
4134 }
4135
4136 fn generate_map_array_data() -> MapArray {
4137 let keys_builder = UInt32Builder::new();
4138 let values_builder = UInt32Builder::new();
4139
4140 let mut builder = MapBuilder::new(None, keys_builder, values_builder);
4141
4142 for i in 0..100_000 {
4143 for _j in 0..3 {
4144 builder.keys().append_value(i);
4145 builder.values().append_value(i * 2);
4146 }
4147 builder.append(true).unwrap();
4148 }
4149
4150 builder.finish()
4151 }
4152
4153 #[test]
4154 #[cfg_attr(miri, ignore)] fn reencode_offsets_when_first_offset_is_not_zero() {
4156 let original_list = generate_list_data::<i32>();
4157 let original_data = original_list.into_data();
4158 let slice_data = original_data.slice(75, 7);
4159 let (new_offsets, original_start, length) =
4160 reencode_offsets::<i32>(&slice_data.buffers()[0], &slice_data);
4161 assert_eq!(
4162 vec![0, 3, 6, 9, 12, 15, 18, 21],
4163 new_offsets.typed_data::<i32>()
4164 );
4165 assert_eq!(225, original_start);
4166 assert_eq!(21, length);
4167 }
4168
4169 #[test]
4170 fn reencode_offsets_when_first_offset_is_zero() {
4171 let mut ls = GenericListBuilder::<i32, _>::new(UInt32Builder::new());
4172 ls.append(true);
4174 ls.values().append_value(35);
4175 ls.values().append_value(42);
4176 ls.append(true);
4177 let original_list = ls.finish();
4178 let original_data = original_list.into_data();
4179
4180 let slice_data = original_data.slice(1, 1);
4181 let (new_offsets, original_start, length) =
4182 reencode_offsets::<i32>(&slice_data.buffers()[0], &slice_data);
4183 assert_eq!(vec![0, 2], new_offsets.typed_data::<i32>());
4184 assert_eq!(0, original_start);
4185 assert_eq!(2, length);
4186 }
4187
4188 fn roundtrip_ensure_sliced_smaller(in_batch: RecordBatch, expected_size_factor: usize) {
4191 let in_sliced = in_batch.slice(999, 1);
4193
4194 let bytes_batch = serialize_file(&in_batch);
4195 let bytes_sliced = serialize_file(&in_sliced);
4196
4197 assert!(bytes_sliced.len() < (bytes_batch.len() / expected_size_factor));
4199
4200 let out_batch = deserialize_file(bytes_batch);
4202 assert_eq!(in_batch, out_batch);
4203
4204 let out_sliced = deserialize_file(bytes_sliced);
4205 assert_eq!(in_sliced, out_sliced);
4206 }
4207
4208 #[test]
4209 #[cfg_attr(miri, ignore)] fn encode_lists() {
4211 let val_inner = Field::new_list_field(DataType::UInt32, true);
4212 let val_list_field = Field::new("val", DataType::List(Arc::new(val_inner)), false);
4213 let schema = Arc::new(Schema::new(vec![val_list_field]));
4214
4215 let values = Arc::new(generate_list_data::<i32>());
4216
4217 let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4218 roundtrip_ensure_sliced_smaller(in_batch, 1000);
4219 }
4220
4221 #[test]
4222 #[cfg_attr(miri, ignore)] fn encode_empty_list() {
4224 let val_inner = Field::new_list_field(DataType::UInt32, true);
4225 let val_list_field = Field::new("val", DataType::List(Arc::new(val_inner)), false);
4226 let schema = Arc::new(Schema::new(vec![val_list_field]));
4227
4228 let values = Arc::new(generate_list_data::<i32>());
4229
4230 let in_batch = RecordBatch::try_new(schema, vec![values])
4231 .unwrap()
4232 .slice(999, 0);
4233 let out_batch = deserialize_file(serialize_file(&in_batch));
4234 assert_eq!(in_batch, out_batch);
4235 }
4236
4237 #[test]
4238 #[cfg_attr(miri, ignore)] fn encode_large_lists() {
4240 let val_inner = Field::new_list_field(DataType::UInt32, true);
4241 let val_list_field = Field::new("val", DataType::LargeList(Arc::new(val_inner)), false);
4242 let schema = Arc::new(Schema::new(vec![val_list_field]));
4243
4244 let values = Arc::new(generate_list_data::<i64>());
4245
4246 let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4249 roundtrip_ensure_sliced_smaller(in_batch, 1000);
4250 }
4251
4252 #[test]
4253 #[cfg_attr(miri, ignore)] fn encode_large_lists_non_zero_offset() {
4255 let val_inner = Field::new_list_field(DataType::UInt32, true);
4256 let val_list_field = Field::new("val", DataType::LargeList(Arc::new(val_inner)), false);
4257 let schema = Arc::new(Schema::new(vec![val_list_field]));
4258
4259 let values = Arc::new(generate_list_data::<i64>());
4260
4261 check_sliced_list_array(schema, values);
4262 }
4263
4264 #[test]
4265 #[cfg_attr(miri, ignore)] fn encode_large_lists_string_non_zero_offset() {
4267 let val_inner = Field::new_list_field(DataType::Utf8, true);
4268 let val_list_field = Field::new("val", DataType::LargeList(Arc::new(val_inner)), false);
4269 let schema = Arc::new(Schema::new(vec![val_list_field]));
4270
4271 let values = Arc::new(generate_string_list_data::<i64>());
4272
4273 check_sliced_list_array(schema, values);
4274 }
4275
4276 #[test]
4277 #[cfg_attr(miri, ignore)] fn encode_large_list_string_view_non_zero_offset() {
4279 let val_inner = Field::new_list_field(DataType::Utf8View, true);
4280 let val_list_field = Field::new("val", DataType::LargeList(Arc::new(val_inner)), false);
4281 let schema = Arc::new(Schema::new(vec![val_list_field]));
4282
4283 let values = Arc::new(generate_utf8view_list_data::<i64>());
4284
4285 check_sliced_list_array(schema, values);
4286 }
4287
4288 fn check_sliced_list_array(schema: Arc<Schema>, values: Arc<GenericListArray<i64>>) {
4289 for (offset, len) in [(999, 1), (0, 13), (47, 12), (values.len() - 13, 13)] {
4290 let in_batch = RecordBatch::try_new(schema.clone(), vec![values.clone()])
4291 .unwrap()
4292 .slice(offset, len);
4293 let out_batch = deserialize_file(serialize_file(&in_batch));
4294 assert_eq!(in_batch, out_batch);
4295 }
4296 }
4297
4298 #[test]
4299 #[cfg_attr(miri, ignore)] fn encode_nested_lists() {
4301 let inner_int = Arc::new(Field::new_list_field(DataType::UInt32, true));
4302 let inner_list_field = Arc::new(Field::new_list_field(DataType::List(inner_int), true));
4303 let list_field = Field::new("val", DataType::List(inner_list_field), true);
4304 let schema = Arc::new(Schema::new(vec![list_field]));
4305
4306 let values = Arc::new(generate_nested_list_data::<i32>());
4307
4308 let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4309 roundtrip_ensure_sliced_smaller(in_batch, 1000);
4310 }
4311
4312 #[test]
4313 #[cfg_attr(miri, ignore)] fn encode_nested_lists_starting_at_zero() {
4315 let inner_int = Arc::new(Field::new("item", DataType::UInt32, true));
4316 let inner_list_field = Arc::new(Field::new("item", DataType::List(inner_int), true));
4317 let list_field = Field::new("val", DataType::List(inner_list_field), true);
4318 let schema = Arc::new(Schema::new(vec![list_field]));
4319
4320 let values = Arc::new(generate_nested_list_data_starting_at_zero::<i32>());
4321
4322 let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4323 roundtrip_ensure_sliced_smaller(in_batch, 1);
4324 }
4325
4326 #[test]
4327 #[cfg_attr(miri, ignore)] fn encode_map_array() {
4329 let keys = Arc::new(Field::new(
4330 Field::MAP_KEY_FIELD_DEFAULT_NAME,
4331 DataType::UInt32,
4332 false,
4333 ));
4334 let values = Arc::new(Field::new(
4335 Field::MAP_VALUE_FIELD_DEFAULT_NAME,
4336 DataType::UInt32,
4337 true,
4338 ));
4339 let map_field = Field::new_map(
4340 "map",
4341 Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
4342 keys,
4343 values,
4344 false,
4345 true,
4346 );
4347 let schema = Arc::new(Schema::new(vec![map_field]));
4348
4349 let values = Arc::new(generate_map_array_data());
4350
4351 let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4352 roundtrip_ensure_sliced_smaller(in_batch, 1000);
4353 }
4354
4355 fn generate_list_view_data<O: OffsetSizeTrait>() -> GenericListViewArray<O> {
4356 let mut builder = GenericListViewBuilder::<O, _>::new(UInt32Builder::new());
4357
4358 for i in 0u32..100_000 {
4359 if i.is_multiple_of(10_000) {
4360 builder.append(false);
4361 continue;
4362 }
4363 for value in [i, i, i] {
4364 builder.values().append_value(value);
4365 }
4366 builder.append(true);
4367 }
4368
4369 builder.finish()
4370 }
4371
4372 #[test]
4373 #[cfg_attr(miri, ignore)] fn encode_list_view_arrays() {
4375 let val_inner = Field::new_list_field(DataType::UInt32, true);
4376 let val_field = Field::new("val", DataType::ListView(Arc::new(val_inner)), true);
4377 let schema = Arc::new(Schema::new(vec![val_field]));
4378
4379 let values = Arc::new(generate_list_view_data::<i32>());
4380
4381 let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4382 let out_batch = deserialize_file(serialize_file(&in_batch));
4383 assert_eq!(in_batch, out_batch);
4384 }
4385
4386 #[test]
4387 #[cfg_attr(miri, ignore)] fn encode_large_list_view_arrays() {
4389 let val_inner = Field::new_list_field(DataType::UInt32, true);
4390 let val_field = Field::new("val", DataType::LargeListView(Arc::new(val_inner)), true);
4391 let schema = Arc::new(Schema::new(vec![val_field]));
4392
4393 let values = Arc::new(generate_list_view_data::<i64>());
4394
4395 let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4396 let out_batch = deserialize_file(serialize_file(&in_batch));
4397 assert_eq!(in_batch, out_batch);
4398 }
4399
4400 #[test]
4401 #[cfg_attr(miri, ignore)] fn check_sliced_list_view_array() {
4403 let inner = Field::new_list_field(DataType::UInt32, true);
4404 let field = Field::new("val", DataType::ListView(Arc::new(inner)), true);
4405 let schema = Arc::new(Schema::new(vec![field]));
4406 let values = Arc::new(generate_list_view_data::<i32>());
4407
4408 for (offset, len) in [(999, 1), (0, 13), (47, 12), (values.len() - 13, 13)] {
4409 let in_batch = RecordBatch::try_new(schema.clone(), vec![values.clone()])
4410 .unwrap()
4411 .slice(offset, len);
4412 let out_batch = deserialize_file(serialize_file(&in_batch));
4413 assert_eq!(in_batch, out_batch);
4414 }
4415 }
4416
4417 #[test]
4418 #[cfg_attr(miri, ignore)] fn check_sliced_large_list_view_array() {
4420 let inner = Field::new_list_field(DataType::UInt32, true);
4421 let field = Field::new("val", DataType::LargeListView(Arc::new(inner)), true);
4422 let schema = Arc::new(Schema::new(vec![field]));
4423 let values = Arc::new(generate_list_view_data::<i64>());
4424
4425 for (offset, len) in [(999, 1), (0, 13), (47, 12), (values.len() - 13, 13)] {
4426 let in_batch = RecordBatch::try_new(schema.clone(), vec![values.clone()])
4427 .unwrap()
4428 .slice(offset, len);
4429 let out_batch = deserialize_file(serialize_file(&in_batch));
4430 assert_eq!(in_batch, out_batch);
4431 }
4432 }
4433
4434 fn generate_nested_list_view_data<O: OffsetSizeTrait>() -> GenericListViewArray<O> {
4435 let inner_builder = UInt32Builder::new();
4436 let middle_builder = GenericListViewBuilder::<O, _>::new(inner_builder);
4437 let mut outer_builder = GenericListViewBuilder::<O, _>::new(middle_builder);
4438
4439 for i in 0u32..10_000 {
4440 if i.is_multiple_of(1_000) {
4441 outer_builder.append(false);
4442 continue;
4443 }
4444
4445 for _ in 0..3 {
4446 for value in [i, i + 1, i + 2] {
4447 outer_builder.values().values().append_value(value);
4448 }
4449 outer_builder.values().append(true);
4450 }
4451 outer_builder.append(true);
4452 }
4453
4454 outer_builder.finish()
4455 }
4456
4457 #[test]
4458 #[cfg_attr(miri, ignore)] fn encode_nested_list_views() {
4460 let inner_int = Arc::new(Field::new_list_field(DataType::UInt32, true));
4461 let inner_list_field = Arc::new(Field::new_list_field(DataType::ListView(inner_int), true));
4462 let list_field = Field::new("val", DataType::ListView(inner_list_field), true);
4463 let schema = Arc::new(Schema::new(vec![list_field]));
4464
4465 let values = Arc::new(generate_nested_list_view_data::<i32>());
4466
4467 let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4468 let out_batch = deserialize_file(serialize_file(&in_batch));
4469 assert_eq!(in_batch, out_batch);
4470 }
4471
4472 fn test_roundtrip_list_view_of_dict_impl<OffsetSize: OffsetSizeTrait, U: ArrowNativeType>(
4473 list_data_type: DataType,
4474 offsets: &[U; 5],
4475 sizes: &[U; 4],
4476 ) {
4477 let values = StringArray::from(vec![Some("alpha"), None, Some("beta"), Some("gamma")]);
4478 let keys = Int32Array::from_iter_values([0, 0, 1, 2, 3, 0, 2]);
4479 let dict_array = DictionaryArray::new(keys, Arc::new(values));
4480 let dict_data = dict_array.to_data();
4481
4482 let value_offsets = Buffer::from_slice_ref(offsets);
4483 let value_sizes = Buffer::from_slice_ref(sizes);
4484
4485 let list_data = ArrayData::builder(list_data_type)
4486 .len(4)
4487 .add_buffer(value_offsets)
4488 .add_buffer(value_sizes)
4489 .add_child_data(dict_data)
4490 .build()
4491 .unwrap();
4492 let list_view_array = GenericListViewArray::<OffsetSize>::from(list_data);
4493
4494 let schema = Arc::new(Schema::new(vec![Field::new(
4495 "f1",
4496 list_view_array.data_type().clone(),
4497 false,
4498 )]));
4499 let input_batch = RecordBatch::try_new(schema, vec![Arc::new(list_view_array)]).unwrap();
4500
4501 let output_batch = deserialize_file(serialize_file(&input_batch));
4502 assert_eq!(input_batch, output_batch);
4503
4504 let output_batch = deserialize_stream(serialize_stream(&input_batch));
4505 assert_eq!(input_batch, output_batch);
4506 }
4507
4508 #[test]
4509 fn test_roundtrip_list_view_of_dict() {
4510 #[expect(deprecated)]
4511 let list_data_type = DataType::ListView(Arc::new(Field::new_dict(
4512 "item",
4513 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4514 true,
4515 1,
4516 false,
4517 )));
4518 let offsets: &[i32; 5] = &[0, 2, 4, 4, 7];
4519 let sizes: &[i32; 4] = &[2, 2, 0, 3];
4520 test_roundtrip_list_view_of_dict_impl::<i32, i32>(list_data_type, offsets, sizes);
4521 }
4522
4523 #[test]
4524 fn test_roundtrip_large_list_view_of_dict() {
4525 #[expect(deprecated)]
4526 let list_data_type = DataType::LargeListView(Arc::new(Field::new_dict(
4527 "item",
4528 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4529 true,
4530 2,
4531 false,
4532 )));
4533 let offsets: &[i64; 5] = &[0, 2, 4, 4, 7];
4534 let sizes: &[i64; 4] = &[2, 2, 0, 3];
4535 test_roundtrip_list_view_of_dict_impl::<i64, i64>(list_data_type, offsets, sizes);
4536 }
4537
4538 #[test]
4539 fn test_roundtrip_sliced_list_view_of_dict() {
4540 #[expect(deprecated)]
4541 let list_data_type = DataType::ListView(Arc::new(Field::new_dict(
4542 "item",
4543 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4544 true,
4545 3,
4546 false,
4547 )));
4548
4549 let values = StringArray::from(vec![Some("alpha"), None, Some("beta"), Some("gamma")]);
4550 let keys = Int32Array::from_iter_values([0, 0, 1, 2, 3, 0, 2, 1, 0, 3, 2, 1]);
4551 let dict_array = DictionaryArray::new(keys, Arc::new(values));
4552 let dict_data = dict_array.to_data();
4553
4554 let offsets: &[i32; 7] = &[0, 2, 4, 4, 7, 9, 12];
4555 let sizes: &[i32; 6] = &[2, 2, 0, 3, 2, 3];
4556 let value_offsets = Buffer::from_slice_ref(offsets);
4557 let value_sizes = Buffer::from_slice_ref(sizes);
4558
4559 let list_data = ArrayData::builder(list_data_type)
4560 .len(6)
4561 .add_buffer(value_offsets)
4562 .add_buffer(value_sizes)
4563 .add_child_data(dict_data)
4564 .build()
4565 .unwrap();
4566 let list_view_array = GenericListViewArray::<i32>::from(list_data);
4567
4568 let schema = Arc::new(Schema::new(vec![Field::new(
4569 "f1",
4570 list_view_array.data_type().clone(),
4571 false,
4572 )]));
4573 let input_batch = RecordBatch::try_new(schema, vec![Arc::new(list_view_array)]).unwrap();
4574
4575 let sliced_batch = input_batch.slice(1, 4);
4576
4577 let output_batch = deserialize_file(serialize_file(&sliced_batch));
4578 assert_eq!(sliced_batch, output_batch);
4579
4580 let output_batch = deserialize_stream(serialize_stream(&sliced_batch));
4581 assert_eq!(sliced_batch, output_batch);
4582 }
4583
4584 #[test]
4585 fn test_roundtrip_dense_union_of_dict() {
4586 let values = StringArray::from(vec![Some("alpha"), None, Some("beta"), Some("gamma")]);
4587 let keys = Int32Array::from_iter_values([0, 0, 1, 2, 3, 0, 2]);
4588 let dict_array = DictionaryArray::new(keys, Arc::new(values));
4589
4590 #[expect(deprecated)]
4591 let dict_field = Arc::new(Field::new_dict(
4592 "dict",
4593 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4594 true,
4595 1,
4596 false,
4597 ));
4598 let int_field = Arc::new(Field::new("int", DataType::Int32, false));
4599 let union_fields = UnionFields::try_new(vec![0, 1], vec![dict_field, int_field]).unwrap();
4600
4601 let types = ScalarBuffer::from(vec![0i8, 0, 1, 0, 1, 0, 0]);
4602 let offsets = ScalarBuffer::from(vec![0i32, 1, 0, 2, 1, 3, 4]);
4603
4604 let int_array = Int32Array::from(vec![100, 200]);
4605
4606 let union = UnionArray::try_new(
4607 union_fields.clone(),
4608 types,
4609 Some(offsets),
4610 vec![Arc::new(dict_array), Arc::new(int_array)],
4611 )
4612 .unwrap();
4613
4614 let schema = Arc::new(Schema::new(vec![Field::new(
4615 "union",
4616 DataType::Union(union_fields, UnionMode::Dense),
4617 false,
4618 )]));
4619 let input_batch = RecordBatch::try_new(schema, vec![Arc::new(union)]).unwrap();
4620
4621 let output_batch = deserialize_file(serialize_file(&input_batch));
4622 assert_eq!(input_batch, output_batch);
4623
4624 let output_batch = deserialize_stream(serialize_stream(&input_batch));
4625 assert_eq!(input_batch, output_batch);
4626 }
4627
4628 #[test]
4629 fn test_roundtrip_sparse_union_of_dict() {
4630 let values = StringArray::from(vec![Some("alpha"), None, Some("beta"), Some("gamma")]);
4631 let keys = Int32Array::from_iter_values([0, 0, 1, 2, 3, 0, 2]);
4632 let dict_array = DictionaryArray::new(keys, Arc::new(values));
4633
4634 #[expect(deprecated)]
4635 let dict_field = Arc::new(Field::new_dict(
4636 "dict",
4637 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4638 true,
4639 2,
4640 false,
4641 ));
4642 let int_field = Arc::new(Field::new("int", DataType::Int32, false));
4643 let union_fields = UnionFields::try_new(vec![0, 1], vec![dict_field, int_field]).unwrap();
4644
4645 let types = ScalarBuffer::from(vec![0i8, 0, 1, 0, 1, 0, 0]);
4646
4647 let int_array = Int32Array::from(vec![0, 0, 100, 0, 200, 0, 0]);
4648
4649 let union = UnionArray::try_new(
4650 union_fields.clone(),
4651 types,
4652 None,
4653 vec![Arc::new(dict_array), Arc::new(int_array)],
4654 )
4655 .unwrap();
4656
4657 let schema = Arc::new(Schema::new(vec![Field::new(
4658 "union",
4659 DataType::Union(union_fields, UnionMode::Sparse),
4660 false,
4661 )]));
4662 let input_batch = RecordBatch::try_new(schema, vec![Arc::new(union)]).unwrap();
4663
4664 let output_batch = deserialize_file(serialize_file(&input_batch));
4665 assert_eq!(input_batch, output_batch);
4666
4667 let output_batch = deserialize_stream(serialize_stream(&input_batch));
4668 assert_eq!(input_batch, output_batch);
4669 }
4670
4671 #[test]
4672 fn test_roundtrip_map_with_dict_keys() {
4673 let key_values = StringArray::from(vec!["key_a", "key_b", "key_c"]);
4676 let keys = Int32Array::from_iter_values([0, 1, 2, 0, 1, 0]);
4677 let dict_keys = DictionaryArray::new(keys, Arc::new(key_values));
4678
4679 let values = Int32Array::from(vec![1, 2, 3, 4, 5, 6]);
4680
4681 #[expect(deprecated)]
4682 let entries_field = Arc::new(Field::new(
4683 Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
4684 DataType::Struct(
4685 vec![
4686 Field::new_dict(
4687 Field::MAP_KEY_FIELD_DEFAULT_NAME,
4688 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4689 false,
4690 1,
4691 false,
4692 ),
4693 Field::new(Field::MAP_VALUE_FIELD_DEFAULT_NAME, DataType::Int32, true),
4694 ]
4695 .into(),
4696 ),
4697 false,
4698 ));
4699
4700 let entries = StructArray::from(vec![
4701 (
4702 Arc::new(Field::new(
4703 Field::MAP_KEY_FIELD_DEFAULT_NAME,
4704 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4705 false,
4706 )),
4707 Arc::new(dict_keys) as ArrayRef,
4708 ),
4709 (
4710 Arc::new(Field::new(
4711 Field::MAP_VALUE_FIELD_DEFAULT_NAME,
4712 DataType::Int32,
4713 true,
4714 )),
4715 Arc::new(values) as ArrayRef,
4716 ),
4717 ]);
4718
4719 let offsets = Buffer::from_slice_ref([0i32, 2, 4, 6]);
4720
4721 let map_data = ArrayData::builder(DataType::Map(entries_field, false))
4722 .len(3)
4723 .add_buffer(offsets)
4724 .add_child_data(entries.into_data())
4725 .build()
4726 .unwrap();
4727 let map_array = MapArray::from(map_data);
4728
4729 let schema = Arc::new(Schema::new(vec![Field::new(
4730 "map",
4731 map_array.data_type().clone(),
4732 false,
4733 )]));
4734 let input_batch = RecordBatch::try_new(schema, vec![Arc::new(map_array)]).unwrap();
4735
4736 let output_batch = deserialize_file(serialize_file(&input_batch));
4737 assert_eq!(input_batch, output_batch);
4738
4739 let output_batch = deserialize_stream(serialize_stream(&input_batch));
4740 assert_eq!(input_batch, output_batch);
4741 }
4742
4743 #[test]
4744 fn test_roundtrip_map_with_dict_values() {
4745 let keys = StringArray::from(vec!["a", "b", "c", "d", "e", "f"]);
4748
4749 let value_values = StringArray::from(vec!["val_x", "val_y", "val_z"]);
4750 let value_keys = Int32Array::from_iter_values([0, 1, 2, 0, 1, 0]);
4751 let dict_values = DictionaryArray::new(value_keys, Arc::new(value_values));
4752
4753 #[expect(deprecated)]
4754 let entries_field = Arc::new(Field::new(
4755 Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
4756 DataType::Struct(
4757 vec![
4758 Field::new(Field::MAP_KEY_FIELD_DEFAULT_NAME, DataType::Utf8, false),
4759 Field::new_dict(
4760 Field::MAP_VALUE_FIELD_DEFAULT_NAME,
4761 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4762 true,
4763 2,
4764 false,
4765 ),
4766 ]
4767 .into(),
4768 ),
4769 false,
4770 ));
4771
4772 let entries = StructArray::from(vec![
4773 (
4774 Arc::new(Field::new(
4775 Field::MAP_KEY_FIELD_DEFAULT_NAME,
4776 DataType::Utf8,
4777 false,
4778 )),
4779 Arc::new(keys) as ArrayRef,
4780 ),
4781 (
4782 Arc::new(Field::new(
4783 Field::MAP_VALUE_FIELD_DEFAULT_NAME,
4784 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4785 true,
4786 )),
4787 Arc::new(dict_values) as ArrayRef,
4788 ),
4789 ]);
4790
4791 let offsets = Buffer::from_slice_ref([0i32, 2, 4, 6]);
4792
4793 let map_data = ArrayData::builder(DataType::Map(entries_field, false))
4794 .len(3)
4795 .add_buffer(offsets)
4796 .add_child_data(entries.into_data())
4797 .build()
4798 .unwrap();
4799 let map_array = MapArray::from(map_data);
4800
4801 let schema = Arc::new(Schema::new(vec![Field::new(
4802 "map",
4803 map_array.data_type().clone(),
4804 false,
4805 )]));
4806 let input_batch = RecordBatch::try_new(schema, vec![Arc::new(map_array)]).unwrap();
4807
4808 let output_batch = deserialize_file(serialize_file(&input_batch));
4809 assert_eq!(input_batch, output_batch);
4810
4811 let output_batch = deserialize_stream(serialize_stream(&input_batch));
4812 assert_eq!(input_batch, output_batch);
4813 }
4814
4815 #[test]
4816 fn test_decimal128_alignment16_is_sufficient() {
4817 const IPC_ALIGNMENT: usize = 16;
4818
4819 for num_cols in [1, 2, 3, 17, 50, 73, 99] {
4824 let num_rows = (num_cols * 7 + 11) % 100; let mut fields = Vec::new();
4827 let mut arrays = Vec::new();
4828 for i in 0..num_cols {
4829 let field = Field::new(format!("col_{i}"), DataType::Decimal128(38, 10), true);
4830 let array = Decimal128Array::from(vec![num_cols as i128; num_rows]);
4831 fields.push(field);
4832 arrays.push(Arc::new(array) as Arc<dyn Array>);
4833 }
4834 let schema = Schema::new(fields);
4835 let batch = RecordBatch::try_new(Arc::new(schema), arrays).unwrap();
4836
4837 let mut writer = FileWriter::try_new_with_options(
4838 Vec::new(),
4839 batch.schema_ref(),
4840 IpcWriteOptions::try_new(IPC_ALIGNMENT, false, MetadataVersion::V5).unwrap(),
4841 )
4842 .unwrap();
4843 writer.write(&batch).unwrap();
4844 writer.finish().unwrap();
4845
4846 let out: Vec<u8> = writer.into_inner().unwrap();
4847
4848 let buffer = Buffer::from_slice_ref(out);
4849 let trailer_start = buffer.len() - 10;
4850 let footer_len =
4851 read_footer_length(buffer[trailer_start..].try_into().unwrap()).unwrap();
4852 let footer =
4853 root_as_footer(&buffer[trailer_start - footer_len..trailer_start]).unwrap();
4854
4855 let schema = try_fb_to_schema(footer.schema().unwrap()).unwrap();
4856
4857 let decoder =
4860 FileDecoder::new(Arc::new(schema), footer.version()).with_require_alignment(true);
4861
4862 let batches = footer.recordBatches().unwrap();
4863
4864 let block = batches.get(0);
4865 let block_len = block.bodyLength() as usize + block.metaDataLength() as usize;
4866 let data = buffer.slice_with_length(block.offset() as _, block_len);
4867
4868 let batch2 = decoder.read_record_batch(block, &data).unwrap().unwrap();
4869
4870 assert_eq!(batch, batch2);
4871 }
4872 }
4873
4874 #[test]
4875 fn test_decimal128_alignment8_is_unaligned() {
4876 const IPC_ALIGNMENT: usize = 8;
4877
4878 let num_cols = 2;
4879
4880 let mut fields = Vec::new();
4881 let mut arrays = Vec::new();
4882 for i in 0..num_cols {
4883 let field = Field::new(format!("col_{i}"), DataType::Decimal128(38, 10), true);
4884 let array = Decimal128Array::from(vec![Some(num_cols as i128), None]);
4888 fields.push(field);
4889 arrays.push(Arc::new(array) as Arc<dyn Array>);
4890 }
4891 let schema = Schema::new(fields);
4892 let batch = RecordBatch::try_new(Arc::new(schema), arrays).unwrap();
4893
4894 let mut writer = FileWriter::try_new_with_options(
4895 Vec::new(),
4896 batch.schema_ref(),
4897 IpcWriteOptions::try_new(IPC_ALIGNMENT, false, MetadataVersion::V5).unwrap(),
4898 )
4899 .unwrap();
4900 writer.write(&batch).unwrap();
4901 writer.finish().unwrap();
4902
4903 let out: Vec<u8> = writer.into_inner().unwrap();
4904
4905 let buffer = Buffer::from_slice_ref(out);
4906 let trailer_start = buffer.len() - 10;
4907 let footer_len = read_footer_length(buffer[trailer_start..].try_into().unwrap()).unwrap();
4908 let footer = root_as_footer(&buffer[trailer_start - footer_len..trailer_start]).unwrap();
4909 let schema = try_fb_to_schema(footer.schema().unwrap()).unwrap();
4910
4911 let decoder =
4914 FileDecoder::new(Arc::new(schema), footer.version()).with_require_alignment(true);
4915
4916 let batches = footer.recordBatches().unwrap();
4917
4918 let block = batches.get(0);
4919 let block_len = block.bodyLength() as usize + block.metaDataLength() as usize;
4920 let data = buffer.slice_with_length(block.offset() as _, block_len);
4921
4922 let result = decoder.read_record_batch(block, &data);
4923
4924 let error = result.unwrap_err();
4925 assert_eq!(
4926 error.to_string(),
4927 "Invalid argument error: Misaligned buffers[0] in array of type Decimal128(38, 10), \
4928 offset from expected alignment of 16 by 8"
4929 );
4930 }
4931
4932 #[test]
4933 fn test_flush() {
4934 let num_cols = 2;
4937 let mut fields = Vec::new();
4938 let options = IpcWriteOptions::try_new(8, false, MetadataVersion::V5).unwrap();
4939 for i in 0..num_cols {
4940 let field = Field::new(format!("col_{i}"), DataType::Decimal128(38, 10), true);
4941 fields.push(field);
4942 }
4943 let schema = Schema::new(fields);
4944 let inner_stream_writer = BufWriter::with_capacity(1024, Vec::new());
4945 let inner_file_writer = BufWriter::with_capacity(1024, Vec::new());
4946 let mut stream_writer =
4947 StreamWriter::try_new_with_options(inner_stream_writer, &schema, options.clone())
4948 .unwrap();
4949 let mut file_writer =
4950 FileWriter::try_new_with_options(inner_file_writer, &schema, options).unwrap();
4951
4952 let stream_bytes_written_on_new = stream_writer.get_ref().get_ref().len();
4953 let file_bytes_written_on_new = file_writer.get_ref().get_ref().len();
4954 stream_writer.flush().unwrap();
4955 file_writer.flush().unwrap();
4956 let stream_bytes_written_on_flush = stream_writer.get_ref().get_ref().len();
4957 let file_bytes_written_on_flush = file_writer.get_ref().get_ref().len();
4958 let stream_out = stream_writer.into_inner().unwrap().into_inner().unwrap();
4959 let expected_stream_flushed_bytes = stream_out.len() - 8;
4963 let expected_file_flushed_bytes = expected_stream_flushed_bytes + 8;
4966
4967 assert!(
4968 stream_bytes_written_on_new < stream_bytes_written_on_flush,
4969 "this test makes no sense if flush is not actually required"
4970 );
4971 assert!(
4972 file_bytes_written_on_new < file_bytes_written_on_flush,
4973 "this test makes no sense if flush is not actually required"
4974 );
4975 assert_eq!(stream_bytes_written_on_flush, expected_stream_flushed_bytes);
4976 assert_eq!(file_bytes_written_on_flush, expected_file_flushed_bytes);
4977 }
4978
4979 #[test]
4980 fn test_roundtrip_list_of_fixed_list() -> Result<(), ArrowError> {
4981 let l1_type =
4982 DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, false)), 3);
4983 let l2_type = DataType::List(Arc::new(Field::new("item", l1_type.clone(), false)));
4984
4985 let l0_builder = Float32Builder::new();
4986 let l1_builder = FixedSizeListBuilder::new(l0_builder, 3).with_field(Arc::new(Field::new(
4987 "item",
4988 DataType::Float32,
4989 false,
4990 )));
4991 let mut l2_builder =
4992 ListBuilder::new(l1_builder).with_field(Arc::new(Field::new("item", l1_type, false)));
4993
4994 for point in [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]] {
4995 l2_builder.values().values().append_value(point[0]);
4996 l2_builder.values().values().append_value(point[1]);
4997 l2_builder.values().values().append_value(point[2]);
4998
4999 l2_builder.values().append(true);
5000 }
5001 l2_builder.append(true);
5002
5003 let point = [10., 11., 12.];
5004 l2_builder.values().values().append_value(point[0]);
5005 l2_builder.values().values().append_value(point[1]);
5006 l2_builder.values().values().append_value(point[2]);
5007
5008 l2_builder.values().append(true);
5009 l2_builder.append(true);
5010
5011 let array = Arc::new(l2_builder.finish()) as ArrayRef;
5012
5013 let schema = Arc::new(Schema::new_with_metadata(
5014 vec![Field::new("points", l2_type, false)],
5015 HashMap::default(),
5016 ));
5017
5018 test_slices(&array, &schema, 0, 1)?;
5021 test_slices(&array, &schema, 0, 2)?;
5022 test_slices(&array, &schema, 1, 1)?;
5023
5024 Ok(())
5025 }
5026
5027 #[test]
5028 fn test_roundtrip_list_of_fixed_list_w_nulls() -> Result<(), ArrowError> {
5029 let l0_builder = Float32Builder::new();
5030 let l1_builder = FixedSizeListBuilder::new(l0_builder, 3);
5031 let mut l2_builder = ListBuilder::new(l1_builder);
5032
5033 for point in [
5034 [Some(1.0), Some(2.0), None],
5035 [Some(4.0), Some(5.0), Some(6.0)],
5036 [None, Some(8.0), Some(9.0)],
5037 ] {
5038 for p in point {
5039 match p {
5040 Some(p) => l2_builder.values().values().append_value(p),
5041 None => l2_builder.values().values().append_null(),
5042 }
5043 }
5044
5045 l2_builder.values().append(true);
5046 }
5047 l2_builder.append(true);
5048
5049 let point = [Some(10.), None, None];
5050 for p in point {
5051 match p {
5052 Some(p) => l2_builder.values().values().append_value(p),
5053 None => l2_builder.values().values().append_null(),
5054 }
5055 }
5056
5057 l2_builder.values().append(true);
5058 l2_builder.append(true);
5059
5060 let array = Arc::new(l2_builder.finish()) as ArrayRef;
5061
5062 let schema = Arc::new(Schema::new_with_metadata(
5063 vec![Field::new(
5064 "points",
5065 DataType::List(Arc::new(Field::new(
5066 "item",
5067 DataType::FixedSizeList(
5068 Arc::new(Field::new("item", DataType::Float32, true)),
5069 3,
5070 ),
5071 true,
5072 ))),
5073 true,
5074 )],
5075 HashMap::default(),
5076 ));
5077
5078 test_slices(&array, &schema, 0, 1)?;
5081 test_slices(&array, &schema, 0, 2)?;
5082 test_slices(&array, &schema, 1, 1)?;
5083
5084 Ok(())
5085 }
5086
5087 fn test_slices(
5088 parent_array: &ArrayRef,
5089 schema: &SchemaRef,
5090 offset: usize,
5091 length: usize,
5092 ) -> Result<(), ArrowError> {
5093 let subarray = parent_array.slice(offset, length);
5094 let original_batch = RecordBatch::try_new(schema.clone(), vec![subarray])?;
5095
5096 let mut bytes = Vec::new();
5097 let mut writer = StreamWriter::try_new(&mut bytes, schema)?;
5098 writer.write(&original_batch)?;
5099 writer.finish()?;
5100
5101 let mut cursor = std::io::Cursor::new(bytes);
5102 let mut reader = StreamReader::try_new(&mut cursor, None)?;
5103 let returned_batch = reader.next().unwrap()?;
5104
5105 assert_eq!(original_batch, returned_batch);
5106
5107 Ok(())
5108 }
5109
5110 #[test]
5111 fn test_roundtrip_fixed_list() -> Result<(), ArrowError> {
5112 let int_builder = Int64Builder::new();
5113 let mut fixed_list_builder = FixedSizeListBuilder::new(int_builder, 3)
5114 .with_field(Arc::new(Field::new("item", DataType::Int64, false)));
5115
5116 for point in [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]] {
5117 fixed_list_builder.values().append_value(point[0]);
5118 fixed_list_builder.values().append_value(point[1]);
5119 fixed_list_builder.values().append_value(point[2]);
5120
5121 fixed_list_builder.append(true);
5122 }
5123
5124 let array = Arc::new(fixed_list_builder.finish()) as ArrayRef;
5125
5126 let schema = Arc::new(Schema::new_with_metadata(
5127 vec![Field::new(
5128 "points",
5129 DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Int64, false)), 3),
5130 false,
5131 )],
5132 HashMap::default(),
5133 ));
5134
5135 test_slices(&array, &schema, 0, 4)?;
5138 test_slices(&array, &schema, 0, 2)?;
5139 test_slices(&array, &schema, 1, 3)?;
5140 test_slices(&array, &schema, 2, 1)?;
5141
5142 Ok(())
5143 }
5144
5145 #[test]
5146 fn test_roundtrip_fixed_list_w_nulls() -> Result<(), ArrowError> {
5147 let int_builder = Int64Builder::new();
5148 let mut fixed_list_builder = FixedSizeListBuilder::new(int_builder, 3);
5149
5150 for point in [
5151 [Some(1), Some(2), None],
5152 [Some(4), Some(5), Some(6)],
5153 [None, Some(8), Some(9)],
5154 [Some(10), None, None],
5155 ] {
5156 for p in point {
5157 match p {
5158 Some(p) => fixed_list_builder.values().append_value(p),
5159 None => fixed_list_builder.values().append_null(),
5160 }
5161 }
5162
5163 fixed_list_builder.append(true);
5164 }
5165
5166 let array = Arc::new(fixed_list_builder.finish()) as ArrayRef;
5167
5168 let schema = Arc::new(Schema::new_with_metadata(
5169 vec![Field::new(
5170 "points",
5171 DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Int64, true)), 3),
5172 true,
5173 )],
5174 HashMap::default(),
5175 ));
5176
5177 test_slices(&array, &schema, 0, 4)?;
5180 test_slices(&array, &schema, 0, 2)?;
5181 test_slices(&array, &schema, 1, 3)?;
5182 test_slices(&array, &schema, 2, 1)?;
5183
5184 Ok(())
5185 }
5186
5187 #[test]
5188 fn test_metadata_encoding_ordering() {
5189 fn create_hash() -> u64 {
5190 let metadata: HashMap<String, String> = [
5191 ("a", "1"), ("b", "2"), ("c", "3"), ("d", "4"), ("e", "5"), ]
5197 .into_iter()
5198 .map(|(k, v)| (k.to_owned(), v.to_owned()))
5199 .collect();
5200
5201 let schema = Arc::new(
5203 Schema::new(vec![
5204 Field::new("a", DataType::Int64, true).with_metadata(metadata.clone()),
5205 ])
5206 .with_metadata(metadata)
5207 .clone(),
5208 );
5209 let batch = RecordBatch::new_empty(schema.clone());
5210
5211 let mut bytes = Vec::new();
5212 let mut w = StreamWriter::try_new(&mut bytes, batch.schema_ref()).unwrap();
5213 w.write(&batch).unwrap();
5214 w.finish().unwrap();
5215
5216 let mut h = std::hash::DefaultHasher::new();
5217 h.write(&bytes);
5218 h.finish()
5219 }
5220
5221 let expected = create_hash();
5222
5223 let all_passed = (0..20).all(|_| create_hash() == expected);
5228 assert!(all_passed);
5229 }
5230
5231 #[test]
5232 fn test_dictionary_tracker_reset() {
5233 let data_gen = IpcDataGenerator::default();
5234 let mut dictionary_tracker = DictionaryTracker::new(false);
5235 let writer_options = IpcWriteOptions::default();
5236 let mut compression_ctx = IpcWriteContext::default();
5237
5238 let schema = Arc::new(Schema::new(vec![Field::new(
5239 "a",
5240 DataType::Dictionary(Box::new(DataType::UInt8), Box::new(DataType::Utf8)),
5241 false,
5242 )]));
5243
5244 let mut write_single_batch_stream =
5245 |batch: RecordBatch, dict_tracker: &mut DictionaryTracker| -> Vec<u8> {
5246 let mut buffer = Vec::new();
5247
5248 let stream_header = data_gen.schema_to_bytes_with_dictionary_tracker(
5250 &schema,
5251 dict_tracker,
5252 &writer_options,
5253 );
5254 _ = write_message(&mut buffer, stream_header, &writer_options).unwrap();
5255
5256 let (encoded_dicts, encoded_batch) = data_gen
5257 .encode(&batch, dict_tracker, &writer_options, &mut compression_ctx)
5258 .unwrap();
5259 for encoded_dict in encoded_dicts {
5260 _ = write_message(&mut buffer, encoded_dict, &writer_options).unwrap();
5261 }
5262 _ = write_message(&mut buffer, encoded_batch, &writer_options).unwrap();
5263
5264 buffer
5265 };
5266
5267 let batch1 = RecordBatch::try_new(
5268 schema.clone(),
5269 vec![Arc::new(DictionaryArray::new(
5270 UInt8Array::from_iter_values([0]),
5271 Arc::new(StringArray::from_iter_values(["a"])),
5272 ))],
5273 )
5274 .unwrap();
5275 let buffer = write_single_batch_stream(batch1.clone(), &mut dictionary_tracker);
5276
5277 let mut reader = StreamReader::try_new(Cursor::new(buffer), None).unwrap();
5279 let read_batch = reader.next().unwrap().unwrap();
5280 assert_eq!(read_batch, batch1);
5281
5282 dictionary_tracker.clear();
5284
5285 let batch2 = RecordBatch::try_new(
5287 schema.clone(),
5288 vec![Arc::new(DictionaryArray::new(
5289 UInt8Array::from_iter_values([0]),
5290 Arc::new(StringArray::from_iter_values(["a"])),
5291 ))],
5292 )
5293 .unwrap();
5294 let buffer = write_single_batch_stream(batch2.clone(), &mut dictionary_tracker);
5295 let mut reader = StreamReader::try_new(Cursor::new(buffer), None).unwrap();
5296 let read_batch = reader.next().unwrap().unwrap();
5297 assert_eq!(read_batch, batch2);
5298 }
5299}