1use std::cmp::min;
15use std::collections::HashMap;
16use std::io::{BufWriter, Write};
17use std::mem::size_of;
18use std::sync::Arc;
19
20use flatbuffers::FlatBufferBuilder;
21
22use arrow_array::cast::*;
23use arrow_array::types::{Int16Type, Int32Type, Int64Type, RunEndIndexType};
24use arrow_array::*;
25use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer, ToByteSlice};
26use arrow_data::{ArrayData, ArrayDataBuilder, BufferSpec, layout};
27use arrow_schema::*;
28
29use crate::CONTINUATION_MARKER;
30use crate::compression::{CompressionCodec, Lz4BlockSize};
31#[expect(deprecated)]
32pub use crate::compression::{CompressionContext, IpcWriteContext};
33use crate::convert::IpcSchemaEncoder;
34
35#[derive(Debug, Clone)]
37pub struct IpcWriteOptions {
38 alignment: u8,
41 write_legacy_ipc_format: bool,
43 metadata_version: crate::MetadataVersion,
52 batch_compression_type: Option<crate::CompressionType>,
55 batch_compression_level: Option<i32>,
57 batch_lz4_block_size: Option<Lz4BlockSize>,
59 dictionary_handling: DictionaryHandling,
61}
62
63enum EncodedBuffer {
69 Raw(Buffer),
71 Compressed(Vec<u8>),
73}
74
75impl EncodedBuffer {
76 fn as_slice(&self) -> &[u8] {
77 match self {
78 EncodedBuffer::Raw(b) => b.as_slice(),
79 EncodedBuffer::Compressed(v) => v.as_slice(),
80 }
81 }
82
83 fn len(&self) -> usize {
84 match self {
85 EncodedBuffer::Raw(b) => b.len(),
86 EncodedBuffer::Compressed(v) => v.len(),
87 }
88 }
89}
90#[derive(Default)]
95struct IpcMetadataBuilder {
96 nodes: Vec<crate::FieldNode>,
97 buffers: Vec<crate::Buffer>,
98}
99
100enum IpcBodySink<'a> {
105 Write(&'a mut Vec<u8>),
107 Collect(&'a mut Vec<EncodedBuffer>),
109}
110impl IpcBodySink<'_> {
111 pub fn write(&mut self, pad_len: usize, buffer: EncodedBuffer) {
113 match self {
114 IpcBodySink::Write(vec) => {
115 vec.extend_from_slice(buffer.as_slice());
116 vec.extend_from_slice(&PADDING[..pad_len]);
117 }
118 IpcBodySink::Collect(vec) => {
119 vec.push(buffer);
120 }
121 }
122 }
123}
124
125struct MetadataLayout {
126 padded_header_len: usize,
127 padded_metadata_len: usize,
128 metadata_padding: usize,
129}
130
131impl MetadataLayout {
132 fn new(metadata_len: usize, write_options: &IpcWriteOptions) -> MetadataLayout {
133 let prefix_size = if write_options.write_legacy_ipc_format {
134 4
135 } else {
136 8
137 };
138 let alignment_mask = usize::from(write_options.alignment - 1);
139 let padded_header_len = (metadata_len + prefix_size + alignment_mask) & !alignment_mask;
140 let padded_metadata_len = padded_header_len - prefix_size;
141 let metadata_padding = padded_metadata_len - metadata_len;
142
143 MetadataLayout {
144 padded_header_len,
145 padded_metadata_len,
146 metadata_padding,
147 }
148 }
149}
150
151trait IpcMessageSink {
157 fn write_slice(&mut self, bytes: &[u8]) -> Result<(), ArrowError>;
158
159 fn write_vec(&mut self, bytes: Vec<u8>) -> Result<(), ArrowError> {
160 self.write_slice(&bytes)
161 }
162
163 fn write_encoded_buffer(&mut self, buffer: EncodedBuffer) -> Result<(), ArrowError> {
164 self.write_slice(buffer.as_slice())
165 }
166}
167
168trait IpcMessageSinkExt: IpcMessageSink {
170 fn write_padding(&mut self, len: usize) -> Result<(), ArrowError> {
171 self.write_slice(&PADDING[..len])
172 }
173
174 fn write_continuation(
176 &mut self,
177 write_options: &IpcWriteOptions,
178 metadata_len: i32,
179 ) -> Result<(), ArrowError> {
180 let mut buffer = [0; 8];
181 let len = match write_options.metadata_version {
182 crate::MetadataVersion::V1
183 | crate::MetadataVersion::V2
184 | crate::MetadataVersion::V3 => {
185 unreachable!("Options with the metadata version cannot be created")
186 }
187 crate::MetadataVersion::V4 => {
188 let metadata_len_bytes = metadata_len.to_le_bytes();
189 if !write_options.write_legacy_ipc_format {
190 buffer[..4].copy_from_slice(&CONTINUATION_MARKER);
192 buffer[4..].copy_from_slice(&metadata_len_bytes);
193 8
194 } else {
195 buffer[..4].copy_from_slice(&metadata_len_bytes);
196 4
197 }
198 }
199 crate::MetadataVersion::V5 => {
200 buffer[..4].copy_from_slice(&CONTINUATION_MARKER);
201 buffer[4..].copy_from_slice(&metadata_len.to_le_bytes());
202 8
203 }
204 z => panic!("Unsupported crate::MetadataVersion {z:?}"),
205 };
206 self.write_slice(&buffer[..len])
207 }
208
209 fn write_body_data(&mut self, data: Vec<u8>, alignment: u8) -> Result<usize, ArrowError> {
210 let len = data.len();
211 let pad_len = pad_to_alignment(alignment, len);
212 self.write_vec(data)?;
213 self.write_padding(pad_len)?;
214 Ok(len + pad_len)
215 }
216
217 fn write_encoded_data(
222 &mut self,
223 encoded: EncodedData,
224 write_options: &IpcWriteOptions,
225 ) -> Result<(usize, usize), ArrowError> {
226 let arrow_data_len = encoded.arrow_data.len();
227 if !arrow_data_len.is_multiple_of(usize::from(write_options.alignment)) {
228 return Err(ArrowError::MemoryError(
229 "Arrow data not aligned".to_string(),
230 ));
231 }
232
233 let metadata = encoded.ipc_message;
234 let metadata_len = metadata.len();
235 let layout = MetadataLayout::new(metadata_len, write_options);
236
237 self.write_continuation(write_options, layout.padded_metadata_len as i32)?;
238 self.write_vec(metadata)?;
239 self.write_padding(layout.metadata_padding)?;
240
241 let body_len = if arrow_data_len > 0 {
242 self.write_body_data(encoded.arrow_data, write_options.alignment)?
243 } else {
244 0
245 };
246
247 Ok((layout.padded_header_len, body_len))
248 }
249
250 fn write_eos(&mut self, write_options: &IpcWriteOptions) -> Result<(), ArrowError> {
252 self.write_continuation(write_options, 0)?;
253 Ok(())
254 }
255}
256
257impl<T: IpcMessageSink + ?Sized> IpcMessageSinkExt for T {}
258
259trait IpcRecordBatchSink: IpcMessageSinkExt {
261 fn write_record_batch(
275 &mut self,
276 metadata: &[u8],
277 encoded_buffers: &mut Vec<EncodedBuffer>,
278 body_len: usize,
279 write_options: &IpcWriteOptions,
280 ) -> Result<(usize, usize), ArrowError> {
281 let alignment = write_options.alignment;
282 let layout = MetadataLayout::new(metadata.len(), write_options);
283
284 self.write_continuation(write_options, layout.padded_metadata_len as i32)?;
285 self.write_slice(metadata)?;
286 self.write_padding(layout.metadata_padding)?;
287 for enc in encoded_buffers.drain(..) {
288 let len = enc.len();
289 self.write_encoded_buffer(enc)?;
290 self.write_padding(pad_to_alignment(alignment, len))?;
291 }
292
293 Ok((layout.padded_header_len, body_len))
294 }
295}
296
297impl<W> IpcMessageSink for W
298where
299 W: Write,
300{
301 fn write_slice(&mut self, bytes: &[u8]) -> Result<(), ArrowError> {
302 if !bytes.is_empty() {
303 self.write_all(bytes)?;
304 }
305 Ok(())
306 }
307}
308
309impl<W> IpcRecordBatchSink for W
310where
311 W: Write,
312{
313 fn write_record_batch(
314 &mut self,
315 metadata: &[u8],
316 encoded_buffers: &mut Vec<EncodedBuffer>,
317 body_len: usize,
318 write_options: &IpcWriteOptions,
319 ) -> Result<(usize, usize), ArrowError> {
320 let alignment = write_options.alignment;
321 let layout = MetadataLayout::new(metadata.len(), write_options);
322
323 self.write_continuation(write_options, layout.padded_metadata_len as i32)?;
324 self.write_all(metadata)?;
325 self.write_all(&PADDING[..layout.metadata_padding])?;
326 for enc in encoded_buffers.iter_mut() {
327 self.write_all(enc.as_slice())?;
328 self.write_all(&PADDING[..pad_to_alignment(alignment, enc.len())])?;
329 }
330 encoded_buffers.clear();
336
337 Ok((layout.padded_header_len, body_len))
338 }
339}
340
341struct Buffers<'a> {
343 out: &'a mut Vec<Buffer>,
344}
345
346impl IpcMessageSink for Buffers<'_> {
347 fn write_slice(&mut self, bytes: &[u8]) -> Result<(), ArrowError> {
348 if !bytes.is_empty() {
349 self.out.push(Buffer::from(bytes));
350 }
351 Ok(())
352 }
353
354 fn write_vec(&mut self, bytes: Vec<u8>) -> Result<(), ArrowError> {
355 if !bytes.is_empty() {
356 self.out.push(Buffer::from(bytes));
357 }
358 Ok(())
359 }
360
361 fn write_encoded_buffer(&mut self, buffer: EncodedBuffer) -> Result<(), ArrowError> {
362 match buffer {
363 EncodedBuffer::Raw(buffer) => self.out.push(buffer),
364 EncodedBuffer::Compressed(bytes) => self.out.push(Buffer::from(bytes)),
365 }
366 Ok(())
367 }
368}
369
370impl IpcRecordBatchSink for Buffers<'_> {}
371
372struct IpcWriteMetadata {
377 dictionary_block_sizes: Vec<(usize, usize)>,
380 padded_header_len: usize,
382 body_len: usize,
384}
385
386impl IpcWriteOptions {
387 pub fn try_with_compression(
392 mut self,
393 batch_compression_type: Option<crate::CompressionType>,
394 ) -> Result<Self, ArrowError> {
395 if matches!(batch_compression_type, Some(crate::CompressionType::ZSTD))
396 && self.batch_lz4_block_size.is_some()
397 {
398 return Err(ArrowError::InvalidArgumentError(
399 "lz4 frame block size does not apply to ZSTD compression".to_string(),
400 ));
401 }
402
403 self.batch_compression_type = batch_compression_type;
404
405 if self.batch_compression_type.is_some()
406 && self.metadata_version < crate::MetadataVersion::V5
407 {
408 return Err(ArrowError::InvalidArgumentError(
409 "Compression only supported in metadata v5 and above".to_string(),
410 ));
411 }
412 Ok(self)
413 }
414
415 pub fn try_with_compression_level(
420 mut self,
421 batch_compression_level: Option<i32>,
422 ) -> Result<Self, ArrowError> {
423 self.batch_compression_level = batch_compression_level;
424
425 if self.batch_compression_level.is_some()
426 && self.metadata_version < crate::MetadataVersion::V5
427 {
428 return Err(ArrowError::InvalidArgumentError(
429 "Compression only supported in metadata v5 and above".to_string(),
430 ));
431 }
432
433 match (self.batch_compression_type, self.batch_compression_level) {
434 (Some(crate::CompressionType::ZSTD), Some(level)) => {
435 return self.check_zstd_level(level);
436 }
437 (Some(crate::CompressionType::LZ4_FRAME), Some(_)) => {
438 return Err(ArrowError::InvalidArgumentError(
439 "LZ4 Frame compression does not support configurable compression levels"
440 .to_string(),
441 ));
442 }
443 _ => {}
444 }
445
446 Ok(self)
447 }
448
449 pub fn try_with_lz4_block_size(
459 mut self,
460 block_size: Option<usize>,
461 ) -> Result<Self, ArrowError> {
462 let block_size = block_size.map(|bytes| {
463 Lz4BlockSize::from_bytes(bytes).ok_or_else(|| {
464 ArrowError::InvalidArgumentError(format!(
465 "Unsupported lz4 block size {bytes}, must be one of 65536, 262144, 1048576, or 4194304 bytes"
466 ))
467 })
468 }).transpose()?;
469
470 if block_size.is_some() {
471 if self.metadata_version < crate::MetadataVersion::V5 {
472 return Err(ArrowError::InvalidArgumentError(
473 "Compression only supported in metadata v5 and above".to_string(),
474 ));
475 }
476
477 if matches!(
478 self.batch_compression_type,
479 Some(crate::CompressionType::ZSTD)
480 ) {
481 return Err(ArrowError::InvalidArgumentError(
482 "lz4 frame block size does not apply to ZSTD compression".to_string(),
483 ));
484 }
485 }
486
487 self.batch_lz4_block_size = block_size;
488 Ok(self)
489 }
490
491 pub(crate) fn lz4_block_size(&self) -> Lz4BlockSize {
493 self.batch_lz4_block_size.unwrap_or(Lz4BlockSize::DEFAULT)
494 }
495
496 #[cfg(not(feature = "zstd"))]
497 fn check_zstd_level(self, _level: i32) -> Result<Self, ArrowError> {
498 Err(ArrowError::InvalidArgumentError(
499 "zstd IPC compression requires the zstd feature".to_string(),
500 ))
501 }
502
503 #[cfg(feature = "zstd")]
504 fn check_zstd_level(self, level: i32) -> Result<Self, ArrowError> {
505 let range = zstd::compression_level_range();
506 if !range.contains(&level) {
507 return Err(ArrowError::InvalidArgumentError(format!(
508 "ZSTD compression level must be between {} and {}, got {}",
509 range.start(),
510 range.end(),
511 level,
512 )));
513 }
514
515 Ok(self)
516 }
517
518 pub fn try_new(
520 alignment: usize,
521 write_legacy_ipc_format: bool,
522 metadata_version: crate::MetadataVersion,
523 ) -> Result<Self, ArrowError> {
524 let alignment: u8 = match alignment {
525 8 => 8,
526 16 => 16,
527 32 => 32,
528 64 => 64,
529 _ => {
530 return Err(ArrowError::InvalidArgumentError(
531 "Alignment should be 8, 16, 32, or 64.".to_string(),
532 ));
533 }
534 };
535 match metadata_version {
536 crate::MetadataVersion::V1
537 | crate::MetadataVersion::V2
538 | crate::MetadataVersion::V3 => Err(ArrowError::InvalidArgumentError(
539 "Writing IPC metadata version 3 and lower not supported".to_string(),
540 )),
541 crate::MetadataVersion::V4 => Ok(Self {
542 alignment,
543 write_legacy_ipc_format,
544 metadata_version,
545 batch_compression_type: None,
546 batch_compression_level: None,
547 batch_lz4_block_size: None,
548 dictionary_handling: DictionaryHandling::default(),
549 }),
550 crate::MetadataVersion::V5 => {
551 if write_legacy_ipc_format {
552 Err(ArrowError::InvalidArgumentError(
553 "Legacy IPC format only supported on metadata version 4".to_string(),
554 ))
555 } else {
556 Ok(Self {
557 alignment,
558 write_legacy_ipc_format,
559 metadata_version,
560 batch_compression_type: None,
561 batch_compression_level: None,
562 batch_lz4_block_size: None,
563 dictionary_handling: DictionaryHandling::default(),
564 })
565 }
566 }
567 z => Err(ArrowError::InvalidArgumentError(format!(
568 "Unsupported crate::MetadataVersion {z:?}"
569 ))),
570 }
571 }
572
573 pub fn with_dictionary_handling(mut self, dictionary_handling: DictionaryHandling) -> Self {
575 self.dictionary_handling = dictionary_handling;
576 self
577 }
578}
579
580impl Default for IpcWriteOptions {
581 fn default() -> Self {
582 Self {
583 alignment: 64,
584 write_legacy_ipc_format: false,
585 metadata_version: crate::MetadataVersion::V5,
586 batch_compression_type: None,
587 batch_compression_level: None,
588 batch_lz4_block_size: None,
589 dictionary_handling: DictionaryHandling::default(),
590 }
591 }
592}
593
594#[derive(Debug, Default)]
595pub struct IpcDataGenerator {}
629
630struct DictionaryToEncode {
632 id: i64,
633 data: ArrayData,
634 is_delta: bool,
635}
636
637struct EncodedRecordBatchMeta {
644 fb_offset: flatbuffers::WIPOffset<crate::RecordBatch<'static>>,
646 body_len: usize,
649}
650
651impl IpcDataGenerator {
652 pub fn schema_to_bytes_with_dictionary_tracker(
655 &self,
656 schema: &Schema,
657 dictionary_tracker: &mut DictionaryTracker,
658 write_options: &IpcWriteOptions,
659 ) -> EncodedData {
660 let mut fbb = FlatBufferBuilder::new();
661 let schema = {
662 let fb = IpcSchemaEncoder::new()
663 .with_dictionary_tracker(dictionary_tracker)
664 .schema_to_fb_offset(&mut fbb, schema);
665 fb.as_union_value()
666 };
667
668 let mut message = crate::MessageBuilder::new(&mut fbb);
669 message.add_version(write_options.metadata_version);
670 message.add_header_type(crate::MessageHeader::Schema);
671 message.add_bodyLength(0);
672 message.add_header(schema);
673 let root = message.finish();
675 fbb.finish(root, None);
676
677 let metadata = fbb.finished_data();
678 EncodedData {
679 ipc_message: metadata.to_vec(),
680 arrow_data: vec![],
681 }
682 }
683
684 fn _collect_dict_updates<I: Iterator<Item = i64>>(
685 &self,
686 column: &ArrayRef,
687 dictionaries_to_encode: &mut Vec<DictionaryToEncode>,
688 dictionary_tracker: &mut DictionaryTracker,
689 write_options: &IpcWriteOptions,
690 dict_id: &mut I,
691 ) -> Result<(), ArrowError> {
692 match column.data_type() {
693 DataType::Struct(fields) => {
694 let s = as_struct_array(column);
695 for (field, column) in fields.iter().zip(s.columns()) {
696 self.collect_dict_updates(
697 field,
698 column,
699 dictionaries_to_encode,
700 dictionary_tracker,
701 write_options,
702 dict_id,
703 )?;
704 }
705 }
706 DataType::RunEndEncoded(_, values) => {
707 let data = column.to_data();
708 if data.child_data().len() != 2 {
709 return Err(ArrowError::InvalidArgumentError(format!(
710 "The run encoded array should have exactly two child arrays. Found {}",
711 data.child_data().len()
712 )));
713 }
714 let values_array = make_array(data.child_data()[1].clone());
717 self.collect_dict_updates(
718 values,
719 &values_array,
720 dictionaries_to_encode,
721 dictionary_tracker,
722 write_options,
723 dict_id,
724 )?;
725 }
726 DataType::List(field) => {
727 let list = as_list_array(column);
728 self.collect_dict_updates(
729 field,
730 list.values(),
731 dictionaries_to_encode,
732 dictionary_tracker,
733 write_options,
734 dict_id,
735 )?;
736 }
737 DataType::LargeList(field) => {
738 let list = as_large_list_array(column);
739 self.collect_dict_updates(
740 field,
741 list.values(),
742 dictionaries_to_encode,
743 dictionary_tracker,
744 write_options,
745 dict_id,
746 )?;
747 }
748 DataType::ListView(field) => {
749 let list = column.as_list_view::<i32>();
750 self.collect_dict_updates(
751 field,
752 list.values(),
753 dictionaries_to_encode,
754 dictionary_tracker,
755 write_options,
756 dict_id,
757 )?;
758 }
759 DataType::LargeListView(field) => {
760 let list = column.as_list_view::<i64>();
761 self.collect_dict_updates(
762 field,
763 list.values(),
764 dictionaries_to_encode,
765 dictionary_tracker,
766 write_options,
767 dict_id,
768 )?;
769 }
770 DataType::FixedSizeList(field, _) => {
771 let list = column
772 .as_any()
773 .downcast_ref::<FixedSizeListArray>()
774 .expect("Unable to downcast to fixed size list array");
775 self.collect_dict_updates(
776 field,
777 list.values(),
778 dictionaries_to_encode,
779 dictionary_tracker,
780 write_options,
781 dict_id,
782 )?;
783 }
784 DataType::Map(field, _) => {
785 let map_array = as_map_array(column);
786
787 let (keys, values) = match field.data_type() {
788 DataType::Struct(fields) if fields.len() == 2 => (&fields[0], &fields[1]),
789 _ => panic!("Incorrect field data type {:?}", field.data_type()),
790 };
791
792 self.collect_dict_updates(
794 keys,
795 map_array.keys(),
796 dictionaries_to_encode,
797 dictionary_tracker,
798 write_options,
799 dict_id,
800 )?;
801
802 self.collect_dict_updates(
804 values,
805 map_array.values(),
806 dictionaries_to_encode,
807 dictionary_tracker,
808 write_options,
809 dict_id,
810 )?;
811 }
812 DataType::Union(fields, _) => {
813 let union = as_union_array(column);
814 for (type_id, field) in fields.iter() {
815 let column = union.child(type_id);
816 self.collect_dict_updates(
817 field,
818 column,
819 dictionaries_to_encode,
820 dictionary_tracker,
821 write_options,
822 dict_id,
823 )?;
824 }
825 }
826 _ => (),
827 }
828
829 Ok(())
830 }
831
832 fn collect_dict_updates<I: Iterator<Item = i64>>(
836 &self,
837 field: &Field,
838 column: &ArrayRef,
839 dictionaries_to_encode: &mut Vec<DictionaryToEncode>,
840 dictionary_tracker: &mut DictionaryTracker,
841 write_options: &IpcWriteOptions,
842 dict_id_seq: &mut I,
843 ) -> Result<(), ArrowError> {
844 match column.data_type() {
845 DataType::Dictionary(_key_type, value_type) => {
846 if matches!(value_type.as_ref(), DataType::Dictionary(_, _)) {
847 return Err(ArrowError::InvalidArgumentError(format!(
848 "Arrow IPC field metadata cannot encode direct dictionary-of-dictionary values for field {:?}",
849 field.name()
850 )));
851 }
852
853 let dict_data = column.to_data();
854 let dict_values = &dict_data.child_data()[0];
855
856 let values = make_array(dict_data.child_data()[0].clone());
857
858 self._collect_dict_updates(
859 &values,
860 dictionaries_to_encode,
861 dictionary_tracker,
862 write_options,
863 dict_id_seq,
864 )?;
865
866 let dict_id = dict_id_seq.next().ok_or_else(|| {
870 ArrowError::IpcError(format!(
871 "no dict id for field {:?}: field.data_type={:?}, column.data_type={:?}",
872 field.name(),
873 field.data_type(),
874 column.data_type()
875 ))
876 })?;
877
878 match dictionary_tracker.insert_column(
879 dict_id,
880 column,
881 write_options.dictionary_handling,
882 )? {
883 DictionaryUpdate::None => {}
884 DictionaryUpdate::New | DictionaryUpdate::Replaced => {
885 dictionaries_to_encode.push(DictionaryToEncode {
886 id: dict_id,
887 data: dict_values.clone(),
888 is_delta: false,
889 });
890 }
891 DictionaryUpdate::Delta(data) => {
892 dictionaries_to_encode.push(DictionaryToEncode {
893 id: dict_id,
894 data,
895 is_delta: true,
896 });
897 }
898 }
899 }
900 _ => self._collect_dict_updates(
901 column,
902 dictionaries_to_encode,
903 dictionary_tracker,
904 write_options,
905 dict_id_seq,
906 )?,
907 }
908
909 Ok(())
910 }
911
912 pub fn encode(
916 &self,
917 batch: &RecordBatch,
918 dictionary_tracker: &mut DictionaryTracker,
919 write_options: &IpcWriteOptions,
920 ipc_write_context: &mut IpcWriteContext,
921 ) -> Result<(Vec<EncodedData>, EncodedData), ArrowError> {
922 let dictionaries = self.collect_all_dicts(batch, dictionary_tracker, write_options)?;
923 let mut encoded_dictionaries = Vec::with_capacity(dictionaries.len());
924 for dict in dictionaries {
925 encoded_dictionaries.push(self.dictionary_batch_to_bytes(
926 dict,
927 write_options,
928 ipc_write_context,
929 )?);
930 }
931 let mut arrow_data = ipc_write_context.scratch();
932 self.record_batch_to_bytes(
933 batch,
934 write_options,
935 ipc_write_context,
936 &mut IpcBodySink::Write(&mut arrow_data),
937 )?;
938 ipc_write_context.reserve_scratch_with_capacity(arrow_data.capacity());
939 Ok((
940 encoded_dictionaries,
941 EncodedData {
942 ipc_message: ipc_write_context.mut_fbb().finished_data().to_vec(),
943 arrow_data,
944 },
945 ))
946 }
947
948 fn collect_all_dicts(
950 &self,
951 batch: &RecordBatch,
952 dictionary_tracker: &mut DictionaryTracker,
953 write_options: &IpcWriteOptions,
954 ) -> Result<Vec<DictionaryToEncode>, ArrowError> {
955 let schema = batch.schema();
956 let mut dictionaries = Vec::with_capacity(schema.flattened_fields().len());
957 let mut dict_id = dictionary_tracker.dict_ids.clone().into_iter();
958 for (i, field) in schema.fields().iter().enumerate() {
959 self.collect_dict_updates(
960 field,
961 batch.column(i),
962 &mut dictionaries,
963 dictionary_tracker,
964 write_options,
965 &mut dict_id,
966 )?;
967 }
968 Ok(dictionaries)
969 }
970
971 fn write<W: Write>(
975 &self,
976 batch: &RecordBatch,
977 dictionary_tracker: &mut DictionaryTracker,
978 write_options: &IpcWriteOptions,
979 ipc_write_context: &mut IpcWriteContext,
980 writer: &mut W,
981 ) -> Result<IpcWriteMetadata, ArrowError> {
982 self.write_to_sink(
983 batch,
984 dictionary_tracker,
985 write_options,
986 ipc_write_context,
987 writer,
988 )
989 }
990
991 fn encode_to_buffers(
994 &self,
995 batch: &RecordBatch,
996 dictionary_tracker: &mut DictionaryTracker,
997 write_options: &IpcWriteOptions,
998 ipc_write_context: &mut IpcWriteContext,
999 out: &mut Vec<Buffer>,
1000 ) -> Result<IpcWriteMetadata, ArrowError> {
1001 let mut sink = Buffers { out };
1002 self.write_to_sink(
1003 batch,
1004 dictionary_tracker,
1005 write_options,
1006 ipc_write_context,
1007 &mut sink,
1008 )
1009 }
1010
1011 fn write_to_sink<S: IpcRecordBatchSink>(
1012 &self,
1013 batch: &RecordBatch,
1014 dictionary_tracker: &mut DictionaryTracker,
1015 write_options: &IpcWriteOptions,
1016 ipc_write_context: &mut IpcWriteContext,
1017 sink: &mut S,
1018 ) -> Result<IpcWriteMetadata, ArrowError> {
1019 let dictionaries = self.collect_all_dicts(batch, dictionary_tracker, write_options)?;
1020 let mut dictionary_block_sizes = Vec::with_capacity(dictionaries.len());
1021
1022 let capacity = batch
1023 .columns()
1024 .iter()
1025 .map(|a| estimate_encoded_buffer_count(a.data_type()))
1026 .sum();
1027 let mut encoded_buffers: Vec<EncodedBuffer> = Vec::with_capacity(capacity);
1028
1029 for dict in &dictionaries {
1030 encoded_buffers.clear();
1031
1032 let body_len = self.dictionary_batch_to_sink(
1033 dict,
1034 write_options,
1035 ipc_write_context,
1036 &mut IpcBodySink::Collect(&mut encoded_buffers),
1037 )?;
1038
1039 dictionary_block_sizes.push(sink.write_record_batch(
1040 ipc_write_context.mut_fbb().finished_data(),
1041 &mut encoded_buffers,
1042 body_len,
1043 write_options,
1044 )?);
1045 }
1046
1047 encoded_buffers.clear();
1048 let body_len = self.record_batch_to_bytes(
1049 batch,
1050 write_options,
1051 ipc_write_context,
1052 &mut IpcBodySink::Collect(&mut encoded_buffers),
1053 )?;
1054
1055 let (padded_header_len, body_len) = sink.write_record_batch(
1056 ipc_write_context.mut_fbb().finished_data(),
1057 &mut encoded_buffers,
1058 body_len,
1059 write_options,
1060 )?;
1061
1062 Ok(IpcWriteMetadata {
1063 dictionary_block_sizes,
1064 padded_header_len,
1065 body_len,
1066 })
1067 }
1068
1069 #[deprecated(since = "57.0.0", note = "Use `encode` instead")]
1073 pub fn encoded_batch(
1074 &self,
1075 batch: &RecordBatch,
1076 dictionary_tracker: &mut DictionaryTracker,
1077 write_options: &IpcWriteOptions,
1078 ) -> Result<(Vec<EncodedData>, EncodedData), ArrowError> {
1079 self.encode(
1080 batch,
1081 dictionary_tracker,
1082 write_options,
1083 &mut Default::default(),
1084 )
1085 }
1086
1087 fn record_batch_to_bytes(
1098 &self,
1099 batch: &RecordBatch,
1100 write_options: &IpcWriteOptions,
1101 ipc_write_context: &mut IpcWriteContext,
1102 sink: &mut IpcBodySink<'_>,
1103 ) -> Result<usize, ArrowError> {
1104 ipc_write_context.mut_fbb().reset();
1106
1107 let EncodedRecordBatchMeta {
1108 fb_offset: record_batch,
1109 body_len,
1110 } = self.encode_record_batch_data(
1111 batch.columns().iter().map(|array| array.to_data()),
1112 batch.num_rows() as i64,
1113 write_options,
1114 ipc_write_context,
1115 sink,
1116 )?;
1117
1118 let fbb = ipc_write_context.mut_fbb();
1119 let mut message = crate::MessageBuilder::new(fbb);
1120 message.add_version(write_options.metadata_version);
1121 message.add_header_type(crate::MessageHeader::RecordBatch);
1122 message.add_bodyLength(body_len as i64);
1123 message.add_header(record_batch.as_union_value());
1124 let root = message.finish();
1125 fbb.finish(root, None);
1126
1127 Ok(body_len)
1128 }
1129
1130 fn dictionary_batch_to_bytes(
1133 &self,
1134 dict: DictionaryToEncode,
1135 write_options: &IpcWriteOptions,
1136 ipc_write_context: &mut IpcWriteContext,
1137 ) -> Result<EncodedData, ArrowError> {
1138 let mut arrow_data: Vec<u8> = vec![];
1139 self.dictionary_batch_to_sink(
1140 &dict,
1141 write_options,
1142 ipc_write_context,
1143 &mut IpcBodySink::Write(&mut arrow_data),
1144 )?;
1145
1146 Ok(EncodedData {
1147 ipc_message: ipc_write_context.mut_fbb().finished_data().to_vec(),
1148 arrow_data,
1149 })
1150 }
1151
1152 fn dictionary_batch_to_sink(
1159 &self,
1160 dict: &DictionaryToEncode,
1161 write_options: &IpcWriteOptions,
1162 ipc_write_context: &mut IpcWriteContext,
1163 sink: &mut IpcBodySink<'_>,
1164 ) -> Result<usize, ArrowError> {
1165 ipc_write_context.mut_fbb().reset();
1167
1168 let EncodedRecordBatchMeta {
1172 fb_offset: record_batch,
1173 body_len,
1174 } = self.encode_record_batch_data(
1175 std::iter::once(dict.data.clone()),
1176 dict.data.len() as i64,
1177 write_options,
1178 ipc_write_context,
1179 sink,
1180 )?;
1181
1182 let fbb = ipc_write_context.mut_fbb();
1183 let root = {
1184 let mut batch_builder = crate::DictionaryBatchBuilder::new(fbb);
1185 batch_builder.add_id(dict.id);
1186 batch_builder.add_data(record_batch);
1187 batch_builder.add_isDelta(dict.is_delta);
1188 batch_builder.finish().as_union_value()
1189 };
1190
1191 let root = {
1192 let mut message_builder = crate::MessageBuilder::new(fbb);
1193 message_builder.add_version(write_options.metadata_version);
1194 message_builder.add_header_type(crate::MessageHeader::DictionaryBatch);
1195 message_builder.add_bodyLength(body_len as i64);
1196 message_builder.add_header(root);
1197 message_builder.finish()
1198 };
1199
1200 fbb.finish(root, None);
1201
1202 Ok(body_len)
1203 }
1204
1205 #[inline]
1218 fn encode_record_batch_data(
1219 &self,
1220 columns: impl IntoIterator<Item = ArrayData>,
1221 row_count: i64,
1222 write_options: &IpcWriteOptions,
1223 ipc_write_context: &mut IpcWriteContext,
1224 sink: &mut IpcBodySink<'_>,
1225 ) -> Result<EncodedRecordBatchMeta, ArrowError> {
1226 let batch_compression_type = write_options.batch_compression_type;
1227
1228 ipc_write_context.set_lz4_block_size(write_options.lz4_block_size());
1231
1232 let compression = batch_compression_type.map(|batch_compression_type| {
1233 let fbb = ipc_write_context.mut_fbb();
1234 let mut c = crate::BodyCompressionBuilder::new(fbb);
1235 c.add_method(crate::BodyCompressionMethod::BUFFER);
1236 c.add_codec(batch_compression_type);
1237 c.finish()
1238 });
1239
1240 let batch_compression_level = write_options.batch_compression_level;
1241 let compression_codec: Option<CompressionCodec> = batch_compression_type
1242 .map(|compression_type| match batch_compression_level {
1243 Some(level) => {
1244 CompressionCodec::try_new_with_compression_level(compression_type, level)
1245 }
1246 None => compression_type.try_into(),
1247 })
1248 .transpose()?;
1249
1250 let mut meta = IpcMetadataBuilder::default();
1251 let mut variadic_buffer_counts: Vec<i64> = vec![];
1252 let mut offset = 0i64;
1253
1254 for array_data in columns {
1255 offset = write_array_data(
1256 &array_data,
1257 &mut meta,
1258 sink,
1259 offset,
1260 compression_codec,
1261 ipc_write_context,
1262 write_options,
1263 )?;
1264 append_variadic_buffer_counts(&mut variadic_buffer_counts, &array_data);
1265 }
1266
1267 debug_assert_eq!(offset % write_options.alignment as i64, 0);
1270 let body_len = offset as usize;
1271
1272 let fbb = ipc_write_context.mut_fbb();
1273 let buffers = fbb.create_vector(&meta.buffers);
1274 let nodes = fbb.create_vector(&meta.nodes);
1275 let variadic_buffer = if variadic_buffer_counts.is_empty() {
1276 None
1277 } else {
1278 Some(fbb.create_vector(&variadic_buffer_counts))
1279 };
1280
1281 let mut batch_builder = crate::RecordBatchBuilder::new(fbb);
1282 batch_builder.add_length(row_count);
1283 batch_builder.add_nodes(nodes);
1284 batch_builder.add_buffers(buffers);
1285 if let Some(c) = compression {
1286 batch_builder.add_compression(c);
1287 }
1288 if let Some(v) = variadic_buffer {
1289 batch_builder.add_variadicBufferCounts(v);
1290 }
1291 let record_batch = batch_builder.finish();
1292
1293 Ok(EncodedRecordBatchMeta {
1294 fb_offset: record_batch,
1295 body_len,
1296 })
1297 }
1298}
1299
1300fn ensure_supported_ipc_schema(schema: &Schema) -> Result<(), ArrowError> {
1301 schema
1302 .fields()
1303 .iter()
1304 .try_for_each(|field| ensure_supported_ipc_data_type(field.name(), field.data_type()))
1305}
1306
1307fn ensure_supported_ipc_data_type(
1308 field_name: &str,
1309 data_type: &DataType,
1310) -> Result<(), ArrowError> {
1311 match data_type {
1312 DataType::Dictionary(_, value_type)
1313 if matches!(value_type.as_ref(), DataType::Dictionary(_, _)) =>
1314 {
1315 Err(ArrowError::InvalidArgumentError(format!(
1316 "Arrow IPC field metadata cannot encode direct dictionary-of-dictionary values for field {field_name:?}"
1317 )))
1318 }
1319 DataType::Dictionary(_, value_type) => {
1320 ensure_supported_ipc_data_type(field_name, value_type)
1321 }
1322 DataType::Struct(fields) => fields
1323 .iter()
1324 .try_for_each(|field| ensure_supported_ipc_data_type(field.name(), field.data_type())),
1325 DataType::RunEndEncoded(_, field)
1326 | DataType::List(field)
1327 | DataType::LargeList(field)
1328 | DataType::ListView(field)
1329 | DataType::LargeListView(field)
1330 | DataType::FixedSizeList(field, _)
1331 | DataType::Map(field, _) => {
1332 ensure_supported_ipc_data_type(field.name(), field.data_type())
1333 }
1334 DataType::Union(fields, _) => fields.iter().try_for_each(|(_, field)| {
1335 ensure_supported_ipc_data_type(field.name(), field.data_type())
1336 }),
1337 _ => Ok(()),
1338 }
1339}
1340
1341fn append_variadic_buffer_counts(counts: &mut Vec<i64>, array: &ArrayData) {
1342 match array.data_type() {
1343 DataType::BinaryView | DataType::Utf8View => {
1344 counts.push(array.buffers().len() as i64 - 1);
1347 }
1348 DataType::Dictionary(_, _) => {
1349 }
1352 _ => {
1353 for child in array.child_data() {
1354 append_variadic_buffer_counts(counts, child)
1355 }
1356 }
1357 }
1358}
1359
1360pub(crate) fn unslice_run_array(arr: ArrayData) -> Result<ArrayData, ArrowError> {
1361 match arr.data_type() {
1362 DataType::RunEndEncoded(k, _) => match k.data_type() {
1363 DataType::Int16 => {
1364 Ok(into_zero_offset_run_array(RunArray::<Int16Type>::from(arr))?.into_data())
1365 }
1366 DataType::Int32 => {
1367 Ok(into_zero_offset_run_array(RunArray::<Int32Type>::from(arr))?.into_data())
1368 }
1369 DataType::Int64 => {
1370 Ok(into_zero_offset_run_array(RunArray::<Int64Type>::from(arr))?.into_data())
1371 }
1372 d => unreachable!("Unexpected data type {d}"),
1373 },
1374 d => Err(ArrowError::InvalidArgumentError(format!(
1375 "The given array is not a run array. Data type of given array: {d}"
1376 ))),
1377 }
1378}
1379
1380fn into_zero_offset_run_array<R: RunEndIndexType>(
1383 run_array: RunArray<R>,
1384) -> Result<RunArray<R>, ArrowError> {
1385 if run_array.is_empty() {
1387 return Ok(ArrayData::new_empty(run_array.data_type()).into());
1388 }
1389
1390 let run_ends = run_array.run_ends();
1391 if run_ends.offset() == 0 && run_ends.max_value() == run_ends.len() {
1392 return Ok(run_array);
1393 }
1394
1395 let start_physical_index = run_ends.get_start_physical_index();
1397
1398 let end_physical_index = run_ends.get_end_physical_index();
1400
1401 let physical_length = end_physical_index - start_physical_index + 1;
1402
1403 let offset = R::Native::usize_as(run_ends.offset());
1405 let mut run_ends_values = Vec::<R::Native>::with_capacity(physical_length);
1406 for run_end_value in &run_ends.values()[start_physical_index..end_physical_index] {
1407 run_ends_values.push(run_end_value.sub_wrapping(offset));
1408 }
1409 run_ends_values.push(R::Native::from_usize(run_array.len()).unwrap());
1410 let offset_buffer = Buffer::from_vec(run_ends_values);
1411 let new_run_ends = unsafe {
1412 ArrayDataBuilder::new(R::DATA_TYPE)
1415 .len(physical_length)
1416 .add_buffer(offset_buffer)
1417 .build_unchecked()
1418 };
1419
1420 let new_values = run_array
1422 .values()
1423 .slice(start_physical_index, physical_length)
1424 .into_data();
1425
1426 let builder = ArrayDataBuilder::new(run_array.data_type().clone())
1427 .len(run_array.len())
1428 .add_child_data(new_run_ends)
1429 .add_child_data(new_values);
1430 let array_data = unsafe {
1431 builder.build_unchecked()
1434 };
1435 Ok(array_data.into())
1436}
1437
1438#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
1440pub enum DictionaryHandling {
1441 #[default]
1443 Resend,
1444 Delta,
1450}
1451
1452#[derive(Debug, Clone)]
1454pub enum DictionaryUpdate {
1455 None,
1458 New,
1460 Replaced,
1462 Delta(ArrayData),
1464}
1465
1466#[derive(Debug)]
1472pub struct DictionaryTracker {
1473 written: HashMap<i64, ArrayData>,
1475 dict_ids: Vec<i64>,
1476 error_on_replacement: bool,
1477}
1478
1479impl DictionaryTracker {
1480 pub fn new(error_on_replacement: bool) -> Self {
1486 Self {
1487 written: HashMap::new(),
1488 dict_ids: Vec::new(),
1489 error_on_replacement,
1490 }
1491 }
1492
1493 pub fn next_dict_id(&mut self) -> i64 {
1495 let next = self
1496 .dict_ids
1497 .last()
1498 .copied()
1499 .map(|i| i + 1)
1500 .unwrap_or_default();
1501
1502 self.dict_ids.push(next);
1503 next
1504 }
1505
1506 pub fn dict_id(&mut self) -> &[i64] {
1509 &self.dict_ids
1510 }
1511
1512 #[deprecated(since = "56.1.0", note = "Use `insert_column` instead")]
1522 pub fn insert(&mut self, dict_id: i64, column: &ArrayRef) -> Result<bool, ArrowError> {
1523 let dict_data = column.to_data();
1524 let dict_values = &dict_data.child_data()[0];
1525
1526 if let Some(last) = self.written.get(&dict_id) {
1528 if ArrayData::ptr_eq(&last.child_data()[0], dict_values) {
1529 return Ok(false);
1531 }
1532 if self.error_on_replacement {
1533 if last.child_data()[0] == *dict_values {
1535 return Ok(false);
1537 }
1538 return Err(ArrowError::InvalidArgumentError(
1539 "Dictionary replacement detected when writing IPC file format. \
1540 Arrow IPC files only support a single dictionary for a given field \
1541 across all batches."
1542 .to_string(),
1543 ));
1544 }
1545 }
1546
1547 self.written.insert(dict_id, dict_data);
1548 Ok(true)
1549 }
1550
1551 pub fn insert_column(
1567 &mut self,
1568 dict_id: i64,
1569 column: &ArrayRef,
1570 dict_handling: DictionaryHandling,
1571 ) -> Result<DictionaryUpdate, ArrowError> {
1572 let new_data = column.to_data();
1573 let new_values = &new_data.child_data()[0];
1574
1575 let Some(old) = self.written.get(&dict_id) else {
1577 self.written.insert(dict_id, new_data);
1578 return Ok(DictionaryUpdate::New);
1579 };
1580
1581 let old_values = &old.child_data()[0];
1584 if ArrayData::ptr_eq(old_values, new_values) {
1585 return Ok(DictionaryUpdate::None);
1586 }
1587
1588 let comparison = compare_dictionaries(old_values, new_values);
1590 if matches!(comparison, DictionaryComparison::Equal) {
1591 return Ok(DictionaryUpdate::None);
1592 }
1593
1594 const REPLACEMENT_ERROR: &str = "Dictionary replacement detected when writing IPC file format. \
1595 Arrow IPC files only support a single dictionary for a given field \
1596 across all batches.";
1597
1598 match comparison {
1599 DictionaryComparison::NotEqual => {
1600 if self.error_on_replacement {
1601 return Err(ArrowError::InvalidArgumentError(
1602 REPLACEMENT_ERROR.to_string(),
1603 ));
1604 }
1605
1606 self.written.insert(dict_id, new_data);
1607 Ok(DictionaryUpdate::Replaced)
1608 }
1609 DictionaryComparison::Delta => match dict_handling {
1610 DictionaryHandling::Resend => {
1611 if self.error_on_replacement {
1612 return Err(ArrowError::InvalidArgumentError(
1613 REPLACEMENT_ERROR.to_string(),
1614 ));
1615 }
1616
1617 self.written.insert(dict_id, new_data);
1618 Ok(DictionaryUpdate::Replaced)
1619 }
1620 DictionaryHandling::Delta => {
1621 let delta =
1622 new_values.slice(old_values.len(), new_values.len() - old_values.len());
1623 self.written.insert(dict_id, new_data);
1624 Ok(DictionaryUpdate::Delta(delta))
1625 }
1626 },
1627 DictionaryComparison::Equal => unreachable!("Already checked equal case"),
1628 }
1629 }
1630
1631 pub fn clear(&mut self) {
1637 self.dict_ids.clear();
1638 self.written.clear();
1639 }
1640}
1641
1642#[derive(Debug, Clone)]
1644enum DictionaryComparison {
1645 NotEqual,
1647 Equal,
1649 Delta,
1652}
1653
1654fn compare_dictionaries(old: &ArrayData, new: &ArrayData) -> DictionaryComparison {
1656 let existing_len = old.len();
1658 let new_len = new.len();
1659 if existing_len == new_len {
1660 return if *old == *new {
1661 DictionaryComparison::Equal
1662 } else {
1663 DictionaryComparison::NotEqual
1664 };
1665 }
1666
1667 if new_len < existing_len {
1669 return DictionaryComparison::NotEqual;
1670 }
1671
1672 if new.slice(0, existing_len) == *old {
1674 return DictionaryComparison::Delta;
1675 }
1676
1677 DictionaryComparison::NotEqual
1678}
1679
1680pub struct FileWriter<W> {
1703 writer: W,
1705 write_options: IpcWriteOptions,
1707 schema: SchemaRef,
1709 block_offsets: usize,
1711 dictionary_blocks: Vec<crate::Block>,
1713 record_blocks: Vec<crate::Block>,
1715 finished: bool,
1717 dictionary_tracker: DictionaryTracker,
1719 custom_metadata: Metadata,
1721
1722 data_gen: IpcDataGenerator,
1723
1724 ipc_write_context: IpcWriteContext,
1728}
1729
1730impl<W: Write> FileWriter<BufWriter<W>> {
1731 pub fn try_new_buffered(writer: W, schema: &Schema) -> Result<Self, ArrowError> {
1735 Self::try_new(BufWriter::new(writer), schema)
1736 }
1737}
1738
1739impl<W: Write> FileWriter<W> {
1740 pub fn try_new(writer: W, schema: &Schema) -> Result<Self, ArrowError> {
1748 let write_options = IpcWriteOptions::default();
1749 Self::try_new_with_options(writer, schema, write_options)
1750 }
1751
1752 pub fn try_new_with_options(
1760 mut writer: W,
1761 schema: &Schema,
1762 write_options: IpcWriteOptions,
1763 ) -> Result<Self, ArrowError> {
1764 ensure_supported_ipc_schema(schema)?;
1765
1766 let data_gen = IpcDataGenerator::default();
1767 let pad_len = pad_to_alignment(write_options.alignment, super::ARROW_MAGIC.len());
1769 let header_size = super::ARROW_MAGIC.len() + pad_len;
1770 writer.write_all(&super::ARROW_MAGIC)?;
1771 writer.write_all(&PADDING[..pad_len])?;
1772 let mut dictionary_tracker = DictionaryTracker::new(true);
1774 let encoded_message = data_gen.schema_to_bytes_with_dictionary_tracker(
1775 schema,
1776 &mut dictionary_tracker,
1777 &write_options,
1778 );
1779 let (meta, data) = write_message(&mut writer, encoded_message, &write_options)?;
1780 Ok(Self {
1781 writer,
1782 write_options,
1783 schema: Arc::new(schema.clone()),
1784 block_offsets: meta + data + header_size,
1785 dictionary_blocks: vec![],
1786 record_blocks: vec![],
1787 finished: false,
1788 dictionary_tracker,
1789 custom_metadata: Default::default(),
1790 data_gen,
1791 ipc_write_context: IpcWriteContext::default(),
1792 })
1793 }
1794
1795 pub fn write_metadata(&mut self, key: impl Into<String>, value: impl Into<String>) {
1797 self.custom_metadata.insert(key.into(), value.into());
1798 }
1799
1800 pub fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
1802 if self.finished {
1803 return Err(ArrowError::IpcError(
1804 "Cannot write record batch to file writer as it is closed".to_string(),
1805 ));
1806 }
1807
1808 let meta = self.data_gen.write(
1809 batch,
1810 &mut self.dictionary_tracker,
1811 &self.write_options,
1812 &mut self.ipc_write_context,
1813 &mut self.writer,
1814 )?;
1815
1816 for (header_len, body_len) in meta.dictionary_block_sizes {
1817 let block = crate::Block::new(
1818 self.block_offsets as i64,
1819 header_len as i32,
1820 body_len as i64,
1821 );
1822 self.dictionary_blocks.push(block);
1823 self.block_offsets += header_len + body_len;
1824 }
1825
1826 let block = crate::Block::new(
1828 self.block_offsets as i64,
1829 meta.padded_header_len as i32,
1830 meta.body_len as i64,
1831 );
1832 self.record_blocks.push(block);
1833 self.block_offsets += meta.padded_header_len + meta.body_len;
1834 Ok(())
1835 }
1836
1837 pub fn finish(&mut self) -> Result<(), ArrowError> {
1839 if self.finished {
1840 return Err(ArrowError::IpcError(
1841 "Cannot write footer to file writer as it is closed".to_string(),
1842 ));
1843 }
1844
1845 self.writer.write_eos(&self.write_options)?;
1847
1848 let mut fbb = FlatBufferBuilder::new();
1849 let dictionaries = fbb.create_vector(&self.dictionary_blocks);
1850 let record_batches = fbb.create_vector(&self.record_blocks);
1851
1852 self.dictionary_tracker.clear();
1854 let schema = IpcSchemaEncoder::new()
1855 .with_dictionary_tracker(&mut self.dictionary_tracker)
1856 .schema_to_fb_offset(&mut fbb, &self.schema);
1857 let fb_custom_metadata = (!self.custom_metadata.is_empty())
1858 .then(|| crate::convert::metadata_to_fb(&mut fbb, &self.custom_metadata));
1859
1860 let root = {
1861 let mut footer_builder = crate::FooterBuilder::new(&mut fbb);
1862 footer_builder.add_version(self.write_options.metadata_version);
1863 footer_builder.add_schema(schema);
1864 footer_builder.add_dictionaries(dictionaries);
1865 footer_builder.add_recordBatches(record_batches);
1866 if let Some(fb_custom_metadata) = fb_custom_metadata {
1867 footer_builder.add_custom_metadata(fb_custom_metadata);
1868 }
1869 footer_builder.finish()
1870 };
1871 fbb.finish(root, None);
1872 let footer_data = fbb.finished_data();
1873 self.writer.write_all(footer_data)?;
1874 self.writer
1875 .write_all(&(footer_data.len() as i32).to_le_bytes())?;
1876 self.writer.write_all(&super::ARROW_MAGIC)?;
1877 self.writer.flush()?;
1878 self.finished = true;
1879
1880 Ok(())
1881 }
1882
1883 pub fn schema(&self) -> &SchemaRef {
1885 &self.schema
1886 }
1887
1888 pub fn get_ref(&self) -> &W {
1890 &self.writer
1891 }
1892
1893 pub fn get_mut(&mut self) -> &mut W {
1897 &mut self.writer
1898 }
1899
1900 pub fn flush(&mut self) -> Result<(), ArrowError> {
1904 self.writer.flush()?;
1905 Ok(())
1906 }
1907
1908 pub fn into_inner(mut self) -> Result<W, ArrowError> {
1917 if !self.finished {
1918 self.finish()?;
1920 }
1921 Ok(self.writer)
1922 }
1923}
1924
1925impl<W: Write> RecordBatchWriter for FileWriter<W> {
1926 fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
1927 self.write(batch)
1928 }
1929
1930 fn close(mut self) -> Result<(), ArrowError> {
1931 self.finish()
1932 }
1933}
1934
1935pub struct StreamEncoder {
1964 schema: Schema,
1965 write_options: IpcWriteOptions,
1967 schema_encoded: bool,
1969 dictionary_tracker: DictionaryTracker,
1971 data_gen: IpcDataGenerator,
1972 ipc_write_context: IpcWriteContext,
1973}
1974
1975impl StreamEncoder {
1976 pub fn try_new(schema: &Schema) -> Result<Self, ArrowError> {
1978 let write_options = IpcWriteOptions::default();
1979 Self::try_new_with_options(schema, write_options)
1980 }
1981
1982 pub fn try_new_with_options(
1984 schema: &Schema,
1985 write_options: IpcWriteOptions,
1986 ) -> Result<Self, ArrowError> {
1987 ensure_supported_ipc_schema(schema)?;
1988
1989 Ok(Self {
1990 schema: schema.clone(),
1991 write_options,
1992 schema_encoded: false,
1993 dictionary_tracker: DictionaryTracker::new(false),
1994 data_gen: IpcDataGenerator::default(),
1995 ipc_write_context: IpcWriteContext::default(),
1996 })
1997 }
1998
1999 pub fn encode(&mut self, batch: &RecordBatch) -> Result<Vec<Buffer>, ArrowError> {
2009 let mut out = vec![];
2010 self.encode_schema(&mut out)?;
2011 self.data_gen.encode_to_buffers(
2012 batch,
2013 &mut self.dictionary_tracker,
2014 &self.write_options,
2015 &mut self.ipc_write_context,
2016 &mut out,
2017 )?;
2018 Ok(out)
2019 }
2020
2021 pub fn finish(mut self) -> Result<Vec<Buffer>, ArrowError> {
2030 let mut out = vec![];
2031 self.encode_schema(&mut out)?;
2032 let mut sink = Buffers { out: &mut out };
2033 sink.write_eos(&self.write_options)?;
2034 Ok(out)
2035 }
2036
2037 fn encode_schema(&mut self, out: &mut Vec<Buffer>) -> Result<(), ArrowError> {
2038 if !self.schema_encoded {
2039 let encoded_message = self.data_gen.schema_to_bytes_with_dictionary_tracker(
2040 &self.schema,
2041 &mut self.dictionary_tracker,
2042 &self.write_options,
2043 );
2044 let mut sink = Buffers { out };
2045 sink.write_encoded_data(encoded_message, &self.write_options)?;
2046 self.schema_encoded = true;
2047 }
2048 Ok(())
2049 }
2050}
2051
2052pub struct StreamWriter<W> {
2126 writer: W,
2128 write_options: IpcWriteOptions,
2130 finished: bool,
2132 dictionary_tracker: DictionaryTracker,
2134
2135 data_gen: IpcDataGenerator,
2136
2137 ipc_write_context: IpcWriteContext,
2139}
2140
2141impl<W: Write> StreamWriter<BufWriter<W>> {
2142 pub fn try_new_buffered(writer: W, schema: &Schema) -> Result<Self, ArrowError> {
2146 Self::try_new(BufWriter::new(writer), schema)
2147 }
2148}
2149
2150impl<W: Write> StreamWriter<W> {
2151 pub fn try_new(writer: W, schema: &Schema) -> Result<Self, ArrowError> {
2159 let write_options = IpcWriteOptions::default();
2160 Self::try_new_with_options(writer, schema, write_options)
2161 }
2162
2163 pub fn try_new_with_options(
2169 mut writer: W,
2170 schema: &Schema,
2171 write_options: IpcWriteOptions,
2172 ) -> Result<Self, ArrowError> {
2173 ensure_supported_ipc_schema(schema)?;
2174
2175 let data_gen = IpcDataGenerator::default();
2176 let mut dictionary_tracker = DictionaryTracker::new(false);
2177
2178 let encoded_message = data_gen.schema_to_bytes_with_dictionary_tracker(
2180 schema,
2181 &mut dictionary_tracker,
2182 &write_options,
2183 );
2184 write_message(&mut writer, encoded_message, &write_options)?;
2185 Ok(Self {
2186 writer,
2187 write_options,
2188 finished: false,
2189 dictionary_tracker,
2190 data_gen,
2191 ipc_write_context: IpcWriteContext::default(),
2192 })
2193 }
2194
2195 pub fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
2197 if self.finished {
2198 return Err(ArrowError::IpcError(
2199 "Cannot write record batch to stream writer as it is closed".to_string(),
2200 ));
2201 }
2202
2203 self.data_gen.write(
2204 batch,
2205 &mut self.dictionary_tracker,
2206 &self.write_options,
2207 &mut self.ipc_write_context,
2208 &mut self.writer,
2209 )?;
2210 Ok(())
2211 }
2212
2213 pub fn finish(&mut self) -> Result<(), ArrowError> {
2215 if self.finished {
2216 return Err(ArrowError::IpcError(
2217 "Cannot write footer to stream writer as it is closed".to_string(),
2218 ));
2219 }
2220
2221 self.writer.write_eos(&self.write_options)?;
2222 self.writer.flush()?;
2223
2224 self.finished = true;
2225
2226 Ok(())
2227 }
2228
2229 pub fn get_ref(&self) -> &W {
2231 &self.writer
2232 }
2233
2234 pub fn get_mut(&mut self) -> &mut W {
2238 &mut self.writer
2239 }
2240
2241 pub fn flush(&mut self) -> Result<(), ArrowError> {
2245 self.writer.flush()?;
2246 Ok(())
2247 }
2248
2249 pub fn into_inner(mut self) -> Result<W, ArrowError> {
2287 if !self.finished {
2288 self.finish()?;
2290 }
2291 Ok(self.writer)
2292 }
2293}
2294
2295impl<W: Write> RecordBatchWriter for StreamWriter<W> {
2296 fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
2297 self.write(batch)
2298 }
2299
2300 fn close(mut self) -> Result<(), ArrowError> {
2301 self.finish()
2302 }
2303}
2304
2305pub struct EncodedData {
2307 pub ipc_message: Vec<u8>,
2309 pub arrow_data: Vec<u8>,
2311}
2312
2313pub fn write_message<W: Write>(
2315 mut writer: W,
2316 encoded: EncodedData,
2317 write_options: &IpcWriteOptions,
2318) -> Result<(usize, usize), ArrowError> {
2319 writer.write_encoded_data(encoded, write_options)
2320}
2321
2322fn has_validity_bitmap(data_type: &DataType, write_options: &IpcWriteOptions) -> bool {
2325 if write_options.metadata_version < crate::MetadataVersion::V5 {
2326 !matches!(data_type, DataType::Null | DataType::RunEndEncoded(_, _))
2327 } else {
2328 !matches!(
2329 data_type,
2330 DataType::Null | DataType::Union(_, _) | DataType::RunEndEncoded(_, _)
2331 )
2332 }
2333}
2334
2335#[inline]
2337fn buffer_need_truncate(
2338 array_offset: usize,
2339 buffer: &Buffer,
2340 spec: &BufferSpec,
2341 min_length: usize,
2342) -> bool {
2343 spec != &BufferSpec::AlwaysNull && (array_offset != 0 || min_length < buffer.len())
2344}
2345
2346#[inline]
2348fn get_buffer_element_width(spec: &BufferSpec) -> usize {
2349 match spec {
2350 BufferSpec::FixedWidth { byte_width, .. } => *byte_width,
2351 _ => 0,
2352 }
2353}
2354
2355fn reencode_offsets<O: OffsetSizeTrait>(
2358 offsets: &Buffer,
2359 data: &ArrayData,
2360) -> (Buffer, usize, usize) {
2361 let offsets_slice: &[O] = offsets.typed_data::<O>();
2362 let offset_slice = &offsets_slice[data.offset()..data.offset() + data.len() + 1];
2363
2364 let start_offset = offset_slice.first().unwrap();
2365 let end_offset = offset_slice.last().unwrap();
2366
2367 let offsets = match start_offset.as_usize() {
2368 0 => {
2369 let size = size_of::<O>();
2370 offsets.slice_with_length(data.offset() * size, (data.len() + 1) * size)
2371 }
2372 _ => offset_slice.iter().map(|x| *x - *start_offset).collect(),
2373 };
2374
2375 let start_offset = start_offset.as_usize();
2376 let end_offset = end_offset.as_usize();
2377
2378 (offsets, start_offset, end_offset - start_offset)
2379}
2380
2381fn get_byte_array_buffers<O: OffsetSizeTrait>(data: &ArrayData) -> [Buffer; 2] {
2388 if data.is_empty() {
2389 let mut offsets = MutableBuffer::new(size_of::<O>());
2392 offsets.extend_from_slice(O::usize_as(0).to_byte_slice());
2393 return [offsets.into(), MutableBuffer::new(0).into()];
2394 }
2395
2396 let (offsets, original_start_offset, len) = reencode_offsets::<O>(&data.buffers()[0], data);
2397 let values = data.buffers()[1].slice_with_length(original_start_offset, len);
2398 [offsets, values]
2399}
2400
2401fn get_list_array_buffers<O: OffsetSizeTrait>(data: &ArrayData) -> (Buffer, ArrayData) {
2404 if data.is_empty() {
2405 let mut offsets = MutableBuffer::new(size_of::<O>());
2408 offsets.extend_from_slice(O::usize_as(0).to_byte_slice());
2409 return (offsets.into(), data.child_data()[0].slice(0, 0));
2410 }
2411
2412 let (offsets, original_start_offset, len) = reencode_offsets::<O>(&data.buffers()[0], data);
2413 let child_data = data.child_data()[0].slice(original_start_offset, len);
2414 (offsets, child_data)
2415}
2416
2417fn get_list_view_array_buffers<O: OffsetSizeTrait>(
2423 data: &ArrayData,
2424) -> (Buffer, Buffer, ArrayData) {
2425 if data.is_empty() {
2426 return (
2427 MutableBuffer::new(0).into(),
2428 MutableBuffer::new(0).into(),
2429 data.child_data()[0].slice(0, 0),
2430 );
2431 }
2432
2433 let offsets = &data.buffers()[0];
2434 let sizes = &data.buffers()[1];
2435
2436 let element_size = std::mem::size_of::<O>();
2437 let offsets_slice =
2438 offsets.slice_with_length(data.offset() * element_size, data.len() * element_size);
2439 let sizes_slice =
2440 sizes.slice_with_length(data.offset() * element_size, data.len() * element_size);
2441
2442 let child_data = data.child_data()[0].clone();
2443
2444 (offsets_slice, sizes_slice, child_data)
2445}
2446
2447fn get_or_truncate_buffer(array_data: &ArrayData) -> Buffer {
2454 let buffer = &array_data.buffers()[0];
2455 let layout = layout(array_data.data_type());
2456 let spec = &layout.buffers[0];
2457
2458 let byte_width = get_buffer_element_width(spec);
2459 let min_length = array_data.len() * byte_width;
2460 if buffer_need_truncate(array_data.offset(), buffer, spec, min_length) {
2461 let byte_offset = array_data.offset() * byte_width;
2462 let buffer_length = min(min_length, buffer.len() - byte_offset);
2463 buffer.slice_with_length(byte_offset, buffer_length)
2464 } else {
2465 buffer.clone()
2466 }
2467}
2468
2469fn write_array_data(
2475 array_data: &ArrayData,
2476 meta: &mut IpcMetadataBuilder,
2477 sink: &mut IpcBodySink<'_>,
2478 offset: i64,
2479 compression_codec: Option<CompressionCodec>,
2480 ipc_write_context: &mut IpcWriteContext,
2481 write_options: &IpcWriteOptions,
2482) -> Result<i64, ArrowError> {
2483 let mut offset = offset;
2484 let num_rows = array_data.len();
2485 let null_count = array_data.null_count();
2486 if !matches!(array_data.data_type(), DataType::Null) {
2487 meta.nodes
2488 .push(crate::FieldNode::new(num_rows as i64, null_count as i64));
2489 } else {
2490 meta.nodes
2492 .push(crate::FieldNode::new(num_rows as i64, num_rows as i64));
2493 }
2494 if has_validity_bitmap(array_data.data_type(), write_options) {
2495 let null_buffer = match array_data.nulls() {
2498 Some(buffer) if null_count > 0 => buffer.inner().sliced(),
2499 _ => MutableBuffer::new(0).into(),
2500 };
2501
2502 offset = encode_sink_buffer(
2503 null_buffer,
2504 meta,
2505 sink,
2506 offset,
2507 compression_codec,
2508 ipc_write_context,
2509 write_options.alignment,
2510 )?;
2511 }
2512
2513 let data_type = array_data.data_type();
2514 if matches!(data_type, DataType::Binary | DataType::Utf8) {
2515 for buffer in get_byte_array_buffers::<i32>(array_data) {
2516 offset = encode_sink_buffer(
2517 buffer,
2518 meta,
2519 sink,
2520 offset,
2521 compression_codec,
2522 ipc_write_context,
2523 write_options.alignment,
2524 )?;
2525 }
2526 } else if matches!(data_type, DataType::BinaryView | DataType::Utf8View) {
2527 let views = get_or_truncate_buffer(array_data);
2534 offset = encode_sink_buffer(
2535 views,
2536 meta,
2537 sink,
2538 offset,
2539 compression_codec,
2540 ipc_write_context,
2541 write_options.alignment,
2542 )?;
2543
2544 for buffer in array_data.buffers().iter().skip(1) {
2545 offset = encode_sink_buffer(
2546 buffer.clone(),
2547 meta,
2548 sink,
2549 offset,
2550 compression_codec,
2551 ipc_write_context,
2552 write_options.alignment,
2553 )?;
2554 }
2555 } else if matches!(data_type, DataType::LargeBinary | DataType::LargeUtf8) {
2556 for buffer in get_byte_array_buffers::<i64>(array_data) {
2557 offset = encode_sink_buffer(
2558 buffer,
2559 meta,
2560 sink,
2561 offset,
2562 compression_codec,
2563 ipc_write_context,
2564 write_options.alignment,
2565 )?;
2566 }
2567 } else if DataType::is_numeric(data_type)
2568 || DataType::is_temporal(data_type)
2569 || matches!(
2570 array_data.data_type(),
2571 DataType::FixedSizeBinary(_) | DataType::Dictionary(_, _)
2572 )
2573 {
2574 assert_eq!(array_data.buffers().len(), 1);
2576
2577 let buffer = get_or_truncate_buffer(array_data);
2578 offset = encode_sink_buffer(
2579 buffer,
2580 meta,
2581 sink,
2582 offset,
2583 compression_codec,
2584 ipc_write_context,
2585 write_options.alignment,
2586 )?;
2587 } else if matches!(data_type, DataType::Boolean) {
2588 assert_eq!(array_data.buffers().len(), 1);
2591
2592 let buffer = &array_data.buffers()[0];
2593 let buffer = buffer.bit_slice(array_data.offset(), array_data.len());
2594 offset = encode_sink_buffer(
2595 buffer,
2596 meta,
2597 sink,
2598 offset,
2599 compression_codec,
2600 ipc_write_context,
2601 write_options.alignment,
2602 )?;
2603 } else if matches!(
2604 data_type,
2605 DataType::List(_) | DataType::LargeList(_) | DataType::Map(_, _)
2606 ) {
2607 assert_eq!(array_data.buffers().len(), 1);
2608 assert_eq!(array_data.child_data().len(), 1);
2609
2610 let (offsets, sliced_child_data) = match data_type {
2612 DataType::List(_) => get_list_array_buffers::<i32>(array_data),
2613 DataType::Map(_, _) => get_list_array_buffers::<i32>(array_data),
2614 DataType::LargeList(_) => get_list_array_buffers::<i64>(array_data),
2615 _ => unreachable!(),
2616 };
2617 offset = encode_sink_buffer(
2618 offsets,
2619 meta,
2620 sink,
2621 offset,
2622 compression_codec,
2623 ipc_write_context,
2624 write_options.alignment,
2625 )?;
2626 offset = write_array_data(
2627 &sliced_child_data,
2628 meta,
2629 sink,
2630 offset,
2631 compression_codec,
2632 ipc_write_context,
2633 write_options,
2634 )?;
2635 return Ok(offset);
2636 } else if matches!(
2637 data_type,
2638 DataType::ListView(_) | DataType::LargeListView(_)
2639 ) {
2640 assert_eq!(array_data.buffers().len(), 2); assert_eq!(array_data.child_data().len(), 1);
2642
2643 let (offsets, sizes, child_data) = match data_type {
2644 DataType::ListView(_) => get_list_view_array_buffers::<i32>(array_data),
2645 DataType::LargeListView(_) => get_list_view_array_buffers::<i64>(array_data),
2646 _ => unreachable!(),
2647 };
2648
2649 offset = encode_sink_buffer(
2650 offsets,
2651 meta,
2652 sink,
2653 offset,
2654 compression_codec,
2655 ipc_write_context,
2656 write_options.alignment,
2657 )?;
2658 offset = encode_sink_buffer(
2659 sizes,
2660 meta,
2661 sink,
2662 offset,
2663 compression_codec,
2664 ipc_write_context,
2665 write_options.alignment,
2666 )?;
2667
2668 offset = write_array_data(
2669 &child_data,
2670 meta,
2671 sink,
2672 offset,
2673 compression_codec,
2674 ipc_write_context,
2675 write_options,
2676 )?;
2677 return Ok(offset);
2678 } else if let DataType::FixedSizeList(_, fixed_size) = data_type {
2679 assert_eq!(array_data.child_data().len(), 1);
2680 let fixed_size = *fixed_size as usize;
2681
2682 let child_offset = array_data.offset() * fixed_size;
2683 let child_length = array_data.len() * fixed_size;
2684 let child_data = array_data.child_data()[0].slice(child_offset, child_length);
2685
2686 offset = write_array_data(
2687 &child_data,
2688 meta,
2689 sink,
2690 offset,
2691 compression_codec,
2692 ipc_write_context,
2693 write_options,
2694 )?;
2695 return Ok(offset);
2696 } else if matches!(data_type, DataType::Union(_, _)) {
2697 for (buffer, byte_width) in array_data
2698 .buffers()
2699 .iter()
2700 .zip([size_of::<i8>(), size_of::<i32>()])
2701 {
2702 let buffer =
2703 buffer.slice_with_length(array_data.offset() * byte_width, num_rows * byte_width);
2704 offset = encode_sink_buffer(
2705 buffer,
2706 meta,
2707 sink,
2708 offset,
2709 compression_codec,
2710 ipc_write_context,
2711 write_options.alignment,
2712 )?;
2713 }
2714 } else {
2715 for buffer in array_data.buffers() {
2716 offset = encode_sink_buffer(
2717 buffer.clone(),
2718 meta,
2719 sink,
2720 offset,
2721 compression_codec,
2722 ipc_write_context,
2723 write_options.alignment,
2724 )?;
2725 }
2726 }
2727
2728 match array_data.data_type() {
2729 DataType::Dictionary(_, _) => {}
2730 DataType::RunEndEncoded(_, _) => {
2731 let arr = unslice_run_array(array_data.clone())?;
2733 for data_ref in arr.child_data() {
2735 offset = write_array_data(
2737 data_ref,
2738 meta,
2739 sink,
2740 offset,
2741 compression_codec,
2742 ipc_write_context,
2743 write_options,
2744 )?;
2745 }
2746 }
2747 DataType::Union(_, UnionMode::Sparse) => {
2748 for data_ref in array_data.child_data() {
2749 let child_data = data_ref.slice(array_data.offset(), num_rows);
2750 offset = write_array_data(
2751 &child_data,
2752 meta,
2753 sink,
2754 offset,
2755 compression_codec,
2756 ipc_write_context,
2757 write_options,
2758 )?;
2759 }
2760 }
2761 _ => {
2762 for data_ref in array_data.child_data() {
2764 offset = write_array_data(
2766 data_ref,
2767 meta,
2768 sink,
2769 offset,
2770 compression_codec,
2771 ipc_write_context,
2772 write_options,
2773 )?;
2774 }
2775 }
2776 }
2777 Ok(offset)
2778}
2779
2780fn encode_sink_buffer(
2794 buffer: Buffer,
2795 ipc_meta_data: &mut IpcMetadataBuilder,
2796 sink: &mut IpcBodySink<'_>,
2797 offset: i64,
2798 compression_codec: Option<CompressionCodec>,
2799 ipc_write_context: &mut IpcWriteContext,
2800 alignment: u8,
2801) -> Result<i64, ArrowError> {
2802 let (encoded, len) = match compression_codec {
2803 None => {
2804 let len = buffer.len() as i64;
2805 (EncodedBuffer::Raw(buffer), len)
2806 }
2807 Some(codec) => {
2808 let mut scratch = Vec::new();
2809 let written =
2810 codec.compress_to_vec(buffer.as_slice(), &mut scratch, ipc_write_context)?;
2811 let len = i64::try_from(written)
2812 .map_err(|e| ArrowError::InvalidArgumentError(format!("{e}")))?;
2813 (EncodedBuffer::Compressed(scratch), len)
2814 }
2815 };
2816
2817 let pad_len = pad_to_alignment(alignment, len as usize);
2818 sink.write(pad_len, encoded);
2819 ipc_meta_data.buffers.push(crate::Buffer::new(offset, len));
2820 Ok(offset + len + pad_len as i64)
2821}
2822
2823const PADDING: [u8; 64] = [0; 64];
2824
2825#[inline]
2831fn estimate_encoded_buffer_count(dt: &DataType) -> usize {
2832 match dt {
2833 DataType::Null => 0,
2834
2835 DataType::Binary | DataType::Utf8 | DataType::LargeBinary | DataType::LargeUtf8 => 3,
2836
2837 DataType::BinaryView | DataType::Utf8View => 3,
2838
2839 DataType::List(f) | DataType::LargeList(f) | DataType::Map(f, _) => {
2840 2 + estimate_encoded_buffer_count(f.data_type())
2841 }
2842
2843 DataType::ListView(f) | DataType::LargeListView(f) => {
2844 3 + estimate_encoded_buffer_count(f.data_type())
2845 }
2846
2847 DataType::FixedSizeList(f, _) => 1 + estimate_encoded_buffer_count(f.data_type()),
2848
2849 DataType::Struct(fields) => {
2850 1 + fields
2851 .iter()
2852 .map(|f| estimate_encoded_buffer_count(f.data_type()))
2853 .sum::<usize>()
2854 }
2855
2856 DataType::Dictionary(_, _) => 2,
2858
2859 DataType::Union(fields, UnionMode::Sparse) => {
2860 1 + fields
2861 .iter()
2862 .map(|(_, f)| estimate_encoded_buffer_count(f.data_type()))
2863 .sum::<usize>()
2864 }
2865 DataType::Union(fields, UnionMode::Dense) => {
2866 2 + fields
2867 .iter()
2868 .map(|(_, f)| estimate_encoded_buffer_count(f.data_type()))
2869 .sum::<usize>()
2870 }
2871
2872 DataType::RunEndEncoded(run_ends, values) => {
2873 estimate_encoded_buffer_count(run_ends.data_type())
2874 + estimate_encoded_buffer_count(values.data_type())
2875 }
2876 _ => 2,
2878 }
2879}
2880
2881#[inline]
2883fn pad_to_alignment(alignment: u8, len: usize) -> usize {
2884 let a = usize::from(alignment - 1);
2885 ((len + a) & !a) - len
2886}
2887
2888#[cfg(test)]
2889mod tests {
2890 use std::hash::Hasher;
2891 use std::io::Cursor;
2892 use std::io::Seek;
2893
2894 use arrow_array::builder::FixedSizeListBuilder;
2895 use arrow_array::builder::Float32Builder;
2896 use arrow_array::builder::Int64Builder;
2897 use arrow_array::builder::MapBuilder;
2898 use arrow_array::builder::StringViewBuilder;
2899 use arrow_array::builder::UnionBuilder;
2900 use arrow_array::builder::{
2901 GenericListBuilder, GenericListViewBuilder, ListBuilder, StringBuilder,
2902 };
2903 use arrow_array::builder::{PrimitiveRunBuilder, UInt32Builder};
2904 use arrow_array::types::*;
2905 use arrow_buffer::ScalarBuffer;
2906
2907 use crate::MetadataVersion;
2908 use crate::convert::try_fb_to_schema;
2909 use crate::reader::*;
2910 use crate::root_as_footer;
2911
2912 use super::*;
2913
2914 #[test]
2915 fn test_roundtrip_sliced_list_of_union() {
2916 fn decoded(array: &ArrayRef) -> Vec<Vec<Option<i32>>> {
2917 (0..array.len())
2918 .map(|row| {
2919 let values = array.as_list::<i32>().value(row);
2920 let union = values.as_union();
2921 (0..union.len())
2922 .map(|index| {
2923 union
2924 .value(index)
2925 .as_primitive::<Int32Type>()
2926 .iter()
2927 .next()
2928 .unwrap()
2929 })
2930 .collect()
2931 })
2932 .collect()
2933 }
2934
2935 for mode in [UnionMode::Dense, UnionMode::Sparse] {
2936 let fields = UnionFields::try_new(
2937 [3, 7],
2938 [
2939 Field::new("a", DataType::Int32, true),
2940 Field::new("b", DataType::Int32, true),
2941 ],
2942 )
2943 .unwrap();
2944 let offsets = (mode == UnionMode::Dense).then(|| vec![2, 0, 1].into());
2945 let union: ArrayRef = Arc::new(
2946 UnionArray::try_new(
2947 fields,
2948 vec![3_i8, 7, 3].into(),
2949 offsets,
2950 vec![
2951 Arc::new(Int32Array::from(vec![Some(11), None, Some(33)])),
2952 Arc::new(Int32Array::from(vec![Some(101), Some(202), None])),
2953 ],
2954 )
2955 .unwrap(),
2956 );
2957 let item = Arc::new(Field::new("item", union.data_type().clone(), true));
2958 let list: ArrayRef = Arc::new(ListArray::new(
2959 item,
2960 arrow_buffer::OffsetBuffer::from_lengths([1, 2]),
2961 union,
2962 None,
2963 ));
2964 for (offset, len) in [(0, 2), (1, 1), (2, 0)] {
2965 let array = list.slice(offset, len);
2966 array.to_data().validate_full().unwrap();
2967 let expected = decoded(&array);
2968 let schema = Arc::new(Schema::new(vec![Field::new(
2969 "v",
2970 array.data_type().clone(),
2971 true,
2972 )]));
2973 let batch = RecordBatch::try_new(schema, vec![array]).unwrap();
2974 for output in [
2975 deserialize_file(serialize_file(&batch)),
2976 deserialize_stream(serialize_stream(&batch)),
2977 ] {
2978 output.column(0).to_data().validate_full().unwrap();
2979 assert_eq!(decoded(output.column(0)), expected);
2980 }
2981 }
2982 }
2983 }
2984
2985 fn serialize_file(rb: &RecordBatch) -> Vec<u8> {
2986 let mut writer = FileWriter::try_new(vec![], rb.schema_ref()).unwrap();
2987 writer.write(rb).unwrap();
2988 writer.finish().unwrap();
2989 writer.into_inner().unwrap()
2990 }
2991
2992 fn deserialize_file(bytes: Vec<u8>) -> RecordBatch {
2993 let mut reader = FileReader::try_new(Cursor::new(bytes), None).unwrap();
2994 reader.next().unwrap().unwrap()
2995 }
2996
2997 fn serialize_stream(record: &RecordBatch) -> Vec<u8> {
2998 const IPC_ALIGNMENT: usize = 8;
3002
3003 let mut stream_writer = StreamWriter::try_new_with_options(
3004 vec![],
3005 record.schema_ref(),
3006 IpcWriteOptions::try_new(IPC_ALIGNMENT, false, MetadataVersion::V5).unwrap(),
3007 )
3008 .unwrap();
3009 stream_writer.write(record).unwrap();
3010 stream_writer.finish().unwrap();
3011 stream_writer.into_inner().unwrap()
3012 }
3013
3014 fn deserialize_stream(bytes: Vec<u8>) -> RecordBatch {
3015 let mut stream_reader = StreamReader::try_new(Cursor::new(bytes), None).unwrap();
3016 stream_reader.next().unwrap().unwrap()
3017 }
3018
3019 fn encode_stream(
3024 schema: &Schema,
3025 batches: &[RecordBatch],
3026 options: IpcWriteOptions,
3027 ) -> Vec<u8> {
3028 let mut encoder = StreamEncoder::try_new_with_options(schema, options).unwrap();
3029 let mut bytes = Vec::new();
3030 for batch in batches {
3031 for buffer in encoder.encode(batch).unwrap() {
3032 bytes.write_all(buffer.as_slice()).unwrap();
3033 }
3034 }
3035 for buffer in encoder.finish().unwrap() {
3036 bytes.write_all(buffer.as_slice()).unwrap();
3037 }
3038 bytes
3039 }
3040
3041 fn write_stream(schema: &Schema, batches: &[RecordBatch], options: IpcWriteOptions) -> Vec<u8> {
3042 let mut bytes = Vec::new();
3043 let mut writer = StreamWriter::try_new_with_options(&mut bytes, schema, options).unwrap();
3044 for batch in batches {
3045 writer.write(batch).unwrap();
3046 }
3047 writer.finish().unwrap();
3048 bytes
3049 }
3050
3051 #[tokio::test]
3054 async fn test_stream_encoder_async_writer_matches_stream_writer() {
3055 use tokio::io::{AsyncReadExt, AsyncWriteExt};
3056
3057 let batch = record_batch!(("a", Int32, [1, 2, 3]), ("b", Utf8, ["x", "y", "z"])).unwrap();
3058 let options = IpcWriteOptions::default();
3059 let expected = write_stream(
3060 batch.schema_ref(),
3061 std::slice::from_ref(&batch),
3062 options.clone(),
3063 );
3064
3065 let (mut sink, mut source) = tokio::io::duplex(64);
3066 let read = tokio::spawn(async move {
3067 let mut bytes = Vec::new();
3068 source.read_to_end(&mut bytes).await.unwrap();
3069 bytes
3070 });
3071
3072 let mut encoder = StreamEncoder::try_new_with_options(batch.schema_ref(), options).unwrap();
3073 for buffer in encoder.encode(&batch).unwrap() {
3074 sink.write_all(buffer.as_slice()).await.unwrap();
3075 }
3076 for buffer in encoder.finish().unwrap() {
3077 sink.write_all(buffer.as_slice()).await.unwrap();
3078 }
3079 sink.shutdown().await.unwrap();
3080
3081 let encoded = read.await.unwrap();
3082 assert_eq!(encoded, expected);
3083
3084 let mut reader = StreamReader::try_new(Cursor::new(encoded), None).unwrap();
3085 assert_eq!(reader.next().unwrap().unwrap(), batch);
3086 assert!(reader.next().is_none());
3087 }
3088
3089 #[test]
3090 fn test_stream_encoder_empty_stream_matches_stream_writer() {
3091 let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);
3092 let options = IpcWriteOptions::default();
3093 let encoded = encode_stream(&schema, &[], options.clone());
3094 let written = write_stream(&schema, &[], options);
3095
3096 assert_eq!(encoded, written);
3097
3098 let mut reader = StreamReader::try_new(Cursor::new(encoded), None).unwrap();
3099 assert!(reader.next().is_none());
3100 }
3101
3102 #[test]
3103 fn test_stream_encoder_dictionary_batches_match_stream_writer() {
3104 let schema = Arc::new(Schema::new(vec![Field::new(
3105 "a",
3106 DataType::Dictionary(Box::new(DataType::UInt8), Box::new(DataType::Utf8)),
3107 false,
3108 )]));
3109 let batch = RecordBatch::try_new(
3110 schema.clone(),
3111 vec![Arc::new(DictionaryArray::new(
3112 UInt8Array::from_iter_values([0, 1, 0]),
3113 Arc::new(StringArray::from_iter_values(["a", "b"])),
3114 ))],
3115 )
3116 .unwrap();
3117 let options = IpcWriteOptions::default();
3118 let encoded = encode_stream(&schema, std::slice::from_ref(&batch), options.clone());
3119 let written = write_stream(&schema, std::slice::from_ref(&batch), options);
3120
3121 assert_eq!(encoded, written);
3122
3123 let mut reader = StreamReader::try_new(Cursor::new(encoded), None).unwrap();
3124 assert_eq!(reader.next().unwrap().unwrap(), batch);
3125 assert!(reader.next().is_none());
3126 }
3127
3128 #[test]
3129 #[cfg(feature = "lz4")]
3130 fn test_write_empty_record_batch_lz4_compression() {
3131 let schema = Schema::new(vec![Field::new("field1", DataType::Int32, true)]);
3132 let values: Vec<Option<i32>> = vec![];
3133 let array = Int32Array::from(values);
3134 let record_batch =
3135 RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(array)]).unwrap();
3136
3137 let mut file = tempfile::tempfile().unwrap();
3138
3139 {
3140 let write_option = IpcWriteOptions::try_new(8, false, crate::MetadataVersion::V5)
3141 .unwrap()
3142 .try_with_compression(Some(crate::CompressionType::LZ4_FRAME))
3143 .unwrap();
3144
3145 let mut writer =
3146 FileWriter::try_new_with_options(&mut file, &schema, write_option).unwrap();
3147 writer.write(&record_batch).unwrap();
3148 writer.finish().unwrap();
3149 }
3150 file.rewind().unwrap();
3151 {
3152 let reader = FileReader::try_new(file, None).unwrap();
3154 for read_batch in reader {
3155 read_batch
3156 .unwrap()
3157 .columns()
3158 .iter()
3159 .zip(record_batch.columns())
3160 .for_each(|(a, b)| {
3161 assert_eq!(a.data_type(), b.data_type());
3162 assert_eq!(a.len(), b.len());
3163 assert_eq!(a.null_count(), b.null_count());
3164 });
3165 }
3166 }
3167 }
3168
3169 #[test]
3170 #[cfg(feature = "lz4")]
3171 fn test_write_file_with_lz4_compression() {
3172 let schema = Schema::new(vec![Field::new("field1", DataType::Int32, true)]);
3173 let values: Vec<Option<i32>> = vec![Some(12), Some(1)];
3174 let array = Int32Array::from(values);
3175 let record_batch =
3176 RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(array)]).unwrap();
3177
3178 let mut file = tempfile::tempfile().unwrap();
3179 {
3180 let write_option = IpcWriteOptions::try_new(8, false, crate::MetadataVersion::V5)
3181 .unwrap()
3182 .try_with_compression(Some(crate::CompressionType::LZ4_FRAME))
3183 .unwrap();
3184
3185 let mut writer =
3186 FileWriter::try_new_with_options(&mut file, &schema, write_option).unwrap();
3187 writer.write(&record_batch).unwrap();
3188 writer.finish().unwrap();
3189 }
3190 file.rewind().unwrap();
3191 {
3192 let reader = FileReader::try_new(file, None).unwrap();
3194 for read_batch in reader {
3195 read_batch
3196 .unwrap()
3197 .columns()
3198 .iter()
3199 .zip(record_batch.columns())
3200 .for_each(|(a, b)| {
3201 assert_eq!(a.data_type(), b.data_type());
3202 assert_eq!(a.len(), b.len());
3203 assert_eq!(a.null_count(), b.null_count());
3204 });
3205 }
3206 }
3207 }
3208
3209 fn lz4_block_size_batch() -> RecordBatch {
3212 let schema = Schema::new(vec![
3213 Field::new("ints", DataType::Int32, true),
3214 Field::new(
3215 "words",
3216 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
3217 false,
3218 ),
3219 ]);
3220 let ints = Int32Array::from(
3221 (0..30_000)
3222 .map(|i| if i % 13 == 0 { None } else { Some(i) })
3223 .collect::<Vec<Option<i32>>>(),
3224 );
3225 let keys = Int32Array::from((0..30_000).map(|i| i % 5).collect::<Vec<i32>>());
3226 let values = StringArray::from(vec!["zero", "one", "two", "three", "four"]);
3227 let words = DictionaryArray::new(keys, Arc::new(values));
3228 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(ints), Arc::new(words)]).unwrap()
3229 }
3230
3231 #[test]
3232 fn test_lz4_block_size_options() {
3233 let options = || IpcWriteOptions::try_new(8, false, MetadataVersion::V5).unwrap();
3234
3235 assert_eq!(options().lz4_block_size(), Lz4BlockSize::DEFAULT);
3236 assert_eq!(
3237 options()
3238 .try_with_lz4_block_size(None)
3239 .unwrap()
3240 .lz4_block_size(),
3241 Lz4BlockSize::DEFAULT
3242 );
3243 assert_eq!(
3244 options()
3245 .try_with_lz4_block_size(Some(1024 * 1024))
3246 .unwrap()
3247 .lz4_block_size(),
3248 Lz4BlockSize::Max1MiB
3249 );
3250
3251 for block_size in [0, 32 * 1024, 128 * 1024, 2 * 1024 * 1024, 8 * 1024 * 1024] {
3252 let err = options()
3253 .try_with_lz4_block_size(Some(block_size))
3254 .expect_err("unsupported block size must be rejected");
3255 assert!(
3256 err.to_string().contains("Unsupported lz4 block size"),
3257 "unexpected error: {err}"
3258 );
3259 }
3260
3261 let options = IpcWriteOptions::try_new(8, false, MetadataVersion::V4).unwrap();
3262 let err = options
3263 .try_with_lz4_block_size(Some(64 * 1024))
3264 .expect_err("block size below metadata V5 must be rejected");
3265 assert!(
3266 err.to_string().contains("metadata v5"),
3267 "unexpected error: {err}"
3268 );
3269
3270 let options = IpcWriteOptions::try_new(8, false, MetadataVersion::V5)
3271 .unwrap()
3272 .try_with_compression(Some(crate::CompressionType::ZSTD))
3273 .unwrap();
3274 let err = options
3275 .try_with_lz4_block_size(Some(64 * 1024))
3276 .expect_err("lz4 block size must be rejected for ZSTD");
3277 assert!(err.to_string().contains("ZSTD"), "unexpected error: {err}");
3278
3279 let err = IpcWriteOptions::try_new(8, false, MetadataVersion::V5)
3280 .unwrap()
3281 .try_with_lz4_block_size(Some(64 * 1024))
3282 .unwrap()
3283 .try_with_compression(Some(crate::CompressionType::ZSTD))
3284 .expect_err("ZSTD must be rejected after setting an lz4 block size");
3285 assert!(err.to_string().contains("ZSTD"), "unexpected error: {err}");
3286 }
3287
3288 #[test]
3289 #[cfg(feature = "lz4")]
3290 fn test_lz4_block_size_configurable_end_to_end() {
3291 let batch = lz4_block_size_batch();
3292 let schema = batch.schema();
3293 let data_gen = IpcDataGenerator::default();
3294 let mut ipc_write_context = IpcWriteContext::default();
3295
3296 let sizes: [Option<usize>; 5] = [
3297 None,
3298 Some(64 * 1024),
3299 Some(256 * 1024),
3300 Some(1024 * 1024),
3301 Some(4 * 1024 * 1024),
3302 ];
3303 let mut outputs = Vec::with_capacity(sizes.len());
3304 for block_size in sizes {
3305 let options = IpcWriteOptions::try_new(8, false, MetadataVersion::V5)
3306 .unwrap()
3307 .try_with_compression(Some(crate::CompressionType::LZ4_FRAME))
3308 .unwrap()
3309 .try_with_lz4_block_size(block_size)
3310 .unwrap();
3311
3312 let mut dictionary_tracker = DictionaryTracker::new(false);
3313 let mut encoded = Vec::new();
3314 let schema_data = data_gen.schema_to_bytes_with_dictionary_tracker(
3315 &schema,
3316 &mut dictionary_tracker,
3317 &options,
3318 );
3319 write_message(&mut encoded, schema_data, &options).unwrap();
3320
3321 let (encoded_dictionaries, encoded_batch) = data_gen
3322 .encode(
3323 &batch,
3324 &mut dictionary_tracker,
3325 &options,
3326 &mut ipc_write_context,
3327 )
3328 .unwrap();
3329 for dictionary in encoded_dictionaries {
3330 write_message(&mut encoded, dictionary, &options).unwrap();
3331 }
3332 write_message(&mut encoded, encoded_batch, &options).unwrap();
3333
3334 let mut reader = StreamReader::try_new(Cursor::new(encoded.as_slice()), None).unwrap();
3336 assert_eq!(reader.next().unwrap().unwrap(), batch, "{block_size:?}");
3337 assert!(reader.next().is_none());
3338
3339 outputs.push(encoded);
3340 }
3341
3342 assert_eq!(
3346 outputs[0], outputs[2],
3347 "explicit 256 KiB must match the default"
3348 );
3349 assert_ne!(outputs[0], outputs[1], "64 KiB must differ from default");
3350 assert_ne!(outputs[0], outputs[3], "1 MiB must differ from default");
3351 assert_ne!(outputs[0], outputs[4], "4 MiB must differ from default");
3352 }
3353
3354 #[test]
3355 #[cfg(feature = "zstd")]
3356 fn test_write_file_with_zstd_compression() {
3357 let schema = Schema::new(vec![Field::new("field1", DataType::Int32, true)]);
3358 let values: Vec<Option<i32>> = vec![Some(12), Some(1)];
3359 let array = Int32Array::from(values);
3360 let record_batch =
3361 RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(array)]).unwrap();
3362 let mut file = tempfile::tempfile().unwrap();
3363 {
3364 let write_option = IpcWriteOptions::try_new(8, false, crate::MetadataVersion::V5)
3365 .unwrap()
3366 .try_with_compression(Some(crate::CompressionType::ZSTD))
3367 .unwrap()
3368 .try_with_compression_level(Some(1))
3369 .unwrap();
3370
3371 let mut writer =
3372 FileWriter::try_new_with_options(&mut file, &schema, write_option).unwrap();
3373 writer.write(&record_batch).unwrap();
3374 writer.finish().unwrap();
3375 }
3376 file.rewind().unwrap();
3377 {
3378 let reader = FileReader::try_new(file, None).unwrap();
3380 for read_batch in reader {
3381 read_batch
3382 .unwrap()
3383 .columns()
3384 .iter()
3385 .zip(record_batch.columns())
3386 .for_each(|(a, b)| {
3387 assert_eq!(a.data_type(), b.data_type());
3388 assert_eq!(a.len(), b.len());
3389 assert_eq!(a.null_count(), b.null_count());
3390 });
3391 }
3392 }
3393 }
3394
3395 #[test]
3396 fn test_write_file() {
3397 let schema = Schema::new(vec![Field::new("field1", DataType::UInt32, true)]);
3398 let values: Vec<Option<u32>> = vec![
3399 Some(999),
3400 None,
3401 Some(235),
3402 Some(123),
3403 None,
3404 None,
3405 None,
3406 None,
3407 None,
3408 ];
3409 let array1 = UInt32Array::from(values);
3410 let batch =
3411 RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(array1) as ArrayRef])
3412 .unwrap();
3413 let mut file = tempfile::tempfile().unwrap();
3414 {
3415 let mut writer = FileWriter::try_new(&mut file, &schema).unwrap();
3416
3417 writer.write(&batch).unwrap();
3418 writer.finish().unwrap();
3419 }
3420 file.rewind().unwrap();
3421
3422 {
3423 let mut reader = FileReader::try_new(file, None).unwrap();
3424 while let Some(Ok(read_batch)) = reader.next() {
3425 read_batch
3426 .columns()
3427 .iter()
3428 .zip(batch.columns())
3429 .for_each(|(a, b)| {
3430 assert_eq!(a.data_type(), b.data_type());
3431 assert_eq!(a.len(), b.len());
3432 assert_eq!(a.null_count(), b.null_count());
3433 });
3434 }
3435 }
3436 }
3437
3438 #[test]
3439 fn test_empty_utf8_ipc_writes_nonempty_offsets_buffer() {
3440 let name = StringArray::from(Vec::<String>::new());
3441 let [offsets, values] = get_byte_array_buffers::<i32>(&name.to_data());
3442
3443 assert_eq!(name.len(), 0);
3444 assert_eq!(
3445 offsets.len(),
3446 std::mem::size_of::<i32>(),
3447 "offsets buffer should contain one zero i32 offset"
3448 );
3449 assert_eq!(values.len(), 0, "values buffer should remain empty");
3450 }
3451
3452 #[test]
3453 fn test_empty_large_utf8_ipc_writes_nonempty_offsets_buffer() {
3454 let name = LargeStringArray::from(Vec::<String>::new());
3455 let [offsets, values] = get_byte_array_buffers::<i64>(&name.to_data());
3456
3457 assert_eq!(name.len(), 0);
3458 assert_eq!(
3459 offsets.len(),
3460 std::mem::size_of::<i64>(),
3461 "offsets buffer should contain one zero i64 offset"
3462 );
3463 assert_eq!(values.len(), 0, "values buffer should remain empty");
3464 }
3465
3466 #[test]
3467 fn test_empty_list_ipc_writes_nonempty_offsets_buffer() {
3468 let list = GenericListBuilder::<i32, _>::new(UInt32Builder::new()).finish();
3469 let (offsets, child_data) = get_list_array_buffers::<i32>(&list.to_data());
3470
3471 assert_eq!(list.len(), 0);
3472 assert_eq!(
3473 offsets.len(),
3474 std::mem::size_of::<i32>(),
3475 "offsets buffer should contain one zero i32 offset"
3476 );
3477 assert_eq!(child_data.len(), 0, "child data should remain empty");
3478 }
3479
3480 #[test]
3481 fn test_empty_large_list_ipc_writes_nonempty_offsets_buffer() {
3482 let list = GenericListBuilder::<i64, _>::new(UInt32Builder::new()).finish();
3483 let (offsets, child_data) = get_list_array_buffers::<i64>(&list.to_data());
3484
3485 assert_eq!(list.len(), 0);
3486 assert_eq!(
3487 offsets.len(),
3488 std::mem::size_of::<i64>(),
3489 "offsets buffer should contain one zero i64 offset"
3490 );
3491 assert_eq!(child_data.len(), 0, "child data should remain empty");
3492 }
3493
3494 fn write_null_file(options: IpcWriteOptions) {
3495 let schema = Schema::new(vec![
3496 Field::new("nulls", DataType::Null, true),
3497 Field::new("int32s", DataType::Int32, false),
3498 Field::new("nulls2", DataType::Null, true),
3499 Field::new("f64s", DataType::Float64, false),
3500 ]);
3501 let array1 = NullArray::new(32);
3502 let array2 = Int32Array::from(vec![1; 32]);
3503 let array3 = NullArray::new(32);
3504 let array4 = Float64Array::from(vec![f64::NAN; 32]);
3505 let batch = RecordBatch::try_new(
3506 Arc::new(schema.clone()),
3507 vec![
3508 Arc::new(array1) as ArrayRef,
3509 Arc::new(array2) as ArrayRef,
3510 Arc::new(array3) as ArrayRef,
3511 Arc::new(array4) as ArrayRef,
3512 ],
3513 )
3514 .unwrap();
3515 let mut file = tempfile::tempfile().unwrap();
3516 {
3517 let mut writer = FileWriter::try_new_with_options(&mut file, &schema, options).unwrap();
3518
3519 writer.write(&batch).unwrap();
3520 writer.finish().unwrap();
3521 }
3522
3523 file.rewind().unwrap();
3524
3525 {
3526 let reader = FileReader::try_new(file, None).unwrap();
3527 reader.for_each(|maybe_batch| {
3528 maybe_batch
3529 .unwrap()
3530 .columns()
3531 .iter()
3532 .zip(batch.columns())
3533 .for_each(|(a, b)| {
3534 assert_eq!(a.data_type(), b.data_type());
3535 assert_eq!(a.len(), b.len());
3536 assert_eq!(a.null_count(), b.null_count());
3537 });
3538 });
3539 }
3540 }
3541 #[test]
3542 fn test_write_null_file_v4() {
3543 write_null_file(IpcWriteOptions::try_new(8, false, MetadataVersion::V4).unwrap());
3544 write_null_file(IpcWriteOptions::try_new(8, true, MetadataVersion::V4).unwrap());
3545 write_null_file(IpcWriteOptions::try_new(64, false, MetadataVersion::V4).unwrap());
3546 write_null_file(IpcWriteOptions::try_new(64, true, MetadataVersion::V4).unwrap());
3547 }
3548
3549 #[test]
3550 fn test_write_null_file_v5() {
3551 write_null_file(IpcWriteOptions::try_new(8, false, MetadataVersion::V5).unwrap());
3552 write_null_file(IpcWriteOptions::try_new(64, false, MetadataVersion::V5).unwrap());
3553 }
3554
3555 #[test]
3556 fn track_union_nested_dict() {
3557 let inner: DictionaryArray<Int32Type> = vec!["a", "b", "a"].into_iter().collect();
3558
3559 let array = Arc::new(inner) as ArrayRef;
3560
3561 #[expect(deprecated)]
3563 let dctfield = Field::new_dict("dict", array.data_type().clone(), false, 0, false);
3564 let union_fields = std::iter::once((0, Arc::new(dctfield))).collect();
3565
3566 let types = [0, 0, 0].into_iter().collect::<ScalarBuffer<i8>>();
3567 let offsets = [0, 1, 2].into_iter().collect::<ScalarBuffer<i32>>();
3568
3569 let union = UnionArray::try_new(union_fields, types, Some(offsets), vec![array]).unwrap();
3570
3571 let schema = Arc::new(Schema::new(vec![Field::new(
3572 "union",
3573 union.data_type().clone(),
3574 false,
3575 )]));
3576
3577 let r#gen = IpcDataGenerator::default();
3578 let mut dict_tracker = DictionaryTracker::new(false);
3579 r#gen.schema_to_bytes_with_dictionary_tracker(
3580 &schema,
3581 &mut dict_tracker,
3582 &IpcWriteOptions::default(),
3583 );
3584
3585 let batch = RecordBatch::try_new(schema, vec![Arc::new(union)]).unwrap();
3586
3587 r#gen
3588 .encode(
3589 &batch,
3590 &mut dict_tracker,
3591 &Default::default(),
3592 &mut Default::default(),
3593 )
3594 .unwrap();
3595
3596 assert!(dict_tracker.written.contains_key(&0));
3599 }
3600
3601 #[test]
3602 fn track_struct_nested_dict() {
3603 let inner: DictionaryArray<Int32Type> = vec!["a", "b", "a"].into_iter().collect();
3604
3605 let array = Arc::new(inner) as ArrayRef;
3606
3607 #[expect(deprecated)]
3609 let dctfield = Arc::new(Field::new_dict(
3610 "dict",
3611 array.data_type().clone(),
3612 false,
3613 2,
3614 false,
3615 ));
3616
3617 let s = StructArray::from(vec![(dctfield, array)]);
3618 let struct_array = Arc::new(s) as ArrayRef;
3619
3620 let schema = Arc::new(Schema::new(vec![Field::new(
3621 "struct",
3622 struct_array.data_type().clone(),
3623 false,
3624 )]));
3625
3626 let r#gen = IpcDataGenerator::default();
3627 let mut dict_tracker = DictionaryTracker::new(false);
3628 r#gen.schema_to_bytes_with_dictionary_tracker(
3629 &schema,
3630 &mut dict_tracker,
3631 &IpcWriteOptions::default(),
3632 );
3633
3634 let batch = RecordBatch::try_new(schema, vec![struct_array]).unwrap();
3635
3636 r#gen
3637 .encode(
3638 &batch,
3639 &mut dict_tracker,
3640 &Default::default(),
3641 &mut Default::default(),
3642 )
3643 .unwrap();
3644
3645 assert!(dict_tracker.written.contains_key(&0));
3646 }
3647
3648 fn write_union_file(options: IpcWriteOptions) {
3649 let schema = Schema::new(vec![Field::new_union(
3650 "union",
3651 vec![0, 1],
3652 vec![
3653 Field::new("a", DataType::Int32, false),
3654 Field::new("c", DataType::Float64, false),
3655 ],
3656 UnionMode::Sparse,
3657 )]);
3658 let mut builder = UnionBuilder::with_capacity_sparse(5);
3659 builder.append::<Int32Type>("a", 1).unwrap();
3660 builder.append_null::<Int32Type>("a").unwrap();
3661 builder.append::<Float64Type>("c", 3.0).unwrap();
3662 builder.append_null::<Float64Type>("c").unwrap();
3663 builder.append::<Int32Type>("a", 4).unwrap();
3664 let union = builder.build().unwrap();
3665
3666 let batch =
3667 RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(union) as ArrayRef])
3668 .unwrap();
3669
3670 let mut file = tempfile::tempfile().unwrap();
3671 {
3672 let mut writer = FileWriter::try_new_with_options(&mut file, &schema, options).unwrap();
3673
3674 writer.write(&batch).unwrap();
3675 writer.finish().unwrap();
3676 }
3677 file.rewind().unwrap();
3678
3679 {
3680 let reader = FileReader::try_new(file, None).unwrap();
3681 reader.for_each(|maybe_batch| {
3682 maybe_batch
3683 .unwrap()
3684 .columns()
3685 .iter()
3686 .zip(batch.columns())
3687 .for_each(|(a, b)| {
3688 assert_eq!(a.data_type(), b.data_type());
3689 assert_eq!(a.len(), b.len());
3690 assert_eq!(a.null_count(), b.null_count());
3691 });
3692 });
3693 }
3694 }
3695
3696 #[test]
3697 fn test_write_union_file_v4_v5() {
3698 write_union_file(IpcWriteOptions::try_new(8, false, MetadataVersion::V4).unwrap());
3699 write_union_file(IpcWriteOptions::try_new(8, false, MetadataVersion::V5).unwrap());
3700 }
3701
3702 #[test]
3703 fn test_write_view_types() {
3704 const LONG_TEST_STRING: &str =
3705 "This is a long string to make sure binary view array handles it";
3706 let schema = Schema::new(vec![
3707 Field::new("field1", DataType::BinaryView, true),
3708 Field::new("field2", DataType::Utf8View, true),
3709 ]);
3710 let values: Vec<Option<&[u8]>> = vec![
3711 Some(b"foo"),
3712 Some(b"bar"),
3713 Some(LONG_TEST_STRING.as_bytes()),
3714 ];
3715 let binary_array = BinaryViewArray::from_iter(values);
3716 let utf8_array =
3717 StringViewArray::from_iter(vec![Some("foo"), Some("bar"), Some(LONG_TEST_STRING)]);
3718 let record_batch = RecordBatch::try_new(
3719 Arc::new(schema.clone()),
3720 vec![Arc::new(binary_array), Arc::new(utf8_array)],
3721 )
3722 .unwrap();
3723
3724 let mut file = tempfile::tempfile().unwrap();
3725 {
3726 let mut writer = FileWriter::try_new(&mut file, &schema).unwrap();
3727 writer.write(&record_batch).unwrap();
3728 writer.finish().unwrap();
3729 }
3730 file.rewind().unwrap();
3731 {
3732 let mut reader = FileReader::try_new(&file, None).unwrap();
3733 let read_batch = reader.next().unwrap().unwrap();
3734 read_batch
3735 .columns()
3736 .iter()
3737 .zip(record_batch.columns())
3738 .for_each(|(a, b)| {
3739 assert_eq!(a, b);
3740 });
3741 }
3742 file.rewind().unwrap();
3743 {
3744 let mut reader = FileReader::try_new(&file, Some(vec![0])).unwrap();
3745 let read_batch = reader.next().unwrap().unwrap();
3746 assert_eq!(read_batch.num_columns(), 1);
3747 let read_array = read_batch.column(0);
3748 let write_array = record_batch.column(0);
3749 assert_eq!(read_array, write_array);
3750 }
3751 }
3752
3753 #[test]
3754 #[cfg_attr(miri, ignore)] fn truncate_ipc_record_batch() {
3756 fn create_batch(rows: usize) -> RecordBatch {
3757 let schema = Schema::new(vec![
3758 Field::new("a", DataType::Int32, false),
3759 Field::new("b", DataType::Utf8, false),
3760 ]);
3761
3762 let a = Int32Array::from_iter_values(0..rows as i32);
3763 let b = StringArray::from_iter_values((0..rows).map(|i| i.to_string()));
3764
3765 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]).unwrap()
3766 }
3767
3768 let big_record_batch = create_batch(65536);
3769
3770 let length = 5;
3771 let small_record_batch = create_batch(length);
3772
3773 let offset = 2;
3774 let record_batch_slice = big_record_batch.slice(offset, length);
3775 assert!(
3776 serialize_stream(&big_record_batch).len() > serialize_stream(&small_record_batch).len()
3777 );
3778 assert_eq!(
3779 serialize_stream(&small_record_batch).len(),
3780 serialize_stream(&record_batch_slice).len()
3781 );
3782
3783 assert_eq!(
3784 deserialize_stream(serialize_stream(&record_batch_slice)),
3785 record_batch_slice
3786 );
3787 }
3788
3789 #[test]
3790 fn truncate_ipc_record_batch_with_nulls() {
3791 fn create_batch() -> RecordBatch {
3792 let schema = Schema::new(vec![
3793 Field::new("a", DataType::Int32, true),
3794 Field::new("b", DataType::Utf8, true),
3795 ]);
3796
3797 let a = Int32Array::from(vec![Some(1), None, Some(1), None, Some(1)]);
3798 let b = StringArray::from(vec![None, Some("a"), Some("a"), None, Some("a")]);
3799
3800 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]).unwrap()
3801 }
3802
3803 let record_batch = create_batch();
3804 let record_batch_slice = record_batch.slice(1, 2);
3805 let deserialized_batch = deserialize_stream(serialize_stream(&record_batch_slice));
3806
3807 assert!(
3808 serialize_stream(&record_batch).len() > serialize_stream(&record_batch_slice).len()
3809 );
3810
3811 assert!(deserialized_batch.column(0).is_null(0));
3812 assert!(deserialized_batch.column(0).is_valid(1));
3813 assert!(deserialized_batch.column(1).is_valid(0));
3814 assert!(deserialized_batch.column(1).is_valid(1));
3815
3816 assert_eq!(record_batch_slice, deserialized_batch);
3817 }
3818
3819 #[test]
3820 fn test_write_empty_validity_buffer_without_nulls() {
3821 let batch = RecordBatch::try_from_iter([
3822 ("a", Arc::new(Int32Array::from(vec![1, 2])) as _),
3823 (
3825 "b",
3826 Arc::new(Int32Array::from(vec![None, Some(2), Some(3)]).slice(1, 2)) as _,
3827 ),
3828 ("c", Arc::new(Int32Array::from(vec![Some(1), None])) as _),
3829 ])
3830 .unwrap();
3831 assert_eq!(batch.column(1).nulls().map(|n| n.null_count()), Some(0));
3832
3833 let (_, encoded) = IpcDataGenerator::default()
3834 .encode(
3835 &batch,
3836 &mut DictionaryTracker::new(false),
3837 &Default::default(),
3838 &mut Default::default(),
3839 )
3840 .unwrap();
3841 let message = crate::root_as_message(&encoded.ipc_message).unwrap();
3842 let buffers = message.header_as_record_batch().unwrap().buffers().unwrap();
3843 let validity_lengths: Vec<_> = buffers.iter().step_by(2).map(|b| b.length()).collect();
3845 assert_eq!(validity_lengths, [0, 0, 1]);
3846
3847 assert_eq!(deserialize_stream(serialize_stream(&batch)), batch);
3848 }
3849
3850 #[test]
3851 fn truncate_ipc_dictionary_array() {
3852 fn create_batch() -> RecordBatch {
3853 let values: StringArray = [Some("foo"), Some("bar"), Some("baz")]
3854 .into_iter()
3855 .collect();
3856 let keys: Int32Array = [Some(0), Some(2), None, Some(1)].into_iter().collect();
3857
3858 let array = DictionaryArray::new(keys, Arc::new(values));
3859
3860 let schema = Schema::new(vec![Field::new("dict", array.data_type().clone(), true)]);
3861
3862 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap()
3863 }
3864
3865 let record_batch = create_batch();
3866 let record_batch_slice = record_batch.slice(1, 2);
3867 let deserialized_batch = deserialize_stream(serialize_stream(&record_batch_slice));
3868
3869 assert!(
3870 serialize_stream(&record_batch).len() > serialize_stream(&record_batch_slice).len()
3871 );
3872
3873 assert!(deserialized_batch.column(0).is_valid(0));
3874 assert!(deserialized_batch.column(0).is_null(1));
3875
3876 assert_eq!(record_batch_slice, deserialized_batch);
3877 }
3878
3879 #[test]
3880 fn truncate_ipc_struct_array() {
3881 fn create_batch() -> RecordBatch {
3882 let strings: StringArray = [Some("foo"), None, Some("bar"), Some("baz")]
3883 .into_iter()
3884 .collect();
3885 let ints: Int32Array = [Some(0), Some(2), None, Some(1)].into_iter().collect();
3886
3887 let struct_array = StructArray::from(vec![
3888 (
3889 Arc::new(Field::new("s", DataType::Utf8, true)),
3890 Arc::new(strings) as ArrayRef,
3891 ),
3892 (
3893 Arc::new(Field::new("c", DataType::Int32, true)),
3894 Arc::new(ints) as ArrayRef,
3895 ),
3896 ]);
3897
3898 let schema = Schema::new(vec![Field::new(
3899 "struct_array",
3900 struct_array.data_type().clone(),
3901 true,
3902 )]);
3903
3904 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(struct_array)]).unwrap()
3905 }
3906
3907 let record_batch = create_batch();
3908 let record_batch_slice = record_batch.slice(1, 2);
3909 let deserialized_batch = deserialize_stream(serialize_stream(&record_batch_slice));
3910
3911 assert!(
3912 serialize_stream(&record_batch).len() > serialize_stream(&record_batch_slice).len()
3913 );
3914
3915 let structs = deserialized_batch
3916 .column(0)
3917 .as_any()
3918 .downcast_ref::<StructArray>()
3919 .unwrap();
3920
3921 assert!(structs.column(0).is_null(0));
3922 assert!(structs.column(0).is_valid(1));
3923 assert!(structs.column(1).is_valid(0));
3924 assert!(structs.column(1).is_null(1));
3925 assert_eq!(record_batch_slice, deserialized_batch);
3926 }
3927
3928 #[test]
3929 fn truncate_ipc_string_array_with_all_empty_string() {
3930 fn create_batch() -> RecordBatch {
3931 let schema = Schema::new(vec![Field::new("a", DataType::Utf8, true)]);
3932 let a = StringArray::from(vec![Some(""), Some(""), Some(""), Some(""), Some("")]);
3933 RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a)]).unwrap()
3934 }
3935
3936 let record_batch = create_batch();
3937 let record_batch_slice = record_batch.slice(0, 1);
3938 let deserialized_batch = deserialize_stream(serialize_stream(&record_batch_slice));
3939
3940 assert!(
3941 serialize_stream(&record_batch).len() > serialize_stream(&record_batch_slice).len()
3942 );
3943 assert_eq!(record_batch_slice, deserialized_batch);
3944 }
3945
3946 #[test]
3947 fn test_stream_writer_writes_array_slice() {
3948 let array = UInt32Array::from(vec![Some(1), Some(2), Some(3)]);
3949 assert_eq!(
3950 vec![Some(1), Some(2), Some(3)],
3951 array.iter().collect::<Vec<_>>()
3952 );
3953
3954 let sliced = array.slice(1, 2);
3955 assert_eq!(vec![Some(2), Some(3)], sliced.iter().collect::<Vec<_>>());
3956
3957 let batch = RecordBatch::try_new(
3958 Arc::new(Schema::new(vec![Field::new("a", DataType::UInt32, true)])),
3959 vec![Arc::new(sliced)],
3960 )
3961 .expect("new batch");
3962
3963 let mut writer = StreamWriter::try_new(vec![], batch.schema_ref()).expect("new writer");
3964 writer.write(&batch).expect("write");
3965 let outbuf = writer.into_inner().expect("inner");
3966
3967 let mut reader = StreamReader::try_new(&outbuf[..], None).expect("new reader");
3968 let read_batch = reader.next().unwrap().expect("read batch");
3969
3970 let read_array: &UInt32Array = read_batch.column(0).as_primitive();
3971 assert_eq!(
3972 vec![Some(2), Some(3)],
3973 read_array.iter().collect::<Vec<_>>()
3974 );
3975 }
3976
3977 #[test]
3978 fn test_large_slice_uint32() {
3979 ensure_roundtrip(Arc::new(UInt32Array::from_iter(
3980 (0..8000).map(|i| if i % 2 == 0 { Some(i) } else { None }),
3981 )));
3982 }
3983
3984 #[test]
3985 #[cfg_attr(miri, ignore)] fn test_large_slice_string() {
3987 let strings: Vec<_> = (0..8000)
3988 .map(|i| {
3989 if i % 2 == 0 {
3990 Some(format!("value{i}"))
3991 } else {
3992 None
3993 }
3994 })
3995 .collect();
3996
3997 ensure_roundtrip(Arc::new(StringArray::from(strings)));
3998 }
3999
4000 #[test]
4001 #[cfg_attr(miri, ignore)] fn test_large_slice_string_list() {
4003 let mut ls = ListBuilder::new(StringBuilder::new());
4004
4005 let mut s = String::new();
4006 for row_number in 0..8000 {
4007 if row_number % 2 == 0 {
4008 for list_element in 0..1000 {
4009 s.clear();
4010 use std::fmt::Write;
4011 write!(&mut s, "value{row_number}-{list_element}").unwrap();
4012 ls.values().append_value(&s);
4013 }
4014 ls.append(true)
4015 } else {
4016 ls.append(false); }
4018 }
4019
4020 ensure_roundtrip(Arc::new(ls.finish()));
4021 }
4022
4023 #[test]
4024 #[cfg_attr(miri, ignore)] fn test_large_slice_string_list_of_lists() {
4026 let mut ls = ListBuilder::new(ListBuilder::new(StringBuilder::new()));
4030
4031 for _ in 0..4000 {
4032 ls.values().append(true);
4033 ls.append(true)
4034 }
4035
4036 let mut s = String::new();
4037 for row_number in 0..4000 {
4038 if row_number % 2 == 0 {
4039 for list_element in 0..1000 {
4040 s.clear();
4041 use std::fmt::Write;
4042 write!(&mut s, "value{row_number}-{list_element}").unwrap();
4043 ls.values().values().append_value(&s);
4044 }
4045 ls.values().append(true);
4046 ls.append(true)
4047 } else {
4048 ls.append(false); }
4050 }
4051
4052 ensure_roundtrip(Arc::new(ls.finish()));
4053 }
4054
4055 fn ensure_roundtrip(array: ArrayRef) {
4057 let num_rows = array.len();
4058 let orig_batch = RecordBatch::try_from_iter(vec![("a", array)]).unwrap();
4059 let sliced_batch = orig_batch.slice(1, num_rows - 1);
4061
4062 let schema = orig_batch.schema();
4063 let stream_data = {
4064 let mut writer = StreamWriter::try_new(vec![], &schema).unwrap();
4065 writer.write(&sliced_batch).unwrap();
4066 writer.into_inner().unwrap()
4067 };
4068 let read_batch = {
4069 let projection = None;
4070 let mut reader = StreamReader::try_new(Cursor::new(stream_data), projection).unwrap();
4071 reader
4072 .next()
4073 .expect("expect no errors reading batch")
4074 .expect("expect batch")
4075 };
4076 assert_eq!(sliced_batch, read_batch);
4077
4078 let file_data = {
4079 let mut writer = FileWriter::try_new_buffered(vec![], &schema).unwrap();
4080 writer.write(&sliced_batch).unwrap();
4081 writer.into_inner().unwrap().into_inner().unwrap()
4082 };
4083 let read_batch = {
4084 let projection = None;
4085 let mut reader = FileReader::try_new(Cursor::new(file_data), projection).unwrap();
4086 reader
4087 .next()
4088 .expect("expect no errors reading batch")
4089 .expect("expect batch")
4090 };
4091 assert_eq!(sliced_batch, read_batch);
4092
4093 }
4095
4096 #[test]
4097 fn encode_bools_slice() {
4098 assert_bool_roundtrip([true, false], 1, 1);
4100
4101 assert_bool_roundtrip(
4103 [
4104 true, false, true, true, false, false, true, true, true, false, false, false, true,
4105 true, true, true, false, false, false, false, true, true, true, true, true, false,
4106 false, false, false, false,
4107 ],
4108 13,
4109 17,
4110 );
4111
4112 assert_bool_roundtrip(
4114 [
4115 true, false, true, true, false, false, true, true, true, false, false, false,
4116 ],
4117 8,
4118 2,
4119 );
4120
4121 assert_bool_roundtrip(
4123 [
4124 true, false, true, true, false, false, true, true, true, false, false, false, true,
4125 true, true, true, true, false, false, false, false, false,
4126 ],
4127 8,
4128 8,
4129 );
4130 }
4131
4132 fn assert_bool_roundtrip<const N: usize>(bools: [bool; N], offset: usize, length: usize) {
4133 let val_bool_field = Field::new("val", DataType::Boolean, false);
4134
4135 let schema = Arc::new(Schema::new(vec![val_bool_field]));
4136
4137 let bools = BooleanArray::from(bools.to_vec());
4138
4139 let batch = RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(bools)]).unwrap();
4140 let batch = batch.slice(offset, length);
4141
4142 let data = serialize_stream(&batch);
4143 let batch2 = deserialize_stream(data);
4144 assert_eq!(batch, batch2);
4145 }
4146
4147 fn assert_empty_run_array_roundtrip<R: RunEndIndexType>() {
4148 let run_ends = PrimitiveArray::<R>::from_iter_values(
4149 [2, 5]
4150 .into_iter()
4151 .map(|v| R::Native::from_usize(v).unwrap()),
4152 );
4153 let values = Int32Array::from(vec![10, 20]);
4154 let array = RunArray::<R>::try_new(&run_ends, &values).unwrap();
4155 let empty = RunArray::<R>::from(ArrayData::new_empty(array.data_type()));
4156
4157 for source in [&array, &empty] {
4158 for offset in 0..=source.len() {
4159 let sliced = source.slice(offset, 0);
4160 let batch = RecordBatch::try_from_iter(vec![("run", Arc::new(sliced) as ArrayRef)])
4161 .unwrap();
4162 for decoded in [
4163 deserialize_stream(serialize_stream(&batch)),
4164 deserialize_file(serialize_file(&batch)),
4165 ] {
4166 assert_eq!(decoded, batch);
4167 let data = decoded.column(0).to_data();
4168 data.validate_full().unwrap();
4169 assert_eq!(data.offset(), 0);
4170 assert!(data.child_data().iter().all(ArrayData::is_empty));
4171 }
4172
4173 let normalized = into_zero_offset_run_array(source.slice(offset, 0)).unwrap();
4174 normalized.to_data().validate_full().unwrap();
4175 assert_eq!(normalized.offset(), 0);
4176 assert!(normalized.run_ends().values().is_empty());
4177 assert!(normalized.values().is_empty());
4178 }
4179 }
4180 }
4181
4182 #[test]
4183 fn test_empty_run_array_roundtrip_i16() {
4184 assert_empty_run_array_roundtrip::<Int16Type>();
4185 }
4186
4187 #[test]
4188 fn test_empty_run_array_roundtrip_i32() {
4189 assert_empty_run_array_roundtrip::<Int32Type>();
4190 }
4191
4192 #[test]
4193 fn test_empty_run_array_roundtrip_i64() {
4194 assert_empty_run_array_roundtrip::<Int64Type>();
4195 }
4196
4197 #[test]
4198 fn test_run_array_unslice() {
4199 let total_len = 80;
4200 let vals: Vec<Option<i32>> = vec![Some(1), None, Some(2), Some(3), Some(4), None, Some(5)];
4201 let repeats: Vec<usize> = vec![3, 4, 1, 2];
4202 let mut input_array: Vec<Option<i32>> = Vec::with_capacity(total_len);
4203 for ix in 0_usize..32 {
4204 let repeat: usize = repeats[ix % repeats.len()];
4205 let val: Option<i32> = vals[ix % vals.len()];
4206 input_array.resize(input_array.len() + repeat, val);
4207 }
4208
4209 let mut builder =
4211 PrimitiveRunBuilder::<Int16Type, Int32Type>::with_capacity(input_array.len());
4212 builder.extend(input_array.iter().copied());
4213 let run_array = builder.finish();
4214
4215 for slice_len in 1..=total_len {
4217 let sliced_run_array: RunArray<Int16Type> =
4219 run_array.slice(0, slice_len).into_data().into();
4220
4221 let unsliced_run_array = into_zero_offset_run_array(sliced_run_array).unwrap();
4223 let typed = unsliced_run_array
4224 .downcast::<PrimitiveArray<Int32Type>>()
4225 .unwrap();
4226 let expected: Vec<Option<i32>> = input_array.iter().take(slice_len).copied().collect();
4227 let actual: Vec<Option<i32>> = typed.into_iter().collect();
4228 assert_eq!(expected, actual);
4229
4230 let sliced_run_array: RunArray<Int16Type> = run_array
4232 .slice(total_len - slice_len, slice_len)
4233 .into_data()
4234 .into();
4235
4236 let unsliced_run_array = into_zero_offset_run_array(sliced_run_array).unwrap();
4238 let typed = unsliced_run_array
4239 .downcast::<PrimitiveArray<Int32Type>>()
4240 .unwrap();
4241 let expected: Vec<Option<i32>> = input_array
4242 .iter()
4243 .skip(total_len - slice_len)
4244 .copied()
4245 .collect();
4246 let actual: Vec<Option<i32>> = typed.into_iter().collect();
4247 assert_eq!(expected, actual);
4248 }
4249 }
4250
4251 fn generate_list_data<O: OffsetSizeTrait>() -> GenericListArray<O> {
4252 let mut ls = GenericListBuilder::<O, _>::new(UInt32Builder::new());
4253
4254 for i in 0..100_000 {
4255 for value in [i, i, i] {
4256 ls.values().append_value(value);
4257 }
4258 ls.append(true)
4259 }
4260
4261 ls.finish()
4262 }
4263
4264 fn generate_utf8view_list_data<O: OffsetSizeTrait>() -> GenericListArray<O> {
4265 let mut ls = GenericListBuilder::<O, _>::new(StringViewBuilder::new());
4266
4267 for i in 0..100_000 {
4268 for value in [
4269 format!("value{i}"),
4270 format!("value{i}"),
4271 format!("value{i}"),
4272 ] {
4273 ls.values().append_value(&value);
4274 }
4275 ls.append(true)
4276 }
4277
4278 ls.finish()
4279 }
4280
4281 fn generate_string_list_data<O: OffsetSizeTrait>() -> GenericListArray<O> {
4282 let mut ls = GenericListBuilder::<O, _>::new(StringBuilder::new());
4283
4284 for i in 0..100_000 {
4285 for value in [
4286 format!("value{i}"),
4287 format!("value{i}"),
4288 format!("value{i}"),
4289 ] {
4290 ls.values().append_value(&value);
4291 }
4292 ls.append(true)
4293 }
4294
4295 ls.finish()
4296 }
4297
4298 fn generate_nested_list_data<O: OffsetSizeTrait>() -> GenericListArray<O> {
4299 let mut ls =
4300 GenericListBuilder::<O, _>::new(GenericListBuilder::<O, _>::new(UInt32Builder::new()));
4301
4302 for _i in 0..10_000 {
4303 for j in 0..10 {
4304 for value in [j, j, j, j] {
4305 ls.values().values().append_value(value);
4306 }
4307 ls.values().append(true)
4308 }
4309 ls.append(true);
4310 }
4311
4312 ls.finish()
4313 }
4314
4315 fn generate_nested_list_data_starting_at_zero<O: OffsetSizeTrait>() -> GenericListArray<O> {
4316 let mut ls =
4317 GenericListBuilder::<O, _>::new(GenericListBuilder::<O, _>::new(UInt32Builder::new()));
4318
4319 for _i in 0..999 {
4320 ls.values().append(true);
4321 ls.append(true);
4322 }
4323
4324 for j in 0..10 {
4325 for value in [j, j, j, j] {
4326 ls.values().values().append_value(value);
4327 }
4328 ls.values().append(true)
4329 }
4330 ls.append(true);
4331
4332 for i in 0..9_000 {
4333 for j in 0..10 {
4334 for value in [i + j, i + j, i + j, i + j] {
4335 ls.values().values().append_value(value);
4336 }
4337 ls.values().append(true)
4338 }
4339 ls.append(true);
4340 }
4341
4342 ls.finish()
4343 }
4344
4345 fn generate_map_array_data() -> MapArray {
4346 let keys_builder = UInt32Builder::new();
4347 let values_builder = UInt32Builder::new();
4348
4349 let mut builder = MapBuilder::new(None, keys_builder, values_builder);
4350
4351 for i in 0..100_000 {
4352 for _j in 0..3 {
4353 builder.keys().append_value(i);
4354 builder.values().append_value(i * 2);
4355 }
4356 builder.append(true).unwrap();
4357 }
4358
4359 builder.finish()
4360 }
4361
4362 #[test]
4363 #[cfg_attr(miri, ignore)] fn reencode_offsets_when_first_offset_is_not_zero() {
4365 let original_list = generate_list_data::<i32>();
4366 let original_data = original_list.into_data();
4367 let slice_data = original_data.slice(75, 7);
4368 let (new_offsets, original_start, length) =
4369 reencode_offsets::<i32>(&slice_data.buffers()[0], &slice_data);
4370 assert_eq!(
4371 vec![0, 3, 6, 9, 12, 15, 18, 21],
4372 new_offsets.typed_data::<i32>()
4373 );
4374 assert_eq!(225, original_start);
4375 assert_eq!(21, length);
4376 }
4377
4378 #[test]
4379 fn reencode_offsets_when_first_offset_is_zero() {
4380 let mut ls = GenericListBuilder::<i32, _>::new(UInt32Builder::new());
4381 ls.append(true);
4383 ls.values().append_value(35);
4384 ls.values().append_value(42);
4385 ls.append(true);
4386 let original_list = ls.finish();
4387 let original_data = original_list.into_data();
4388
4389 let slice_data = original_data.slice(1, 1);
4390 let (new_offsets, original_start, length) =
4391 reencode_offsets::<i32>(&slice_data.buffers()[0], &slice_data);
4392 assert_eq!(vec![0, 2], new_offsets.typed_data::<i32>());
4393 assert_eq!(0, original_start);
4394 assert_eq!(2, length);
4395 }
4396
4397 fn roundtrip_ensure_sliced_smaller(in_batch: RecordBatch, expected_size_factor: usize) {
4400 let in_sliced = in_batch.slice(999, 1);
4402
4403 let bytes_batch = serialize_file(&in_batch);
4404 let bytes_sliced = serialize_file(&in_sliced);
4405
4406 assert!(bytes_sliced.len() < (bytes_batch.len() / expected_size_factor));
4408
4409 let out_batch = deserialize_file(bytes_batch);
4411 assert_eq!(in_batch, out_batch);
4412
4413 let out_sliced = deserialize_file(bytes_sliced);
4414 assert_eq!(in_sliced, out_sliced);
4415 }
4416
4417 #[test]
4418 #[cfg_attr(miri, ignore)] fn encode_lists() {
4420 let val_inner = Field::new_list_field(DataType::UInt32, true);
4421 let val_list_field = Field::new("val", DataType::List(Arc::new(val_inner)), false);
4422 let schema = Arc::new(Schema::new(vec![val_list_field]));
4423
4424 let values = Arc::new(generate_list_data::<i32>());
4425
4426 let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4427 roundtrip_ensure_sliced_smaller(in_batch, 1000);
4428 }
4429
4430 #[test]
4431 #[cfg_attr(miri, ignore)] fn encode_empty_list() {
4433 let val_inner = Field::new_list_field(DataType::UInt32, true);
4434 let val_list_field = Field::new("val", DataType::List(Arc::new(val_inner)), false);
4435 let schema = Arc::new(Schema::new(vec![val_list_field]));
4436
4437 let values = Arc::new(generate_list_data::<i32>());
4438
4439 let in_batch = RecordBatch::try_new(schema, vec![values])
4440 .unwrap()
4441 .slice(999, 0);
4442 let out_batch = deserialize_file(serialize_file(&in_batch));
4443 assert_eq!(in_batch, out_batch);
4444 }
4445
4446 #[test]
4447 #[cfg_attr(miri, ignore)] fn encode_large_lists() {
4449 let val_inner = Field::new_list_field(DataType::UInt32, true);
4450 let val_list_field = Field::new("val", DataType::LargeList(Arc::new(val_inner)), false);
4451 let schema = Arc::new(Schema::new(vec![val_list_field]));
4452
4453 let values = Arc::new(generate_list_data::<i64>());
4454
4455 let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4458 roundtrip_ensure_sliced_smaller(in_batch, 1000);
4459 }
4460
4461 #[test]
4462 #[cfg_attr(miri, ignore)] fn encode_large_lists_non_zero_offset() {
4464 let val_inner = Field::new_list_field(DataType::UInt32, true);
4465 let val_list_field = Field::new("val", DataType::LargeList(Arc::new(val_inner)), false);
4466 let schema = Arc::new(Schema::new(vec![val_list_field]));
4467
4468 let values = Arc::new(generate_list_data::<i64>());
4469
4470 check_sliced_list_array(schema, values);
4471 }
4472
4473 #[test]
4474 #[cfg_attr(miri, ignore)] fn encode_large_lists_string_non_zero_offset() {
4476 let val_inner = Field::new_list_field(DataType::Utf8, true);
4477 let val_list_field = Field::new("val", DataType::LargeList(Arc::new(val_inner)), false);
4478 let schema = Arc::new(Schema::new(vec![val_list_field]));
4479
4480 let values = Arc::new(generate_string_list_data::<i64>());
4481
4482 check_sliced_list_array(schema, values);
4483 }
4484
4485 #[test]
4486 #[cfg_attr(miri, ignore)] fn encode_large_list_string_view_non_zero_offset() {
4488 let val_inner = Field::new_list_field(DataType::Utf8View, true);
4489 let val_list_field = Field::new("val", DataType::LargeList(Arc::new(val_inner)), false);
4490 let schema = Arc::new(Schema::new(vec![val_list_field]));
4491
4492 let values = Arc::new(generate_utf8view_list_data::<i64>());
4493
4494 check_sliced_list_array(schema, values);
4495 }
4496
4497 fn check_sliced_list_array(schema: Arc<Schema>, values: Arc<GenericListArray<i64>>) {
4498 for (offset, len) in [(999, 1), (0, 13), (47, 12), (values.len() - 13, 13)] {
4499 let in_batch = RecordBatch::try_new(schema.clone(), vec![values.clone()])
4500 .unwrap()
4501 .slice(offset, len);
4502 let out_batch = deserialize_file(serialize_file(&in_batch));
4503 assert_eq!(in_batch, out_batch);
4504 }
4505 }
4506
4507 #[test]
4508 #[cfg_attr(miri, ignore)] fn encode_nested_lists() {
4510 let inner_int = Arc::new(Field::new_list_field(DataType::UInt32, true));
4511 let inner_list_field = Arc::new(Field::new_list_field(DataType::List(inner_int), true));
4512 let list_field = Field::new("val", DataType::List(inner_list_field), true);
4513 let schema = Arc::new(Schema::new(vec![list_field]));
4514
4515 let values = Arc::new(generate_nested_list_data::<i32>());
4516
4517 let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4518 roundtrip_ensure_sliced_smaller(in_batch, 1000);
4519 }
4520
4521 #[test]
4522 #[cfg_attr(miri, ignore)] fn encode_nested_lists_starting_at_zero() {
4524 let inner_int = Arc::new(Field::new("item", DataType::UInt32, true));
4525 let inner_list_field = Arc::new(Field::new("item", DataType::List(inner_int), true));
4526 let list_field = Field::new("val", DataType::List(inner_list_field), true);
4527 let schema = Arc::new(Schema::new(vec![list_field]));
4528
4529 let values = Arc::new(generate_nested_list_data_starting_at_zero::<i32>());
4530
4531 let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4532 roundtrip_ensure_sliced_smaller(in_batch, 1);
4533 }
4534
4535 #[test]
4536 #[cfg_attr(miri, ignore)] fn encode_map_array() {
4538 let keys = Arc::new(Field::new(
4539 Field::MAP_KEY_FIELD_DEFAULT_NAME,
4540 DataType::UInt32,
4541 false,
4542 ));
4543 let values = Arc::new(Field::new(
4544 Field::MAP_VALUE_FIELD_DEFAULT_NAME,
4545 DataType::UInt32,
4546 true,
4547 ));
4548 let map_field = Field::new_map(
4549 "map",
4550 Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
4551 keys,
4552 values,
4553 false,
4554 true,
4555 );
4556 let schema = Arc::new(Schema::new(vec![map_field]));
4557
4558 let values = Arc::new(generate_map_array_data());
4559
4560 let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4561 roundtrip_ensure_sliced_smaller(in_batch, 1000);
4562 }
4563
4564 fn generate_list_view_data<O: OffsetSizeTrait>() -> GenericListViewArray<O> {
4565 let mut builder = GenericListViewBuilder::<O, _>::new(UInt32Builder::new());
4566
4567 for i in 0u32..100_000 {
4568 if i.is_multiple_of(10_000) {
4569 builder.append(false);
4570 continue;
4571 }
4572 for value in [i, i, i] {
4573 builder.values().append_value(value);
4574 }
4575 builder.append(true);
4576 }
4577
4578 builder.finish()
4579 }
4580
4581 #[test]
4582 #[cfg_attr(miri, ignore)] fn encode_list_view_arrays() {
4584 let val_inner = Field::new_list_field(DataType::UInt32, true);
4585 let val_field = Field::new("val", DataType::ListView(Arc::new(val_inner)), true);
4586 let schema = Arc::new(Schema::new(vec![val_field]));
4587
4588 let values = Arc::new(generate_list_view_data::<i32>());
4589
4590 let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4591 let out_batch = deserialize_file(serialize_file(&in_batch));
4592 assert_eq!(in_batch, out_batch);
4593 }
4594
4595 #[test]
4596 #[cfg_attr(miri, ignore)] fn encode_large_list_view_arrays() {
4598 let val_inner = Field::new_list_field(DataType::UInt32, true);
4599 let val_field = Field::new("val", DataType::LargeListView(Arc::new(val_inner)), true);
4600 let schema = Arc::new(Schema::new(vec![val_field]));
4601
4602 let values = Arc::new(generate_list_view_data::<i64>());
4603
4604 let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4605 let out_batch = deserialize_file(serialize_file(&in_batch));
4606 assert_eq!(in_batch, out_batch);
4607 }
4608
4609 #[test]
4610 #[cfg_attr(miri, ignore)] fn check_sliced_list_view_array() {
4612 let inner = Field::new_list_field(DataType::UInt32, true);
4613 let field = Field::new("val", DataType::ListView(Arc::new(inner)), true);
4614 let schema = Arc::new(Schema::new(vec![field]));
4615 let values = Arc::new(generate_list_view_data::<i32>());
4616
4617 for (offset, len) in [(999, 1), (0, 13), (47, 12), (values.len() - 13, 13)] {
4618 let in_batch = RecordBatch::try_new(schema.clone(), vec![values.clone()])
4619 .unwrap()
4620 .slice(offset, len);
4621 let out_batch = deserialize_file(serialize_file(&in_batch));
4622 assert_eq!(in_batch, out_batch);
4623 }
4624 }
4625
4626 #[test]
4627 #[cfg_attr(miri, ignore)] fn check_sliced_large_list_view_array() {
4629 let inner = Field::new_list_field(DataType::UInt32, true);
4630 let field = Field::new("val", DataType::LargeListView(Arc::new(inner)), true);
4631 let schema = Arc::new(Schema::new(vec![field]));
4632 let values = Arc::new(generate_list_view_data::<i64>());
4633
4634 for (offset, len) in [(999, 1), (0, 13), (47, 12), (values.len() - 13, 13)] {
4635 let in_batch = RecordBatch::try_new(schema.clone(), vec![values.clone()])
4636 .unwrap()
4637 .slice(offset, len);
4638 let out_batch = deserialize_file(serialize_file(&in_batch));
4639 assert_eq!(in_batch, out_batch);
4640 }
4641 }
4642
4643 fn generate_nested_list_view_data<O: OffsetSizeTrait>() -> GenericListViewArray<O> {
4644 let inner_builder = UInt32Builder::new();
4645 let middle_builder = GenericListViewBuilder::<O, _>::new(inner_builder);
4646 let mut outer_builder = GenericListViewBuilder::<O, _>::new(middle_builder);
4647
4648 for i in 0u32..10_000 {
4649 if i.is_multiple_of(1_000) {
4650 outer_builder.append(false);
4651 continue;
4652 }
4653
4654 for _ in 0..3 {
4655 for value in [i, i + 1, i + 2] {
4656 outer_builder.values().values().append_value(value);
4657 }
4658 outer_builder.values().append(true);
4659 }
4660 outer_builder.append(true);
4661 }
4662
4663 outer_builder.finish()
4664 }
4665
4666 #[test]
4667 #[cfg_attr(miri, ignore)] fn encode_nested_list_views() {
4669 let inner_int = Arc::new(Field::new_list_field(DataType::UInt32, true));
4670 let inner_list_field = Arc::new(Field::new_list_field(DataType::ListView(inner_int), true));
4671 let list_field = Field::new("val", DataType::ListView(inner_list_field), true);
4672 let schema = Arc::new(Schema::new(vec![list_field]));
4673
4674 let values = Arc::new(generate_nested_list_view_data::<i32>());
4675
4676 let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4677 let out_batch = deserialize_file(serialize_file(&in_batch));
4678 assert_eq!(in_batch, out_batch);
4679 }
4680
4681 fn test_roundtrip_list_view_of_dict_impl<OffsetSize: OffsetSizeTrait, U: ArrowNativeType>(
4682 list_data_type: DataType,
4683 offsets: &[U; 5],
4684 sizes: &[U; 4],
4685 ) {
4686 let values = StringArray::from(vec![Some("alpha"), None, Some("beta"), Some("gamma")]);
4687 let keys = Int32Array::from_iter_values([0, 0, 1, 2, 3, 0, 2]);
4688 let dict_array = DictionaryArray::new(keys, Arc::new(values));
4689 let dict_data = dict_array.to_data();
4690
4691 let value_offsets = Buffer::from_slice_ref(offsets);
4692 let value_sizes = Buffer::from_slice_ref(sizes);
4693
4694 let list_data = ArrayData::builder(list_data_type)
4695 .len(4)
4696 .add_buffer(value_offsets)
4697 .add_buffer(value_sizes)
4698 .add_child_data(dict_data)
4699 .build()
4700 .unwrap();
4701 let list_view_array = GenericListViewArray::<OffsetSize>::from(list_data);
4702
4703 let schema = Arc::new(Schema::new(vec![Field::new(
4704 "f1",
4705 list_view_array.data_type().clone(),
4706 false,
4707 )]));
4708 let input_batch = RecordBatch::try_new(schema, vec![Arc::new(list_view_array)]).unwrap();
4709
4710 let output_batch = deserialize_file(serialize_file(&input_batch));
4711 assert_eq!(input_batch, output_batch);
4712
4713 let output_batch = deserialize_stream(serialize_stream(&input_batch));
4714 assert_eq!(input_batch, output_batch);
4715 }
4716
4717 #[test]
4718 fn test_roundtrip_list_view_of_dict() {
4719 #[expect(deprecated)]
4720 let list_data_type = DataType::ListView(Arc::new(Field::new_dict(
4721 "item",
4722 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4723 true,
4724 1,
4725 false,
4726 )));
4727 let offsets: &[i32; 5] = &[0, 2, 4, 4, 7];
4728 let sizes: &[i32; 4] = &[2, 2, 0, 3];
4729 test_roundtrip_list_view_of_dict_impl::<i32, i32>(list_data_type, offsets, sizes);
4730 }
4731
4732 #[test]
4733 fn test_roundtrip_large_list_view_of_dict() {
4734 #[expect(deprecated)]
4735 let list_data_type = DataType::LargeListView(Arc::new(Field::new_dict(
4736 "item",
4737 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4738 true,
4739 2,
4740 false,
4741 )));
4742 let offsets: &[i64; 5] = &[0, 2, 4, 4, 7];
4743 let sizes: &[i64; 4] = &[2, 2, 0, 3];
4744 test_roundtrip_list_view_of_dict_impl::<i64, i64>(list_data_type, offsets, sizes);
4745 }
4746
4747 #[test]
4748 fn test_roundtrip_sliced_list_view_of_dict() {
4749 #[expect(deprecated)]
4750 let list_data_type = DataType::ListView(Arc::new(Field::new_dict(
4751 "item",
4752 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4753 true,
4754 3,
4755 false,
4756 )));
4757
4758 let values = StringArray::from(vec![Some("alpha"), None, Some("beta"), Some("gamma")]);
4759 let keys = Int32Array::from_iter_values([0, 0, 1, 2, 3, 0, 2, 1, 0, 3, 2, 1]);
4760 let dict_array = DictionaryArray::new(keys, Arc::new(values));
4761 let dict_data = dict_array.to_data();
4762
4763 let offsets: &[i32; 7] = &[0, 2, 4, 4, 7, 9, 12];
4764 let sizes: &[i32; 6] = &[2, 2, 0, 3, 2, 3];
4765 let value_offsets = Buffer::from_slice_ref(offsets);
4766 let value_sizes = Buffer::from_slice_ref(sizes);
4767
4768 let list_data = ArrayData::builder(list_data_type)
4769 .len(6)
4770 .add_buffer(value_offsets)
4771 .add_buffer(value_sizes)
4772 .add_child_data(dict_data)
4773 .build()
4774 .unwrap();
4775 let list_view_array = GenericListViewArray::<i32>::from(list_data);
4776
4777 let schema = Arc::new(Schema::new(vec![Field::new(
4778 "f1",
4779 list_view_array.data_type().clone(),
4780 false,
4781 )]));
4782 let input_batch = RecordBatch::try_new(schema, vec![Arc::new(list_view_array)]).unwrap();
4783
4784 let sliced_batch = input_batch.slice(1, 4);
4785
4786 let output_batch = deserialize_file(serialize_file(&sliced_batch));
4787 assert_eq!(sliced_batch, output_batch);
4788
4789 let output_batch = deserialize_stream(serialize_stream(&sliced_batch));
4790 assert_eq!(sliced_batch, output_batch);
4791 }
4792
4793 #[test]
4794 fn test_roundtrip_dense_union_of_dict() {
4795 let values = StringArray::from(vec![Some("alpha"), None, Some("beta"), Some("gamma")]);
4796 let keys = Int32Array::from_iter_values([0, 0, 1, 2, 3, 0, 2]);
4797 let dict_array = DictionaryArray::new(keys, Arc::new(values));
4798
4799 #[expect(deprecated)]
4800 let dict_field = Arc::new(Field::new_dict(
4801 "dict",
4802 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4803 true,
4804 1,
4805 false,
4806 ));
4807 let int_field = Arc::new(Field::new("int", DataType::Int32, false));
4808 let union_fields = UnionFields::try_new(vec![0, 1], vec![dict_field, int_field]).unwrap();
4809
4810 let types = ScalarBuffer::from(vec![0i8, 0, 1, 0, 1, 0, 0]);
4811 let offsets = ScalarBuffer::from(vec![0i32, 1, 0, 2, 1, 3, 4]);
4812
4813 let int_array = Int32Array::from(vec![100, 200]);
4814
4815 let union = UnionArray::try_new(
4816 union_fields.clone(),
4817 types,
4818 Some(offsets),
4819 vec![Arc::new(dict_array), Arc::new(int_array)],
4820 )
4821 .unwrap();
4822
4823 let schema = Arc::new(Schema::new(vec![Field::new(
4824 "union",
4825 DataType::Union(union_fields, UnionMode::Dense),
4826 false,
4827 )]));
4828 let input_batch = RecordBatch::try_new(schema, vec![Arc::new(union)]).unwrap();
4829
4830 let output_batch = deserialize_file(serialize_file(&input_batch));
4831 assert_eq!(input_batch, output_batch);
4832
4833 let output_batch = deserialize_stream(serialize_stream(&input_batch));
4834 assert_eq!(input_batch, output_batch);
4835 }
4836
4837 #[test]
4838 fn test_roundtrip_sparse_union_of_dict() {
4839 let values = StringArray::from(vec![Some("alpha"), None, Some("beta"), Some("gamma")]);
4840 let keys = Int32Array::from_iter_values([0, 0, 1, 2, 3, 0, 2]);
4841 let dict_array = DictionaryArray::new(keys, Arc::new(values));
4842
4843 #[expect(deprecated)]
4844 let dict_field = Arc::new(Field::new_dict(
4845 "dict",
4846 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4847 true,
4848 2,
4849 false,
4850 ));
4851 let int_field = Arc::new(Field::new("int", DataType::Int32, false));
4852 let union_fields = UnionFields::try_new(vec![0, 1], vec![dict_field, int_field]).unwrap();
4853
4854 let types = ScalarBuffer::from(vec![0i8, 0, 1, 0, 1, 0, 0]);
4855
4856 let int_array = Int32Array::from(vec![0, 0, 100, 0, 200, 0, 0]);
4857
4858 let union = UnionArray::try_new(
4859 union_fields.clone(),
4860 types,
4861 None,
4862 vec![Arc::new(dict_array), Arc::new(int_array)],
4863 )
4864 .unwrap();
4865
4866 let schema = Arc::new(Schema::new(vec![Field::new(
4867 "union",
4868 DataType::Union(union_fields, UnionMode::Sparse),
4869 false,
4870 )]));
4871 let input_batch = RecordBatch::try_new(schema, vec![Arc::new(union)]).unwrap();
4872
4873 let output_batch = deserialize_file(serialize_file(&input_batch));
4874 assert_eq!(input_batch, output_batch);
4875
4876 let output_batch = deserialize_stream(serialize_stream(&input_batch));
4877 assert_eq!(input_batch, output_batch);
4878 }
4879
4880 #[test]
4881 fn test_roundtrip_map_with_dict_keys() {
4882 let key_values = StringArray::from(vec!["key_a", "key_b", "key_c"]);
4885 let keys = Int32Array::from_iter_values([0, 1, 2, 0, 1, 0]);
4886 let dict_keys = DictionaryArray::new(keys, Arc::new(key_values));
4887
4888 let values = Int32Array::from(vec![1, 2, 3, 4, 5, 6]);
4889
4890 #[expect(deprecated)]
4891 let entries_field = Arc::new(Field::new(
4892 Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
4893 DataType::Struct(
4894 vec![
4895 Field::new_dict(
4896 Field::MAP_KEY_FIELD_DEFAULT_NAME,
4897 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4898 false,
4899 1,
4900 false,
4901 ),
4902 Field::new(Field::MAP_VALUE_FIELD_DEFAULT_NAME, DataType::Int32, true),
4903 ]
4904 .into(),
4905 ),
4906 false,
4907 ));
4908
4909 let entries = StructArray::from(vec![
4910 (
4911 Arc::new(Field::new(
4912 Field::MAP_KEY_FIELD_DEFAULT_NAME,
4913 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4914 false,
4915 )),
4916 Arc::new(dict_keys) as ArrayRef,
4917 ),
4918 (
4919 Arc::new(Field::new(
4920 Field::MAP_VALUE_FIELD_DEFAULT_NAME,
4921 DataType::Int32,
4922 true,
4923 )),
4924 Arc::new(values) as ArrayRef,
4925 ),
4926 ]);
4927
4928 let offsets = Buffer::from_slice_ref([0i32, 2, 4, 6]);
4929
4930 let map_data = ArrayData::builder(DataType::Map(entries_field, false))
4931 .len(3)
4932 .add_buffer(offsets)
4933 .add_child_data(entries.into_data())
4934 .build()
4935 .unwrap();
4936 let map_array = MapArray::from(map_data);
4937
4938 let schema = Arc::new(Schema::new(vec![Field::new(
4939 "map",
4940 map_array.data_type().clone(),
4941 false,
4942 )]));
4943 let input_batch = RecordBatch::try_new(schema, vec![Arc::new(map_array)]).unwrap();
4944
4945 let output_batch = deserialize_file(serialize_file(&input_batch));
4946 assert_eq!(input_batch, output_batch);
4947
4948 let output_batch = deserialize_stream(serialize_stream(&input_batch));
4949 assert_eq!(input_batch, output_batch);
4950 }
4951
4952 #[test]
4953 fn test_roundtrip_map_with_dict_values() {
4954 let keys = StringArray::from(vec!["a", "b", "c", "d", "e", "f"]);
4957
4958 let value_values = StringArray::from(vec!["val_x", "val_y", "val_z"]);
4959 let value_keys = Int32Array::from_iter_values([0, 1, 2, 0, 1, 0]);
4960 let dict_values = DictionaryArray::new(value_keys, Arc::new(value_values));
4961
4962 #[expect(deprecated)]
4963 let entries_field = Arc::new(Field::new(
4964 Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
4965 DataType::Struct(
4966 vec![
4967 Field::new(Field::MAP_KEY_FIELD_DEFAULT_NAME, DataType::Utf8, false),
4968 Field::new_dict(
4969 Field::MAP_VALUE_FIELD_DEFAULT_NAME,
4970 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4971 true,
4972 2,
4973 false,
4974 ),
4975 ]
4976 .into(),
4977 ),
4978 false,
4979 ));
4980
4981 let entries = StructArray::from(vec![
4982 (
4983 Arc::new(Field::new(
4984 Field::MAP_KEY_FIELD_DEFAULT_NAME,
4985 DataType::Utf8,
4986 false,
4987 )),
4988 Arc::new(keys) as ArrayRef,
4989 ),
4990 (
4991 Arc::new(Field::new(
4992 Field::MAP_VALUE_FIELD_DEFAULT_NAME,
4993 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4994 true,
4995 )),
4996 Arc::new(dict_values) as ArrayRef,
4997 ),
4998 ]);
4999
5000 let offsets = Buffer::from_slice_ref([0i32, 2, 4, 6]);
5001
5002 let map_data = ArrayData::builder(DataType::Map(entries_field, false))
5003 .len(3)
5004 .add_buffer(offsets)
5005 .add_child_data(entries.into_data())
5006 .build()
5007 .unwrap();
5008 let map_array = MapArray::from(map_data);
5009
5010 let schema = Arc::new(Schema::new(vec![Field::new(
5011 "map",
5012 map_array.data_type().clone(),
5013 false,
5014 )]));
5015 let input_batch = RecordBatch::try_new(schema, vec![Arc::new(map_array)]).unwrap();
5016
5017 let output_batch = deserialize_file(serialize_file(&input_batch));
5018 assert_eq!(input_batch, output_batch);
5019
5020 let output_batch = deserialize_stream(serialize_stream(&input_batch));
5021 assert_eq!(input_batch, output_batch);
5022 }
5023
5024 #[test]
5025 fn test_decimal128_alignment16_is_sufficient() {
5026 const IPC_ALIGNMENT: usize = 16;
5027
5028 for num_cols in [1, 2, 3, 17, 50, 73, 99] {
5033 let num_rows = (num_cols * 7 + 11) % 100; let mut fields = Vec::new();
5036 let mut arrays = Vec::new();
5037 for i in 0..num_cols {
5038 let field = Field::new(format!("col_{i}"), DataType::Decimal128(38, 10), true);
5039 let array = Decimal128Array::from(vec![num_cols as i128; num_rows]);
5040 fields.push(field);
5041 arrays.push(Arc::new(array) as Arc<dyn Array>);
5042 }
5043 let schema = Schema::new(fields);
5044 let batch = RecordBatch::try_new(Arc::new(schema), arrays).unwrap();
5045
5046 let mut writer = FileWriter::try_new_with_options(
5047 Vec::new(),
5048 batch.schema_ref(),
5049 IpcWriteOptions::try_new(IPC_ALIGNMENT, false, MetadataVersion::V5).unwrap(),
5050 )
5051 .unwrap();
5052 writer.write(&batch).unwrap();
5053 writer.finish().unwrap();
5054
5055 let out: Vec<u8> = writer.into_inner().unwrap();
5056
5057 let buffer = Buffer::from_slice_ref(out);
5058 let trailer_start = buffer.len() - 10;
5059 let footer_len =
5060 read_footer_length(buffer[trailer_start..].try_into().unwrap()).unwrap();
5061 let footer =
5062 root_as_footer(&buffer[trailer_start - footer_len..trailer_start]).unwrap();
5063
5064 let schema = try_fb_to_schema(footer.schema().unwrap()).unwrap();
5065
5066 let decoder =
5069 FileDecoder::new(Arc::new(schema), footer.version()).with_require_alignment(true);
5070
5071 let batches = footer.recordBatches().unwrap();
5072
5073 let block = batches.get(0);
5074 let block_len = block.bodyLength() as usize + block.metaDataLength() as usize;
5075 let data = buffer.slice_with_length(block.offset() as _, block_len);
5076
5077 let batch2 = decoder.read_record_batch(block, &data).unwrap().unwrap();
5078
5079 assert_eq!(batch, batch2);
5080 }
5081 }
5082
5083 #[test]
5084 fn test_decimal128_alignment8_is_unaligned() {
5085 const IPC_ALIGNMENT: usize = 8;
5086
5087 let num_cols = 2;
5088
5089 let mut fields = Vec::new();
5090 let mut arrays = Vec::new();
5091 for i in 0..num_cols {
5092 let field = Field::new(format!("col_{i}"), DataType::Decimal128(38, 10), true);
5093 let array = Decimal128Array::from(vec![Some(num_cols as i128), None]);
5097 fields.push(field);
5098 arrays.push(Arc::new(array) as Arc<dyn Array>);
5099 }
5100 let schema = Schema::new(fields);
5101 let batch = RecordBatch::try_new(Arc::new(schema), arrays).unwrap();
5102
5103 let mut writer = FileWriter::try_new_with_options(
5104 Vec::new(),
5105 batch.schema_ref(),
5106 IpcWriteOptions::try_new(IPC_ALIGNMENT, false, MetadataVersion::V5).unwrap(),
5107 )
5108 .unwrap();
5109 writer.write(&batch).unwrap();
5110 writer.finish().unwrap();
5111
5112 let out: Vec<u8> = writer.into_inner().unwrap();
5113
5114 let buffer = Buffer::from_slice_ref(out);
5115 let trailer_start = buffer.len() - 10;
5116 let footer_len = read_footer_length(buffer[trailer_start..].try_into().unwrap()).unwrap();
5117 let footer = root_as_footer(&buffer[trailer_start - footer_len..trailer_start]).unwrap();
5118 let schema = try_fb_to_schema(footer.schema().unwrap()).unwrap();
5119
5120 let decoder =
5123 FileDecoder::new(Arc::new(schema), footer.version()).with_require_alignment(true);
5124
5125 let batches = footer.recordBatches().unwrap();
5126
5127 let block = batches.get(0);
5128 let block_len = block.bodyLength() as usize + block.metaDataLength() as usize;
5129 let data = buffer.slice_with_length(block.offset() as _, block_len);
5130
5131 let result = decoder.read_record_batch(block, &data);
5132
5133 let error = result.unwrap_err();
5134 assert_eq!(
5135 error.to_string(),
5136 "Invalid argument error: Misaligned buffers[0] in array of type Decimal128(38, 10), \
5137 offset from expected alignment of 16 by 8"
5138 );
5139 }
5140
5141 #[test]
5142 fn test_flush() {
5143 let num_cols = 2;
5146 let mut fields = Vec::new();
5147 let options = IpcWriteOptions::try_new(8, false, MetadataVersion::V5).unwrap();
5148 for i in 0..num_cols {
5149 let field = Field::new(format!("col_{i}"), DataType::Decimal128(38, 10), true);
5150 fields.push(field);
5151 }
5152 let schema = Schema::new(fields);
5153 let inner_stream_writer = BufWriter::with_capacity(1024, Vec::new());
5154 let inner_file_writer = BufWriter::with_capacity(1024, Vec::new());
5155 let mut stream_writer =
5156 StreamWriter::try_new_with_options(inner_stream_writer, &schema, options.clone())
5157 .unwrap();
5158 let mut file_writer =
5159 FileWriter::try_new_with_options(inner_file_writer, &schema, options).unwrap();
5160
5161 let stream_bytes_written_on_new = stream_writer.get_ref().get_ref().len();
5162 let file_bytes_written_on_new = file_writer.get_ref().get_ref().len();
5163 stream_writer.flush().unwrap();
5164 file_writer.flush().unwrap();
5165 let stream_bytes_written_on_flush = stream_writer.get_ref().get_ref().len();
5166 let file_bytes_written_on_flush = file_writer.get_ref().get_ref().len();
5167 let stream_out = stream_writer.into_inner().unwrap().into_inner().unwrap();
5168 let expected_stream_flushed_bytes = stream_out.len() - 8;
5172 let expected_file_flushed_bytes = expected_stream_flushed_bytes + 8;
5175
5176 assert!(
5177 stream_bytes_written_on_new < stream_bytes_written_on_flush,
5178 "this test makes no sense if flush is not actually required"
5179 );
5180 assert!(
5181 file_bytes_written_on_new < file_bytes_written_on_flush,
5182 "this test makes no sense if flush is not actually required"
5183 );
5184 assert_eq!(stream_bytes_written_on_flush, expected_stream_flushed_bytes);
5185 assert_eq!(file_bytes_written_on_flush, expected_file_flushed_bytes);
5186 }
5187
5188 #[test]
5189 fn test_roundtrip_list_of_fixed_list() -> Result<(), ArrowError> {
5190 let l1_type =
5191 DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, false)), 3);
5192 let l2_type = DataType::List(Arc::new(Field::new("item", l1_type.clone(), false)));
5193
5194 let l0_builder = Float32Builder::new();
5195 let l1_builder = FixedSizeListBuilder::new(l0_builder, 3).with_field(Arc::new(Field::new(
5196 "item",
5197 DataType::Float32,
5198 false,
5199 )));
5200 let mut l2_builder =
5201 ListBuilder::new(l1_builder).with_field(Arc::new(Field::new("item", l1_type, false)));
5202
5203 for point in [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]] {
5204 l2_builder.values().values().append_value(point[0]);
5205 l2_builder.values().values().append_value(point[1]);
5206 l2_builder.values().values().append_value(point[2]);
5207
5208 l2_builder.values().append(true);
5209 }
5210 l2_builder.append(true);
5211
5212 let point = [10., 11., 12.];
5213 l2_builder.values().values().append_value(point[0]);
5214 l2_builder.values().values().append_value(point[1]);
5215 l2_builder.values().values().append_value(point[2]);
5216
5217 l2_builder.values().append(true);
5218 l2_builder.append(true);
5219
5220 let array = Arc::new(l2_builder.finish()) as ArrayRef;
5221
5222 let schema = Arc::new(Schema::new_with_metadata(
5223 vec![Field::new("points", l2_type, false)],
5224 HashMap::default(),
5225 ));
5226
5227 test_slices(&array, &schema, 0, 1)?;
5230 test_slices(&array, &schema, 0, 2)?;
5231 test_slices(&array, &schema, 1, 1)?;
5232
5233 Ok(())
5234 }
5235
5236 #[test]
5237 fn test_roundtrip_list_of_fixed_list_w_nulls() -> Result<(), ArrowError> {
5238 let l0_builder = Float32Builder::new();
5239 let l1_builder = FixedSizeListBuilder::new(l0_builder, 3);
5240 let mut l2_builder = ListBuilder::new(l1_builder);
5241
5242 for point in [
5243 [Some(1.0), Some(2.0), None],
5244 [Some(4.0), Some(5.0), Some(6.0)],
5245 [None, Some(8.0), Some(9.0)],
5246 ] {
5247 for p in point {
5248 match p {
5249 Some(p) => l2_builder.values().values().append_value(p),
5250 None => l2_builder.values().values().append_null(),
5251 }
5252 }
5253
5254 l2_builder.values().append(true);
5255 }
5256 l2_builder.append(true);
5257
5258 let point = [Some(10.), None, None];
5259 for p in point {
5260 match p {
5261 Some(p) => l2_builder.values().values().append_value(p),
5262 None => l2_builder.values().values().append_null(),
5263 }
5264 }
5265
5266 l2_builder.values().append(true);
5267 l2_builder.append(true);
5268
5269 let array = Arc::new(l2_builder.finish()) as ArrayRef;
5270
5271 let schema = Arc::new(Schema::new_with_metadata(
5272 vec![Field::new(
5273 "points",
5274 DataType::List(Arc::new(Field::new(
5275 "item",
5276 DataType::FixedSizeList(
5277 Arc::new(Field::new("item", DataType::Float32, true)),
5278 3,
5279 ),
5280 true,
5281 ))),
5282 true,
5283 )],
5284 HashMap::default(),
5285 ));
5286
5287 test_slices(&array, &schema, 0, 1)?;
5290 test_slices(&array, &schema, 0, 2)?;
5291 test_slices(&array, &schema, 1, 1)?;
5292
5293 Ok(())
5294 }
5295
5296 fn test_slices(
5297 parent_array: &ArrayRef,
5298 schema: &SchemaRef,
5299 offset: usize,
5300 length: usize,
5301 ) -> Result<(), ArrowError> {
5302 let subarray = parent_array.slice(offset, length);
5303 let original_batch = RecordBatch::try_new(schema.clone(), vec![subarray])?;
5304
5305 let mut bytes = Vec::new();
5306 let mut writer = StreamWriter::try_new(&mut bytes, schema)?;
5307 writer.write(&original_batch)?;
5308 writer.finish()?;
5309
5310 let mut cursor = std::io::Cursor::new(bytes);
5311 let mut reader = StreamReader::try_new(&mut cursor, None)?;
5312 let returned_batch = reader.next().unwrap()?;
5313
5314 assert_eq!(original_batch, returned_batch);
5315
5316 Ok(())
5317 }
5318
5319 #[test]
5320 fn test_roundtrip_fixed_list() -> Result<(), ArrowError> {
5321 let int_builder = Int64Builder::new();
5322 let mut fixed_list_builder = FixedSizeListBuilder::new(int_builder, 3)
5323 .with_field(Arc::new(Field::new("item", DataType::Int64, false)));
5324
5325 for point in [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]] {
5326 fixed_list_builder.values().append_value(point[0]);
5327 fixed_list_builder.values().append_value(point[1]);
5328 fixed_list_builder.values().append_value(point[2]);
5329
5330 fixed_list_builder.append(true);
5331 }
5332
5333 let array = Arc::new(fixed_list_builder.finish()) as ArrayRef;
5334
5335 let schema = Arc::new(Schema::new_with_metadata(
5336 vec![Field::new(
5337 "points",
5338 DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Int64, false)), 3),
5339 false,
5340 )],
5341 HashMap::default(),
5342 ));
5343
5344 test_slices(&array, &schema, 0, 4)?;
5347 test_slices(&array, &schema, 0, 2)?;
5348 test_slices(&array, &schema, 1, 3)?;
5349 test_slices(&array, &schema, 2, 1)?;
5350
5351 Ok(())
5352 }
5353
5354 #[test]
5355 fn test_roundtrip_fixed_list_w_nulls() -> Result<(), ArrowError> {
5356 let int_builder = Int64Builder::new();
5357 let mut fixed_list_builder = FixedSizeListBuilder::new(int_builder, 3);
5358
5359 for point in [
5360 [Some(1), Some(2), None],
5361 [Some(4), Some(5), Some(6)],
5362 [None, Some(8), Some(9)],
5363 [Some(10), None, None],
5364 ] {
5365 for p in point {
5366 match p {
5367 Some(p) => fixed_list_builder.values().append_value(p),
5368 None => fixed_list_builder.values().append_null(),
5369 }
5370 }
5371
5372 fixed_list_builder.append(true);
5373 }
5374
5375 let array = Arc::new(fixed_list_builder.finish()) as ArrayRef;
5376
5377 let schema = Arc::new(Schema::new_with_metadata(
5378 vec![Field::new(
5379 "points",
5380 DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Int64, true)), 3),
5381 true,
5382 )],
5383 HashMap::default(),
5384 ));
5385
5386 test_slices(&array, &schema, 0, 4)?;
5389 test_slices(&array, &schema, 0, 2)?;
5390 test_slices(&array, &schema, 1, 3)?;
5391 test_slices(&array, &schema, 2, 1)?;
5392
5393 Ok(())
5394 }
5395
5396 #[test]
5397 fn test_metadata_encoding_ordering() {
5398 fn create_hash() -> u64 {
5399 let metadata: HashMap<String, String> = [
5400 ("a", "1"), ("b", "2"), ("c", "3"), ("d", "4"), ("e", "5"), ]
5406 .into_iter()
5407 .map(|(k, v)| (k.to_owned(), v.to_owned()))
5408 .collect();
5409
5410 let schema = Arc::new(
5412 Schema::new(vec![
5413 Field::new("a", DataType::Int64, true).with_metadata(metadata.clone()),
5414 ])
5415 .with_metadata(metadata)
5416 .clone(),
5417 );
5418 let batch = RecordBatch::new_empty(schema.clone());
5419
5420 let mut bytes = Vec::new();
5421 let mut w = StreamWriter::try_new(&mut bytes, batch.schema_ref()).unwrap();
5422 w.write(&batch).unwrap();
5423 w.finish().unwrap();
5424
5425 let mut h = std::hash::DefaultHasher::new();
5426 h.write(&bytes);
5427 h.finish()
5428 }
5429
5430 let expected = create_hash();
5431
5432 let all_passed = (0..20).all(|_| create_hash() == expected);
5437 assert!(all_passed);
5438 }
5439
5440 #[test]
5441 fn test_dictionary_tracker_reset() {
5442 let data_gen = IpcDataGenerator::default();
5443 let mut dictionary_tracker = DictionaryTracker::new(false);
5444 let writer_options = IpcWriteOptions::default();
5445 let mut compression_ctx = IpcWriteContext::default();
5446
5447 let schema = Arc::new(Schema::new(vec![Field::new(
5448 "a",
5449 DataType::Dictionary(Box::new(DataType::UInt8), Box::new(DataType::Utf8)),
5450 false,
5451 )]));
5452
5453 let mut write_single_batch_stream =
5454 |batch: RecordBatch, dict_tracker: &mut DictionaryTracker| -> Vec<u8> {
5455 let mut buffer = Vec::new();
5456
5457 let stream_header = data_gen.schema_to_bytes_with_dictionary_tracker(
5459 &schema,
5460 dict_tracker,
5461 &writer_options,
5462 );
5463 _ = write_message(&mut buffer, stream_header, &writer_options).unwrap();
5464
5465 let (encoded_dicts, encoded_batch) = data_gen
5466 .encode(&batch, dict_tracker, &writer_options, &mut compression_ctx)
5467 .unwrap();
5468 for encoded_dict in encoded_dicts {
5469 _ = write_message(&mut buffer, encoded_dict, &writer_options).unwrap();
5470 }
5471 _ = write_message(&mut buffer, encoded_batch, &writer_options).unwrap();
5472
5473 buffer
5474 };
5475
5476 let batch1 = RecordBatch::try_new(
5477 schema.clone(),
5478 vec![Arc::new(DictionaryArray::new(
5479 UInt8Array::from_iter_values([0]),
5480 Arc::new(StringArray::from_iter_values(["a"])),
5481 ))],
5482 )
5483 .unwrap();
5484 let buffer = write_single_batch_stream(batch1.clone(), &mut dictionary_tracker);
5485
5486 let mut reader = StreamReader::try_new(Cursor::new(buffer), None).unwrap();
5488 let read_batch = reader.next().unwrap().unwrap();
5489 assert_eq!(read_batch, batch1);
5490
5491 dictionary_tracker.clear();
5493
5494 let batch2 = RecordBatch::try_new(
5496 schema.clone(),
5497 vec![Arc::new(DictionaryArray::new(
5498 UInt8Array::from_iter_values([0]),
5499 Arc::new(StringArray::from_iter_values(["a"])),
5500 ))],
5501 )
5502 .unwrap();
5503 let buffer = write_single_batch_stream(batch2.clone(), &mut dictionary_tracker);
5504 let mut reader = StreamReader::try_new(Cursor::new(buffer), None).unwrap();
5505 let read_batch = reader.next().unwrap().unwrap();
5506 assert_eq!(read_batch, batch2);
5507 }
5508}