Skip to main content

arrow_ipc/
writer.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Arrow IPC File and Stream Writers
6//!
7//! # Notes
8//!
9//! [`FileWriter`] and [`StreamWriter`] have similar interfaces,
10//! however the [`FileWriter`] expects a reader that supports [`Seek`]ing
11//!
12//! [`Seek`]: std::io::Seek
13
14use std::cmp::min;
15use std::collections::HashMap;
16use std::io::{BufWriter, Write};
17use std::mem::size_of;
18use std::sync::Arc;
19
20use flatbuffers::FlatBufferBuilder;
21
22use arrow_array::cast::*;
23use arrow_array::types::{Int16Type, Int32Type, Int64Type, RunEndIndexType};
24use arrow_array::*;
25use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer, ToByteSlice};
26use arrow_data::{ArrayData, ArrayDataBuilder, BufferSpec, layout};
27use arrow_schema::*;
28
29use crate::CONTINUATION_MARKER;
30use crate::compression::{CompressionCodec, Lz4BlockSize};
31#[expect(deprecated)]
32pub use crate::compression::{CompressionContext, IpcWriteContext};
33use crate::convert::IpcSchemaEncoder;
34
35/// IPC write options used to control the behaviour of the [`IpcDataGenerator`]
36#[derive(Debug, Clone)]
37pub struct IpcWriteOptions {
38    /// Write padding after memory buffers to this multiple of bytes.
39    /// Must be 8, 16, 32, or 64 - defaults to 64.
40    alignment: u8,
41    /// The legacy format is for releases before 0.15.0, and uses metadata V4
42    write_legacy_ipc_format: bool,
43    /// The metadata version to write. The Rust IPC writer supports V4+
44    ///
45    /// *Default versions per crate*
46    ///
47    /// When creating the default IpcWriteOptions, the following metadata versions are used:
48    ///
49    /// version 2.0.0: V4, with legacy format enabled
50    /// version 4.0.0: V5
51    metadata_version: crate::MetadataVersion,
52    /// Compression, if desired. Will result in a runtime error
53    /// if the corresponding feature is not enabled
54    batch_compression_type: Option<crate::CompressionType>,
55    // Compression level
56    batch_compression_level: Option<i32>,
57    /// The lz4 frame block size; `None` selects 256 KiB.
58    batch_lz4_block_size: Option<Lz4BlockSize>,
59    /// How to handle updating dictionaries in IPC messages
60    dictionary_handling: DictionaryHandling,
61}
62
63/// A single buffer segment ready to be written to the output stream.
64///
65/// For the uncompressed path the original Arc-backed [`Buffer`] is stored
66/// directly (zero copy). For the compressed path the compressed bytes are
67/// owned by a scratch `Vec<u8>`.
68enum EncodedBuffer {
69    /// Uncompressed: Arc-backed reference to the original array buffer.
70    Raw(Buffer),
71    /// Compressed: owned scratch bytes produced by the codec.
72    Compressed(Vec<u8>),
73}
74
75impl EncodedBuffer {
76    fn as_slice(&self) -> &[u8] {
77        match self {
78            EncodedBuffer::Raw(b) => b.as_slice(),
79            EncodedBuffer::Compressed(v) => v.as_slice(),
80        }
81    }
82
83    fn len(&self) -> usize {
84        match self {
85            EncodedBuffer::Raw(b) => b.len(),
86            EncodedBuffer::Compressed(v) => v.len(),
87        }
88    }
89}
90/// Accumulates the IPC metadata produced by [`write_array_data`].
91///
92/// `nodes` and `buffers` are serialised into the flatbuffer `RecordBatch` (or `DictionaryBatch`)
93/// header. The companion [`IpcBodySink`] holds the actual encoded bytes.
94#[derive(Default)]
95struct IpcMetadataBuilder {
96    nodes: Vec<crate::FieldNode>,
97    buffers: Vec<crate::Buffer>,
98}
99
100/// Destination for the raw Arrow data bytes (the IPC message body) produced by [`write_array_data`].
101///
102/// The companion [`IpcMetadataBuilder`] accumulates the flatbuffer metadata
103/// (offset + length of each buffer in the body); together they form a complete IPC message.
104enum IpcBodySink<'a> {
105    /// Serialize buffer bytes (with padding) into a contiguous byte vec.
106    Write(&'a mut Vec<u8>),
107    /// Accumulate pre-encoded buffer segments for deferred zero-copy streaming.
108    Collect(&'a mut Vec<EncodedBuffer>),
109}
110impl IpcBodySink<'_> {
111    /// Writes the encoded buffer to the sink.
112    pub fn write(&mut self, pad_len: usize, buffer: EncodedBuffer) {
113        match self {
114            IpcBodySink::Write(vec) => {
115                vec.extend_from_slice(buffer.as_slice());
116                vec.extend_from_slice(&PADDING[..pad_len]);
117            }
118            IpcBodySink::Collect(vec) => {
119                vec.push(buffer);
120            }
121        }
122    }
123}
124
125struct MetadataLayout {
126    padded_header_len: usize,
127    padded_metadata_len: usize,
128    metadata_padding: usize,
129}
130
131impl MetadataLayout {
132    fn new(metadata_len: usize, write_options: &IpcWriteOptions) -> MetadataLayout {
133        let prefix_size = if write_options.write_legacy_ipc_format {
134            4
135        } else {
136            8
137        };
138        let alignment_mask = usize::from(write_options.alignment - 1);
139        let padded_header_len = (metadata_len + prefix_size + alignment_mask) & !alignment_mask;
140        let padded_metadata_len = padded_header_len - prefix_size;
141        let metadata_padding = padded_metadata_len - metadata_len;
142
143        MetadataLayout {
144            padded_header_len,
145            padded_metadata_len,
146            metadata_padding,
147        }
148    }
149}
150
151/// Destination-specific byte writing for a framed IPC message.
152///
153/// The default owned-buffer methods borrow their bytes and call `write_slice`,
154/// which can copy the data. Sink implementations that can take ownership of
155/// generated bytes should override `write_vec` or `write_encoded_buffer`.
156trait IpcMessageSink {
157    fn write_slice(&mut self, bytes: &[u8]) -> Result<(), ArrowError>;
158
159    fn write_vec(&mut self, bytes: Vec<u8>) -> Result<(), ArrowError> {
160        self.write_slice(&bytes)
161    }
162
163    fn write_encoded_buffer(&mut self, buffer: EncodedBuffer) -> Result<(), ArrowError> {
164        self.write_slice(buffer.as_slice())
165    }
166}
167
168/// Shared IPC framing helpers for [`IpcMessageSink`].
169trait IpcMessageSinkExt: IpcMessageSink {
170    fn write_padding(&mut self, len: usize) -> Result<(), ArrowError> {
171        self.write_slice(&PADDING[..len])
172    }
173
174    /// Writes the IPC continuation marker and metadata length prefix.
175    fn write_continuation(
176        &mut self,
177        write_options: &IpcWriteOptions,
178        metadata_len: i32,
179    ) -> Result<(), ArrowError> {
180        let mut buffer = [0; 8];
181        let len = match write_options.metadata_version {
182            crate::MetadataVersion::V1
183            | crate::MetadataVersion::V2
184            | crate::MetadataVersion::V3 => {
185                unreachable!("Options with the metadata version cannot be created")
186            }
187            crate::MetadataVersion::V4 => {
188                let metadata_len_bytes = metadata_len.to_le_bytes();
189                if !write_options.write_legacy_ipc_format {
190                    // v0.15.0 format
191                    buffer[..4].copy_from_slice(&CONTINUATION_MARKER);
192                    buffer[4..].copy_from_slice(&metadata_len_bytes);
193                    8
194                } else {
195                    buffer[..4].copy_from_slice(&metadata_len_bytes);
196                    4
197                }
198            }
199            crate::MetadataVersion::V5 => {
200                buffer[..4].copy_from_slice(&CONTINUATION_MARKER);
201                buffer[4..].copy_from_slice(&metadata_len.to_le_bytes());
202                8
203            }
204            z => panic!("Unsupported crate::MetadataVersion {z:?}"),
205        };
206        self.write_slice(&buffer[..len])
207    }
208
209    fn write_body_data(&mut self, data: Vec<u8>, alignment: u8) -> Result<usize, ArrowError> {
210        let len = data.len();
211        let pad_len = pad_to_alignment(alignment, len);
212        self.write_vec(data)?;
213        self.write_padding(pad_len)?;
214        Ok(len + pad_len)
215    }
216
217    /// Writes an already encoded IPC message with optional contiguous body data.
218    ///
219    /// This is used for schema and dictionary messages represented by [`EncodedData`].
220    /// Returns the padded metadata length and body length written.
221    fn write_encoded_data(
222        &mut self,
223        encoded: EncodedData,
224        write_options: &IpcWriteOptions,
225    ) -> Result<(usize, usize), ArrowError> {
226        let arrow_data_len = encoded.arrow_data.len();
227        if !arrow_data_len.is_multiple_of(usize::from(write_options.alignment)) {
228            return Err(ArrowError::MemoryError(
229                "Arrow data not aligned".to_string(),
230            ));
231        }
232
233        let metadata = encoded.ipc_message;
234        let metadata_len = metadata.len();
235        let layout = MetadataLayout::new(metadata_len, write_options);
236
237        self.write_continuation(write_options, layout.padded_metadata_len as i32)?;
238        self.write_vec(metadata)?;
239        self.write_padding(layout.metadata_padding)?;
240
241        let body_len = if arrow_data_len > 0 {
242            self.write_body_data(encoded.arrow_data, write_options.alignment)?
243        } else {
244            0
245        };
246
247        Ok((layout.padded_header_len, body_len))
248    }
249
250    /// Writes the IPC end-of-stream marker.
251    fn write_eos(&mut self, write_options: &IpcWriteOptions) -> Result<(), ArrowError> {
252        self.write_continuation(write_options, 0)?;
253        Ok(())
254    }
255}
256
257impl<T: IpcMessageSink + ?Sized> IpcMessageSinkExt for T {}
258
259/// Optional hot-path hook for record batch messages.
260trait IpcRecordBatchSink: IpcMessageSinkExt {
261    /// Writes a record batch (or dictionary batch) message from its encoded
262    /// metadata and body buffers.
263    ///
264    /// `metadata` is the raw flatbuffer [`crate::Message`] (without continuation
265    /// prefix), borrowed from the reused [`FlatBufferBuilder`]. The body buffers are
266    /// already materialized as [`EncodedBuffer`] segments, allowing buffer output to
267    /// preserve uncompressed Arrow buffers. They are drained out of
268    /// `encoded_buffers` so the caller can reuse its allocation.
269    ///
270    /// Each body buffer is padded to the alignment as it is written, so the body
271    /// needs no trailing padding.
272    ///
273    /// Returns the padded metadata length and body length written.
274    fn write_record_batch(
275        &mut self,
276        metadata: &[u8],
277        encoded_buffers: &mut Vec<EncodedBuffer>,
278        body_len: usize,
279        write_options: &IpcWriteOptions,
280    ) -> Result<(usize, usize), ArrowError> {
281        let alignment = write_options.alignment;
282        let layout = MetadataLayout::new(metadata.len(), write_options);
283
284        self.write_continuation(write_options, layout.padded_metadata_len as i32)?;
285        self.write_slice(metadata)?;
286        self.write_padding(layout.metadata_padding)?;
287        for enc in encoded_buffers.drain(..) {
288            let len = enc.len();
289            self.write_encoded_buffer(enc)?;
290            self.write_padding(pad_to_alignment(alignment, len))?;
291        }
292
293        Ok((layout.padded_header_len, body_len))
294    }
295}
296
297impl<W> IpcMessageSink for W
298where
299    W: Write,
300{
301    fn write_slice(&mut self, bytes: &[u8]) -> Result<(), ArrowError> {
302        if !bytes.is_empty() {
303            self.write_all(bytes)?;
304        }
305        Ok(())
306    }
307}
308
309impl<W> IpcRecordBatchSink for W
310where
311    W: Write,
312{
313    fn write_record_batch(
314        &mut self,
315        metadata: &[u8],
316        encoded_buffers: &mut Vec<EncodedBuffer>,
317        body_len: usize,
318        write_options: &IpcWriteOptions,
319    ) -> Result<(usize, usize), ArrowError> {
320        let alignment = write_options.alignment;
321        let layout = MetadataLayout::new(metadata.len(), write_options);
322
323        self.write_continuation(write_options, layout.padded_metadata_len as i32)?;
324        self.write_all(metadata)?;
325        self.write_all(&PADDING[..layout.metadata_padding])?;
326        for enc in encoded_buffers.iter_mut() {
327            self.write_all(enc.as_slice())?;
328            self.write_all(&PADDING[..pad_to_alignment(alignment, enc.len())])?;
329        }
330        // Clearing the buffers after the loop instead of draining the vec is
331        // a deliberate choice. It seems to perform much better on certain
332        // ARM chips.
333        //
334        // See: https://github.com/apache/arrow-rs/pull/10128#issuecomment-5406019272
335        encoded_buffers.clear();
336
337        Ok((layout.padded_header_len, body_len))
338    }
339}
340
341/// Accumulates complete framed IPC messages as ordered buffers.
342struct Buffers<'a> {
343    out: &'a mut Vec<Buffer>,
344}
345
346impl IpcMessageSink for Buffers<'_> {
347    fn write_slice(&mut self, bytes: &[u8]) -> Result<(), ArrowError> {
348        if !bytes.is_empty() {
349            self.out.push(Buffer::from(bytes));
350        }
351        Ok(())
352    }
353
354    fn write_vec(&mut self, bytes: Vec<u8>) -> Result<(), ArrowError> {
355        if !bytes.is_empty() {
356            self.out.push(Buffer::from(bytes));
357        }
358        Ok(())
359    }
360
361    fn write_encoded_buffer(&mut self, buffer: EncodedBuffer) -> Result<(), ArrowError> {
362        match buffer {
363            EncodedBuffer::Raw(buffer) => self.out.push(buffer),
364            EncodedBuffer::Compressed(bytes) => self.out.push(Buffer::from(bytes)),
365        }
366        Ok(())
367    }
368}
369
370impl IpcRecordBatchSink for Buffers<'_> {}
371
372/// Per-message sizes produced by [`IpcDataGenerator::write`].
373///
374/// [`FileWriter`] uses these to build the Block index entries required by the IPC footer for
375/// random-access reads.
376struct IpcWriteMetadata {
377    /// Per-dictionary `(padded_header_len, body_len)` for each dictionary batch written
378    /// before the record batch.
379    dictionary_block_sizes: Vec<(usize, usize)>,
380    /// Flatbuffer header size including continuation prefix and alignment padding.
381    padded_header_len: usize,
382    /// Total length of the record-batch body including alignment padding
383    body_len: usize,
384}
385
386impl IpcWriteOptions {
387    /// Configures compression when writing IPC files.
388    ///
389    /// Will result in a runtime error if the corresponding feature
390    /// is not enabled
391    pub fn try_with_compression(
392        mut self,
393        batch_compression_type: Option<crate::CompressionType>,
394    ) -> Result<Self, ArrowError> {
395        if matches!(batch_compression_type, Some(crate::CompressionType::ZSTD))
396            && self.batch_lz4_block_size.is_some()
397        {
398            return Err(ArrowError::InvalidArgumentError(
399                "lz4 frame block size does not apply to ZSTD compression".to_string(),
400            ));
401        }
402
403        self.batch_compression_type = batch_compression_type;
404
405        if self.batch_compression_type.is_some()
406            && self.metadata_version < crate::MetadataVersion::V5
407        {
408            return Err(ArrowError::InvalidArgumentError(
409                "Compression only supported in metadata v5 and above".to_string(),
410            ));
411        }
412        Ok(self)
413    }
414
415    /// Configures the compression level used when writing compressed IPC batches.
416    ///
417    /// Compression levels require metadata V5 or newer and are currently only
418    /// supported for ZSTD compression.
419    pub fn try_with_compression_level(
420        mut self,
421        batch_compression_level: Option<i32>,
422    ) -> Result<Self, ArrowError> {
423        self.batch_compression_level = batch_compression_level;
424
425        if self.batch_compression_level.is_some()
426            && self.metadata_version < crate::MetadataVersion::V5
427        {
428            return Err(ArrowError::InvalidArgumentError(
429                "Compression only supported in metadata v5 and above".to_string(),
430            ));
431        }
432
433        match (self.batch_compression_type, self.batch_compression_level) {
434            (Some(crate::CompressionType::ZSTD), Some(level)) => {
435                return self.check_zstd_level(level);
436            }
437            (Some(crate::CompressionType::LZ4_FRAME), Some(_)) => {
438                return Err(ArrowError::InvalidArgumentError(
439                    "LZ4 Frame compression does not support configurable compression levels"
440                        .to_string(),
441                ));
442            }
443            _ => {}
444        }
445
446        Ok(self)
447    }
448
449    /// Configures the lz4 frame block size used when compressing batches with
450    /// [`crate::CompressionType::LZ4_FRAME`].
451    ///
452    /// `None` restores the default of 256 KiB, which balances compression throughput against the
453    /// buffers retained by the reusable encoder. The LZ4 frame header can only describe the
454    /// block sizes 64 KiB, 256 KiB, 1 MiB and 4 MiB; any other value returns an
455    /// [`ArrowError::InvalidArgumentError`]. Like the other compression settings this requires
456    /// metadata version V5 or newer, and it has no effect when another compression codec is
457    /// selected.
458    pub fn try_with_lz4_block_size(
459        mut self,
460        block_size: Option<usize>,
461    ) -> Result<Self, ArrowError> {
462        let block_size = block_size.map(|bytes| {
463            Lz4BlockSize::from_bytes(bytes).ok_or_else(|| {
464                ArrowError::InvalidArgumentError(format!(
465                    "Unsupported lz4 block size {bytes}, must be one of 65536, 262144, 1048576, or 4194304 bytes"
466                ))
467            })
468        }).transpose()?;
469
470        if block_size.is_some() {
471            if self.metadata_version < crate::MetadataVersion::V5 {
472                return Err(ArrowError::InvalidArgumentError(
473                    "Compression only supported in metadata v5 and above".to_string(),
474                ));
475            }
476
477            if matches!(
478                self.batch_compression_type,
479                Some(crate::CompressionType::ZSTD)
480            ) {
481                return Err(ArrowError::InvalidArgumentError(
482                    "lz4 frame block size does not apply to ZSTD compression".to_string(),
483                ));
484            }
485        }
486
487        self.batch_lz4_block_size = block_size;
488        Ok(self)
489    }
490
491    /// The lz4 frame block size to compress with, in bytes.
492    pub(crate) fn lz4_block_size(&self) -> Lz4BlockSize {
493        self.batch_lz4_block_size.unwrap_or(Lz4BlockSize::DEFAULT)
494    }
495
496    #[cfg(not(feature = "zstd"))]
497    fn check_zstd_level(self, _level: i32) -> Result<Self, ArrowError> {
498        Err(ArrowError::InvalidArgumentError(
499            "zstd IPC compression requires the zstd feature".to_string(),
500        ))
501    }
502
503    #[cfg(feature = "zstd")]
504    fn check_zstd_level(self, level: i32) -> Result<Self, ArrowError> {
505        let range = zstd::compression_level_range();
506        if !range.contains(&level) {
507            return Err(ArrowError::InvalidArgumentError(format!(
508                "ZSTD compression level must be between {} and {}, got {}",
509                range.start(),
510                range.end(),
511                level,
512            )));
513        }
514
515        Ok(self)
516    }
517
518    /// Try to create IpcWriteOptions, checking for incompatible settings
519    pub fn try_new(
520        alignment: usize,
521        write_legacy_ipc_format: bool,
522        metadata_version: crate::MetadataVersion,
523    ) -> Result<Self, ArrowError> {
524        let alignment: u8 = match alignment {
525            8 => 8,
526            16 => 16,
527            32 => 32,
528            64 => 64,
529            _ => {
530                return Err(ArrowError::InvalidArgumentError(
531                    "Alignment should be 8, 16, 32, or 64.".to_string(),
532                ));
533            }
534        };
535        match metadata_version {
536            crate::MetadataVersion::V1
537            | crate::MetadataVersion::V2
538            | crate::MetadataVersion::V3 => Err(ArrowError::InvalidArgumentError(
539                "Writing IPC metadata version 3 and lower not supported".to_string(),
540            )),
541            crate::MetadataVersion::V4 => Ok(Self {
542                alignment,
543                write_legacy_ipc_format,
544                metadata_version,
545                batch_compression_type: None,
546                batch_compression_level: None,
547                batch_lz4_block_size: None,
548                dictionary_handling: DictionaryHandling::default(),
549            }),
550            crate::MetadataVersion::V5 => {
551                if write_legacy_ipc_format {
552                    Err(ArrowError::InvalidArgumentError(
553                        "Legacy IPC format only supported on metadata version 4".to_string(),
554                    ))
555                } else {
556                    Ok(Self {
557                        alignment,
558                        write_legacy_ipc_format,
559                        metadata_version,
560                        batch_compression_type: None,
561                        batch_compression_level: None,
562                        batch_lz4_block_size: None,
563                        dictionary_handling: DictionaryHandling::default(),
564                    })
565                }
566            }
567            z => Err(ArrowError::InvalidArgumentError(format!(
568                "Unsupported crate::MetadataVersion {z:?}"
569            ))),
570        }
571    }
572
573    /// Configure how dictionaries are handled in IPC messages
574    pub fn with_dictionary_handling(mut self, dictionary_handling: DictionaryHandling) -> Self {
575        self.dictionary_handling = dictionary_handling;
576        self
577    }
578}
579
580impl Default for IpcWriteOptions {
581    fn default() -> Self {
582        Self {
583            alignment: 64,
584            write_legacy_ipc_format: false,
585            metadata_version: crate::MetadataVersion::V5,
586            batch_compression_type: None,
587            batch_compression_level: None,
588            batch_lz4_block_size: None,
589            dictionary_handling: DictionaryHandling::default(),
590        }
591    }
592}
593
594#[derive(Debug, Default)]
595/// Handles low level details of encoding [`Array`] and [`Schema`] into the
596/// [Arrow IPC Format].
597///
598/// # Example
599/// ```
600/// # fn run() {
601/// # use std::sync::Arc;
602/// # use arrow_array::UInt64Array;
603/// # use arrow_array::RecordBatch;
604/// # use arrow_ipc::writer::{IpcWriteContext, DictionaryTracker, IpcDataGenerator, IpcWriteOptions};
605///
606/// // Create a record batch
607/// let batch = RecordBatch::try_from_iter(vec![
608///  ("col2", Arc::new(UInt64Array::from_iter([10, 23, 33])) as _)
609/// ]).unwrap();
610///
611/// // Error of dictionary ids are replaced.
612/// let error_on_replacement = true;
613/// let options = IpcWriteOptions::default();
614/// let mut dictionary_tracker = DictionaryTracker::new(error_on_replacement);
615///
616/// let mut ipc_write_context = IpcWriteContext::default();
617///
618/// // encode the batch into zero or more encoded dictionaries
619/// // and the data for the actual array.
620/// let data_gen = IpcDataGenerator::default();
621/// let (encoded_dictionaries, encoded_message) = data_gen
622///   .encode(&batch, &mut dictionary_tracker, &options, &mut ipc_write_context)
623///   .unwrap();
624/// # }
625/// ```
626///
627/// [Arrow IPC Format]: https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc
628pub struct IpcDataGenerator {}
629
630/// A dictionary determined to need sending as a part of a record batch.
631struct DictionaryToEncode {
632    id: i64,
633    data: ArrayData,
634    is_delta: bool,
635}
636
637/// The result of encoding a record batch's body and flatbuffer `RecordBatch` table
638/// via [`IpcDataGenerator::encode_record_batch_data`].
639///
640/// The `RecordBatch` table is left in-progress in the caller's [`FlatBufferBuilder`];
641/// the caller wraps `record_batch` in a `Message` (directly for a record batch
642/// message, or inside a `DictionaryBatch` for a dictionary message).
643struct EncodedRecordBatchMeta {
644    /// In-progress flatbuffer offset of the `RecordBatch` table.
645    fb_offset: flatbuffers::WIPOffset<crate::RecordBatch<'static>>,
646    /// Total body length written to the sink (the sum of each buffer's encoded
647    /// length and its alignment padding).
648    body_len: usize,
649}
650
651impl IpcDataGenerator {
652    /// Converts a schema to an IPC message along with `dictionary_tracker`
653    /// and returns it encoded inside [EncodedData] as a flatbuffer.
654    pub fn schema_to_bytes_with_dictionary_tracker(
655        &self,
656        schema: &Schema,
657        dictionary_tracker: &mut DictionaryTracker,
658        write_options: &IpcWriteOptions,
659    ) -> EncodedData {
660        let mut fbb = FlatBufferBuilder::new();
661        let schema = {
662            let fb = IpcSchemaEncoder::new()
663                .with_dictionary_tracker(dictionary_tracker)
664                .schema_to_fb_offset(&mut fbb, schema);
665            fb.as_union_value()
666        };
667
668        let mut message = crate::MessageBuilder::new(&mut fbb);
669        message.add_version(write_options.metadata_version);
670        message.add_header_type(crate::MessageHeader::Schema);
671        message.add_bodyLength(0);
672        message.add_header(schema);
673        // TODO: custom metadata
674        let root = message.finish();
675        fbb.finish(root, None);
676
677        let metadata = fbb.finished_data();
678        EncodedData {
679            ipc_message: metadata.to_vec(),
680            arrow_data: vec![],
681        }
682    }
683
684    fn _collect_dict_updates<I: Iterator<Item = i64>>(
685        &self,
686        column: &ArrayRef,
687        dictionaries_to_encode: &mut Vec<DictionaryToEncode>,
688        dictionary_tracker: &mut DictionaryTracker,
689        write_options: &IpcWriteOptions,
690        dict_id: &mut I,
691    ) -> Result<(), ArrowError> {
692        match column.data_type() {
693            DataType::Struct(fields) => {
694                let s = as_struct_array(column);
695                for (field, column) in fields.iter().zip(s.columns()) {
696                    self.collect_dict_updates(
697                        field,
698                        column,
699                        dictionaries_to_encode,
700                        dictionary_tracker,
701                        write_options,
702                        dict_id,
703                    )?;
704                }
705            }
706            DataType::RunEndEncoded(_, values) => {
707                let data = column.to_data();
708                if data.child_data().len() != 2 {
709                    return Err(ArrowError::InvalidArgumentError(format!(
710                        "The run encoded array should have exactly two child arrays. Found {}",
711                        data.child_data().len()
712                    )));
713                }
714                // The run_ends array is not expected to be dictionary encoded. Hence encode dictionaries
715                // only for values array.
716                let values_array = make_array(data.child_data()[1].clone());
717                self.collect_dict_updates(
718                    values,
719                    &values_array,
720                    dictionaries_to_encode,
721                    dictionary_tracker,
722                    write_options,
723                    dict_id,
724                )?;
725            }
726            DataType::List(field) => {
727                let list = as_list_array(column);
728                self.collect_dict_updates(
729                    field,
730                    list.values(),
731                    dictionaries_to_encode,
732                    dictionary_tracker,
733                    write_options,
734                    dict_id,
735                )?;
736            }
737            DataType::LargeList(field) => {
738                let list = as_large_list_array(column);
739                self.collect_dict_updates(
740                    field,
741                    list.values(),
742                    dictionaries_to_encode,
743                    dictionary_tracker,
744                    write_options,
745                    dict_id,
746                )?;
747            }
748            DataType::ListView(field) => {
749                let list = column.as_list_view::<i32>();
750                self.collect_dict_updates(
751                    field,
752                    list.values(),
753                    dictionaries_to_encode,
754                    dictionary_tracker,
755                    write_options,
756                    dict_id,
757                )?;
758            }
759            DataType::LargeListView(field) => {
760                let list = column.as_list_view::<i64>();
761                self.collect_dict_updates(
762                    field,
763                    list.values(),
764                    dictionaries_to_encode,
765                    dictionary_tracker,
766                    write_options,
767                    dict_id,
768                )?;
769            }
770            DataType::FixedSizeList(field, _) => {
771                let list = column
772                    .as_any()
773                    .downcast_ref::<FixedSizeListArray>()
774                    .expect("Unable to downcast to fixed size list array");
775                self.collect_dict_updates(
776                    field,
777                    list.values(),
778                    dictionaries_to_encode,
779                    dictionary_tracker,
780                    write_options,
781                    dict_id,
782                )?;
783            }
784            DataType::Map(field, _) => {
785                let map_array = as_map_array(column);
786
787                let (keys, values) = match field.data_type() {
788                    DataType::Struct(fields) if fields.len() == 2 => (&fields[0], &fields[1]),
789                    _ => panic!("Incorrect field data type {:?}", field.data_type()),
790                };
791
792                // keys
793                self.collect_dict_updates(
794                    keys,
795                    map_array.keys(),
796                    dictionaries_to_encode,
797                    dictionary_tracker,
798                    write_options,
799                    dict_id,
800                )?;
801
802                // values
803                self.collect_dict_updates(
804                    values,
805                    map_array.values(),
806                    dictionaries_to_encode,
807                    dictionary_tracker,
808                    write_options,
809                    dict_id,
810                )?;
811            }
812            DataType::Union(fields, _) => {
813                let union = as_union_array(column);
814                for (type_id, field) in fields.iter() {
815                    let column = union.child(type_id);
816                    self.collect_dict_updates(
817                        field,
818                        column,
819                        dictionaries_to_encode,
820                        dictionary_tracker,
821                        write_options,
822                        dict_id,
823                    )?;
824                }
825            }
826            _ => (),
827        }
828
829        Ok(())
830    }
831
832    // Collect all dicts that need a dictionary message i.e. ones that were
833    // either not in the tracker previously or ones that were but need a
834    // replacement or delta.
835    fn collect_dict_updates<I: Iterator<Item = i64>>(
836        &self,
837        field: &Field,
838        column: &ArrayRef,
839        dictionaries_to_encode: &mut Vec<DictionaryToEncode>,
840        dictionary_tracker: &mut DictionaryTracker,
841        write_options: &IpcWriteOptions,
842        dict_id_seq: &mut I,
843    ) -> Result<(), ArrowError> {
844        match column.data_type() {
845            DataType::Dictionary(_key_type, value_type) => {
846                if matches!(value_type.as_ref(), DataType::Dictionary(_, _)) {
847                    return Err(ArrowError::InvalidArgumentError(format!(
848                        "Arrow IPC field metadata cannot encode direct dictionary-of-dictionary values for field {:?}",
849                        field.name()
850                    )));
851                }
852
853                let dict_data = column.to_data();
854                let dict_values = &dict_data.child_data()[0];
855
856                let values = make_array(dict_data.child_data()[0].clone());
857
858                self._collect_dict_updates(
859                    &values,
860                    dictionaries_to_encode,
861                    dictionary_tracker,
862                    write_options,
863                    dict_id_seq,
864                )?;
865
866                // It's important to only take the dict_id at this point, because the dict ID
867                // sequence is assigned depth-first, so we need to first encode children and have
868                // them take their assigned dict IDs before we take the dict ID for this field.
869                let dict_id = dict_id_seq.next().ok_or_else(|| {
870                    ArrowError::IpcError(format!(
871                        "no dict id for field {:?}: field.data_type={:?}, column.data_type={:?}",
872                        field.name(),
873                        field.data_type(),
874                        column.data_type()
875                    ))
876                })?;
877
878                match dictionary_tracker.insert_column(
879                    dict_id,
880                    column,
881                    write_options.dictionary_handling,
882                )? {
883                    DictionaryUpdate::None => {}
884                    DictionaryUpdate::New | DictionaryUpdate::Replaced => {
885                        dictionaries_to_encode.push(DictionaryToEncode {
886                            id: dict_id,
887                            data: dict_values.clone(),
888                            is_delta: false,
889                        });
890                    }
891                    DictionaryUpdate::Delta(data) => {
892                        dictionaries_to_encode.push(DictionaryToEncode {
893                            id: dict_id,
894                            data,
895                            is_delta: true,
896                        });
897                    }
898                }
899            }
900            _ => self._collect_dict_updates(
901                column,
902                dictionaries_to_encode,
903                dictionary_tracker,
904                write_options,
905                dict_id_seq,
906            )?,
907        }
908
909        Ok(())
910    }
911
912    /// Encodes a batch to a number of [EncodedData] items (dictionary batches + the record batch).
913    /// The [DictionaryTracker] keeps track of dictionaries with new `dict_id`s  (so they are only sent once)
914    /// Make sure the [DictionaryTracker] is initialized at the start of the stream.
915    pub fn encode(
916        &self,
917        batch: &RecordBatch,
918        dictionary_tracker: &mut DictionaryTracker,
919        write_options: &IpcWriteOptions,
920        ipc_write_context: &mut IpcWriteContext,
921    ) -> Result<(Vec<EncodedData>, EncodedData), ArrowError> {
922        let dictionaries = self.collect_all_dicts(batch, dictionary_tracker, write_options)?;
923        let mut encoded_dictionaries = Vec::with_capacity(dictionaries.len());
924        for dict in dictionaries {
925            encoded_dictionaries.push(self.dictionary_batch_to_bytes(
926                dict,
927                write_options,
928                ipc_write_context,
929            )?);
930        }
931        let mut arrow_data = ipc_write_context.scratch();
932        self.record_batch_to_bytes(
933            batch,
934            write_options,
935            ipc_write_context,
936            &mut IpcBodySink::Write(&mut arrow_data),
937        )?;
938        ipc_write_context.reserve_scratch_with_capacity(arrow_data.capacity());
939        Ok((
940            encoded_dictionaries,
941            EncodedData {
942                ipc_message: ipc_write_context.mut_fbb().finished_data().to_vec(),
943                arrow_data,
944            },
945        ))
946    }
947
948    /// Walk the record batch and collect dictionaries for later encoding.
949    fn collect_all_dicts(
950        &self,
951        batch: &RecordBatch,
952        dictionary_tracker: &mut DictionaryTracker,
953        write_options: &IpcWriteOptions,
954    ) -> Result<Vec<DictionaryToEncode>, ArrowError> {
955        let schema = batch.schema();
956        let mut dictionaries = Vec::with_capacity(schema.flattened_fields().len());
957        let mut dict_id = dictionary_tracker.dict_ids.clone().into_iter();
958        for (i, field) in schema.fields().iter().enumerate() {
959            self.collect_dict_updates(
960                field,
961                batch.column(i),
962                &mut dictionaries,
963                dictionary_tracker,
964                write_options,
965                &mut dict_id,
966            )?;
967        }
968        Ok(dictionaries)
969    }
970
971    /// Write dictionary batches and the record batch directly to `writer`, skipping the
972    /// intermediate body `Vec<u8>` allocations
973    /// Returns [`IpcWriteMetadata`] with the sizes needed to build footer blocks.
974    fn write<W: Write>(
975        &self,
976        batch: &RecordBatch,
977        dictionary_tracker: &mut DictionaryTracker,
978        write_options: &IpcWriteOptions,
979        ipc_write_context: &mut IpcWriteContext,
980        writer: &mut W,
981    ) -> Result<IpcWriteMetadata, ArrowError> {
982        self.write_to_sink(
983            batch,
984            dictionary_tracker,
985            write_options,
986            ipc_write_context,
987            writer,
988        )
989    }
990
991    /// Encode dictionary batches and the record batch to output buffers, skipping the
992    /// intermediate body `Vec<u8>` allocations for uncompressed record batch buffers.
993    fn encode_to_buffers(
994        &self,
995        batch: &RecordBatch,
996        dictionary_tracker: &mut DictionaryTracker,
997        write_options: &IpcWriteOptions,
998        ipc_write_context: &mut IpcWriteContext,
999        out: &mut Vec<Buffer>,
1000    ) -> Result<IpcWriteMetadata, ArrowError> {
1001        let mut sink = Buffers { out };
1002        self.write_to_sink(
1003            batch,
1004            dictionary_tracker,
1005            write_options,
1006            ipc_write_context,
1007            &mut sink,
1008        )
1009    }
1010
1011    fn write_to_sink<S: IpcRecordBatchSink>(
1012        &self,
1013        batch: &RecordBatch,
1014        dictionary_tracker: &mut DictionaryTracker,
1015        write_options: &IpcWriteOptions,
1016        ipc_write_context: &mut IpcWriteContext,
1017        sink: &mut S,
1018    ) -> Result<IpcWriteMetadata, ArrowError> {
1019        let dictionaries = self.collect_all_dicts(batch, dictionary_tracker, write_options)?;
1020        let mut dictionary_block_sizes = Vec::with_capacity(dictionaries.len());
1021
1022        let capacity = batch
1023            .columns()
1024            .iter()
1025            .map(|a| estimate_encoded_buffer_count(a.data_type()))
1026            .sum();
1027        let mut encoded_buffers: Vec<EncodedBuffer> = Vec::with_capacity(capacity);
1028
1029        for dict in &dictionaries {
1030            encoded_buffers.clear();
1031
1032            let body_len = self.dictionary_batch_to_sink(
1033                dict,
1034                write_options,
1035                ipc_write_context,
1036                &mut IpcBodySink::Collect(&mut encoded_buffers),
1037            )?;
1038
1039            dictionary_block_sizes.push(sink.write_record_batch(
1040                ipc_write_context.mut_fbb().finished_data(),
1041                &mut encoded_buffers,
1042                body_len,
1043                write_options,
1044            )?);
1045        }
1046
1047        encoded_buffers.clear();
1048        let body_len = self.record_batch_to_bytes(
1049            batch,
1050            write_options,
1051            ipc_write_context,
1052            &mut IpcBodySink::Collect(&mut encoded_buffers),
1053        )?;
1054
1055        let (padded_header_len, body_len) = sink.write_record_batch(
1056            ipc_write_context.mut_fbb().finished_data(),
1057            &mut encoded_buffers,
1058            body_len,
1059            write_options,
1060        )?;
1061
1062        Ok(IpcWriteMetadata {
1063            dictionary_block_sizes,
1064            padded_header_len,
1065            body_len,
1066        })
1067    }
1068
1069    /// Encodes a batch to a number of [EncodedData] items (dictionary batches + the record batch).
1070    /// The [DictionaryTracker] keeps track of dictionaries with new `dict_id`s  (so they are only sent once)
1071    /// Make sure the [DictionaryTracker] is initialized at the start of the stream.
1072    #[deprecated(since = "57.0.0", note = "Use `encode` instead")]
1073    pub fn encoded_batch(
1074        &self,
1075        batch: &RecordBatch,
1076        dictionary_tracker: &mut DictionaryTracker,
1077        write_options: &IpcWriteOptions,
1078    ) -> Result<(Vec<EncodedData>, EncodedData), ArrowError> {
1079        self.encode(
1080            batch,
1081            dictionary_tracker,
1082            write_options,
1083            &mut Default::default(),
1084        )
1085    }
1086
1087    /// Encodes a `RecordBatch` into a flatbuffer IPC message and fills `sink` with the
1088    /// serialised buffer data.
1089    ///
1090    /// Returns the total body length written to `sink` (including per-buffer alignment
1091    /// padding).
1092    ///
1093    /// The FlatBuffer [`crate::Message`] is located in `ipc_write_context`'s
1094    /// [`FlatBufferBuilder`] finished bytes. A successful Result from this function
1095    /// guarantees the builder is in a finished state to call
1096    /// [`FlatBufferBuilder::finished_data`].
1097    fn record_batch_to_bytes(
1098        &self,
1099        batch: &RecordBatch,
1100        write_options: &IpcWriteOptions,
1101        ipc_write_context: &mut IpcWriteContext,
1102        sink: &mut IpcBodySink<'_>,
1103    ) -> Result<usize, ArrowError> {
1104        // Reset the fbb
1105        ipc_write_context.mut_fbb().reset();
1106
1107        let EncodedRecordBatchMeta {
1108            fb_offset: record_batch,
1109            body_len,
1110        } = self.encode_record_batch_data(
1111            batch.columns().iter().map(|array| array.to_data()),
1112            batch.num_rows() as i64,
1113            write_options,
1114            ipc_write_context,
1115            sink,
1116        )?;
1117
1118        let fbb = ipc_write_context.mut_fbb();
1119        let mut message = crate::MessageBuilder::new(fbb);
1120        message.add_version(write_options.metadata_version);
1121        message.add_header_type(crate::MessageHeader::RecordBatch);
1122        message.add_bodyLength(body_len as i64);
1123        message.add_header(record_batch.as_union_value());
1124        let root = message.finish();
1125        fbb.finish(root, None);
1126
1127        Ok(body_len)
1128    }
1129
1130    /// Write dictionary values into two sets of bytes, one for the header (crate::Message) and the
1131    /// other for the data
1132    fn dictionary_batch_to_bytes(
1133        &self,
1134        dict: DictionaryToEncode,
1135        write_options: &IpcWriteOptions,
1136        ipc_write_context: &mut IpcWriteContext,
1137    ) -> Result<EncodedData, ArrowError> {
1138        let mut arrow_data: Vec<u8> = vec![];
1139        self.dictionary_batch_to_sink(
1140            &dict,
1141            write_options,
1142            ipc_write_context,
1143            &mut IpcBodySink::Write(&mut arrow_data),
1144        )?;
1145
1146        Ok(EncodedData {
1147            ipc_message: ipc_write_context.mut_fbb().finished_data().to_vec(),
1148            arrow_data,
1149        })
1150    }
1151
1152    /// Encodes a dictionary batch's flatbuffer metadata into `ipc_write_context`'s
1153    /// builder (read back via `ipc_write_context.mut_fbb().finished_data()`) and fills
1154    /// `sink` with its body buffers.
1155    ///
1156    /// Returns the total body length written to `sink` (including per-buffer alignment
1157    /// padding).
1158    fn dictionary_batch_to_sink(
1159        &self,
1160        dict: &DictionaryToEncode,
1161        write_options: &IpcWriteOptions,
1162        ipc_write_context: &mut IpcWriteContext,
1163        sink: &mut IpcBodySink<'_>,
1164    ) -> Result<usize, ArrowError> {
1165        // Reset fbb
1166        ipc_write_context.mut_fbb().reset();
1167
1168        // A dictionary batch is a record batch (the single column of dictionary
1169        // values) wrapped in a DictionaryBatch, so we share the record batch body
1170        // and table encoding and only differ in the message framing.
1171        let EncodedRecordBatchMeta {
1172            fb_offset: record_batch,
1173            body_len,
1174        } = self.encode_record_batch_data(
1175            std::iter::once(dict.data.clone()),
1176            dict.data.len() as i64,
1177            write_options,
1178            ipc_write_context,
1179            sink,
1180        )?;
1181
1182        let fbb = ipc_write_context.mut_fbb();
1183        let root = {
1184            let mut batch_builder = crate::DictionaryBatchBuilder::new(fbb);
1185            batch_builder.add_id(dict.id);
1186            batch_builder.add_data(record_batch);
1187            batch_builder.add_isDelta(dict.is_delta);
1188            batch_builder.finish().as_union_value()
1189        };
1190
1191        let root = {
1192            let mut message_builder = crate::MessageBuilder::new(fbb);
1193            message_builder.add_version(write_options.metadata_version);
1194            message_builder.add_header_type(crate::MessageHeader::DictionaryBatch);
1195            message_builder.add_bodyLength(body_len as i64);
1196            message_builder.add_header(root);
1197            message_builder.finish()
1198        };
1199
1200        fbb.finish(root, None);
1201
1202        Ok(body_len)
1203    }
1204
1205    /// Encode the body buffers of a single record batch into `sink` and build the
1206    /// flatbuffer `RecordBatch` table into `ipc_write_context`'s builder.
1207    ///
1208    /// This is the shared core of [`Self::record_batch_to_bytes`] and
1209    /// [`Self::dictionary_batch_to_sink`]: a dictionary message embeds a record
1210    /// batch (its single column of values), so both encode the same `RecordBatch`
1211    /// table and only differ in how they wrap the returned offset in a message.
1212    ///
1213    /// `columns` yields the column data in IPC buffer order and `row_count` is the
1214    /// logical length recorded in the `RecordBatch` header. The caller is responsible
1215    /// for wrapping the returned [`EncodedRecordBatchMeta::fb_offset`] in a `Message`
1216    /// and calling [`FlatBufferBuilder::finish`].
1217    #[inline]
1218    fn encode_record_batch_data(
1219        &self,
1220        columns: impl IntoIterator<Item = ArrayData>,
1221        row_count: i64,
1222        write_options: &IpcWriteOptions,
1223        ipc_write_context: &mut IpcWriteContext,
1224        sink: &mut IpcBodySink<'_>,
1225    ) -> Result<EncodedRecordBatchMeta, ArrowError> {
1226        let batch_compression_type = write_options.batch_compression_type;
1227
1228        // Keep the reusable lz4 encoder cached in `ipc_write_context` in sync with these
1229        // options: the context may outlive a single writer or option set.
1230        ipc_write_context.set_lz4_block_size(write_options.lz4_block_size());
1231
1232        let compression = batch_compression_type.map(|batch_compression_type| {
1233            let fbb = ipc_write_context.mut_fbb();
1234            let mut c = crate::BodyCompressionBuilder::new(fbb);
1235            c.add_method(crate::BodyCompressionMethod::BUFFER);
1236            c.add_codec(batch_compression_type);
1237            c.finish()
1238        });
1239
1240        let batch_compression_level = write_options.batch_compression_level;
1241        let compression_codec: Option<CompressionCodec> = batch_compression_type
1242            .map(|compression_type| match batch_compression_level {
1243                Some(level) => {
1244                    CompressionCodec::try_new_with_compression_level(compression_type, level)
1245                }
1246                None => compression_type.try_into(),
1247            })
1248            .transpose()?;
1249
1250        let mut meta = IpcMetadataBuilder::default();
1251        let mut variadic_buffer_counts: Vec<i64> = vec![];
1252        let mut offset = 0i64;
1253
1254        for array_data in columns {
1255            offset = write_array_data(
1256                &array_data,
1257                &mut meta,
1258                sink,
1259                offset,
1260                compression_codec,
1261                ipc_write_context,
1262                write_options,
1263            )?;
1264            append_variadic_buffer_counts(&mut variadic_buffer_counts, &array_data);
1265        }
1266
1267        // Each buffer is padded to the alignment as it is written, so `offset` is
1268        // already a multiple of the alignment -- the body needs no trailing padding.
1269        debug_assert_eq!(offset % write_options.alignment as i64, 0);
1270        let body_len = offset as usize;
1271
1272        let fbb = ipc_write_context.mut_fbb();
1273        let buffers = fbb.create_vector(&meta.buffers);
1274        let nodes = fbb.create_vector(&meta.nodes);
1275        let variadic_buffer = if variadic_buffer_counts.is_empty() {
1276            None
1277        } else {
1278            Some(fbb.create_vector(&variadic_buffer_counts))
1279        };
1280
1281        let mut batch_builder = crate::RecordBatchBuilder::new(fbb);
1282        batch_builder.add_length(row_count);
1283        batch_builder.add_nodes(nodes);
1284        batch_builder.add_buffers(buffers);
1285        if let Some(c) = compression {
1286            batch_builder.add_compression(c);
1287        }
1288        if let Some(v) = variadic_buffer {
1289            batch_builder.add_variadicBufferCounts(v);
1290        }
1291        let record_batch = batch_builder.finish();
1292
1293        Ok(EncodedRecordBatchMeta {
1294            fb_offset: record_batch,
1295            body_len,
1296        })
1297    }
1298}
1299
1300fn ensure_supported_ipc_schema(schema: &Schema) -> Result<(), ArrowError> {
1301    schema
1302        .fields()
1303        .iter()
1304        .try_for_each(|field| ensure_supported_ipc_data_type(field.name(), field.data_type()))
1305}
1306
1307fn ensure_supported_ipc_data_type(
1308    field_name: &str,
1309    data_type: &DataType,
1310) -> Result<(), ArrowError> {
1311    match data_type {
1312        DataType::Dictionary(_, value_type)
1313            if matches!(value_type.as_ref(), DataType::Dictionary(_, _)) =>
1314        {
1315            Err(ArrowError::InvalidArgumentError(format!(
1316                "Arrow IPC field metadata cannot encode direct dictionary-of-dictionary values for field {field_name:?}"
1317            )))
1318        }
1319        DataType::Dictionary(_, value_type) => {
1320            ensure_supported_ipc_data_type(field_name, value_type)
1321        }
1322        DataType::Struct(fields) => fields
1323            .iter()
1324            .try_for_each(|field| ensure_supported_ipc_data_type(field.name(), field.data_type())),
1325        DataType::RunEndEncoded(_, field)
1326        | DataType::List(field)
1327        | DataType::LargeList(field)
1328        | DataType::ListView(field)
1329        | DataType::LargeListView(field)
1330        | DataType::FixedSizeList(field, _)
1331        | DataType::Map(field, _) => {
1332            ensure_supported_ipc_data_type(field.name(), field.data_type())
1333        }
1334        DataType::Union(fields, _) => fields.iter().try_for_each(|(_, field)| {
1335            ensure_supported_ipc_data_type(field.name(), field.data_type())
1336        }),
1337        _ => Ok(()),
1338    }
1339}
1340
1341fn append_variadic_buffer_counts(counts: &mut Vec<i64>, array: &ArrayData) {
1342    match array.data_type() {
1343        DataType::BinaryView | DataType::Utf8View => {
1344            // The spec documents the counts only includes the variadic buffers, not the view/null buffers.
1345            // https://arrow.apache.org/docs/format/Columnar.html#variadic-buffers
1346            counts.push(array.buffers().len() as i64 - 1);
1347        }
1348        DataType::Dictionary(_, _) => {
1349            // Do nothing
1350            // Dictionary values are handled in special dictionary messages
1351        }
1352        _ => {
1353            for child in array.child_data() {
1354                append_variadic_buffer_counts(counts, child)
1355            }
1356        }
1357    }
1358}
1359
1360pub(crate) fn unslice_run_array(arr: ArrayData) -> Result<ArrayData, ArrowError> {
1361    match arr.data_type() {
1362        DataType::RunEndEncoded(k, _) => match k.data_type() {
1363            DataType::Int16 => {
1364                Ok(into_zero_offset_run_array(RunArray::<Int16Type>::from(arr))?.into_data())
1365            }
1366            DataType::Int32 => {
1367                Ok(into_zero_offset_run_array(RunArray::<Int32Type>::from(arr))?.into_data())
1368            }
1369            DataType::Int64 => {
1370                Ok(into_zero_offset_run_array(RunArray::<Int64Type>::from(arr))?.into_data())
1371            }
1372            d => unreachable!("Unexpected data type {d}"),
1373        },
1374        d => Err(ArrowError::InvalidArgumentError(format!(
1375            "The given array is not a run array. Data type of given array: {d}"
1376        ))),
1377    }
1378}
1379
1380// Returns a `RunArray` with zero offset and length matching the last value
1381// in run_ends array.
1382fn into_zero_offset_run_array<R: RunEndIndexType>(
1383    run_array: RunArray<R>,
1384) -> Result<RunArray<R>, ArrowError> {
1385    // Empty slices have no physical runs, regardless of their offset.
1386    if run_array.is_empty() {
1387        return Ok(ArrayData::new_empty(run_array.data_type()).into());
1388    }
1389
1390    let run_ends = run_array.run_ends();
1391    if run_ends.offset() == 0 && run_ends.max_value() == run_ends.len() {
1392        return Ok(run_array);
1393    }
1394
1395    // The physical index of original run_ends array from which the `ArrayData`is sliced.
1396    let start_physical_index = run_ends.get_start_physical_index();
1397
1398    // The physical index of original run_ends array until which the `ArrayData`is sliced.
1399    let end_physical_index = run_ends.get_end_physical_index();
1400
1401    let physical_length = end_physical_index - start_physical_index + 1;
1402
1403    // build new run_ends array by subtracting offset from run ends.
1404    let offset = R::Native::usize_as(run_ends.offset());
1405    let mut run_ends_values = Vec::<R::Native>::with_capacity(physical_length);
1406    for run_end_value in &run_ends.values()[start_physical_index..end_physical_index] {
1407        run_ends_values.push(run_end_value.sub_wrapping(offset));
1408    }
1409    run_ends_values.push(R::Native::from_usize(run_array.len()).unwrap());
1410    let offset_buffer = Buffer::from_vec(run_ends_values);
1411    let new_run_ends = unsafe {
1412        // Safety:
1413        // The function builds a valid run_ends array and hence need not be validated.
1414        ArrayDataBuilder::new(R::DATA_TYPE)
1415            .len(physical_length)
1416            .add_buffer(offset_buffer)
1417            .build_unchecked()
1418    };
1419
1420    // build new values by slicing physical indices.
1421    let new_values = run_array
1422        .values()
1423        .slice(start_physical_index, physical_length)
1424        .into_data();
1425
1426    let builder = ArrayDataBuilder::new(run_array.data_type().clone())
1427        .len(run_array.len())
1428        .add_child_data(new_run_ends)
1429        .add_child_data(new_values);
1430    let array_data = unsafe {
1431        // Safety:
1432        //  This function builds a valid run array and hence can skip validation.
1433        builder.build_unchecked()
1434    };
1435    Ok(array_data.into())
1436}
1437
1438/// Controls how dictionaries are handled in Arrow IPC messages
1439#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
1440pub enum DictionaryHandling {
1441    /// Send the entire dictionary every time it is encountered (default)
1442    #[default]
1443    Resend,
1444    /// Send only new dictionary values since the last batch (delta encoding)
1445    ///
1446    /// When a dictionary is first encountered, the entire dictionary is sent.
1447    /// For subsequent batches, only values that are new (not previously sent)
1448    /// are transmitted with the `isDelta` flag set to true.
1449    Delta,
1450}
1451
1452/// Describes what kind of update took place after a call to [`DictionaryTracker::insert`].
1453#[derive(Debug, Clone)]
1454pub enum DictionaryUpdate {
1455    /// No dictionary was written, the dictionary was identical to what was already
1456    /// in the tracker.
1457    None,
1458    /// No dictionary was present in the tracker
1459    New,
1460    /// Dictionary was replaced with the new data
1461    Replaced,
1462    /// Dictionary was updated, ArrayData is the delta between old and new
1463    Delta(ArrayData),
1464}
1465
1466/// Keeps track of dictionaries that have been written, to avoid emitting the same dictionary
1467/// multiple times.
1468///
1469/// Can optionally error if an update to an existing dictionary is attempted, which
1470/// isn't allowed in the `FileWriter`.
1471#[derive(Debug)]
1472pub struct DictionaryTracker {
1473    // NOTE: When adding fields, update the clear() method accordingly.
1474    written: HashMap<i64, ArrayData>,
1475    dict_ids: Vec<i64>,
1476    error_on_replacement: bool,
1477}
1478
1479impl DictionaryTracker {
1480    /// Create a new [`DictionaryTracker`].
1481    ///
1482    /// If `error_on_replacement`
1483    /// is true, an error will be generated if an update to an
1484    /// existing dictionary is attempted.
1485    pub fn new(error_on_replacement: bool) -> Self {
1486        Self {
1487            written: HashMap::new(),
1488            dict_ids: Vec::new(),
1489            error_on_replacement,
1490        }
1491    }
1492
1493    /// Record and return the next dictionary ID.
1494    pub fn next_dict_id(&mut self) -> i64 {
1495        let next = self
1496            .dict_ids
1497            .last()
1498            .copied()
1499            .map(|i| i + 1)
1500            .unwrap_or_default();
1501
1502        self.dict_ids.push(next);
1503        next
1504    }
1505
1506    /// Return the sequence of dictionary IDs in the order they should be observed while
1507    /// traversing the schema
1508    pub fn dict_id(&mut self) -> &[i64] {
1509        &self.dict_ids
1510    }
1511
1512    /// Keep track of the dictionary with the given ID and values. Behavior:
1513    ///
1514    /// * If this ID has been written already and has the same data, return `Ok(false)` to indicate
1515    ///   that the dictionary was not actually inserted (because it's already been seen).
1516    /// * If this ID has been written already but with different data, and this tracker is
1517    ///   configured to return an error, return an error.
1518    /// * If the tracker has not been configured to error on replacement or this dictionary
1519    ///   has never been seen before, return `Ok(true)` to indicate that the dictionary was just
1520    ///   inserted.
1521    #[deprecated(since = "56.1.0", note = "Use `insert_column` instead")]
1522    pub fn insert(&mut self, dict_id: i64, column: &ArrayRef) -> Result<bool, ArrowError> {
1523        let dict_data = column.to_data();
1524        let dict_values = &dict_data.child_data()[0];
1525
1526        // If a dictionary with this id was already emitted, check if it was the same.
1527        if let Some(last) = self.written.get(&dict_id) {
1528            if ArrayData::ptr_eq(&last.child_data()[0], dict_values) {
1529                // Same dictionary values => no need to emit it again
1530                return Ok(false);
1531            }
1532            if self.error_on_replacement {
1533                // If error on replacement perform a logical comparison
1534                if last.child_data()[0] == *dict_values {
1535                    // Same dictionary values => no need to emit it again
1536                    return Ok(false);
1537                }
1538                return Err(ArrowError::InvalidArgumentError(
1539                    "Dictionary replacement detected when writing IPC file format. \
1540                     Arrow IPC files only support a single dictionary for a given field \
1541                     across all batches."
1542                        .to_string(),
1543                ));
1544            }
1545        }
1546
1547        self.written.insert(dict_id, dict_data);
1548        Ok(true)
1549    }
1550
1551    /// Keep track of the dictionary with the given ID and values. The return
1552    /// value indicates what, if any, update to the internal map took place
1553    /// and how it should be interpreted based on the `dict_handling` parameter.
1554    ///
1555    /// # Returns
1556    ///
1557    /// * `Ok(Dictionary::New)` - If the dictionary was not previously written
1558    /// * `Ok(Dictionary::Replaced)` - If the dictionary was previously written
1559    ///   with completely different data, or if the data is a delta of the existing,
1560    ///   but with `dict_handling` set to `DictionaryHandling::Resend`
1561    /// * `Ok(Dictionary::Delta)` - If the dictionary was previously written, but
1562    ///   the new data is a delta of the old and the `dict_handling` is set to
1563    ///   `DictionaryHandling::Delta`
1564    /// * `Err(e)` - If the dictionary was previously written with different data,
1565    ///   and `error_on_replacement` is set to `true`.
1566    pub fn insert_column(
1567        &mut self,
1568        dict_id: i64,
1569        column: &ArrayRef,
1570        dict_handling: DictionaryHandling,
1571    ) -> Result<DictionaryUpdate, ArrowError> {
1572        let new_data = column.to_data();
1573        let new_values = &new_data.child_data()[0];
1574
1575        // If there is no existing dictionary with this ID, we always insert
1576        let Some(old) = self.written.get(&dict_id) else {
1577            self.written.insert(dict_id, new_data);
1578            return Ok(DictionaryUpdate::New);
1579        };
1580
1581        // Fast path - If the array data points to the same buffer as the
1582        // existing then they're the same.
1583        let old_values = &old.child_data()[0];
1584        if ArrayData::ptr_eq(old_values, new_values) {
1585            return Ok(DictionaryUpdate::None);
1586        }
1587
1588        // Slow path - Compare the dictionaries value by value
1589        let comparison = compare_dictionaries(old_values, new_values);
1590        if matches!(comparison, DictionaryComparison::Equal) {
1591            return Ok(DictionaryUpdate::None);
1592        }
1593
1594        const REPLACEMENT_ERROR: &str = "Dictionary replacement detected when writing IPC file format. \
1595                 Arrow IPC files only support a single dictionary for a given field \
1596                 across all batches.";
1597
1598        match comparison {
1599            DictionaryComparison::NotEqual => {
1600                if self.error_on_replacement {
1601                    return Err(ArrowError::InvalidArgumentError(
1602                        REPLACEMENT_ERROR.to_string(),
1603                    ));
1604                }
1605
1606                self.written.insert(dict_id, new_data);
1607                Ok(DictionaryUpdate::Replaced)
1608            }
1609            DictionaryComparison::Delta => match dict_handling {
1610                DictionaryHandling::Resend => {
1611                    if self.error_on_replacement {
1612                        return Err(ArrowError::InvalidArgumentError(
1613                            REPLACEMENT_ERROR.to_string(),
1614                        ));
1615                    }
1616
1617                    self.written.insert(dict_id, new_data);
1618                    Ok(DictionaryUpdate::Replaced)
1619                }
1620                DictionaryHandling::Delta => {
1621                    let delta =
1622                        new_values.slice(old_values.len(), new_values.len() - old_values.len());
1623                    self.written.insert(dict_id, new_data);
1624                    Ok(DictionaryUpdate::Delta(delta))
1625                }
1626            },
1627            DictionaryComparison::Equal => unreachable!("Already checked equal case"),
1628        }
1629    }
1630
1631    /// Clears the state of the dictionary tracker.
1632    ///
1633    /// This allows the dictionary tracker to be reused for a new IPC stream while avoiding the
1634    /// allocation cost of creating a new instance. This method should not be called if
1635    /// the dictionary tracker will be used to continue writing to an existing IPC stream.
1636    pub fn clear(&mut self) {
1637        self.dict_ids.clear();
1638        self.written.clear();
1639    }
1640}
1641
1642/// Describes how two dictionary arrays compare to each other.
1643#[derive(Debug, Clone)]
1644enum DictionaryComparison {
1645    /// Neither a delta, nor an exact match
1646    NotEqual,
1647    /// Exact element-wise match
1648    Equal,
1649    /// The two arrays are dictionary deltas of each other, meaning the first
1650    /// is a prefix of the second.
1651    Delta,
1652}
1653
1654// Compares two dictionaries and returns a [`DictionaryComparison`].
1655fn compare_dictionaries(old: &ArrayData, new: &ArrayData) -> DictionaryComparison {
1656    // Check for exact match
1657    let existing_len = old.len();
1658    let new_len = new.len();
1659    if existing_len == new_len {
1660        return if *old == *new {
1661            DictionaryComparison::Equal
1662        } else {
1663            DictionaryComparison::NotEqual
1664        };
1665    }
1666
1667    // Can't be a delta if the new is shorter than the existing
1668    if new_len < existing_len {
1669        return DictionaryComparison::NotEqual;
1670    }
1671
1672    // Check for delta
1673    if new.slice(0, existing_len) == *old {
1674        return DictionaryComparison::Delta;
1675    }
1676
1677    DictionaryComparison::NotEqual
1678}
1679
1680/// Arrow File Writer
1681///
1682/// Writes Arrow [`RecordBatch`]es in the [IPC File Format].
1683///
1684/// # See Also
1685///
1686/// * [`StreamWriter`] for writing IPC Streams
1687///
1688/// # Example
1689/// ```
1690/// # use arrow_array::record_batch;
1691/// # use arrow_ipc::writer::FileWriter;
1692/// # let mut file = vec![]; // mimic a file for the example
1693/// let batch = record_batch!(("a", Int32, [1, 2, 3])).unwrap();
1694/// // create a new writer, the schema must be known in advance
1695/// let mut writer = FileWriter::try_new(&mut file, &batch.schema()).unwrap();
1696/// // write each batch to the underlying writer
1697/// writer.write(&batch).unwrap();
1698/// // When all batches are written, call finish to flush all buffers
1699/// writer.finish().unwrap();
1700/// ```
1701/// [IPC File Format]: https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
1702pub struct FileWriter<W> {
1703    /// The object to write to
1704    writer: W,
1705    /// IPC write options
1706    write_options: IpcWriteOptions,
1707    /// A reference to the schema, used in validating record batches
1708    schema: SchemaRef,
1709    /// The number of bytes between each block of bytes, as an offset for random access
1710    block_offsets: usize,
1711    /// Dictionary blocks that will be written as part of the IPC footer
1712    dictionary_blocks: Vec<crate::Block>,
1713    /// Record blocks that will be written as part of the IPC footer
1714    record_blocks: Vec<crate::Block>,
1715    /// Whether the writer footer has been written, and the writer is finished
1716    finished: bool,
1717    /// Keeps track of dictionaries that have been written
1718    dictionary_tracker: DictionaryTracker,
1719    /// User level customized metadata
1720    custom_metadata: Metadata,
1721
1722    data_gen: IpcDataGenerator,
1723
1724    /// Holds reusable scratch state shared across all messages -- including the
1725    /// [`FlatBufferBuilder`] whose internal buffer is reused to avoid reallocating
1726    /// on every batch.
1727    ipc_write_context: IpcWriteContext,
1728}
1729
1730impl<W: Write> FileWriter<BufWriter<W>> {
1731    /// Try to create a new file writer with the writer wrapped in a BufWriter.
1732    ///
1733    /// See [`FileWriter::try_new`] for an unbuffered version.
1734    pub fn try_new_buffered(writer: W, schema: &Schema) -> Result<Self, ArrowError> {
1735        Self::try_new(BufWriter::new(writer), schema)
1736    }
1737}
1738
1739impl<W: Write> FileWriter<W> {
1740    /// Try to create a new writer, with the schema written as part of the header
1741    ///
1742    /// Note the created writer is not buffered. See [`FileWriter::try_new_buffered`] for details.
1743    ///
1744    /// # Errors
1745    ///
1746    /// An [`Err`] may be returned if writing the header to the writer fails.
1747    pub fn try_new(writer: W, schema: &Schema) -> Result<Self, ArrowError> {
1748        let write_options = IpcWriteOptions::default();
1749        Self::try_new_with_options(writer, schema, write_options)
1750    }
1751
1752    /// Try to create a new writer with IpcWriteOptions
1753    ///
1754    /// Note the created writer is not buffered. See [`FileWriter::try_new_buffered`] for details.
1755    ///
1756    /// # Errors
1757    ///
1758    /// An [`Err`] may be returned if writing the header to the writer fails.
1759    pub fn try_new_with_options(
1760        mut writer: W,
1761        schema: &Schema,
1762        write_options: IpcWriteOptions,
1763    ) -> Result<Self, ArrowError> {
1764        ensure_supported_ipc_schema(schema)?;
1765
1766        let data_gen = IpcDataGenerator::default();
1767        // write magic to header aligned on alignment boundary
1768        let pad_len = pad_to_alignment(write_options.alignment, super::ARROW_MAGIC.len());
1769        let header_size = super::ARROW_MAGIC.len() + pad_len;
1770        writer.write_all(&super::ARROW_MAGIC)?;
1771        writer.write_all(&PADDING[..pad_len])?;
1772        // write the schema, set the written bytes to the schema + header
1773        let mut dictionary_tracker = DictionaryTracker::new(true);
1774        let encoded_message = data_gen.schema_to_bytes_with_dictionary_tracker(
1775            schema,
1776            &mut dictionary_tracker,
1777            &write_options,
1778        );
1779        let (meta, data) = write_message(&mut writer, encoded_message, &write_options)?;
1780        Ok(Self {
1781            writer,
1782            write_options,
1783            schema: Arc::new(schema.clone()),
1784            block_offsets: meta + data + header_size,
1785            dictionary_blocks: vec![],
1786            record_blocks: vec![],
1787            finished: false,
1788            dictionary_tracker,
1789            custom_metadata: Default::default(),
1790            data_gen,
1791            ipc_write_context: IpcWriteContext::default(),
1792        })
1793    }
1794
1795    /// Adds a key-value pair to the [FileWriter]'s custom metadata
1796    pub fn write_metadata(&mut self, key: impl Into<String>, value: impl Into<String>) {
1797        self.custom_metadata.insert(key.into(), value.into());
1798    }
1799
1800    /// Write a record batch to the file
1801    pub fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
1802        if self.finished {
1803            return Err(ArrowError::IpcError(
1804                "Cannot write record batch to file writer as it is closed".to_string(),
1805            ));
1806        }
1807
1808        let meta = self.data_gen.write(
1809            batch,
1810            &mut self.dictionary_tracker,
1811            &self.write_options,
1812            &mut self.ipc_write_context,
1813            &mut self.writer,
1814        )?;
1815
1816        for (header_len, body_len) in meta.dictionary_block_sizes {
1817            let block = crate::Block::new(
1818                self.block_offsets as i64,
1819                header_len as i32,
1820                body_len as i64,
1821            );
1822            self.dictionary_blocks.push(block);
1823            self.block_offsets += header_len + body_len;
1824        }
1825
1826        // add a record block for the footer
1827        let block = crate::Block::new(
1828            self.block_offsets as i64,
1829            meta.padded_header_len as i32,
1830            meta.body_len as i64,
1831        );
1832        self.record_blocks.push(block);
1833        self.block_offsets += meta.padded_header_len + meta.body_len;
1834        Ok(())
1835    }
1836
1837    /// Write footer and closing tag, then mark the writer as done
1838    pub fn finish(&mut self) -> Result<(), ArrowError> {
1839        if self.finished {
1840            return Err(ArrowError::IpcError(
1841                "Cannot write footer to file writer as it is closed".to_string(),
1842            ));
1843        }
1844
1845        // write EOS
1846        self.writer.write_eos(&self.write_options)?;
1847
1848        let mut fbb = FlatBufferBuilder::new();
1849        let dictionaries = fbb.create_vector(&self.dictionary_blocks);
1850        let record_batches = fbb.create_vector(&self.record_blocks);
1851
1852        // dictionaries are already written, so we can reset dictionary tracker to reuse for schema
1853        self.dictionary_tracker.clear();
1854        let schema = IpcSchemaEncoder::new()
1855            .with_dictionary_tracker(&mut self.dictionary_tracker)
1856            .schema_to_fb_offset(&mut fbb, &self.schema);
1857        let fb_custom_metadata = (!self.custom_metadata.is_empty())
1858            .then(|| crate::convert::metadata_to_fb(&mut fbb, &self.custom_metadata));
1859
1860        let root = {
1861            let mut footer_builder = crate::FooterBuilder::new(&mut fbb);
1862            footer_builder.add_version(self.write_options.metadata_version);
1863            footer_builder.add_schema(schema);
1864            footer_builder.add_dictionaries(dictionaries);
1865            footer_builder.add_recordBatches(record_batches);
1866            if let Some(fb_custom_metadata) = fb_custom_metadata {
1867                footer_builder.add_custom_metadata(fb_custom_metadata);
1868            }
1869            footer_builder.finish()
1870        };
1871        fbb.finish(root, None);
1872        let footer_data = fbb.finished_data();
1873        self.writer.write_all(footer_data)?;
1874        self.writer
1875            .write_all(&(footer_data.len() as i32).to_le_bytes())?;
1876        self.writer.write_all(&super::ARROW_MAGIC)?;
1877        self.writer.flush()?;
1878        self.finished = true;
1879
1880        Ok(())
1881    }
1882
1883    /// Returns the arrow [`SchemaRef`] for this arrow file.
1884    pub fn schema(&self) -> &SchemaRef {
1885        &self.schema
1886    }
1887
1888    /// Gets a reference to the underlying writer.
1889    pub fn get_ref(&self) -> &W {
1890        &self.writer
1891    }
1892
1893    /// Gets a mutable reference to the underlying writer.
1894    ///
1895    /// It is inadvisable to directly write to the underlying writer.
1896    pub fn get_mut(&mut self) -> &mut W {
1897        &mut self.writer
1898    }
1899
1900    /// Flush the underlying writer.
1901    ///
1902    /// Both the BufWriter and the underlying writer are flushed.
1903    pub fn flush(&mut self) -> Result<(), ArrowError> {
1904        self.writer.flush()?;
1905        Ok(())
1906    }
1907
1908    /// Unwraps the underlying writer.
1909    ///
1910    /// The writer is flushed and the FileWriter is finished before returning.
1911    ///
1912    /// # Errors
1913    ///
1914    /// An [`Err`] may be returned if an error occurs while finishing the StreamWriter
1915    /// or while flushing the writer.
1916    pub fn into_inner(mut self) -> Result<W, ArrowError> {
1917        if !self.finished {
1918            // `finish` flushes the writer.
1919            self.finish()?;
1920        }
1921        Ok(self.writer)
1922    }
1923}
1924
1925impl<W: Write> RecordBatchWriter for FileWriter<W> {
1926    fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
1927        self.write(batch)
1928    }
1929
1930    fn close(mut self) -> Result<(), ArrowError> {
1931        self.finish()
1932    }
1933}
1934
1935/// Arrow IPC stream encoder.
1936///
1937/// Encodes Arrow [`RecordBatch`]es to byte buffers using the [IPC Streaming Format],
1938/// without performing any IO.
1939///
1940/// The returned [`Buffer`]s are ordered and should be written to the destination
1941/// stream in order. Uncompressed record batch body buffers can share the original
1942/// Arrow buffers instead of being copied into an intermediate contiguous buffer.
1943///
1944/// # Example
1945/// ```
1946/// # use arrow_array::record_batch;
1947/// # use arrow_ipc::writer::StreamEncoder;
1948/// # use arrow_schema::ArrowError;
1949/// # fn main() -> Result<(), ArrowError> {
1950/// let batch = record_batch!(("a", Int32, [1, 2, 3]))?;
1951///
1952/// let mut encoder = StreamEncoder::try_new(&batch.schema())?;
1953/// let mut stream = vec![];
1954/// for buffer in encoder.encode(&batch)? {
1955///     stream.extend_from_slice(buffer.as_slice());
1956/// }
1957/// for buffer in encoder.finish()? {
1958///     stream.extend_from_slice(buffer.as_slice());
1959/// }
1960/// # Ok(())
1961/// # }
1962/// ```
1963pub struct StreamEncoder {
1964    schema: Schema,
1965    /// IPC write options
1966    write_options: IpcWriteOptions,
1967    /// Whether the stream schema has been encoded
1968    schema_encoded: bool,
1969    /// Keeps track of dictionaries that have been encoded
1970    dictionary_tracker: DictionaryTracker,
1971    data_gen: IpcDataGenerator,
1972    ipc_write_context: IpcWriteContext,
1973}
1974
1975impl StreamEncoder {
1976    /// Try to create a new stream encoder.
1977    pub fn try_new(schema: &Schema) -> Result<Self, ArrowError> {
1978        let write_options = IpcWriteOptions::default();
1979        Self::try_new_with_options(schema, write_options)
1980    }
1981
1982    /// Try to create a new stream encoder with [`IpcWriteOptions`].
1983    pub fn try_new_with_options(
1984        schema: &Schema,
1985        write_options: IpcWriteOptions,
1986    ) -> Result<Self, ArrowError> {
1987        ensure_supported_ipc_schema(schema)?;
1988
1989        Ok(Self {
1990            schema: schema.clone(),
1991            write_options,
1992            schema_encoded: false,
1993            dictionary_tracker: DictionaryTracker::new(false),
1994            data_gen: IpcDataGenerator::default(),
1995            ipc_write_context: IpcWriteContext::default(),
1996        })
1997    }
1998
1999    /// Encode a [`RecordBatch`] into buffers.
2000    ///
2001    /// The first call also includes the IPC stream schema message before the
2002    /// record batch message. Later calls only include dictionary and record
2003    /// batch messages.
2004    ///
2005    /// # Errors
2006    ///
2007    /// Returns an error if encoding fails.
2008    pub fn encode(&mut self, batch: &RecordBatch) -> Result<Vec<Buffer>, ArrowError> {
2009        let mut out = vec![];
2010        self.encode_schema(&mut out)?;
2011        self.data_gen.encode_to_buffers(
2012            batch,
2013            &mut self.dictionary_tracker,
2014            &self.write_options,
2015            &mut self.ipc_write_context,
2016            &mut out,
2017        )?;
2018        Ok(out)
2019    }
2020
2021    /// Encode the end-of-stream marker.
2022    ///
2023    /// If no batches have been encoded, this also emits the IPC stream schema
2024    /// message so the returned buffers form a valid empty IPC stream.
2025    ///
2026    /// # Errors
2027    ///
2028    /// Returns an error if encoding the schema or end-of-stream marker fails.
2029    pub fn finish(mut self) -> Result<Vec<Buffer>, ArrowError> {
2030        let mut out = vec![];
2031        self.encode_schema(&mut out)?;
2032        let mut sink = Buffers { out: &mut out };
2033        sink.write_eos(&self.write_options)?;
2034        Ok(out)
2035    }
2036
2037    fn encode_schema(&mut self, out: &mut Vec<Buffer>) -> Result<(), ArrowError> {
2038        if !self.schema_encoded {
2039            let encoded_message = self.data_gen.schema_to_bytes_with_dictionary_tracker(
2040                &self.schema,
2041                &mut self.dictionary_tracker,
2042                &self.write_options,
2043            );
2044            let mut sink = Buffers { out };
2045            sink.write_encoded_data(encoded_message, &self.write_options)?;
2046            self.schema_encoded = true;
2047        }
2048        Ok(())
2049    }
2050}
2051
2052/// Arrow Stream Writer
2053///
2054/// Writes Arrow [`RecordBatch`]es to bytes using the [IPC Streaming Format].
2055///
2056/// # See Also
2057///
2058/// * [`FileWriter`] for writing IPC Files
2059///
2060/// # Example - Basic usage
2061/// ```
2062/// # use arrow_array::record_batch;
2063/// # use arrow_ipc::writer::StreamWriter;
2064/// # let mut stream = vec![]; // mimic a stream for the example
2065/// let batch = record_batch!(("a", Int32, [1, 2, 3])).unwrap();
2066/// // create a new writer, the schema must be known in advance
2067/// let mut writer = StreamWriter::try_new(&mut stream, &batch.schema()).unwrap();
2068/// // write each batch to the underlying stream
2069/// writer.write(&batch).unwrap();
2070/// // When all batches are written, call finish to flush all buffers
2071/// writer.finish().unwrap();
2072/// ```
2073/// # Example - Efficient delta dictionaries
2074/// ```
2075/// # use arrow_array::record_batch;
2076/// # use arrow_ipc::writer::{StreamWriter, IpcWriteOptions};
2077/// # use arrow_ipc::writer::DictionaryHandling;
2078/// # use arrow_schema::{DataType, Field, Schema, SchemaRef};
2079/// # use arrow_array::{
2080/// #    builder::StringDictionaryBuilder, types::Int32Type, Array, ArrayRef, DictionaryArray,
2081/// #    RecordBatch, StringArray,
2082/// # };
2083/// # use std::sync::Arc;
2084///
2085/// let schema = Arc::new(Schema::new(vec![Field::new(
2086///    "col1",
2087///    DataType::Dictionary(Box::from(DataType::Int32), Box::from(DataType::Utf8)),
2088///    true,
2089/// )]));
2090///
2091/// let mut builder = StringDictionaryBuilder::<arrow_array::types::Int32Type>::new();
2092///
2093/// // `finish_preserve_values` will keep the dictionary values along with their
2094/// // key assignments so that they can be re-used in the next batch.
2095/// builder.append("a").unwrap();
2096/// builder.append("b").unwrap();
2097/// let array1 = builder.finish_preserve_values();
2098/// let batch1 = RecordBatch::try_new(schema.clone(), vec![Arc::new(array1) as ArrayRef]).unwrap();
2099///
2100/// // In this batch, 'a' will have the same dictionary key as 'a' in the previous batch,
2101/// // and 'd' will take the next available key.
2102/// builder.append("a").unwrap();
2103/// builder.append("d").unwrap();
2104/// let array2 = builder.finish_preserve_values();
2105/// let batch2 = RecordBatch::try_new(schema.clone(), vec![Arc::new(array2) as ArrayRef]).unwrap();
2106///
2107/// let mut stream = vec![];
2108/// // You must set `.with_dictionary_handling(DictionaryHandling::Delta)` to
2109/// // enable delta dictionaries in the writer
2110/// let options = IpcWriteOptions::default().with_dictionary_handling(DictionaryHandling::Delta);
2111/// let mut writer = StreamWriter::try_new_with_options(&mut stream, &schema, options).unwrap();
2112///
2113/// // When writing the first batch, a dictionary message with 'a' and 'b' will be written
2114/// // prior to the record batch.
2115/// writer.write(&batch1).unwrap();
2116/// // With the second batch only a delta dictionary with 'd' will be written
2117/// // prior to the record batch. This is only possible with `finish_preserve_values`.
2118/// // Without it, 'a' and 'd' in this batch would have different keys than the
2119/// // first batch and so we'd have to send a replacement dictionary with new keys
2120/// // for both.
2121/// writer.write(&batch2).unwrap();
2122/// writer.finish().unwrap();
2123/// ```
2124/// [IPC Streaming Format]: https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format
2125pub struct StreamWriter<W> {
2126    /// The object to write to
2127    writer: W,
2128    /// IPC write options
2129    write_options: IpcWriteOptions,
2130    /// Whether the writer footer has been written, and the writer is finished
2131    finished: bool,
2132    /// Keeps track of dictionaries that have been written
2133    dictionary_tracker: DictionaryTracker,
2134
2135    data_gen: IpcDataGenerator,
2136
2137    /// Reusable scratch state shared across all messages.
2138    ipc_write_context: IpcWriteContext,
2139}
2140
2141impl<W: Write> StreamWriter<BufWriter<W>> {
2142    /// Try to create a new stream writer with the writer wrapped in a BufWriter.
2143    ///
2144    /// See [`StreamWriter::try_new`] for an unbuffered version.
2145    pub fn try_new_buffered(writer: W, schema: &Schema) -> Result<Self, ArrowError> {
2146        Self::try_new(BufWriter::new(writer), schema)
2147    }
2148}
2149
2150impl<W: Write> StreamWriter<W> {
2151    /// Try to create a new writer, with the schema written as part of the header.
2152    ///
2153    /// Note that there is no internal buffering. See also [`StreamWriter::try_new_buffered`].
2154    ///
2155    /// # Errors
2156    ///
2157    /// An [`Err`] may be returned if writing the header to the writer fails.
2158    pub fn try_new(writer: W, schema: &Schema) -> Result<Self, ArrowError> {
2159        let write_options = IpcWriteOptions::default();
2160        Self::try_new_with_options(writer, schema, write_options)
2161    }
2162
2163    /// Try to create a new writer with [`IpcWriteOptions`].
2164    ///
2165    /// # Errors
2166    ///
2167    /// An [`Err`] may be returned if writing the header to the writer fails.
2168    pub fn try_new_with_options(
2169        mut writer: W,
2170        schema: &Schema,
2171        write_options: IpcWriteOptions,
2172    ) -> Result<Self, ArrowError> {
2173        ensure_supported_ipc_schema(schema)?;
2174
2175        let data_gen = IpcDataGenerator::default();
2176        let mut dictionary_tracker = DictionaryTracker::new(false);
2177
2178        // write the schema, set the written bytes to the schema
2179        let encoded_message = data_gen.schema_to_bytes_with_dictionary_tracker(
2180            schema,
2181            &mut dictionary_tracker,
2182            &write_options,
2183        );
2184        write_message(&mut writer, encoded_message, &write_options)?;
2185        Ok(Self {
2186            writer,
2187            write_options,
2188            finished: false,
2189            dictionary_tracker,
2190            data_gen,
2191            ipc_write_context: IpcWriteContext::default(),
2192        })
2193    }
2194
2195    /// Write a record batch to the stream
2196    pub fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
2197        if self.finished {
2198            return Err(ArrowError::IpcError(
2199                "Cannot write record batch to stream writer as it is closed".to_string(),
2200            ));
2201        }
2202
2203        self.data_gen.write(
2204            batch,
2205            &mut self.dictionary_tracker,
2206            &self.write_options,
2207            &mut self.ipc_write_context,
2208            &mut self.writer,
2209        )?;
2210        Ok(())
2211    }
2212
2213    /// Write continuation bytes, and mark the stream as done
2214    pub fn finish(&mut self) -> Result<(), ArrowError> {
2215        if self.finished {
2216            return Err(ArrowError::IpcError(
2217                "Cannot write footer to stream writer as it is closed".to_string(),
2218            ));
2219        }
2220
2221        self.writer.write_eos(&self.write_options)?;
2222        self.writer.flush()?;
2223
2224        self.finished = true;
2225
2226        Ok(())
2227    }
2228
2229    /// Gets a reference to the underlying writer.
2230    pub fn get_ref(&self) -> &W {
2231        &self.writer
2232    }
2233
2234    /// Gets a mutable reference to the underlying writer.
2235    ///
2236    /// It is inadvisable to directly write to the underlying writer.
2237    pub fn get_mut(&mut self) -> &mut W {
2238        &mut self.writer
2239    }
2240
2241    /// Flush the underlying writer.
2242    ///
2243    /// Both the BufWriter and the underlying writer are flushed.
2244    pub fn flush(&mut self) -> Result<(), ArrowError> {
2245        self.writer.flush()?;
2246        Ok(())
2247    }
2248
2249    /// Unwraps the the underlying writer.
2250    ///
2251    /// The writer is flushed and the StreamWriter is finished before returning.
2252    ///
2253    /// # Errors
2254    ///
2255    /// An [`Err`] may be returned if an error occurs while finishing the StreamWriter
2256    /// or while flushing the writer.
2257    ///
2258    /// # Example
2259    ///
2260    /// ```
2261    /// # use arrow_ipc::writer::{StreamWriter, IpcWriteOptions};
2262    /// # use arrow_ipc::MetadataVersion;
2263    /// # use arrow_schema::{ArrowError, Schema};
2264    /// # fn main() -> Result<(), ArrowError> {
2265    /// // The result we expect from an empty schema
2266    /// let expected = vec![
2267    ///     255, 255, 255, 255,  48,   0,   0,   0,
2268    ///      16,   0,   0,   0,   0,   0,  10,   0,
2269    ///      12,   0,  10,   0,   9,   0,   4,   0,
2270    ///      10,   0,   0,   0,  16,   0,   0,   0,
2271    ///       0,   1,   4,   0,   8,   0,   8,   0,
2272    ///       0,   0,   4,   0,   8,   0,   0,   0,
2273    ///       4,   0,   0,   0,   0,   0,   0,   0,
2274    ///     255, 255, 255, 255,   0,   0,   0,   0
2275    /// ];
2276    ///
2277    /// let schema = Schema::empty();
2278    /// let buffer: Vec<u8> = Vec::new();
2279    /// let options = IpcWriteOptions::try_new(8, false, MetadataVersion::V5)?;
2280    /// let stream_writer = StreamWriter::try_new_with_options(buffer, &schema, options)?;
2281    ///
2282    /// assert_eq!(stream_writer.into_inner()?, expected);
2283    /// # Ok(())
2284    /// # }
2285    /// ```
2286    pub fn into_inner(mut self) -> Result<W, ArrowError> {
2287        if !self.finished {
2288            // `finish` flushes.
2289            self.finish()?;
2290        }
2291        Ok(self.writer)
2292    }
2293}
2294
2295impl<W: Write> RecordBatchWriter for StreamWriter<W> {
2296    fn write(&mut self, batch: &RecordBatch) -> Result<(), ArrowError> {
2297        self.write(batch)
2298    }
2299
2300    fn close(mut self) -> Result<(), ArrowError> {
2301        self.finish()
2302    }
2303}
2304
2305/// Stores the encoded data, which is an crate::Message, and optional Arrow data
2306pub struct EncodedData {
2307    /// An encoded crate::Message
2308    pub ipc_message: Vec<u8>,
2309    /// Arrow buffers to be written, should be an empty vec for schema messages
2310    pub arrow_data: Vec<u8>,
2311}
2312
2313/// Write a message's IPC data and buffers, returning metadata and buffer data lengths written
2314pub fn write_message<W: Write>(
2315    mut writer: W,
2316    encoded: EncodedData,
2317    write_options: &IpcWriteOptions,
2318) -> Result<(usize, usize), ArrowError> {
2319    writer.write_encoded_data(encoded, write_options)
2320}
2321
2322/// In V4, null and run end encoded types have no validity bitmap
2323/// In V5 and later, null, union and run end encoded types have no validity bitmap
2324fn has_validity_bitmap(data_type: &DataType, write_options: &IpcWriteOptions) -> bool {
2325    if write_options.metadata_version < crate::MetadataVersion::V5 {
2326        !matches!(data_type, DataType::Null | DataType::RunEndEncoded(_, _))
2327    } else {
2328        !matches!(
2329            data_type,
2330            DataType::Null | DataType::Union(_, _) | DataType::RunEndEncoded(_, _)
2331        )
2332    }
2333}
2334
2335/// Whether to truncate the buffer
2336#[inline]
2337fn buffer_need_truncate(
2338    array_offset: usize,
2339    buffer: &Buffer,
2340    spec: &BufferSpec,
2341    min_length: usize,
2342) -> bool {
2343    spec != &BufferSpec::AlwaysNull && (array_offset != 0 || min_length < buffer.len())
2344}
2345
2346/// Returns byte width for a buffer spec. Only for `BufferSpec::FixedWidth`.
2347#[inline]
2348fn get_buffer_element_width(spec: &BufferSpec) -> usize {
2349    match spec {
2350        BufferSpec::FixedWidth { byte_width, .. } => *byte_width,
2351        _ => 0,
2352    }
2353}
2354
2355/// Common functionality for re-encoding offsets. Returns the new offsets as well as
2356/// original start offset and length for use in slicing child data.
2357fn reencode_offsets<O: OffsetSizeTrait>(
2358    offsets: &Buffer,
2359    data: &ArrayData,
2360) -> (Buffer, usize, usize) {
2361    let offsets_slice: &[O] = offsets.typed_data::<O>();
2362    let offset_slice = &offsets_slice[data.offset()..data.offset() + data.len() + 1];
2363
2364    let start_offset = offset_slice.first().unwrap();
2365    let end_offset = offset_slice.last().unwrap();
2366
2367    let offsets = match start_offset.as_usize() {
2368        0 => {
2369            let size = size_of::<O>();
2370            offsets.slice_with_length(data.offset() * size, (data.len() + 1) * size)
2371        }
2372        _ => offset_slice.iter().map(|x| *x - *start_offset).collect(),
2373    };
2374
2375    let start_offset = start_offset.as_usize();
2376    let end_offset = end_offset.as_usize();
2377
2378    (offsets, start_offset, end_offset - start_offset)
2379}
2380
2381/// Returns the values and offsets [`Buffer`] for a ByteArray with offset type `O`
2382///
2383/// In particular, this handles re-encoding the offsets if they don't start at `0`,
2384/// slicing the values buffer as appropriate. This helps reduce the encoded
2385/// size of sliced arrays, as values that have been sliced away are not encoded
2386/// Returns the offsets and values buffers, in that order.
2387fn get_byte_array_buffers<O: OffsetSizeTrait>(data: &ArrayData) -> [Buffer; 2] {
2388    if data.is_empty() {
2389        // As per specification, offsets buffer has N+1 elements.
2390        // So an empty array should still be encoded with a single 0 offset.
2391        let mut offsets = MutableBuffer::new(size_of::<O>());
2392        offsets.extend_from_slice(O::usize_as(0).to_byte_slice());
2393        return [offsets.into(), MutableBuffer::new(0).into()];
2394    }
2395
2396    let (offsets, original_start_offset, len) = reencode_offsets::<O>(&data.buffers()[0], data);
2397    let values = data.buffers()[1].slice_with_length(original_start_offset, len);
2398    [offsets, values]
2399}
2400
2401/// Similar logic as [`get_byte_array_buffers()`] but slices the child array instead
2402/// of a values buffer.
2403fn get_list_array_buffers<O: OffsetSizeTrait>(data: &ArrayData) -> (Buffer, ArrayData) {
2404    if data.is_empty() {
2405        // As per specification, offsets buffer has N+1 elements.
2406        // So an empty array should still be encoded with a single 0 offset.
2407        let mut offsets = MutableBuffer::new(size_of::<O>());
2408        offsets.extend_from_slice(O::usize_as(0).to_byte_slice());
2409        return (offsets.into(), data.child_data()[0].slice(0, 0));
2410    }
2411
2412    let (offsets, original_start_offset, len) = reencode_offsets::<O>(&data.buffers()[0], data);
2413    let child_data = data.child_data()[0].slice(original_start_offset, len);
2414    (offsets, child_data)
2415}
2416
2417/// Returns the offsets, sizes, and child data buffers for a ListView array.
2418///
2419/// Unlike List arrays, ListView arrays store both offsets and sizes explicitly,
2420/// and offsets can be non-monotonic. When slicing, we simply pass through the
2421/// offsets and sizes without re-encoding, and do not slice the child data.
2422fn get_list_view_array_buffers<O: OffsetSizeTrait>(
2423    data: &ArrayData,
2424) -> (Buffer, Buffer, ArrayData) {
2425    if data.is_empty() {
2426        return (
2427            MutableBuffer::new(0).into(),
2428            MutableBuffer::new(0).into(),
2429            data.child_data()[0].slice(0, 0),
2430        );
2431    }
2432
2433    let offsets = &data.buffers()[0];
2434    let sizes = &data.buffers()[1];
2435
2436    let element_size = std::mem::size_of::<O>();
2437    let offsets_slice =
2438        offsets.slice_with_length(data.offset() * element_size, data.len() * element_size);
2439    let sizes_slice =
2440        sizes.slice_with_length(data.offset() * element_size, data.len() * element_size);
2441
2442    let child_data = data.child_data()[0].clone();
2443
2444    (offsets_slice, sizes_slice, child_data)
2445}
2446
2447/// Returns the sliced views [`Buffer`] for a BinaryView/Utf8View array.
2448///
2449/// The views buffer is sliced to only include views in the valid range based on
2450/// the array's offset and length. This helps reduce the encoded size of sliced
2451/// arrays
2452///
2453fn get_or_truncate_buffer(array_data: &ArrayData) -> Buffer {
2454    let buffer = &array_data.buffers()[0];
2455    let layout = layout(array_data.data_type());
2456    let spec = &layout.buffers[0];
2457
2458    let byte_width = get_buffer_element_width(spec);
2459    let min_length = array_data.len() * byte_width;
2460    if buffer_need_truncate(array_data.offset(), buffer, spec, min_length) {
2461        let byte_offset = array_data.offset() * byte_width;
2462        let buffer_length = min(min_length, buffer.len() - byte_offset);
2463        buffer.slice_with_length(byte_offset, buffer_length)
2464    } else {
2465        buffer.clone()
2466    }
2467}
2468
2469/// Recursively encodes `array_data` into its IPC representation.
2470///
2471/// Output goes to two separate channels:
2472/// - `meta`: accumulates IPC metadata (`nodes` and `buffers`) for the flatbuffer header.
2473/// - `sink`: the raw Arrow data bytes that form the IPC message body.
2474fn write_array_data(
2475    array_data: &ArrayData,
2476    meta: &mut IpcMetadataBuilder,
2477    sink: &mut IpcBodySink<'_>,
2478    offset: i64,
2479    compression_codec: Option<CompressionCodec>,
2480    ipc_write_context: &mut IpcWriteContext,
2481    write_options: &IpcWriteOptions,
2482) -> Result<i64, ArrowError> {
2483    let mut offset = offset;
2484    let num_rows = array_data.len();
2485    let null_count = array_data.null_count();
2486    if !matches!(array_data.data_type(), DataType::Null) {
2487        meta.nodes
2488            .push(crate::FieldNode::new(num_rows as i64, null_count as i64));
2489    } else {
2490        // NullArray's null_count equals to len, but ArrayData null_count is always 0.
2491        meta.nodes
2492            .push(crate::FieldNode::new(num_rows as i64, num_rows as i64));
2493    }
2494    if has_validity_bitmap(array_data.data_type(), write_options) {
2495        // Fields with null_count == 0 may write a 0-length validity buffer
2496        // instead of a materialized bitmap (see FieldNode in Message.fbs)
2497        let null_buffer = match array_data.nulls() {
2498            Some(buffer) if null_count > 0 => buffer.inner().sliced(),
2499            _ => MutableBuffer::new(0).into(),
2500        };
2501
2502        offset = encode_sink_buffer(
2503            null_buffer,
2504            meta,
2505            sink,
2506            offset,
2507            compression_codec,
2508            ipc_write_context,
2509            write_options.alignment,
2510        )?;
2511    }
2512
2513    let data_type = array_data.data_type();
2514    if matches!(data_type, DataType::Binary | DataType::Utf8) {
2515        for buffer in get_byte_array_buffers::<i32>(array_data) {
2516            offset = encode_sink_buffer(
2517                buffer,
2518                meta,
2519                sink,
2520                offset,
2521                compression_codec,
2522                ipc_write_context,
2523                write_options.alignment,
2524            )?;
2525        }
2526    } else if matches!(data_type, DataType::BinaryView | DataType::Utf8View) {
2527        // Slicing the views buffer is safe and easy,
2528        // but pruning unneeded data buffers is much more nuanced since it's complicated to prove that no views reference the pruned buffers
2529        //
2530        // Current implementation just serialize the raw arrays as given and not try to optimize anything.
2531        // If users wants to "compact" the arrays prior to sending them over IPC,
2532        // they should consider the gc API suggested in #5513
2533        let views = get_or_truncate_buffer(array_data);
2534        offset = encode_sink_buffer(
2535            views,
2536            meta,
2537            sink,
2538            offset,
2539            compression_codec,
2540            ipc_write_context,
2541            write_options.alignment,
2542        )?;
2543
2544        for buffer in array_data.buffers().iter().skip(1) {
2545            offset = encode_sink_buffer(
2546                buffer.clone(),
2547                meta,
2548                sink,
2549                offset,
2550                compression_codec,
2551                ipc_write_context,
2552                write_options.alignment,
2553            )?;
2554        }
2555    } else if matches!(data_type, DataType::LargeBinary | DataType::LargeUtf8) {
2556        for buffer in get_byte_array_buffers::<i64>(array_data) {
2557            offset = encode_sink_buffer(
2558                buffer,
2559                meta,
2560                sink,
2561                offset,
2562                compression_codec,
2563                ipc_write_context,
2564                write_options.alignment,
2565            )?;
2566        }
2567    } else if DataType::is_numeric(data_type)
2568        || DataType::is_temporal(data_type)
2569        || matches!(
2570            array_data.data_type(),
2571            DataType::FixedSizeBinary(_) | DataType::Dictionary(_, _)
2572        )
2573    {
2574        // Truncate values
2575        assert_eq!(array_data.buffers().len(), 1);
2576
2577        let buffer = get_or_truncate_buffer(array_data);
2578        offset = encode_sink_buffer(
2579            buffer,
2580            meta,
2581            sink,
2582            offset,
2583            compression_codec,
2584            ipc_write_context,
2585            write_options.alignment,
2586        )?;
2587    } else if matches!(data_type, DataType::Boolean) {
2588        // Bools are special because the payload (= 1 bit) is smaller than the physical container elements (= bytes).
2589        // The array data may not start at the physical boundary of the underlying buffer, so we need to shift bits around.
2590        assert_eq!(array_data.buffers().len(), 1);
2591
2592        let buffer = &array_data.buffers()[0];
2593        let buffer = buffer.bit_slice(array_data.offset(), array_data.len());
2594        offset = encode_sink_buffer(
2595            buffer,
2596            meta,
2597            sink,
2598            offset,
2599            compression_codec,
2600            ipc_write_context,
2601            write_options.alignment,
2602        )?;
2603    } else if matches!(
2604        data_type,
2605        DataType::List(_) | DataType::LargeList(_) | DataType::Map(_, _)
2606    ) {
2607        assert_eq!(array_data.buffers().len(), 1);
2608        assert_eq!(array_data.child_data().len(), 1);
2609
2610        // Truncate offsets and the child data to avoid writing unnecessary data
2611        let (offsets, sliced_child_data) = match data_type {
2612            DataType::List(_) => get_list_array_buffers::<i32>(array_data),
2613            DataType::Map(_, _) => get_list_array_buffers::<i32>(array_data),
2614            DataType::LargeList(_) => get_list_array_buffers::<i64>(array_data),
2615            _ => unreachable!(),
2616        };
2617        offset = encode_sink_buffer(
2618            offsets,
2619            meta,
2620            sink,
2621            offset,
2622            compression_codec,
2623            ipc_write_context,
2624            write_options.alignment,
2625        )?;
2626        offset = write_array_data(
2627            &sliced_child_data,
2628            meta,
2629            sink,
2630            offset,
2631            compression_codec,
2632            ipc_write_context,
2633            write_options,
2634        )?;
2635        return Ok(offset);
2636    } else if matches!(
2637        data_type,
2638        DataType::ListView(_) | DataType::LargeListView(_)
2639    ) {
2640        assert_eq!(array_data.buffers().len(), 2); // offsets + sizes
2641        assert_eq!(array_data.child_data().len(), 1);
2642
2643        let (offsets, sizes, child_data) = match data_type {
2644            DataType::ListView(_) => get_list_view_array_buffers::<i32>(array_data),
2645            DataType::LargeListView(_) => get_list_view_array_buffers::<i64>(array_data),
2646            _ => unreachable!(),
2647        };
2648
2649        offset = encode_sink_buffer(
2650            offsets,
2651            meta,
2652            sink,
2653            offset,
2654            compression_codec,
2655            ipc_write_context,
2656            write_options.alignment,
2657        )?;
2658        offset = encode_sink_buffer(
2659            sizes,
2660            meta,
2661            sink,
2662            offset,
2663            compression_codec,
2664            ipc_write_context,
2665            write_options.alignment,
2666        )?;
2667
2668        offset = write_array_data(
2669            &child_data,
2670            meta,
2671            sink,
2672            offset,
2673            compression_codec,
2674            ipc_write_context,
2675            write_options,
2676        )?;
2677        return Ok(offset);
2678    } else if let DataType::FixedSizeList(_, fixed_size) = data_type {
2679        assert_eq!(array_data.child_data().len(), 1);
2680        let fixed_size = *fixed_size as usize;
2681
2682        let child_offset = array_data.offset() * fixed_size;
2683        let child_length = array_data.len() * fixed_size;
2684        let child_data = array_data.child_data()[0].slice(child_offset, child_length);
2685
2686        offset = write_array_data(
2687            &child_data,
2688            meta,
2689            sink,
2690            offset,
2691            compression_codec,
2692            ipc_write_context,
2693            write_options,
2694        )?;
2695        return Ok(offset);
2696    } else if matches!(data_type, DataType::Union(_, _)) {
2697        for (buffer, byte_width) in array_data
2698            .buffers()
2699            .iter()
2700            .zip([size_of::<i8>(), size_of::<i32>()])
2701        {
2702            let buffer =
2703                buffer.slice_with_length(array_data.offset() * byte_width, num_rows * byte_width);
2704            offset = encode_sink_buffer(
2705                buffer,
2706                meta,
2707                sink,
2708                offset,
2709                compression_codec,
2710                ipc_write_context,
2711                write_options.alignment,
2712            )?;
2713        }
2714    } else {
2715        for buffer in array_data.buffers() {
2716            offset = encode_sink_buffer(
2717                buffer.clone(),
2718                meta,
2719                sink,
2720                offset,
2721                compression_codec,
2722                ipc_write_context,
2723                write_options.alignment,
2724            )?;
2725        }
2726    }
2727
2728    match array_data.data_type() {
2729        DataType::Dictionary(_, _) => {}
2730        DataType::RunEndEncoded(_, _) => {
2731            // unslice the run encoded array.
2732            let arr = unslice_run_array(array_data.clone())?;
2733            // recursively write out nested structures
2734            for data_ref in arr.child_data() {
2735                // write the nested data (e.g list data)
2736                offset = write_array_data(
2737                    data_ref,
2738                    meta,
2739                    sink,
2740                    offset,
2741                    compression_codec,
2742                    ipc_write_context,
2743                    write_options,
2744                )?;
2745            }
2746        }
2747        DataType::Union(_, UnionMode::Sparse) => {
2748            for data_ref in array_data.child_data() {
2749                let child_data = data_ref.slice(array_data.offset(), num_rows);
2750                offset = write_array_data(
2751                    &child_data,
2752                    meta,
2753                    sink,
2754                    offset,
2755                    compression_codec,
2756                    ipc_write_context,
2757                    write_options,
2758                )?;
2759            }
2760        }
2761        _ => {
2762            // recursively write out nested structures
2763            for data_ref in array_data.child_data() {
2764                // write the nested data (e.g list data)
2765                offset = write_array_data(
2766                    data_ref,
2767                    meta,
2768                    sink,
2769                    offset,
2770                    compression_codec,
2771                    ipc_write_context,
2772                    write_options,
2773                )?;
2774            }
2775        }
2776    }
2777    Ok(offset)
2778}
2779
2780/// Encodes a single Arrow [`Buffer`] into the IPC body and records its metadata.
2781///
2782/// - `buffer`: the Arrow data buffer to encode (validity bitmap, offsets, values, etc.)
2783/// - `buffers`: in-progress list of IPC `Buffer` metadata entries (body offset + length) that
2784///   will eventually be serialised into the flatbuffer `RecordBatch` header.
2785/// - `sink`: destination for the actual encoded bytes; either a contiguous `Vec<u8>` for
2786///   in-memory writes, or a list of [`EncodedBuffer`] segments for deferred zero-copy streaming.
2787/// - `offset`: running byte offset into the IPC message body, used to compute the metadata entry.
2788/// - `compression_codec` / `ipc_write_context`: if `Some`, the buffer is compressed before
2789///   writing; `ipc_write_context` provides reusable scratch space across calls.
2790/// - `alignment`: each buffer is padded to this many bytes so the next buffer starts aligned.
2791///
2792/// Returns the updated `offset` (advanced by the encoded length plus any alignment padding).
2793fn encode_sink_buffer(
2794    buffer: Buffer,
2795    ipc_meta_data: &mut IpcMetadataBuilder,
2796    sink: &mut IpcBodySink<'_>,
2797    offset: i64,
2798    compression_codec: Option<CompressionCodec>,
2799    ipc_write_context: &mut IpcWriteContext,
2800    alignment: u8,
2801) -> Result<i64, ArrowError> {
2802    let (encoded, len) = match compression_codec {
2803        None => {
2804            let len = buffer.len() as i64;
2805            (EncodedBuffer::Raw(buffer), len)
2806        }
2807        Some(codec) => {
2808            let mut scratch = Vec::new();
2809            let written =
2810                codec.compress_to_vec(buffer.as_slice(), &mut scratch, ipc_write_context)?;
2811            let len = i64::try_from(written)
2812                .map_err(|e| ArrowError::InvalidArgumentError(format!("{e}")))?;
2813            (EncodedBuffer::Compressed(scratch), len)
2814        }
2815    };
2816
2817    let pad_len = pad_to_alignment(alignment, len as usize);
2818    sink.write(pad_len, encoded);
2819    ipc_meta_data.buffers.push(crate::Buffer::new(offset, len));
2820    Ok(offset + len + pad_len as i64)
2821}
2822
2823const PADDING: [u8; 64] = [0; 64];
2824
2825/// Estimates the number of [`EncodedBuffer`] segments that [`write_array_data`]
2826/// will produce for a column of the given type.
2827///
2828/// Based on the Arrow IPC buffer layout
2829/// (<https://arrow.apache.org/docs/format/Columnar.html#recordbatch-message>):
2830#[inline]
2831fn estimate_encoded_buffer_count(dt: &DataType) -> usize {
2832    match dt {
2833        DataType::Null => 0,
2834
2835        DataType::Binary | DataType::Utf8 | DataType::LargeBinary | DataType::LargeUtf8 => 3,
2836
2837        DataType::BinaryView | DataType::Utf8View => 3,
2838
2839        DataType::List(f) | DataType::LargeList(f) | DataType::Map(f, _) => {
2840            2 + estimate_encoded_buffer_count(f.data_type())
2841        }
2842
2843        DataType::ListView(f) | DataType::LargeListView(f) => {
2844            3 + estimate_encoded_buffer_count(f.data_type())
2845        }
2846
2847        DataType::FixedSizeList(f, _) => 1 + estimate_encoded_buffer_count(f.data_type()),
2848
2849        DataType::Struct(fields) => {
2850            1 + fields
2851                .iter()
2852                .map(|f| estimate_encoded_buffer_count(f.data_type()))
2853                .sum::<usize>()
2854        }
2855
2856        // Dictionary indices only; dictionary body is a separate IPC message.
2857        DataType::Dictionary(_, _) => 2,
2858
2859        DataType::Union(fields, UnionMode::Sparse) => {
2860            1 + fields
2861                .iter()
2862                .map(|(_, f)| estimate_encoded_buffer_count(f.data_type()))
2863                .sum::<usize>()
2864        }
2865        DataType::Union(fields, UnionMode::Dense) => {
2866            2 + fields
2867                .iter()
2868                .map(|(_, f)| estimate_encoded_buffer_count(f.data_type()))
2869                .sum::<usize>()
2870        }
2871
2872        DataType::RunEndEncoded(run_ends, values) => {
2873            estimate_encoded_buffer_count(run_ends.data_type())
2874                + estimate_encoded_buffer_count(values.data_type())
2875        }
2876        // Primitive, Bool, temporal, Decimal*, FixedSizeBinary: validity + values.
2877        _ => 2,
2878    }
2879}
2880
2881/// Calculate an alignment boundary and return the number of bytes needed to pad to the alignment boundary
2882#[inline]
2883fn pad_to_alignment(alignment: u8, len: usize) -> usize {
2884    let a = usize::from(alignment - 1);
2885    ((len + a) & !a) - len
2886}
2887
2888#[cfg(test)]
2889mod tests {
2890    use std::hash::Hasher;
2891    use std::io::Cursor;
2892    use std::io::Seek;
2893
2894    use arrow_array::builder::FixedSizeListBuilder;
2895    use arrow_array::builder::Float32Builder;
2896    use arrow_array::builder::Int64Builder;
2897    use arrow_array::builder::MapBuilder;
2898    use arrow_array::builder::StringViewBuilder;
2899    use arrow_array::builder::UnionBuilder;
2900    use arrow_array::builder::{
2901        GenericListBuilder, GenericListViewBuilder, ListBuilder, StringBuilder,
2902    };
2903    use arrow_array::builder::{PrimitiveRunBuilder, UInt32Builder};
2904    use arrow_array::types::*;
2905    use arrow_buffer::ScalarBuffer;
2906
2907    use crate::MetadataVersion;
2908    use crate::convert::try_fb_to_schema;
2909    use crate::reader::*;
2910    use crate::root_as_footer;
2911
2912    use super::*;
2913
2914    #[test]
2915    fn test_roundtrip_sliced_list_of_union() {
2916        fn decoded(array: &ArrayRef) -> Vec<Vec<Option<i32>>> {
2917            (0..array.len())
2918                .map(|row| {
2919                    let values = array.as_list::<i32>().value(row);
2920                    let union = values.as_union();
2921                    (0..union.len())
2922                        .map(|index| {
2923                            union
2924                                .value(index)
2925                                .as_primitive::<Int32Type>()
2926                                .iter()
2927                                .next()
2928                                .unwrap()
2929                        })
2930                        .collect()
2931                })
2932                .collect()
2933        }
2934
2935        for mode in [UnionMode::Dense, UnionMode::Sparse] {
2936            let fields = UnionFields::try_new(
2937                [3, 7],
2938                [
2939                    Field::new("a", DataType::Int32, true),
2940                    Field::new("b", DataType::Int32, true),
2941                ],
2942            )
2943            .unwrap();
2944            let offsets = (mode == UnionMode::Dense).then(|| vec![2, 0, 1].into());
2945            let union: ArrayRef = Arc::new(
2946                UnionArray::try_new(
2947                    fields,
2948                    vec![3_i8, 7, 3].into(),
2949                    offsets,
2950                    vec![
2951                        Arc::new(Int32Array::from(vec![Some(11), None, Some(33)])),
2952                        Arc::new(Int32Array::from(vec![Some(101), Some(202), None])),
2953                    ],
2954                )
2955                .unwrap(),
2956            );
2957            let item = Arc::new(Field::new("item", union.data_type().clone(), true));
2958            let list: ArrayRef = Arc::new(ListArray::new(
2959                item,
2960                arrow_buffer::OffsetBuffer::from_lengths([1, 2]),
2961                union,
2962                None,
2963            ));
2964            for (offset, len) in [(0, 2), (1, 1), (2, 0)] {
2965                let array = list.slice(offset, len);
2966                array.to_data().validate_full().unwrap();
2967                let expected = decoded(&array);
2968                let schema = Arc::new(Schema::new(vec![Field::new(
2969                    "v",
2970                    array.data_type().clone(),
2971                    true,
2972                )]));
2973                let batch = RecordBatch::try_new(schema, vec![array]).unwrap();
2974                for output in [
2975                    deserialize_file(serialize_file(&batch)),
2976                    deserialize_stream(serialize_stream(&batch)),
2977                ] {
2978                    output.column(0).to_data().validate_full().unwrap();
2979                    assert_eq!(decoded(output.column(0)), expected);
2980                }
2981            }
2982        }
2983    }
2984
2985    fn serialize_file(rb: &RecordBatch) -> Vec<u8> {
2986        let mut writer = FileWriter::try_new(vec![], rb.schema_ref()).unwrap();
2987        writer.write(rb).unwrap();
2988        writer.finish().unwrap();
2989        writer.into_inner().unwrap()
2990    }
2991
2992    fn deserialize_file(bytes: Vec<u8>) -> RecordBatch {
2993        let mut reader = FileReader::try_new(Cursor::new(bytes), None).unwrap();
2994        reader.next().unwrap().unwrap()
2995    }
2996
2997    fn serialize_stream(record: &RecordBatch) -> Vec<u8> {
2998        // Use 8-byte alignment so that the various `truncate_*` tests can be compactly written,
2999        // without needing to construct a giant array to spill over the 64-byte default alignment
3000        // boundary.
3001        const IPC_ALIGNMENT: usize = 8;
3002
3003        let mut stream_writer = StreamWriter::try_new_with_options(
3004            vec![],
3005            record.schema_ref(),
3006            IpcWriteOptions::try_new(IPC_ALIGNMENT, false, MetadataVersion::V5).unwrap(),
3007        )
3008        .unwrap();
3009        stream_writer.write(record).unwrap();
3010        stream_writer.finish().unwrap();
3011        stream_writer.into_inner().unwrap()
3012    }
3013
3014    fn deserialize_stream(bytes: Vec<u8>) -> RecordBatch {
3015        let mut stream_reader = StreamReader::try_new(Cursor::new(bytes), None).unwrap();
3016        stream_reader.next().unwrap().unwrap()
3017    }
3018
3019    /// Encodes record batches with [`StreamEncoder`] into one contiguous byte vector.
3020    ///
3021    /// This still exercises the sans-IO encoder path; `Vec<u8>` is only used
3022    /// as a convenient [`Write`] sink for comparing the resulting byte stream.
3023    fn encode_stream(
3024        schema: &Schema,
3025        batches: &[RecordBatch],
3026        options: IpcWriteOptions,
3027    ) -> Vec<u8> {
3028        let mut encoder = StreamEncoder::try_new_with_options(schema, options).unwrap();
3029        let mut bytes = Vec::new();
3030        for batch in batches {
3031            for buffer in encoder.encode(batch).unwrap() {
3032                bytes.write_all(buffer.as_slice()).unwrap();
3033            }
3034        }
3035        for buffer in encoder.finish().unwrap() {
3036            bytes.write_all(buffer.as_slice()).unwrap();
3037        }
3038        bytes
3039    }
3040
3041    fn write_stream(schema: &Schema, batches: &[RecordBatch], options: IpcWriteOptions) -> Vec<u8> {
3042        let mut bytes = Vec::new();
3043        let mut writer = StreamWriter::try_new_with_options(&mut bytes, schema, options).unwrap();
3044        for batch in batches {
3045            writer.write(batch).unwrap();
3046        }
3047        writer.finish().unwrap();
3048        bytes
3049    }
3050
3051    // StreamEncoder and StreamWriter currently use separate encoding paths, so these tests
3052    // verify the new sans-IO API preserves the existing IPC stream byte layout.
3053    #[tokio::test]
3054    async fn test_stream_encoder_async_writer_matches_stream_writer() {
3055        use tokio::io::{AsyncReadExt, AsyncWriteExt};
3056
3057        let batch = record_batch!(("a", Int32, [1, 2, 3]), ("b", Utf8, ["x", "y", "z"])).unwrap();
3058        let options = IpcWriteOptions::default();
3059        let expected = write_stream(
3060            batch.schema_ref(),
3061            std::slice::from_ref(&batch),
3062            options.clone(),
3063        );
3064
3065        let (mut sink, mut source) = tokio::io::duplex(64);
3066        let read = tokio::spawn(async move {
3067            let mut bytes = Vec::new();
3068            source.read_to_end(&mut bytes).await.unwrap();
3069            bytes
3070        });
3071
3072        let mut encoder = StreamEncoder::try_new_with_options(batch.schema_ref(), options).unwrap();
3073        for buffer in encoder.encode(&batch).unwrap() {
3074            sink.write_all(buffer.as_slice()).await.unwrap();
3075        }
3076        for buffer in encoder.finish().unwrap() {
3077            sink.write_all(buffer.as_slice()).await.unwrap();
3078        }
3079        sink.shutdown().await.unwrap();
3080
3081        let encoded = read.await.unwrap();
3082        assert_eq!(encoded, expected);
3083
3084        let mut reader = StreamReader::try_new(Cursor::new(encoded), None).unwrap();
3085        assert_eq!(reader.next().unwrap().unwrap(), batch);
3086        assert!(reader.next().is_none());
3087    }
3088
3089    #[test]
3090    fn test_stream_encoder_empty_stream_matches_stream_writer() {
3091        let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);
3092        let options = IpcWriteOptions::default();
3093        let encoded = encode_stream(&schema, &[], options.clone());
3094        let written = write_stream(&schema, &[], options);
3095
3096        assert_eq!(encoded, written);
3097
3098        let mut reader = StreamReader::try_new(Cursor::new(encoded), None).unwrap();
3099        assert!(reader.next().is_none());
3100    }
3101
3102    #[test]
3103    fn test_stream_encoder_dictionary_batches_match_stream_writer() {
3104        let schema = Arc::new(Schema::new(vec![Field::new(
3105            "a",
3106            DataType::Dictionary(Box::new(DataType::UInt8), Box::new(DataType::Utf8)),
3107            false,
3108        )]));
3109        let batch = RecordBatch::try_new(
3110            schema.clone(),
3111            vec![Arc::new(DictionaryArray::new(
3112                UInt8Array::from_iter_values([0, 1, 0]),
3113                Arc::new(StringArray::from_iter_values(["a", "b"])),
3114            ))],
3115        )
3116        .unwrap();
3117        let options = IpcWriteOptions::default();
3118        let encoded = encode_stream(&schema, std::slice::from_ref(&batch), options.clone());
3119        let written = write_stream(&schema, std::slice::from_ref(&batch), options);
3120
3121        assert_eq!(encoded, written);
3122
3123        let mut reader = StreamReader::try_new(Cursor::new(encoded), None).unwrap();
3124        assert_eq!(reader.next().unwrap().unwrap(), batch);
3125        assert!(reader.next().is_none());
3126    }
3127
3128    #[test]
3129    #[cfg(feature = "lz4")]
3130    fn test_write_empty_record_batch_lz4_compression() {
3131        let schema = Schema::new(vec![Field::new("field1", DataType::Int32, true)]);
3132        let values: Vec<Option<i32>> = vec![];
3133        let array = Int32Array::from(values);
3134        let record_batch =
3135            RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(array)]).unwrap();
3136
3137        let mut file = tempfile::tempfile().unwrap();
3138
3139        {
3140            let write_option = IpcWriteOptions::try_new(8, false, crate::MetadataVersion::V5)
3141                .unwrap()
3142                .try_with_compression(Some(crate::CompressionType::LZ4_FRAME))
3143                .unwrap();
3144
3145            let mut writer =
3146                FileWriter::try_new_with_options(&mut file, &schema, write_option).unwrap();
3147            writer.write(&record_batch).unwrap();
3148            writer.finish().unwrap();
3149        }
3150        file.rewind().unwrap();
3151        {
3152            // read file
3153            let reader = FileReader::try_new(file, None).unwrap();
3154            for read_batch in reader {
3155                read_batch
3156                    .unwrap()
3157                    .columns()
3158                    .iter()
3159                    .zip(record_batch.columns())
3160                    .for_each(|(a, b)| {
3161                        assert_eq!(a.data_type(), b.data_type());
3162                        assert_eq!(a.len(), b.len());
3163                        assert_eq!(a.null_count(), b.null_count());
3164                    });
3165            }
3166        }
3167    }
3168
3169    #[test]
3170    #[cfg(feature = "lz4")]
3171    fn test_write_file_with_lz4_compression() {
3172        let schema = Schema::new(vec![Field::new("field1", DataType::Int32, true)]);
3173        let values: Vec<Option<i32>> = vec![Some(12), Some(1)];
3174        let array = Int32Array::from(values);
3175        let record_batch =
3176            RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(array)]).unwrap();
3177
3178        let mut file = tempfile::tempfile().unwrap();
3179        {
3180            let write_option = IpcWriteOptions::try_new(8, false, crate::MetadataVersion::V5)
3181                .unwrap()
3182                .try_with_compression(Some(crate::CompressionType::LZ4_FRAME))
3183                .unwrap();
3184
3185            let mut writer =
3186                FileWriter::try_new_with_options(&mut file, &schema, write_option).unwrap();
3187            writer.write(&record_batch).unwrap();
3188            writer.finish().unwrap();
3189        }
3190        file.rewind().unwrap();
3191        {
3192            // read file
3193            let reader = FileReader::try_new(file, None).unwrap();
3194            for read_batch in reader {
3195                read_batch
3196                    .unwrap()
3197                    .columns()
3198                    .iter()
3199                    .zip(record_batch.columns())
3200                    .for_each(|(a, b)| {
3201                        assert_eq!(a.data_type(), b.data_type());
3202                        assert_eq!(a.len(), b.len());
3203                        assert_eq!(a.null_count(), b.null_count());
3204                    });
3205            }
3206        }
3207    }
3208
3209    /// Batch exercising nulls, a dictionary column, and enough bytes that lz4 emits several
3210    /// blocks even at the smallest configurable block size.
3211    fn lz4_block_size_batch() -> RecordBatch {
3212        let schema = Schema::new(vec![
3213            Field::new("ints", DataType::Int32, true),
3214            Field::new(
3215                "words",
3216                DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
3217                false,
3218            ),
3219        ]);
3220        let ints = Int32Array::from(
3221            (0..30_000)
3222                .map(|i| if i % 13 == 0 { None } else { Some(i) })
3223                .collect::<Vec<Option<i32>>>(),
3224        );
3225        let keys = Int32Array::from((0..30_000).map(|i| i % 5).collect::<Vec<i32>>());
3226        let values = StringArray::from(vec!["zero", "one", "two", "three", "four"]);
3227        let words = DictionaryArray::new(keys, Arc::new(values));
3228        RecordBatch::try_new(Arc::new(schema), vec![Arc::new(ints), Arc::new(words)]).unwrap()
3229    }
3230
3231    #[test]
3232    fn test_lz4_block_size_options() {
3233        let options = || IpcWriteOptions::try_new(8, false, MetadataVersion::V5).unwrap();
3234
3235        assert_eq!(options().lz4_block_size(), Lz4BlockSize::DEFAULT);
3236        assert_eq!(
3237            options()
3238                .try_with_lz4_block_size(None)
3239                .unwrap()
3240                .lz4_block_size(),
3241            Lz4BlockSize::DEFAULT
3242        );
3243        assert_eq!(
3244            options()
3245                .try_with_lz4_block_size(Some(1024 * 1024))
3246                .unwrap()
3247                .lz4_block_size(),
3248            Lz4BlockSize::Max1MiB
3249        );
3250
3251        for block_size in [0, 32 * 1024, 128 * 1024, 2 * 1024 * 1024, 8 * 1024 * 1024] {
3252            let err = options()
3253                .try_with_lz4_block_size(Some(block_size))
3254                .expect_err("unsupported block size must be rejected");
3255            assert!(
3256                err.to_string().contains("Unsupported lz4 block size"),
3257                "unexpected error: {err}"
3258            );
3259        }
3260
3261        let options = IpcWriteOptions::try_new(8, false, MetadataVersion::V4).unwrap();
3262        let err = options
3263            .try_with_lz4_block_size(Some(64 * 1024))
3264            .expect_err("block size below metadata V5 must be rejected");
3265        assert!(
3266            err.to_string().contains("metadata v5"),
3267            "unexpected error: {err}"
3268        );
3269
3270        let options = IpcWriteOptions::try_new(8, false, MetadataVersion::V5)
3271            .unwrap()
3272            .try_with_compression(Some(crate::CompressionType::ZSTD))
3273            .unwrap();
3274        let err = options
3275            .try_with_lz4_block_size(Some(64 * 1024))
3276            .expect_err("lz4 block size must be rejected for ZSTD");
3277        assert!(err.to_string().contains("ZSTD"), "unexpected error: {err}");
3278
3279        let err = IpcWriteOptions::try_new(8, false, MetadataVersion::V5)
3280            .unwrap()
3281            .try_with_lz4_block_size(Some(64 * 1024))
3282            .unwrap()
3283            .try_with_compression(Some(crate::CompressionType::ZSTD))
3284            .expect_err("ZSTD must be rejected after setting an lz4 block size");
3285        assert!(err.to_string().contains("ZSTD"), "unexpected error: {err}");
3286    }
3287
3288    #[test]
3289    #[cfg(feature = "lz4")]
3290    fn test_lz4_block_size_configurable_end_to_end() {
3291        let batch = lz4_block_size_batch();
3292        let schema = batch.schema();
3293        let data_gen = IpcDataGenerator::default();
3294        let mut ipc_write_context = IpcWriteContext::default();
3295
3296        let sizes: [Option<usize>; 5] = [
3297            None,
3298            Some(64 * 1024),
3299            Some(256 * 1024),
3300            Some(1024 * 1024),
3301            Some(4 * 1024 * 1024),
3302        ];
3303        let mut outputs = Vec::with_capacity(sizes.len());
3304        for block_size in sizes {
3305            let options = IpcWriteOptions::try_new(8, false, MetadataVersion::V5)
3306                .unwrap()
3307                .try_with_compression(Some(crate::CompressionType::LZ4_FRAME))
3308                .unwrap()
3309                .try_with_lz4_block_size(block_size)
3310                .unwrap();
3311
3312            let mut dictionary_tracker = DictionaryTracker::new(false);
3313            let mut encoded = Vec::new();
3314            let schema_data = data_gen.schema_to_bytes_with_dictionary_tracker(
3315                &schema,
3316                &mut dictionary_tracker,
3317                &options,
3318            );
3319            write_message(&mut encoded, schema_data, &options).unwrap();
3320
3321            let (encoded_dictionaries, encoded_batch) = data_gen
3322                .encode(
3323                    &batch,
3324                    &mut dictionary_tracker,
3325                    &options,
3326                    &mut ipc_write_context,
3327                )
3328                .unwrap();
3329            for dictionary in encoded_dictionaries {
3330                write_message(&mut encoded, dictionary, &options).unwrap();
3331            }
3332            write_message(&mut encoded, encoded_batch, &options).unwrap();
3333
3334            // Reading must not depend on the block size the writer chose.
3335            let mut reader = StreamReader::try_new(Cursor::new(encoded.as_slice()), None).unwrap();
3336            assert_eq!(reader.next().unwrap().unwrap(), batch, "{block_size:?}");
3337            assert!(reader.next().is_none());
3338
3339            outputs.push(encoded);
3340        }
3341
3342        // The option must reach the encoder: an explicitly configured default is byte
3343        // identical to the untouched default, while every other size changes the frame
3344        // header and therefore the output.
3345        assert_eq!(
3346            outputs[0], outputs[2],
3347            "explicit 256 KiB must match the default"
3348        );
3349        assert_ne!(outputs[0], outputs[1], "64 KiB must differ from default");
3350        assert_ne!(outputs[0], outputs[3], "1 MiB must differ from default");
3351        assert_ne!(outputs[0], outputs[4], "4 MiB must differ from default");
3352    }
3353
3354    #[test]
3355    #[cfg(feature = "zstd")]
3356    fn test_write_file_with_zstd_compression() {
3357        let schema = Schema::new(vec![Field::new("field1", DataType::Int32, true)]);
3358        let values: Vec<Option<i32>> = vec![Some(12), Some(1)];
3359        let array = Int32Array::from(values);
3360        let record_batch =
3361            RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(array)]).unwrap();
3362        let mut file = tempfile::tempfile().unwrap();
3363        {
3364            let write_option = IpcWriteOptions::try_new(8, false, crate::MetadataVersion::V5)
3365                .unwrap()
3366                .try_with_compression(Some(crate::CompressionType::ZSTD))
3367                .unwrap()
3368                .try_with_compression_level(Some(1))
3369                .unwrap();
3370
3371            let mut writer =
3372                FileWriter::try_new_with_options(&mut file, &schema, write_option).unwrap();
3373            writer.write(&record_batch).unwrap();
3374            writer.finish().unwrap();
3375        }
3376        file.rewind().unwrap();
3377        {
3378            // read file
3379            let reader = FileReader::try_new(file, None).unwrap();
3380            for read_batch in reader {
3381                read_batch
3382                    .unwrap()
3383                    .columns()
3384                    .iter()
3385                    .zip(record_batch.columns())
3386                    .for_each(|(a, b)| {
3387                        assert_eq!(a.data_type(), b.data_type());
3388                        assert_eq!(a.len(), b.len());
3389                        assert_eq!(a.null_count(), b.null_count());
3390                    });
3391            }
3392        }
3393    }
3394
3395    #[test]
3396    fn test_write_file() {
3397        let schema = Schema::new(vec![Field::new("field1", DataType::UInt32, true)]);
3398        let values: Vec<Option<u32>> = vec![
3399            Some(999),
3400            None,
3401            Some(235),
3402            Some(123),
3403            None,
3404            None,
3405            None,
3406            None,
3407            None,
3408        ];
3409        let array1 = UInt32Array::from(values);
3410        let batch =
3411            RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(array1) as ArrayRef])
3412                .unwrap();
3413        let mut file = tempfile::tempfile().unwrap();
3414        {
3415            let mut writer = FileWriter::try_new(&mut file, &schema).unwrap();
3416
3417            writer.write(&batch).unwrap();
3418            writer.finish().unwrap();
3419        }
3420        file.rewind().unwrap();
3421
3422        {
3423            let mut reader = FileReader::try_new(file, None).unwrap();
3424            while let Some(Ok(read_batch)) = reader.next() {
3425                read_batch
3426                    .columns()
3427                    .iter()
3428                    .zip(batch.columns())
3429                    .for_each(|(a, b)| {
3430                        assert_eq!(a.data_type(), b.data_type());
3431                        assert_eq!(a.len(), b.len());
3432                        assert_eq!(a.null_count(), b.null_count());
3433                    });
3434            }
3435        }
3436    }
3437
3438    #[test]
3439    fn test_empty_utf8_ipc_writes_nonempty_offsets_buffer() {
3440        let name = StringArray::from(Vec::<String>::new());
3441        let [offsets, values] = get_byte_array_buffers::<i32>(&name.to_data());
3442
3443        assert_eq!(name.len(), 0);
3444        assert_eq!(
3445            offsets.len(),
3446            std::mem::size_of::<i32>(),
3447            "offsets buffer should contain one zero i32 offset"
3448        );
3449        assert_eq!(values.len(), 0, "values buffer should remain empty");
3450    }
3451
3452    #[test]
3453    fn test_empty_large_utf8_ipc_writes_nonempty_offsets_buffer() {
3454        let name = LargeStringArray::from(Vec::<String>::new());
3455        let [offsets, values] = get_byte_array_buffers::<i64>(&name.to_data());
3456
3457        assert_eq!(name.len(), 0);
3458        assert_eq!(
3459            offsets.len(),
3460            std::mem::size_of::<i64>(),
3461            "offsets buffer should contain one zero i64 offset"
3462        );
3463        assert_eq!(values.len(), 0, "values buffer should remain empty");
3464    }
3465
3466    #[test]
3467    fn test_empty_list_ipc_writes_nonempty_offsets_buffer() {
3468        let list = GenericListBuilder::<i32, _>::new(UInt32Builder::new()).finish();
3469        let (offsets, child_data) = get_list_array_buffers::<i32>(&list.to_data());
3470
3471        assert_eq!(list.len(), 0);
3472        assert_eq!(
3473            offsets.len(),
3474            std::mem::size_of::<i32>(),
3475            "offsets buffer should contain one zero i32 offset"
3476        );
3477        assert_eq!(child_data.len(), 0, "child data should remain empty");
3478    }
3479
3480    #[test]
3481    fn test_empty_large_list_ipc_writes_nonempty_offsets_buffer() {
3482        let list = GenericListBuilder::<i64, _>::new(UInt32Builder::new()).finish();
3483        let (offsets, child_data) = get_list_array_buffers::<i64>(&list.to_data());
3484
3485        assert_eq!(list.len(), 0);
3486        assert_eq!(
3487            offsets.len(),
3488            std::mem::size_of::<i64>(),
3489            "offsets buffer should contain one zero i64 offset"
3490        );
3491        assert_eq!(child_data.len(), 0, "child data should remain empty");
3492    }
3493
3494    fn write_null_file(options: IpcWriteOptions) {
3495        let schema = Schema::new(vec![
3496            Field::new("nulls", DataType::Null, true),
3497            Field::new("int32s", DataType::Int32, false),
3498            Field::new("nulls2", DataType::Null, true),
3499            Field::new("f64s", DataType::Float64, false),
3500        ]);
3501        let array1 = NullArray::new(32);
3502        let array2 = Int32Array::from(vec![1; 32]);
3503        let array3 = NullArray::new(32);
3504        let array4 = Float64Array::from(vec![f64::NAN; 32]);
3505        let batch = RecordBatch::try_new(
3506            Arc::new(schema.clone()),
3507            vec![
3508                Arc::new(array1) as ArrayRef,
3509                Arc::new(array2) as ArrayRef,
3510                Arc::new(array3) as ArrayRef,
3511                Arc::new(array4) as ArrayRef,
3512            ],
3513        )
3514        .unwrap();
3515        let mut file = tempfile::tempfile().unwrap();
3516        {
3517            let mut writer = FileWriter::try_new_with_options(&mut file, &schema, options).unwrap();
3518
3519            writer.write(&batch).unwrap();
3520            writer.finish().unwrap();
3521        }
3522
3523        file.rewind().unwrap();
3524
3525        {
3526            let reader = FileReader::try_new(file, None).unwrap();
3527            reader.for_each(|maybe_batch| {
3528                maybe_batch
3529                    .unwrap()
3530                    .columns()
3531                    .iter()
3532                    .zip(batch.columns())
3533                    .for_each(|(a, b)| {
3534                        assert_eq!(a.data_type(), b.data_type());
3535                        assert_eq!(a.len(), b.len());
3536                        assert_eq!(a.null_count(), b.null_count());
3537                    });
3538            });
3539        }
3540    }
3541    #[test]
3542    fn test_write_null_file_v4() {
3543        write_null_file(IpcWriteOptions::try_new(8, false, MetadataVersion::V4).unwrap());
3544        write_null_file(IpcWriteOptions::try_new(8, true, MetadataVersion::V4).unwrap());
3545        write_null_file(IpcWriteOptions::try_new(64, false, MetadataVersion::V4).unwrap());
3546        write_null_file(IpcWriteOptions::try_new(64, true, MetadataVersion::V4).unwrap());
3547    }
3548
3549    #[test]
3550    fn test_write_null_file_v5() {
3551        write_null_file(IpcWriteOptions::try_new(8, false, MetadataVersion::V5).unwrap());
3552        write_null_file(IpcWriteOptions::try_new(64, false, MetadataVersion::V5).unwrap());
3553    }
3554
3555    #[test]
3556    fn track_union_nested_dict() {
3557        let inner: DictionaryArray<Int32Type> = vec!["a", "b", "a"].into_iter().collect();
3558
3559        let array = Arc::new(inner) as ArrayRef;
3560
3561        // Dict field with id 2
3562        #[expect(deprecated)]
3563        let dctfield = Field::new_dict("dict", array.data_type().clone(), false, 0, false);
3564        let union_fields = std::iter::once((0, Arc::new(dctfield))).collect();
3565
3566        let types = [0, 0, 0].into_iter().collect::<ScalarBuffer<i8>>();
3567        let offsets = [0, 1, 2].into_iter().collect::<ScalarBuffer<i32>>();
3568
3569        let union = UnionArray::try_new(union_fields, types, Some(offsets), vec![array]).unwrap();
3570
3571        let schema = Arc::new(Schema::new(vec![Field::new(
3572            "union",
3573            union.data_type().clone(),
3574            false,
3575        )]));
3576
3577        let r#gen = IpcDataGenerator::default();
3578        let mut dict_tracker = DictionaryTracker::new(false);
3579        r#gen.schema_to_bytes_with_dictionary_tracker(
3580            &schema,
3581            &mut dict_tracker,
3582            &IpcWriteOptions::default(),
3583        );
3584
3585        let batch = RecordBatch::try_new(schema, vec![Arc::new(union)]).unwrap();
3586
3587        r#gen
3588            .encode(
3589                &batch,
3590                &mut dict_tracker,
3591                &Default::default(),
3592                &mut Default::default(),
3593            )
3594            .unwrap();
3595
3596        // The encoder will assign dict IDs itself to ensure uniqueness and ignore the dict ID in the schema
3597        // so we expect the dict will be keyed to 0
3598        assert!(dict_tracker.written.contains_key(&0));
3599    }
3600
3601    #[test]
3602    fn track_struct_nested_dict() {
3603        let inner: DictionaryArray<Int32Type> = vec!["a", "b", "a"].into_iter().collect();
3604
3605        let array = Arc::new(inner) as ArrayRef;
3606
3607        // Dict field with id 2
3608        #[expect(deprecated)]
3609        let dctfield = Arc::new(Field::new_dict(
3610            "dict",
3611            array.data_type().clone(),
3612            false,
3613            2,
3614            false,
3615        ));
3616
3617        let s = StructArray::from(vec![(dctfield, array)]);
3618        let struct_array = Arc::new(s) as ArrayRef;
3619
3620        let schema = Arc::new(Schema::new(vec![Field::new(
3621            "struct",
3622            struct_array.data_type().clone(),
3623            false,
3624        )]));
3625
3626        let r#gen = IpcDataGenerator::default();
3627        let mut dict_tracker = DictionaryTracker::new(false);
3628        r#gen.schema_to_bytes_with_dictionary_tracker(
3629            &schema,
3630            &mut dict_tracker,
3631            &IpcWriteOptions::default(),
3632        );
3633
3634        let batch = RecordBatch::try_new(schema, vec![struct_array]).unwrap();
3635
3636        r#gen
3637            .encode(
3638                &batch,
3639                &mut dict_tracker,
3640                &Default::default(),
3641                &mut Default::default(),
3642            )
3643            .unwrap();
3644
3645        assert!(dict_tracker.written.contains_key(&0));
3646    }
3647
3648    fn write_union_file(options: IpcWriteOptions) {
3649        let schema = Schema::new(vec![Field::new_union(
3650            "union",
3651            vec![0, 1],
3652            vec![
3653                Field::new("a", DataType::Int32, false),
3654                Field::new("c", DataType::Float64, false),
3655            ],
3656            UnionMode::Sparse,
3657        )]);
3658        let mut builder = UnionBuilder::with_capacity_sparse(5);
3659        builder.append::<Int32Type>("a", 1).unwrap();
3660        builder.append_null::<Int32Type>("a").unwrap();
3661        builder.append::<Float64Type>("c", 3.0).unwrap();
3662        builder.append_null::<Float64Type>("c").unwrap();
3663        builder.append::<Int32Type>("a", 4).unwrap();
3664        let union = builder.build().unwrap();
3665
3666        let batch =
3667            RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(union) as ArrayRef])
3668                .unwrap();
3669
3670        let mut file = tempfile::tempfile().unwrap();
3671        {
3672            let mut writer = FileWriter::try_new_with_options(&mut file, &schema, options).unwrap();
3673
3674            writer.write(&batch).unwrap();
3675            writer.finish().unwrap();
3676        }
3677        file.rewind().unwrap();
3678
3679        {
3680            let reader = FileReader::try_new(file, None).unwrap();
3681            reader.for_each(|maybe_batch| {
3682                maybe_batch
3683                    .unwrap()
3684                    .columns()
3685                    .iter()
3686                    .zip(batch.columns())
3687                    .for_each(|(a, b)| {
3688                        assert_eq!(a.data_type(), b.data_type());
3689                        assert_eq!(a.len(), b.len());
3690                        assert_eq!(a.null_count(), b.null_count());
3691                    });
3692            });
3693        }
3694    }
3695
3696    #[test]
3697    fn test_write_union_file_v4_v5() {
3698        write_union_file(IpcWriteOptions::try_new(8, false, MetadataVersion::V4).unwrap());
3699        write_union_file(IpcWriteOptions::try_new(8, false, MetadataVersion::V5).unwrap());
3700    }
3701
3702    #[test]
3703    fn test_write_view_types() {
3704        const LONG_TEST_STRING: &str =
3705            "This is a long string to make sure binary view array handles it";
3706        let schema = Schema::new(vec![
3707            Field::new("field1", DataType::BinaryView, true),
3708            Field::new("field2", DataType::Utf8View, true),
3709        ]);
3710        let values: Vec<Option<&[u8]>> = vec![
3711            Some(b"foo"),
3712            Some(b"bar"),
3713            Some(LONG_TEST_STRING.as_bytes()),
3714        ];
3715        let binary_array = BinaryViewArray::from_iter(values);
3716        let utf8_array =
3717            StringViewArray::from_iter(vec![Some("foo"), Some("bar"), Some(LONG_TEST_STRING)]);
3718        let record_batch = RecordBatch::try_new(
3719            Arc::new(schema.clone()),
3720            vec![Arc::new(binary_array), Arc::new(utf8_array)],
3721        )
3722        .unwrap();
3723
3724        let mut file = tempfile::tempfile().unwrap();
3725        {
3726            let mut writer = FileWriter::try_new(&mut file, &schema).unwrap();
3727            writer.write(&record_batch).unwrap();
3728            writer.finish().unwrap();
3729        }
3730        file.rewind().unwrap();
3731        {
3732            let mut reader = FileReader::try_new(&file, None).unwrap();
3733            let read_batch = reader.next().unwrap().unwrap();
3734            read_batch
3735                .columns()
3736                .iter()
3737                .zip(record_batch.columns())
3738                .for_each(|(a, b)| {
3739                    assert_eq!(a, b);
3740                });
3741        }
3742        file.rewind().unwrap();
3743        {
3744            let mut reader = FileReader::try_new(&file, Some(vec![0])).unwrap();
3745            let read_batch = reader.next().unwrap().unwrap();
3746            assert_eq!(read_batch.num_columns(), 1);
3747            let read_array = read_batch.column(0);
3748            let write_array = record_batch.column(0);
3749            assert_eq!(read_array, write_array);
3750        }
3751    }
3752
3753    #[test]
3754    #[cfg_attr(miri, ignore)] // Takes too long
3755    fn truncate_ipc_record_batch() {
3756        fn create_batch(rows: usize) -> RecordBatch {
3757            let schema = Schema::new(vec![
3758                Field::new("a", DataType::Int32, false),
3759                Field::new("b", DataType::Utf8, false),
3760            ]);
3761
3762            let a = Int32Array::from_iter_values(0..rows as i32);
3763            let b = StringArray::from_iter_values((0..rows).map(|i| i.to_string()));
3764
3765            RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]).unwrap()
3766        }
3767
3768        let big_record_batch = create_batch(65536);
3769
3770        let length = 5;
3771        let small_record_batch = create_batch(length);
3772
3773        let offset = 2;
3774        let record_batch_slice = big_record_batch.slice(offset, length);
3775        assert!(
3776            serialize_stream(&big_record_batch).len() > serialize_stream(&small_record_batch).len()
3777        );
3778        assert_eq!(
3779            serialize_stream(&small_record_batch).len(),
3780            serialize_stream(&record_batch_slice).len()
3781        );
3782
3783        assert_eq!(
3784            deserialize_stream(serialize_stream(&record_batch_slice)),
3785            record_batch_slice
3786        );
3787    }
3788
3789    #[test]
3790    fn truncate_ipc_record_batch_with_nulls() {
3791        fn create_batch() -> RecordBatch {
3792            let schema = Schema::new(vec![
3793                Field::new("a", DataType::Int32, true),
3794                Field::new("b", DataType::Utf8, true),
3795            ]);
3796
3797            let a = Int32Array::from(vec![Some(1), None, Some(1), None, Some(1)]);
3798            let b = StringArray::from(vec![None, Some("a"), Some("a"), None, Some("a")]);
3799
3800            RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a), Arc::new(b)]).unwrap()
3801        }
3802
3803        let record_batch = create_batch();
3804        let record_batch_slice = record_batch.slice(1, 2);
3805        let deserialized_batch = deserialize_stream(serialize_stream(&record_batch_slice));
3806
3807        assert!(
3808            serialize_stream(&record_batch).len() > serialize_stream(&record_batch_slice).len()
3809        );
3810
3811        assert!(deserialized_batch.column(0).is_null(0));
3812        assert!(deserialized_batch.column(0).is_valid(1));
3813        assert!(deserialized_batch.column(1).is_valid(0));
3814        assert!(deserialized_batch.column(1).is_valid(1));
3815
3816        assert_eq!(record_batch_slice, deserialized_batch);
3817    }
3818
3819    #[test]
3820    fn test_write_empty_validity_buffer_without_nulls() {
3821        let batch = RecordBatch::try_from_iter([
3822            ("a", Arc::new(Int32Array::from(vec![1, 2])) as _),
3823            // Keeps a null buffer, but the slice has no nulls
3824            (
3825                "b",
3826                Arc::new(Int32Array::from(vec![None, Some(2), Some(3)]).slice(1, 2)) as _,
3827            ),
3828            ("c", Arc::new(Int32Array::from(vec![Some(1), None])) as _),
3829        ])
3830        .unwrap();
3831        assert_eq!(batch.column(1).nulls().map(|n| n.null_count()), Some(0));
3832
3833        let (_, encoded) = IpcDataGenerator::default()
3834            .encode(
3835                &batch,
3836                &mut DictionaryTracker::new(false),
3837                &Default::default(),
3838                &mut Default::default(),
3839            )
3840            .unwrap();
3841        let message = crate::root_as_message(&encoded.ipc_message).unwrap();
3842        let buffers = message.header_as_record_batch().unwrap().buffers().unwrap();
3843        // Each column writes a validity buffer, then a values buffer
3844        let validity_lengths: Vec<_> = buffers.iter().step_by(2).map(|b| b.length()).collect();
3845        assert_eq!(validity_lengths, [0, 0, 1]);
3846
3847        assert_eq!(deserialize_stream(serialize_stream(&batch)), batch);
3848    }
3849
3850    #[test]
3851    fn truncate_ipc_dictionary_array() {
3852        fn create_batch() -> RecordBatch {
3853            let values: StringArray = [Some("foo"), Some("bar"), Some("baz")]
3854                .into_iter()
3855                .collect();
3856            let keys: Int32Array = [Some(0), Some(2), None, Some(1)].into_iter().collect();
3857
3858            let array = DictionaryArray::new(keys, Arc::new(values));
3859
3860            let schema = Schema::new(vec![Field::new("dict", array.data_type().clone(), true)]);
3861
3862            RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap()
3863        }
3864
3865        let record_batch = create_batch();
3866        let record_batch_slice = record_batch.slice(1, 2);
3867        let deserialized_batch = deserialize_stream(serialize_stream(&record_batch_slice));
3868
3869        assert!(
3870            serialize_stream(&record_batch).len() > serialize_stream(&record_batch_slice).len()
3871        );
3872
3873        assert!(deserialized_batch.column(0).is_valid(0));
3874        assert!(deserialized_batch.column(0).is_null(1));
3875
3876        assert_eq!(record_batch_slice, deserialized_batch);
3877    }
3878
3879    #[test]
3880    fn truncate_ipc_struct_array() {
3881        fn create_batch() -> RecordBatch {
3882            let strings: StringArray = [Some("foo"), None, Some("bar"), Some("baz")]
3883                .into_iter()
3884                .collect();
3885            let ints: Int32Array = [Some(0), Some(2), None, Some(1)].into_iter().collect();
3886
3887            let struct_array = StructArray::from(vec![
3888                (
3889                    Arc::new(Field::new("s", DataType::Utf8, true)),
3890                    Arc::new(strings) as ArrayRef,
3891                ),
3892                (
3893                    Arc::new(Field::new("c", DataType::Int32, true)),
3894                    Arc::new(ints) as ArrayRef,
3895                ),
3896            ]);
3897
3898            let schema = Schema::new(vec![Field::new(
3899                "struct_array",
3900                struct_array.data_type().clone(),
3901                true,
3902            )]);
3903
3904            RecordBatch::try_new(Arc::new(schema), vec![Arc::new(struct_array)]).unwrap()
3905        }
3906
3907        let record_batch = create_batch();
3908        let record_batch_slice = record_batch.slice(1, 2);
3909        let deserialized_batch = deserialize_stream(serialize_stream(&record_batch_slice));
3910
3911        assert!(
3912            serialize_stream(&record_batch).len() > serialize_stream(&record_batch_slice).len()
3913        );
3914
3915        let structs = deserialized_batch
3916            .column(0)
3917            .as_any()
3918            .downcast_ref::<StructArray>()
3919            .unwrap();
3920
3921        assert!(structs.column(0).is_null(0));
3922        assert!(structs.column(0).is_valid(1));
3923        assert!(structs.column(1).is_valid(0));
3924        assert!(structs.column(1).is_null(1));
3925        assert_eq!(record_batch_slice, deserialized_batch);
3926    }
3927
3928    #[test]
3929    fn truncate_ipc_string_array_with_all_empty_string() {
3930        fn create_batch() -> RecordBatch {
3931            let schema = Schema::new(vec![Field::new("a", DataType::Utf8, true)]);
3932            let a = StringArray::from(vec![Some(""), Some(""), Some(""), Some(""), Some("")]);
3933            RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a)]).unwrap()
3934        }
3935
3936        let record_batch = create_batch();
3937        let record_batch_slice = record_batch.slice(0, 1);
3938        let deserialized_batch = deserialize_stream(serialize_stream(&record_batch_slice));
3939
3940        assert!(
3941            serialize_stream(&record_batch).len() > serialize_stream(&record_batch_slice).len()
3942        );
3943        assert_eq!(record_batch_slice, deserialized_batch);
3944    }
3945
3946    #[test]
3947    fn test_stream_writer_writes_array_slice() {
3948        let array = UInt32Array::from(vec![Some(1), Some(2), Some(3)]);
3949        assert_eq!(
3950            vec![Some(1), Some(2), Some(3)],
3951            array.iter().collect::<Vec<_>>()
3952        );
3953
3954        let sliced = array.slice(1, 2);
3955        assert_eq!(vec![Some(2), Some(3)], sliced.iter().collect::<Vec<_>>());
3956
3957        let batch = RecordBatch::try_new(
3958            Arc::new(Schema::new(vec![Field::new("a", DataType::UInt32, true)])),
3959            vec![Arc::new(sliced)],
3960        )
3961        .expect("new batch");
3962
3963        let mut writer = StreamWriter::try_new(vec![], batch.schema_ref()).expect("new writer");
3964        writer.write(&batch).expect("write");
3965        let outbuf = writer.into_inner().expect("inner");
3966
3967        let mut reader = StreamReader::try_new(&outbuf[..], None).expect("new reader");
3968        let read_batch = reader.next().unwrap().expect("read batch");
3969
3970        let read_array: &UInt32Array = read_batch.column(0).as_primitive();
3971        assert_eq!(
3972            vec![Some(2), Some(3)],
3973            read_array.iter().collect::<Vec<_>>()
3974        );
3975    }
3976
3977    #[test]
3978    fn test_large_slice_uint32() {
3979        ensure_roundtrip(Arc::new(UInt32Array::from_iter(
3980            (0..8000).map(|i| if i % 2 == 0 { Some(i) } else { None }),
3981        )));
3982    }
3983
3984    #[test]
3985    #[cfg_attr(miri, ignore)] // Takes too long
3986    fn test_large_slice_string() {
3987        let strings: Vec<_> = (0..8000)
3988            .map(|i| {
3989                if i % 2 == 0 {
3990                    Some(format!("value{i}"))
3991                } else {
3992                    None
3993                }
3994            })
3995            .collect();
3996
3997        ensure_roundtrip(Arc::new(StringArray::from(strings)));
3998    }
3999
4000    #[test]
4001    #[cfg_attr(miri, ignore)] // Takes too long
4002    fn test_large_slice_string_list() {
4003        let mut ls = ListBuilder::new(StringBuilder::new());
4004
4005        let mut s = String::new();
4006        for row_number in 0..8000 {
4007            if row_number % 2 == 0 {
4008                for list_element in 0..1000 {
4009                    s.clear();
4010                    use std::fmt::Write;
4011                    write!(&mut s, "value{row_number}-{list_element}").unwrap();
4012                    ls.values().append_value(&s);
4013                }
4014                ls.append(true)
4015            } else {
4016                ls.append(false); // null
4017            }
4018        }
4019
4020        ensure_roundtrip(Arc::new(ls.finish()));
4021    }
4022
4023    #[test]
4024    #[cfg_attr(miri, ignore)] // Takes too long
4025    fn test_large_slice_string_list_of_lists() {
4026        // The reason for the special test is to verify reencode_offsets which looks both at
4027        // the starting offset and the data offset.  So need a dataset where the starting_offset
4028        // is zero but the data offset is not.
4029        let mut ls = ListBuilder::new(ListBuilder::new(StringBuilder::new()));
4030
4031        for _ in 0..4000 {
4032            ls.values().append(true);
4033            ls.append(true)
4034        }
4035
4036        let mut s = String::new();
4037        for row_number in 0..4000 {
4038            if row_number % 2 == 0 {
4039                for list_element in 0..1000 {
4040                    s.clear();
4041                    use std::fmt::Write;
4042                    write!(&mut s, "value{row_number}-{list_element}").unwrap();
4043                    ls.values().values().append_value(&s);
4044                }
4045                ls.values().append(true);
4046                ls.append(true)
4047            } else {
4048                ls.append(false); // null
4049            }
4050        }
4051
4052        ensure_roundtrip(Arc::new(ls.finish()));
4053    }
4054
4055    /// Read/write a record batch to a File and Stream and ensure it is the same at the output
4056    fn ensure_roundtrip(array: ArrayRef) {
4057        let num_rows = array.len();
4058        let orig_batch = RecordBatch::try_from_iter(vec![("a", array)]).unwrap();
4059        // take off the first element
4060        let sliced_batch = orig_batch.slice(1, num_rows - 1);
4061
4062        let schema = orig_batch.schema();
4063        let stream_data = {
4064            let mut writer = StreamWriter::try_new(vec![], &schema).unwrap();
4065            writer.write(&sliced_batch).unwrap();
4066            writer.into_inner().unwrap()
4067        };
4068        let read_batch = {
4069            let projection = None;
4070            let mut reader = StreamReader::try_new(Cursor::new(stream_data), projection).unwrap();
4071            reader
4072                .next()
4073                .expect("expect no errors reading batch")
4074                .expect("expect batch")
4075        };
4076        assert_eq!(sliced_batch, read_batch);
4077
4078        let file_data = {
4079            let mut writer = FileWriter::try_new_buffered(vec![], &schema).unwrap();
4080            writer.write(&sliced_batch).unwrap();
4081            writer.into_inner().unwrap().into_inner().unwrap()
4082        };
4083        let read_batch = {
4084            let projection = None;
4085            let mut reader = FileReader::try_new(Cursor::new(file_data), projection).unwrap();
4086            reader
4087                .next()
4088                .expect("expect no errors reading batch")
4089                .expect("expect batch")
4090        };
4091        assert_eq!(sliced_batch, read_batch);
4092
4093        // TODO test file writer/reader
4094    }
4095
4096    #[test]
4097    fn encode_bools_slice() {
4098        // Test case for https://github.com/apache/arrow-rs/issues/3496
4099        assert_bool_roundtrip([true, false], 1, 1);
4100
4101        // slice somewhere in the middle
4102        assert_bool_roundtrip(
4103            [
4104                true, false, true, true, false, false, true, true, true, false, false, false, true,
4105                true, true, true, false, false, false, false, true, true, true, true, true, false,
4106                false, false, false, false,
4107            ],
4108            13,
4109            17,
4110        );
4111
4112        // start at byte boundary, end in the middle
4113        assert_bool_roundtrip(
4114            [
4115                true, false, true, true, false, false, true, true, true, false, false, false,
4116            ],
4117            8,
4118            2,
4119        );
4120
4121        // start and stop and byte boundary
4122        assert_bool_roundtrip(
4123            [
4124                true, false, true, true, false, false, true, true, true, false, false, false, true,
4125                true, true, true, true, false, false, false, false, false,
4126            ],
4127            8,
4128            8,
4129        );
4130    }
4131
4132    fn assert_bool_roundtrip<const N: usize>(bools: [bool; N], offset: usize, length: usize) {
4133        let val_bool_field = Field::new("val", DataType::Boolean, false);
4134
4135        let schema = Arc::new(Schema::new(vec![val_bool_field]));
4136
4137        let bools = BooleanArray::from(bools.to_vec());
4138
4139        let batch = RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(bools)]).unwrap();
4140        let batch = batch.slice(offset, length);
4141
4142        let data = serialize_stream(&batch);
4143        let batch2 = deserialize_stream(data);
4144        assert_eq!(batch, batch2);
4145    }
4146
4147    fn assert_empty_run_array_roundtrip<R: RunEndIndexType>() {
4148        let run_ends = PrimitiveArray::<R>::from_iter_values(
4149            [2, 5]
4150                .into_iter()
4151                .map(|v| R::Native::from_usize(v).unwrap()),
4152        );
4153        let values = Int32Array::from(vec![10, 20]);
4154        let array = RunArray::<R>::try_new(&run_ends, &values).unwrap();
4155        let empty = RunArray::<R>::from(ArrayData::new_empty(array.data_type()));
4156
4157        for source in [&array, &empty] {
4158            for offset in 0..=source.len() {
4159                let sliced = source.slice(offset, 0);
4160                let batch = RecordBatch::try_from_iter(vec![("run", Arc::new(sliced) as ArrayRef)])
4161                    .unwrap();
4162                for decoded in [
4163                    deserialize_stream(serialize_stream(&batch)),
4164                    deserialize_file(serialize_file(&batch)),
4165                ] {
4166                    assert_eq!(decoded, batch);
4167                    let data = decoded.column(0).to_data();
4168                    data.validate_full().unwrap();
4169                    assert_eq!(data.offset(), 0);
4170                    assert!(data.child_data().iter().all(ArrayData::is_empty));
4171                }
4172
4173                let normalized = into_zero_offset_run_array(source.slice(offset, 0)).unwrap();
4174                normalized.to_data().validate_full().unwrap();
4175                assert_eq!(normalized.offset(), 0);
4176                assert!(normalized.run_ends().values().is_empty());
4177                assert!(normalized.values().is_empty());
4178            }
4179        }
4180    }
4181
4182    #[test]
4183    fn test_empty_run_array_roundtrip_i16() {
4184        assert_empty_run_array_roundtrip::<Int16Type>();
4185    }
4186
4187    #[test]
4188    fn test_empty_run_array_roundtrip_i32() {
4189        assert_empty_run_array_roundtrip::<Int32Type>();
4190    }
4191
4192    #[test]
4193    fn test_empty_run_array_roundtrip_i64() {
4194        assert_empty_run_array_roundtrip::<Int64Type>();
4195    }
4196
4197    #[test]
4198    fn test_run_array_unslice() {
4199        let total_len = 80;
4200        let vals: Vec<Option<i32>> = vec![Some(1), None, Some(2), Some(3), Some(4), None, Some(5)];
4201        let repeats: Vec<usize> = vec![3, 4, 1, 2];
4202        let mut input_array: Vec<Option<i32>> = Vec::with_capacity(total_len);
4203        for ix in 0_usize..32 {
4204            let repeat: usize = repeats[ix % repeats.len()];
4205            let val: Option<i32> = vals[ix % vals.len()];
4206            input_array.resize(input_array.len() + repeat, val);
4207        }
4208
4209        // Encode the input_array to run array
4210        let mut builder =
4211            PrimitiveRunBuilder::<Int16Type, Int32Type>::with_capacity(input_array.len());
4212        builder.extend(input_array.iter().copied());
4213        let run_array = builder.finish();
4214
4215        // test for all slice lengths.
4216        for slice_len in 1..=total_len {
4217            // test for offset = 0, slice length = slice_len
4218            let sliced_run_array: RunArray<Int16Type> =
4219                run_array.slice(0, slice_len).into_data().into();
4220
4221            // Create unsliced run array.
4222            let unsliced_run_array = into_zero_offset_run_array(sliced_run_array).unwrap();
4223            let typed = unsliced_run_array
4224                .downcast::<PrimitiveArray<Int32Type>>()
4225                .unwrap();
4226            let expected: Vec<Option<i32>> = input_array.iter().take(slice_len).copied().collect();
4227            let actual: Vec<Option<i32>> = typed.into_iter().collect();
4228            assert_eq!(expected, actual);
4229
4230            // test for offset = total_len - slice_len, length = slice_len
4231            let sliced_run_array: RunArray<Int16Type> = run_array
4232                .slice(total_len - slice_len, slice_len)
4233                .into_data()
4234                .into();
4235
4236            // Create unsliced run array.
4237            let unsliced_run_array = into_zero_offset_run_array(sliced_run_array).unwrap();
4238            let typed = unsliced_run_array
4239                .downcast::<PrimitiveArray<Int32Type>>()
4240                .unwrap();
4241            let expected: Vec<Option<i32>> = input_array
4242                .iter()
4243                .skip(total_len - slice_len)
4244                .copied()
4245                .collect();
4246            let actual: Vec<Option<i32>> = typed.into_iter().collect();
4247            assert_eq!(expected, actual);
4248        }
4249    }
4250
4251    fn generate_list_data<O: OffsetSizeTrait>() -> GenericListArray<O> {
4252        let mut ls = GenericListBuilder::<O, _>::new(UInt32Builder::new());
4253
4254        for i in 0..100_000 {
4255            for value in [i, i, i] {
4256                ls.values().append_value(value);
4257            }
4258            ls.append(true)
4259        }
4260
4261        ls.finish()
4262    }
4263
4264    fn generate_utf8view_list_data<O: OffsetSizeTrait>() -> GenericListArray<O> {
4265        let mut ls = GenericListBuilder::<O, _>::new(StringViewBuilder::new());
4266
4267        for i in 0..100_000 {
4268            for value in [
4269                format!("value{i}"),
4270                format!("value{i}"),
4271                format!("value{i}"),
4272            ] {
4273                ls.values().append_value(&value);
4274            }
4275            ls.append(true)
4276        }
4277
4278        ls.finish()
4279    }
4280
4281    fn generate_string_list_data<O: OffsetSizeTrait>() -> GenericListArray<O> {
4282        let mut ls = GenericListBuilder::<O, _>::new(StringBuilder::new());
4283
4284        for i in 0..100_000 {
4285            for value in [
4286                format!("value{i}"),
4287                format!("value{i}"),
4288                format!("value{i}"),
4289            ] {
4290                ls.values().append_value(&value);
4291            }
4292            ls.append(true)
4293        }
4294
4295        ls.finish()
4296    }
4297
4298    fn generate_nested_list_data<O: OffsetSizeTrait>() -> GenericListArray<O> {
4299        let mut ls =
4300            GenericListBuilder::<O, _>::new(GenericListBuilder::<O, _>::new(UInt32Builder::new()));
4301
4302        for _i in 0..10_000 {
4303            for j in 0..10 {
4304                for value in [j, j, j, j] {
4305                    ls.values().values().append_value(value);
4306                }
4307                ls.values().append(true)
4308            }
4309            ls.append(true);
4310        }
4311
4312        ls.finish()
4313    }
4314
4315    fn generate_nested_list_data_starting_at_zero<O: OffsetSizeTrait>() -> GenericListArray<O> {
4316        let mut ls =
4317            GenericListBuilder::<O, _>::new(GenericListBuilder::<O, _>::new(UInt32Builder::new()));
4318
4319        for _i in 0..999 {
4320            ls.values().append(true);
4321            ls.append(true);
4322        }
4323
4324        for j in 0..10 {
4325            for value in [j, j, j, j] {
4326                ls.values().values().append_value(value);
4327            }
4328            ls.values().append(true)
4329        }
4330        ls.append(true);
4331
4332        for i in 0..9_000 {
4333            for j in 0..10 {
4334                for value in [i + j, i + j, i + j, i + j] {
4335                    ls.values().values().append_value(value);
4336                }
4337                ls.values().append(true)
4338            }
4339            ls.append(true);
4340        }
4341
4342        ls.finish()
4343    }
4344
4345    fn generate_map_array_data() -> MapArray {
4346        let keys_builder = UInt32Builder::new();
4347        let values_builder = UInt32Builder::new();
4348
4349        let mut builder = MapBuilder::new(None, keys_builder, values_builder);
4350
4351        for i in 0..100_000 {
4352            for _j in 0..3 {
4353                builder.keys().append_value(i);
4354                builder.values().append_value(i * 2);
4355            }
4356            builder.append(true).unwrap();
4357        }
4358
4359        builder.finish()
4360    }
4361
4362    #[test]
4363    #[cfg_attr(miri, ignore)] // Takes too long
4364    fn reencode_offsets_when_first_offset_is_not_zero() {
4365        let original_list = generate_list_data::<i32>();
4366        let original_data = original_list.into_data();
4367        let slice_data = original_data.slice(75, 7);
4368        let (new_offsets, original_start, length) =
4369            reencode_offsets::<i32>(&slice_data.buffers()[0], &slice_data);
4370        assert_eq!(
4371            vec![0, 3, 6, 9, 12, 15, 18, 21],
4372            new_offsets.typed_data::<i32>()
4373        );
4374        assert_eq!(225, original_start);
4375        assert_eq!(21, length);
4376    }
4377
4378    #[test]
4379    fn reencode_offsets_when_first_offset_is_zero() {
4380        let mut ls = GenericListBuilder::<i32, _>::new(UInt32Builder::new());
4381        // ls = [[], [35, 42]
4382        ls.append(true);
4383        ls.values().append_value(35);
4384        ls.values().append_value(42);
4385        ls.append(true);
4386        let original_list = ls.finish();
4387        let original_data = original_list.into_data();
4388
4389        let slice_data = original_data.slice(1, 1);
4390        let (new_offsets, original_start, length) =
4391            reencode_offsets::<i32>(&slice_data.buffers()[0], &slice_data);
4392        assert_eq!(vec![0, 2], new_offsets.typed_data::<i32>());
4393        assert_eq!(0, original_start);
4394        assert_eq!(2, length);
4395    }
4396
4397    /// Ensure when serde full & sliced versions they are equal to original input.
4398    /// Also ensure serialized sliced version is significantly smaller than serialized full.
4399    fn roundtrip_ensure_sliced_smaller(in_batch: RecordBatch, expected_size_factor: usize) {
4400        // test both full and sliced versions
4401        let in_sliced = in_batch.slice(999, 1);
4402
4403        let bytes_batch = serialize_file(&in_batch);
4404        let bytes_sliced = serialize_file(&in_sliced);
4405
4406        // serializing 1 row should be significantly smaller than serializing 100,000
4407        assert!(bytes_sliced.len() < (bytes_batch.len() / expected_size_factor));
4408
4409        // ensure both are still valid and equal to originals
4410        let out_batch = deserialize_file(bytes_batch);
4411        assert_eq!(in_batch, out_batch);
4412
4413        let out_sliced = deserialize_file(bytes_sliced);
4414        assert_eq!(in_sliced, out_sliced);
4415    }
4416
4417    #[test]
4418    #[cfg_attr(miri, ignore)] // Takes too long
4419    fn encode_lists() {
4420        let val_inner = Field::new_list_field(DataType::UInt32, true);
4421        let val_list_field = Field::new("val", DataType::List(Arc::new(val_inner)), false);
4422        let schema = Arc::new(Schema::new(vec![val_list_field]));
4423
4424        let values = Arc::new(generate_list_data::<i32>());
4425
4426        let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4427        roundtrip_ensure_sliced_smaller(in_batch, 1000);
4428    }
4429
4430    #[test]
4431    #[cfg_attr(miri, ignore)] // Takes too long
4432    fn encode_empty_list() {
4433        let val_inner = Field::new_list_field(DataType::UInt32, true);
4434        let val_list_field = Field::new("val", DataType::List(Arc::new(val_inner)), false);
4435        let schema = Arc::new(Schema::new(vec![val_list_field]));
4436
4437        let values = Arc::new(generate_list_data::<i32>());
4438
4439        let in_batch = RecordBatch::try_new(schema, vec![values])
4440            .unwrap()
4441            .slice(999, 0);
4442        let out_batch = deserialize_file(serialize_file(&in_batch));
4443        assert_eq!(in_batch, out_batch);
4444    }
4445
4446    #[test]
4447    #[cfg_attr(miri, ignore)] // Takes too long
4448    fn encode_large_lists() {
4449        let val_inner = Field::new_list_field(DataType::UInt32, true);
4450        let val_list_field = Field::new("val", DataType::LargeList(Arc::new(val_inner)), false);
4451        let schema = Arc::new(Schema::new(vec![val_list_field]));
4452
4453        let values = Arc::new(generate_list_data::<i64>());
4454
4455        // ensure when serde full & sliced versions they are equal to original input
4456        // also ensure serialized sliced version is significantly smaller than serialized full
4457        let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4458        roundtrip_ensure_sliced_smaller(in_batch, 1000);
4459    }
4460
4461    #[test]
4462    #[cfg_attr(miri, ignore)] // Takes too long
4463    fn encode_large_lists_non_zero_offset() {
4464        let val_inner = Field::new_list_field(DataType::UInt32, true);
4465        let val_list_field = Field::new("val", DataType::LargeList(Arc::new(val_inner)), false);
4466        let schema = Arc::new(Schema::new(vec![val_list_field]));
4467
4468        let values = Arc::new(generate_list_data::<i64>());
4469
4470        check_sliced_list_array(schema, values);
4471    }
4472
4473    #[test]
4474    #[cfg_attr(miri, ignore)] // Takes too long
4475    fn encode_large_lists_string_non_zero_offset() {
4476        let val_inner = Field::new_list_field(DataType::Utf8, true);
4477        let val_list_field = Field::new("val", DataType::LargeList(Arc::new(val_inner)), false);
4478        let schema = Arc::new(Schema::new(vec![val_list_field]));
4479
4480        let values = Arc::new(generate_string_list_data::<i64>());
4481
4482        check_sliced_list_array(schema, values);
4483    }
4484
4485    #[test]
4486    #[cfg_attr(miri, ignore)] // Takes too long
4487    fn encode_large_list_string_view_non_zero_offset() {
4488        let val_inner = Field::new_list_field(DataType::Utf8View, true);
4489        let val_list_field = Field::new("val", DataType::LargeList(Arc::new(val_inner)), false);
4490        let schema = Arc::new(Schema::new(vec![val_list_field]));
4491
4492        let values = Arc::new(generate_utf8view_list_data::<i64>());
4493
4494        check_sliced_list_array(schema, values);
4495    }
4496
4497    fn check_sliced_list_array(schema: Arc<Schema>, values: Arc<GenericListArray<i64>>) {
4498        for (offset, len) in [(999, 1), (0, 13), (47, 12), (values.len() - 13, 13)] {
4499            let in_batch = RecordBatch::try_new(schema.clone(), vec![values.clone()])
4500                .unwrap()
4501                .slice(offset, len);
4502            let out_batch = deserialize_file(serialize_file(&in_batch));
4503            assert_eq!(in_batch, out_batch);
4504        }
4505    }
4506
4507    #[test]
4508    #[cfg_attr(miri, ignore)] // Takes too long
4509    fn encode_nested_lists() {
4510        let inner_int = Arc::new(Field::new_list_field(DataType::UInt32, true));
4511        let inner_list_field = Arc::new(Field::new_list_field(DataType::List(inner_int), true));
4512        let list_field = Field::new("val", DataType::List(inner_list_field), true);
4513        let schema = Arc::new(Schema::new(vec![list_field]));
4514
4515        let values = Arc::new(generate_nested_list_data::<i32>());
4516
4517        let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4518        roundtrip_ensure_sliced_smaller(in_batch, 1000);
4519    }
4520
4521    #[test]
4522    #[cfg_attr(miri, ignore)] // Takes too long
4523    fn encode_nested_lists_starting_at_zero() {
4524        let inner_int = Arc::new(Field::new("item", DataType::UInt32, true));
4525        let inner_list_field = Arc::new(Field::new("item", DataType::List(inner_int), true));
4526        let list_field = Field::new("val", DataType::List(inner_list_field), true);
4527        let schema = Arc::new(Schema::new(vec![list_field]));
4528
4529        let values = Arc::new(generate_nested_list_data_starting_at_zero::<i32>());
4530
4531        let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4532        roundtrip_ensure_sliced_smaller(in_batch, 1);
4533    }
4534
4535    #[test]
4536    #[cfg_attr(miri, ignore)] // Takes too long
4537    fn encode_map_array() {
4538        let keys = Arc::new(Field::new(
4539            Field::MAP_KEY_FIELD_DEFAULT_NAME,
4540            DataType::UInt32,
4541            false,
4542        ));
4543        let values = Arc::new(Field::new(
4544            Field::MAP_VALUE_FIELD_DEFAULT_NAME,
4545            DataType::UInt32,
4546            true,
4547        ));
4548        let map_field = Field::new_map(
4549            "map",
4550            Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
4551            keys,
4552            values,
4553            false,
4554            true,
4555        );
4556        let schema = Arc::new(Schema::new(vec![map_field]));
4557
4558        let values = Arc::new(generate_map_array_data());
4559
4560        let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4561        roundtrip_ensure_sliced_smaller(in_batch, 1000);
4562    }
4563
4564    fn generate_list_view_data<O: OffsetSizeTrait>() -> GenericListViewArray<O> {
4565        let mut builder = GenericListViewBuilder::<O, _>::new(UInt32Builder::new());
4566
4567        for i in 0u32..100_000 {
4568            if i.is_multiple_of(10_000) {
4569                builder.append(false);
4570                continue;
4571            }
4572            for value in [i, i, i] {
4573                builder.values().append_value(value);
4574            }
4575            builder.append(true);
4576        }
4577
4578        builder.finish()
4579    }
4580
4581    #[test]
4582    #[cfg_attr(miri, ignore)] // Takes too long
4583    fn encode_list_view_arrays() {
4584        let val_inner = Field::new_list_field(DataType::UInt32, true);
4585        let val_field = Field::new("val", DataType::ListView(Arc::new(val_inner)), true);
4586        let schema = Arc::new(Schema::new(vec![val_field]));
4587
4588        let values = Arc::new(generate_list_view_data::<i32>());
4589
4590        let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4591        let out_batch = deserialize_file(serialize_file(&in_batch));
4592        assert_eq!(in_batch, out_batch);
4593    }
4594
4595    #[test]
4596    #[cfg_attr(miri, ignore)] // Takes too long
4597    fn encode_large_list_view_arrays() {
4598        let val_inner = Field::new_list_field(DataType::UInt32, true);
4599        let val_field = Field::new("val", DataType::LargeListView(Arc::new(val_inner)), true);
4600        let schema = Arc::new(Schema::new(vec![val_field]));
4601
4602        let values = Arc::new(generate_list_view_data::<i64>());
4603
4604        let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4605        let out_batch = deserialize_file(serialize_file(&in_batch));
4606        assert_eq!(in_batch, out_batch);
4607    }
4608
4609    #[test]
4610    #[cfg_attr(miri, ignore)] // Takes too long
4611    fn check_sliced_list_view_array() {
4612        let inner = Field::new_list_field(DataType::UInt32, true);
4613        let field = Field::new("val", DataType::ListView(Arc::new(inner)), true);
4614        let schema = Arc::new(Schema::new(vec![field]));
4615        let values = Arc::new(generate_list_view_data::<i32>());
4616
4617        for (offset, len) in [(999, 1), (0, 13), (47, 12), (values.len() - 13, 13)] {
4618            let in_batch = RecordBatch::try_new(schema.clone(), vec![values.clone()])
4619                .unwrap()
4620                .slice(offset, len);
4621            let out_batch = deserialize_file(serialize_file(&in_batch));
4622            assert_eq!(in_batch, out_batch);
4623        }
4624    }
4625
4626    #[test]
4627    #[cfg_attr(miri, ignore)] // Takes too long
4628    fn check_sliced_large_list_view_array() {
4629        let inner = Field::new_list_field(DataType::UInt32, true);
4630        let field = Field::new("val", DataType::LargeListView(Arc::new(inner)), true);
4631        let schema = Arc::new(Schema::new(vec![field]));
4632        let values = Arc::new(generate_list_view_data::<i64>());
4633
4634        for (offset, len) in [(999, 1), (0, 13), (47, 12), (values.len() - 13, 13)] {
4635            let in_batch = RecordBatch::try_new(schema.clone(), vec![values.clone()])
4636                .unwrap()
4637                .slice(offset, len);
4638            let out_batch = deserialize_file(serialize_file(&in_batch));
4639            assert_eq!(in_batch, out_batch);
4640        }
4641    }
4642
4643    fn generate_nested_list_view_data<O: OffsetSizeTrait>() -> GenericListViewArray<O> {
4644        let inner_builder = UInt32Builder::new();
4645        let middle_builder = GenericListViewBuilder::<O, _>::new(inner_builder);
4646        let mut outer_builder = GenericListViewBuilder::<O, _>::new(middle_builder);
4647
4648        for i in 0u32..10_000 {
4649            if i.is_multiple_of(1_000) {
4650                outer_builder.append(false);
4651                continue;
4652            }
4653
4654            for _ in 0..3 {
4655                for value in [i, i + 1, i + 2] {
4656                    outer_builder.values().values().append_value(value);
4657                }
4658                outer_builder.values().append(true);
4659            }
4660            outer_builder.append(true);
4661        }
4662
4663        outer_builder.finish()
4664    }
4665
4666    #[test]
4667    #[cfg_attr(miri, ignore)] // Takes too long
4668    fn encode_nested_list_views() {
4669        let inner_int = Arc::new(Field::new_list_field(DataType::UInt32, true));
4670        let inner_list_field = Arc::new(Field::new_list_field(DataType::ListView(inner_int), true));
4671        let list_field = Field::new("val", DataType::ListView(inner_list_field), true);
4672        let schema = Arc::new(Schema::new(vec![list_field]));
4673
4674        let values = Arc::new(generate_nested_list_view_data::<i32>());
4675
4676        let in_batch = RecordBatch::try_new(schema, vec![values]).unwrap();
4677        let out_batch = deserialize_file(serialize_file(&in_batch));
4678        assert_eq!(in_batch, out_batch);
4679    }
4680
4681    fn test_roundtrip_list_view_of_dict_impl<OffsetSize: OffsetSizeTrait, U: ArrowNativeType>(
4682        list_data_type: DataType,
4683        offsets: &[U; 5],
4684        sizes: &[U; 4],
4685    ) {
4686        let values = StringArray::from(vec![Some("alpha"), None, Some("beta"), Some("gamma")]);
4687        let keys = Int32Array::from_iter_values([0, 0, 1, 2, 3, 0, 2]);
4688        let dict_array = DictionaryArray::new(keys, Arc::new(values));
4689        let dict_data = dict_array.to_data();
4690
4691        let value_offsets = Buffer::from_slice_ref(offsets);
4692        let value_sizes = Buffer::from_slice_ref(sizes);
4693
4694        let list_data = ArrayData::builder(list_data_type)
4695            .len(4)
4696            .add_buffer(value_offsets)
4697            .add_buffer(value_sizes)
4698            .add_child_data(dict_data)
4699            .build()
4700            .unwrap();
4701        let list_view_array = GenericListViewArray::<OffsetSize>::from(list_data);
4702
4703        let schema = Arc::new(Schema::new(vec![Field::new(
4704            "f1",
4705            list_view_array.data_type().clone(),
4706            false,
4707        )]));
4708        let input_batch = RecordBatch::try_new(schema, vec![Arc::new(list_view_array)]).unwrap();
4709
4710        let output_batch = deserialize_file(serialize_file(&input_batch));
4711        assert_eq!(input_batch, output_batch);
4712
4713        let output_batch = deserialize_stream(serialize_stream(&input_batch));
4714        assert_eq!(input_batch, output_batch);
4715    }
4716
4717    #[test]
4718    fn test_roundtrip_list_view_of_dict() {
4719        #[expect(deprecated)]
4720        let list_data_type = DataType::ListView(Arc::new(Field::new_dict(
4721            "item",
4722            DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4723            true,
4724            1,
4725            false,
4726        )));
4727        let offsets: &[i32; 5] = &[0, 2, 4, 4, 7];
4728        let sizes: &[i32; 4] = &[2, 2, 0, 3];
4729        test_roundtrip_list_view_of_dict_impl::<i32, i32>(list_data_type, offsets, sizes);
4730    }
4731
4732    #[test]
4733    fn test_roundtrip_large_list_view_of_dict() {
4734        #[expect(deprecated)]
4735        let list_data_type = DataType::LargeListView(Arc::new(Field::new_dict(
4736            "item",
4737            DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4738            true,
4739            2,
4740            false,
4741        )));
4742        let offsets: &[i64; 5] = &[0, 2, 4, 4, 7];
4743        let sizes: &[i64; 4] = &[2, 2, 0, 3];
4744        test_roundtrip_list_view_of_dict_impl::<i64, i64>(list_data_type, offsets, sizes);
4745    }
4746
4747    #[test]
4748    fn test_roundtrip_sliced_list_view_of_dict() {
4749        #[expect(deprecated)]
4750        let list_data_type = DataType::ListView(Arc::new(Field::new_dict(
4751            "item",
4752            DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4753            true,
4754            3,
4755            false,
4756        )));
4757
4758        let values = StringArray::from(vec![Some("alpha"), None, Some("beta"), Some("gamma")]);
4759        let keys = Int32Array::from_iter_values([0, 0, 1, 2, 3, 0, 2, 1, 0, 3, 2, 1]);
4760        let dict_array = DictionaryArray::new(keys, Arc::new(values));
4761        let dict_data = dict_array.to_data();
4762
4763        let offsets: &[i32; 7] = &[0, 2, 4, 4, 7, 9, 12];
4764        let sizes: &[i32; 6] = &[2, 2, 0, 3, 2, 3];
4765        let value_offsets = Buffer::from_slice_ref(offsets);
4766        let value_sizes = Buffer::from_slice_ref(sizes);
4767
4768        let list_data = ArrayData::builder(list_data_type)
4769            .len(6)
4770            .add_buffer(value_offsets)
4771            .add_buffer(value_sizes)
4772            .add_child_data(dict_data)
4773            .build()
4774            .unwrap();
4775        let list_view_array = GenericListViewArray::<i32>::from(list_data);
4776
4777        let schema = Arc::new(Schema::new(vec![Field::new(
4778            "f1",
4779            list_view_array.data_type().clone(),
4780            false,
4781        )]));
4782        let input_batch = RecordBatch::try_new(schema, vec![Arc::new(list_view_array)]).unwrap();
4783
4784        let sliced_batch = input_batch.slice(1, 4);
4785
4786        let output_batch = deserialize_file(serialize_file(&sliced_batch));
4787        assert_eq!(sliced_batch, output_batch);
4788
4789        let output_batch = deserialize_stream(serialize_stream(&sliced_batch));
4790        assert_eq!(sliced_batch, output_batch);
4791    }
4792
4793    #[test]
4794    fn test_roundtrip_dense_union_of_dict() {
4795        let values = StringArray::from(vec![Some("alpha"), None, Some("beta"), Some("gamma")]);
4796        let keys = Int32Array::from_iter_values([0, 0, 1, 2, 3, 0, 2]);
4797        let dict_array = DictionaryArray::new(keys, Arc::new(values));
4798
4799        #[expect(deprecated)]
4800        let dict_field = Arc::new(Field::new_dict(
4801            "dict",
4802            DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4803            true,
4804            1,
4805            false,
4806        ));
4807        let int_field = Arc::new(Field::new("int", DataType::Int32, false));
4808        let union_fields = UnionFields::try_new(vec![0, 1], vec![dict_field, int_field]).unwrap();
4809
4810        let types = ScalarBuffer::from(vec![0i8, 0, 1, 0, 1, 0, 0]);
4811        let offsets = ScalarBuffer::from(vec![0i32, 1, 0, 2, 1, 3, 4]);
4812
4813        let int_array = Int32Array::from(vec![100, 200]);
4814
4815        let union = UnionArray::try_new(
4816            union_fields.clone(),
4817            types,
4818            Some(offsets),
4819            vec![Arc::new(dict_array), Arc::new(int_array)],
4820        )
4821        .unwrap();
4822
4823        let schema = Arc::new(Schema::new(vec![Field::new(
4824            "union",
4825            DataType::Union(union_fields, UnionMode::Dense),
4826            false,
4827        )]));
4828        let input_batch = RecordBatch::try_new(schema, vec![Arc::new(union)]).unwrap();
4829
4830        let output_batch = deserialize_file(serialize_file(&input_batch));
4831        assert_eq!(input_batch, output_batch);
4832
4833        let output_batch = deserialize_stream(serialize_stream(&input_batch));
4834        assert_eq!(input_batch, output_batch);
4835    }
4836
4837    #[test]
4838    fn test_roundtrip_sparse_union_of_dict() {
4839        let values = StringArray::from(vec![Some("alpha"), None, Some("beta"), Some("gamma")]);
4840        let keys = Int32Array::from_iter_values([0, 0, 1, 2, 3, 0, 2]);
4841        let dict_array = DictionaryArray::new(keys, Arc::new(values));
4842
4843        #[expect(deprecated)]
4844        let dict_field = Arc::new(Field::new_dict(
4845            "dict",
4846            DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4847            true,
4848            2,
4849            false,
4850        ));
4851        let int_field = Arc::new(Field::new("int", DataType::Int32, false));
4852        let union_fields = UnionFields::try_new(vec![0, 1], vec![dict_field, int_field]).unwrap();
4853
4854        let types = ScalarBuffer::from(vec![0i8, 0, 1, 0, 1, 0, 0]);
4855
4856        let int_array = Int32Array::from(vec![0, 0, 100, 0, 200, 0, 0]);
4857
4858        let union = UnionArray::try_new(
4859            union_fields.clone(),
4860            types,
4861            None,
4862            vec![Arc::new(dict_array), Arc::new(int_array)],
4863        )
4864        .unwrap();
4865
4866        let schema = Arc::new(Schema::new(vec![Field::new(
4867            "union",
4868            DataType::Union(union_fields, UnionMode::Sparse),
4869            false,
4870        )]));
4871        let input_batch = RecordBatch::try_new(schema, vec![Arc::new(union)]).unwrap();
4872
4873        let output_batch = deserialize_file(serialize_file(&input_batch));
4874        assert_eq!(input_batch, output_batch);
4875
4876        let output_batch = deserialize_stream(serialize_stream(&input_batch));
4877        assert_eq!(input_batch, output_batch);
4878    }
4879
4880    #[test]
4881    fn test_roundtrip_map_with_dict_keys() {
4882        // Building a map array is a bit involved. We first build a struct arary that has a key and
4883        // value field and then use that to build the actual map array.
4884        let key_values = StringArray::from(vec!["key_a", "key_b", "key_c"]);
4885        let keys = Int32Array::from_iter_values([0, 1, 2, 0, 1, 0]);
4886        let dict_keys = DictionaryArray::new(keys, Arc::new(key_values));
4887
4888        let values = Int32Array::from(vec![1, 2, 3, 4, 5, 6]);
4889
4890        #[expect(deprecated)]
4891        let entries_field = Arc::new(Field::new(
4892            Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
4893            DataType::Struct(
4894                vec![
4895                    Field::new_dict(
4896                        Field::MAP_KEY_FIELD_DEFAULT_NAME,
4897                        DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4898                        false,
4899                        1,
4900                        false,
4901                    ),
4902                    Field::new(Field::MAP_VALUE_FIELD_DEFAULT_NAME, DataType::Int32, true),
4903                ]
4904                .into(),
4905            ),
4906            false,
4907        ));
4908
4909        let entries = StructArray::from(vec![
4910            (
4911                Arc::new(Field::new(
4912                    Field::MAP_KEY_FIELD_DEFAULT_NAME,
4913                    DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4914                    false,
4915                )),
4916                Arc::new(dict_keys) as ArrayRef,
4917            ),
4918            (
4919                Arc::new(Field::new(
4920                    Field::MAP_VALUE_FIELD_DEFAULT_NAME,
4921                    DataType::Int32,
4922                    true,
4923                )),
4924                Arc::new(values) as ArrayRef,
4925            ),
4926        ]);
4927
4928        let offsets = Buffer::from_slice_ref([0i32, 2, 4, 6]);
4929
4930        let map_data = ArrayData::builder(DataType::Map(entries_field, false))
4931            .len(3)
4932            .add_buffer(offsets)
4933            .add_child_data(entries.into_data())
4934            .build()
4935            .unwrap();
4936        let map_array = MapArray::from(map_data);
4937
4938        let schema = Arc::new(Schema::new(vec![Field::new(
4939            "map",
4940            map_array.data_type().clone(),
4941            false,
4942        )]));
4943        let input_batch = RecordBatch::try_new(schema, vec![Arc::new(map_array)]).unwrap();
4944
4945        let output_batch = deserialize_file(serialize_file(&input_batch));
4946        assert_eq!(input_batch, output_batch);
4947
4948        let output_batch = deserialize_stream(serialize_stream(&input_batch));
4949        assert_eq!(input_batch, output_batch);
4950    }
4951
4952    #[test]
4953    fn test_roundtrip_map_with_dict_values() {
4954        // Building a map array is a bit involved. We first build a struct arary that has a key and
4955        // value field and then use that to build the actual map array.
4956        let keys = StringArray::from(vec!["a", "b", "c", "d", "e", "f"]);
4957
4958        let value_values = StringArray::from(vec!["val_x", "val_y", "val_z"]);
4959        let value_keys = Int32Array::from_iter_values([0, 1, 2, 0, 1, 0]);
4960        let dict_values = DictionaryArray::new(value_keys, Arc::new(value_values));
4961
4962        #[expect(deprecated)]
4963        let entries_field = Arc::new(Field::new(
4964            Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
4965            DataType::Struct(
4966                vec![
4967                    Field::new(Field::MAP_KEY_FIELD_DEFAULT_NAME, DataType::Utf8, false),
4968                    Field::new_dict(
4969                        Field::MAP_VALUE_FIELD_DEFAULT_NAME,
4970                        DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4971                        true,
4972                        2,
4973                        false,
4974                    ),
4975                ]
4976                .into(),
4977            ),
4978            false,
4979        ));
4980
4981        let entries = StructArray::from(vec![
4982            (
4983                Arc::new(Field::new(
4984                    Field::MAP_KEY_FIELD_DEFAULT_NAME,
4985                    DataType::Utf8,
4986                    false,
4987                )),
4988                Arc::new(keys) as ArrayRef,
4989            ),
4990            (
4991                Arc::new(Field::new(
4992                    Field::MAP_VALUE_FIELD_DEFAULT_NAME,
4993                    DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
4994                    true,
4995                )),
4996                Arc::new(dict_values) as ArrayRef,
4997            ),
4998        ]);
4999
5000        let offsets = Buffer::from_slice_ref([0i32, 2, 4, 6]);
5001
5002        let map_data = ArrayData::builder(DataType::Map(entries_field, false))
5003            .len(3)
5004            .add_buffer(offsets)
5005            .add_child_data(entries.into_data())
5006            .build()
5007            .unwrap();
5008        let map_array = MapArray::from(map_data);
5009
5010        let schema = Arc::new(Schema::new(vec![Field::new(
5011            "map",
5012            map_array.data_type().clone(),
5013            false,
5014        )]));
5015        let input_batch = RecordBatch::try_new(schema, vec![Arc::new(map_array)]).unwrap();
5016
5017        let output_batch = deserialize_file(serialize_file(&input_batch));
5018        assert_eq!(input_batch, output_batch);
5019
5020        let output_batch = deserialize_stream(serialize_stream(&input_batch));
5021        assert_eq!(input_batch, output_batch);
5022    }
5023
5024    #[test]
5025    fn test_decimal128_alignment16_is_sufficient() {
5026        const IPC_ALIGNMENT: usize = 16;
5027
5028        // Test a bunch of different dimensions to ensure alignment is never an issue.
5029        // For example, if we only test `num_cols = 1` then even with alignment 8 this
5030        // test would _happen_ to pass, even though for different dimensions like
5031        // `num_cols = 2` it would fail.
5032        for num_cols in [1, 2, 3, 17, 50, 73, 99] {
5033            let num_rows = (num_cols * 7 + 11) % 100; // Deterministic swizzle
5034
5035            let mut fields = Vec::new();
5036            let mut arrays = Vec::new();
5037            for i in 0..num_cols {
5038                let field = Field::new(format!("col_{i}"), DataType::Decimal128(38, 10), true);
5039                let array = Decimal128Array::from(vec![num_cols as i128; num_rows]);
5040                fields.push(field);
5041                arrays.push(Arc::new(array) as Arc<dyn Array>);
5042            }
5043            let schema = Schema::new(fields);
5044            let batch = RecordBatch::try_new(Arc::new(schema), arrays).unwrap();
5045
5046            let mut writer = FileWriter::try_new_with_options(
5047                Vec::new(),
5048                batch.schema_ref(),
5049                IpcWriteOptions::try_new(IPC_ALIGNMENT, false, MetadataVersion::V5).unwrap(),
5050            )
5051            .unwrap();
5052            writer.write(&batch).unwrap();
5053            writer.finish().unwrap();
5054
5055            let out: Vec<u8> = writer.into_inner().unwrap();
5056
5057            let buffer = Buffer::from_slice_ref(out);
5058            let trailer_start = buffer.len() - 10;
5059            let footer_len =
5060                read_footer_length(buffer[trailer_start..].try_into().unwrap()).unwrap();
5061            let footer =
5062                root_as_footer(&buffer[trailer_start - footer_len..trailer_start]).unwrap();
5063
5064            let schema = try_fb_to_schema(footer.schema().unwrap()).unwrap();
5065
5066            // Importantly we set `require_alignment`, checking that 16-byte alignment is sufficient
5067            // for `read_record_batch` later on to read the data in a zero-copy manner.
5068            let decoder =
5069                FileDecoder::new(Arc::new(schema), footer.version()).with_require_alignment(true);
5070
5071            let batches = footer.recordBatches().unwrap();
5072
5073            let block = batches.get(0);
5074            let block_len = block.bodyLength() as usize + block.metaDataLength() as usize;
5075            let data = buffer.slice_with_length(block.offset() as _, block_len);
5076
5077            let batch2 = decoder.read_record_batch(block, &data).unwrap().unwrap();
5078
5079            assert_eq!(batch, batch2);
5080        }
5081    }
5082
5083    #[test]
5084    fn test_decimal128_alignment8_is_unaligned() {
5085        const IPC_ALIGNMENT: usize = 8;
5086
5087        let num_cols = 2;
5088
5089        let mut fields = Vec::new();
5090        let mut arrays = Vec::new();
5091        for i in 0..num_cols {
5092            let field = Field::new(format!("col_{i}"), DataType::Decimal128(38, 10), true);
5093            // The null makes the writer emit a 1-byte validity buffer. With IPC_ALIGNMENT = 8,
5094            // the writer pads that buffer to 8 bytes, so the decimal values that follow start
5095            // at offset 8 of the body, which is not a multiple of 16.
5096            let array = Decimal128Array::from(vec![Some(num_cols as i128), None]);
5097            fields.push(field);
5098            arrays.push(Arc::new(array) as Arc<dyn Array>);
5099        }
5100        let schema = Schema::new(fields);
5101        let batch = RecordBatch::try_new(Arc::new(schema), arrays).unwrap();
5102
5103        let mut writer = FileWriter::try_new_with_options(
5104            Vec::new(),
5105            batch.schema_ref(),
5106            IpcWriteOptions::try_new(IPC_ALIGNMENT, false, MetadataVersion::V5).unwrap(),
5107        )
5108        .unwrap();
5109        writer.write(&batch).unwrap();
5110        writer.finish().unwrap();
5111
5112        let out: Vec<u8> = writer.into_inner().unwrap();
5113
5114        let buffer = Buffer::from_slice_ref(out);
5115        let trailer_start = buffer.len() - 10;
5116        let footer_len = read_footer_length(buffer[trailer_start..].try_into().unwrap()).unwrap();
5117        let footer = root_as_footer(&buffer[trailer_start - footer_len..trailer_start]).unwrap();
5118        let schema = try_fb_to_schema(footer.schema().unwrap()).unwrap();
5119
5120        // Importantly we set `require_alignment`, otherwise the error later is suppressed due to copying
5121        // to an aligned buffer in `ArrayDataBuilder.build_aligned`.
5122        let decoder =
5123            FileDecoder::new(Arc::new(schema), footer.version()).with_require_alignment(true);
5124
5125        let batches = footer.recordBatches().unwrap();
5126
5127        let block = batches.get(0);
5128        let block_len = block.bodyLength() as usize + block.metaDataLength() as usize;
5129        let data = buffer.slice_with_length(block.offset() as _, block_len);
5130
5131        let result = decoder.read_record_batch(block, &data);
5132
5133        let error = result.unwrap_err();
5134        assert_eq!(
5135            error.to_string(),
5136            "Invalid argument error: Misaligned buffers[0] in array of type Decimal128(38, 10), \
5137             offset from expected alignment of 16 by 8"
5138        );
5139    }
5140
5141    #[test]
5142    fn test_flush() {
5143        // We write a schema which is small enough to fit into a buffer and not get flushed,
5144        // and then force the write with .flush().
5145        let num_cols = 2;
5146        let mut fields = Vec::new();
5147        let options = IpcWriteOptions::try_new(8, false, MetadataVersion::V5).unwrap();
5148        for i in 0..num_cols {
5149            let field = Field::new(format!("col_{i}"), DataType::Decimal128(38, 10), true);
5150            fields.push(field);
5151        }
5152        let schema = Schema::new(fields);
5153        let inner_stream_writer = BufWriter::with_capacity(1024, Vec::new());
5154        let inner_file_writer = BufWriter::with_capacity(1024, Vec::new());
5155        let mut stream_writer =
5156            StreamWriter::try_new_with_options(inner_stream_writer, &schema, options.clone())
5157                .unwrap();
5158        let mut file_writer =
5159            FileWriter::try_new_with_options(inner_file_writer, &schema, options).unwrap();
5160
5161        let stream_bytes_written_on_new = stream_writer.get_ref().get_ref().len();
5162        let file_bytes_written_on_new = file_writer.get_ref().get_ref().len();
5163        stream_writer.flush().unwrap();
5164        file_writer.flush().unwrap();
5165        let stream_bytes_written_on_flush = stream_writer.get_ref().get_ref().len();
5166        let file_bytes_written_on_flush = file_writer.get_ref().get_ref().len();
5167        let stream_out = stream_writer.into_inner().unwrap().into_inner().unwrap();
5168        // Finishing a stream writes the continuation bytes in MetadataVersion::V5 (4 bytes)
5169        // and then a length of 0 (4 bytes) for a total of 8 bytes.
5170        // Everything before that should have been flushed in the .flush() call.
5171        let expected_stream_flushed_bytes = stream_out.len() - 8;
5172        // A file write is the same as the stream write except for the leading magic string
5173        // ARROW1 plus padding, which is 8 bytes.
5174        let expected_file_flushed_bytes = expected_stream_flushed_bytes + 8;
5175
5176        assert!(
5177            stream_bytes_written_on_new < stream_bytes_written_on_flush,
5178            "this test makes no sense if flush is not actually required"
5179        );
5180        assert!(
5181            file_bytes_written_on_new < file_bytes_written_on_flush,
5182            "this test makes no sense if flush is not actually required"
5183        );
5184        assert_eq!(stream_bytes_written_on_flush, expected_stream_flushed_bytes);
5185        assert_eq!(file_bytes_written_on_flush, expected_file_flushed_bytes);
5186    }
5187
5188    #[test]
5189    fn test_roundtrip_list_of_fixed_list() -> Result<(), ArrowError> {
5190        let l1_type =
5191            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, false)), 3);
5192        let l2_type = DataType::List(Arc::new(Field::new("item", l1_type.clone(), false)));
5193
5194        let l0_builder = Float32Builder::new();
5195        let l1_builder = FixedSizeListBuilder::new(l0_builder, 3).with_field(Arc::new(Field::new(
5196            "item",
5197            DataType::Float32,
5198            false,
5199        )));
5200        let mut l2_builder =
5201            ListBuilder::new(l1_builder).with_field(Arc::new(Field::new("item", l1_type, false)));
5202
5203        for point in [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]] {
5204            l2_builder.values().values().append_value(point[0]);
5205            l2_builder.values().values().append_value(point[1]);
5206            l2_builder.values().values().append_value(point[2]);
5207
5208            l2_builder.values().append(true);
5209        }
5210        l2_builder.append(true);
5211
5212        let point = [10., 11., 12.];
5213        l2_builder.values().values().append_value(point[0]);
5214        l2_builder.values().values().append_value(point[1]);
5215        l2_builder.values().values().append_value(point[2]);
5216
5217        l2_builder.values().append(true);
5218        l2_builder.append(true);
5219
5220        let array = Arc::new(l2_builder.finish()) as ArrayRef;
5221
5222        let schema = Arc::new(Schema::new_with_metadata(
5223            vec![Field::new("points", l2_type, false)],
5224            HashMap::default(),
5225        ));
5226
5227        // Test a variety of combinations that include 0 and non-zero offsets
5228        // and also portions or the rest of the array
5229        test_slices(&array, &schema, 0, 1)?;
5230        test_slices(&array, &schema, 0, 2)?;
5231        test_slices(&array, &schema, 1, 1)?;
5232
5233        Ok(())
5234    }
5235
5236    #[test]
5237    fn test_roundtrip_list_of_fixed_list_w_nulls() -> Result<(), ArrowError> {
5238        let l0_builder = Float32Builder::new();
5239        let l1_builder = FixedSizeListBuilder::new(l0_builder, 3);
5240        let mut l2_builder = ListBuilder::new(l1_builder);
5241
5242        for point in [
5243            [Some(1.0), Some(2.0), None],
5244            [Some(4.0), Some(5.0), Some(6.0)],
5245            [None, Some(8.0), Some(9.0)],
5246        ] {
5247            for p in point {
5248                match p {
5249                    Some(p) => l2_builder.values().values().append_value(p),
5250                    None => l2_builder.values().values().append_null(),
5251                }
5252            }
5253
5254            l2_builder.values().append(true);
5255        }
5256        l2_builder.append(true);
5257
5258        let point = [Some(10.), None, None];
5259        for p in point {
5260            match p {
5261                Some(p) => l2_builder.values().values().append_value(p),
5262                None => l2_builder.values().values().append_null(),
5263            }
5264        }
5265
5266        l2_builder.values().append(true);
5267        l2_builder.append(true);
5268
5269        let array = Arc::new(l2_builder.finish()) as ArrayRef;
5270
5271        let schema = Arc::new(Schema::new_with_metadata(
5272            vec![Field::new(
5273                "points",
5274                DataType::List(Arc::new(Field::new(
5275                    "item",
5276                    DataType::FixedSizeList(
5277                        Arc::new(Field::new("item", DataType::Float32, true)),
5278                        3,
5279                    ),
5280                    true,
5281                ))),
5282                true,
5283            )],
5284            HashMap::default(),
5285        ));
5286
5287        // Test a variety of combinations that include 0 and non-zero offsets
5288        // and also portions or the rest of the array
5289        test_slices(&array, &schema, 0, 1)?;
5290        test_slices(&array, &schema, 0, 2)?;
5291        test_slices(&array, &schema, 1, 1)?;
5292
5293        Ok(())
5294    }
5295
5296    fn test_slices(
5297        parent_array: &ArrayRef,
5298        schema: &SchemaRef,
5299        offset: usize,
5300        length: usize,
5301    ) -> Result<(), ArrowError> {
5302        let subarray = parent_array.slice(offset, length);
5303        let original_batch = RecordBatch::try_new(schema.clone(), vec![subarray])?;
5304
5305        let mut bytes = Vec::new();
5306        let mut writer = StreamWriter::try_new(&mut bytes, schema)?;
5307        writer.write(&original_batch)?;
5308        writer.finish()?;
5309
5310        let mut cursor = std::io::Cursor::new(bytes);
5311        let mut reader = StreamReader::try_new(&mut cursor, None)?;
5312        let returned_batch = reader.next().unwrap()?;
5313
5314        assert_eq!(original_batch, returned_batch);
5315
5316        Ok(())
5317    }
5318
5319    #[test]
5320    fn test_roundtrip_fixed_list() -> Result<(), ArrowError> {
5321        let int_builder = Int64Builder::new();
5322        let mut fixed_list_builder = FixedSizeListBuilder::new(int_builder, 3)
5323            .with_field(Arc::new(Field::new("item", DataType::Int64, false)));
5324
5325        for point in [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]] {
5326            fixed_list_builder.values().append_value(point[0]);
5327            fixed_list_builder.values().append_value(point[1]);
5328            fixed_list_builder.values().append_value(point[2]);
5329
5330            fixed_list_builder.append(true);
5331        }
5332
5333        let array = Arc::new(fixed_list_builder.finish()) as ArrayRef;
5334
5335        let schema = Arc::new(Schema::new_with_metadata(
5336            vec![Field::new(
5337                "points",
5338                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Int64, false)), 3),
5339                false,
5340            )],
5341            HashMap::default(),
5342        ));
5343
5344        // Test a variety of combinations that include 0 and non-zero offsets
5345        // and also portions or the rest of the array
5346        test_slices(&array, &schema, 0, 4)?;
5347        test_slices(&array, &schema, 0, 2)?;
5348        test_slices(&array, &schema, 1, 3)?;
5349        test_slices(&array, &schema, 2, 1)?;
5350
5351        Ok(())
5352    }
5353
5354    #[test]
5355    fn test_roundtrip_fixed_list_w_nulls() -> Result<(), ArrowError> {
5356        let int_builder = Int64Builder::new();
5357        let mut fixed_list_builder = FixedSizeListBuilder::new(int_builder, 3);
5358
5359        for point in [
5360            [Some(1), Some(2), None],
5361            [Some(4), Some(5), Some(6)],
5362            [None, Some(8), Some(9)],
5363            [Some(10), None, None],
5364        ] {
5365            for p in point {
5366                match p {
5367                    Some(p) => fixed_list_builder.values().append_value(p),
5368                    None => fixed_list_builder.values().append_null(),
5369                }
5370            }
5371
5372            fixed_list_builder.append(true);
5373        }
5374
5375        let array = Arc::new(fixed_list_builder.finish()) as ArrayRef;
5376
5377        let schema = Arc::new(Schema::new_with_metadata(
5378            vec![Field::new(
5379                "points",
5380                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Int64, true)), 3),
5381                true,
5382            )],
5383            HashMap::default(),
5384        ));
5385
5386        // Test a variety of combinations that include 0 and non-zero offsets
5387        // and also portions or the rest of the array
5388        test_slices(&array, &schema, 0, 4)?;
5389        test_slices(&array, &schema, 0, 2)?;
5390        test_slices(&array, &schema, 1, 3)?;
5391        test_slices(&array, &schema, 2, 1)?;
5392
5393        Ok(())
5394    }
5395
5396    #[test]
5397    fn test_metadata_encoding_ordering() {
5398        fn create_hash() -> u64 {
5399            let metadata: HashMap<String, String> = [
5400                ("a", "1"), //
5401                ("b", "2"), //
5402                ("c", "3"), //
5403                ("d", "4"), //
5404                ("e", "5"), //
5405            ]
5406            .into_iter()
5407            .map(|(k, v)| (k.to_owned(), v.to_owned()))
5408            .collect();
5409
5410            // Set metadata on both the schema and a field within it.
5411            let schema = Arc::new(
5412                Schema::new(vec![
5413                    Field::new("a", DataType::Int64, true).with_metadata(metadata.clone()),
5414                ])
5415                .with_metadata(metadata)
5416                .clone(),
5417            );
5418            let batch = RecordBatch::new_empty(schema.clone());
5419
5420            let mut bytes = Vec::new();
5421            let mut w = StreamWriter::try_new(&mut bytes, batch.schema_ref()).unwrap();
5422            w.write(&batch).unwrap();
5423            w.finish().unwrap();
5424
5425            let mut h = std::hash::DefaultHasher::new();
5426            h.write(&bytes);
5427            h.finish()
5428        }
5429
5430        let expected = create_hash();
5431
5432        // Since there is randomness in the HashMap and we cannot specify our
5433        // own Hasher for the implementation used for metadata, run the above
5434        // code 20x and verify it does not change. This is not perfect but it
5435        // should be good enough.
5436        let all_passed = (0..20).all(|_| create_hash() == expected);
5437        assert!(all_passed);
5438    }
5439
5440    #[test]
5441    fn test_dictionary_tracker_reset() {
5442        let data_gen = IpcDataGenerator::default();
5443        let mut dictionary_tracker = DictionaryTracker::new(false);
5444        let writer_options = IpcWriteOptions::default();
5445        let mut compression_ctx = IpcWriteContext::default();
5446
5447        let schema = Arc::new(Schema::new(vec![Field::new(
5448            "a",
5449            DataType::Dictionary(Box::new(DataType::UInt8), Box::new(DataType::Utf8)),
5450            false,
5451        )]));
5452
5453        let mut write_single_batch_stream =
5454            |batch: RecordBatch, dict_tracker: &mut DictionaryTracker| -> Vec<u8> {
5455                let mut buffer = Vec::new();
5456
5457                // create a new IPC stream:
5458                let stream_header = data_gen.schema_to_bytes_with_dictionary_tracker(
5459                    &schema,
5460                    dict_tracker,
5461                    &writer_options,
5462                );
5463                _ = write_message(&mut buffer, stream_header, &writer_options).unwrap();
5464
5465                let (encoded_dicts, encoded_batch) = data_gen
5466                    .encode(&batch, dict_tracker, &writer_options, &mut compression_ctx)
5467                    .unwrap();
5468                for encoded_dict in encoded_dicts {
5469                    _ = write_message(&mut buffer, encoded_dict, &writer_options).unwrap();
5470                }
5471                _ = write_message(&mut buffer, encoded_batch, &writer_options).unwrap();
5472
5473                buffer
5474            };
5475
5476        let batch1 = RecordBatch::try_new(
5477            schema.clone(),
5478            vec![Arc::new(DictionaryArray::new(
5479                UInt8Array::from_iter_values([0]),
5480                Arc::new(StringArray::from_iter_values(["a"])),
5481            ))],
5482        )
5483        .unwrap();
5484        let buffer = write_single_batch_stream(batch1.clone(), &mut dictionary_tracker);
5485
5486        // ensure we can read the stream back
5487        let mut reader = StreamReader::try_new(Cursor::new(buffer), None).unwrap();
5488        let read_batch = reader.next().unwrap().unwrap();
5489        assert_eq!(read_batch, batch1);
5490
5491        // reset the dictionary tracker so it can be used for next stream
5492        dictionary_tracker.clear();
5493
5494        // now write a 2nd stream and ensure we can also read it:
5495        let batch2 = RecordBatch::try_new(
5496            schema.clone(),
5497            vec![Arc::new(DictionaryArray::new(
5498                UInt8Array::from_iter_values([0]),
5499                Arc::new(StringArray::from_iter_values(["a"])),
5500            ))],
5501        )
5502        .unwrap();
5503        let buffer = write_single_batch_stream(batch2.clone(), &mut dictionary_tracker);
5504        let mut reader = StreamReader::try_new(Cursor::new(buffer), None).unwrap();
5505        let read_batch = reader.next().unwrap().unwrap();
5506        assert_eq!(read_batch, batch2);
5507    }
5508}