Skip to main content

arrow_avro/writer/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Avro writer implementation for the `arrow-avro` crate.
6//!
7//! # Overview
8//!
9//! Use this module to serialize Arrow [`arrow_array::RecordBatch`] values into Avro. Three output
10//! modes are supported:
11//!
12//! * **[`crate::writer::AvroWriter`]** — writes an **Object Container File (OCF)**: a self‑describing
13//!   file with header (schema JSON and metadata), optional compression, data blocks, and
14//!   sync markers. See Avro 1.11.1 "Object Container Files."
15//!   <https://avro.apache.org/docs/1.11.1/specification/#object-container-files>
16//!
17//! * **[`crate::writer::AvroStreamWriter`]** — writes a **Single Object Encoding (SOE) Stream** without
18//!   any container framing. This is useful when the schema is known out‑of‑band (i.e.,
19//!   via a registry) and you want minimal overhead.
20//!
21//! * **[`crate::writer::Encoder`]** — a row-by-row encoder that buffers encoded records into a single
22//!   contiguous byte buffer and returns per-row [`bytes::Bytes`] slices.
23//!   Ideal for publishing individual messages to Kafka, Pulsar, or other message queues
24//!   where each message must be a self-contained Avro payload.
25//!
26//! ## Which writer should you use?
27//!
28//! | Use Case | Recommended Type |
29//! |----------|------------------|
30//! | Write an OCF file to disk | [`crate::writer::AvroWriter`] |
31//! | Stream records continuously to a file/socket | [`crate::writer::AvroStreamWriter`] |
32//! | Publish individual records to Kafka/Pulsar | [`crate::writer::Encoder`] |
33//! | Need per-row byte slices for custom framing | [`crate::writer::Encoder`] |
34//!
35//! ## Per-Record Prefix Formats
36//!
37//! For [`crate::writer::AvroStreamWriter`] and [`crate::writer::Encoder`], each record is automatically prefixed
38//! based on the fingerprint strategy:
39//!
40//! | Strategy | Prefix | Use Case |
41//! |----------|--------|----------|
42//! | `FingerprintStrategy::Rabin` (default) | `0xC3 0x01` + 8-byte LE Rabin fingerprint | Standard Avro SOE |
43//! | `FingerprintStrategy::Id(id)` | `0x00` + 4-byte BE schema ID | [Confluent Schema Registry] |
44//! | `FingerprintStrategy::Id64(id)` | `0x00` + 8-byte BE schema ID | [Apicurio Registry] |
45//!
46//! [Confluent Schema Registry]: https://docs.confluent.io/platform/current/schema-registry/fundamentals/serdes-develop/index.html#wire-format
47//! [Apicurio Registry]: https://www.apicur.io/registry/docs/apicurio-registry/1.3.3.Final/getting-started/assembly-using-kafka-client-serdes.html#registry-serdes-types-avro-registry
48//!
49//! ## Choosing the Avro Schema
50//!
51//! By default, the writer converts your Arrow schema to Avro (including a top‑level record
52//! name). If you already have an Avro schema JSON you want to use verbatim, put it into the
53//! Arrow schema metadata under the [`SCHEMA_METADATA_KEY`]
54//! key before constructing the writer. The builder will use that schema instead of generating
55//! a new one.
56//!
57//! ## Compression
58//!
59//! For OCF ([`crate::writer::AvroWriter`]), you may enable a compression codec via
60//! [`crate::writer::WriterBuilder::with_compression`]. The chosen codec is written into the file header
61//! and used for subsequent blocks. SOE stream writing ([`crate::writer::AvroStreamWriter`], [`crate::writer::Encoder`])
62//! does not apply container‑level compression.
63//!
64//! # Examples
65//!
66//! ## Writing an OCF File
67//!
68//! ```
69//! use std::sync::Arc;
70//! use arrow_array::{ArrayRef, Int64Array, StringArray, RecordBatch};
71//! use arrow_schema::{DataType, Field, Schema};
72//! use arrow_avro::writer::AvroWriter;
73//!
74//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
75//! let schema = Schema::new(vec![
76//!     Field::new("id", DataType::Int64, false),
77//!     Field::new("name", DataType::Utf8, false),
78//! ]);
79//!
80//! let batch = RecordBatch::try_new(
81//!     Arc::new(schema.clone()),
82//!     vec![
83//!         Arc::new(Int64Array::from(vec![1, 2])) as ArrayRef,
84//!         Arc::new(StringArray::from(vec!["alice", "bob"])) as ArrayRef,
85//!     ],
86//! )?;
87//!
88//! let mut writer = AvroWriter::new(Vec::<u8>::new(), schema)?;
89//! writer.write(&batch)?;
90//! writer.finish()?;
91//! let bytes = writer.into_inner();
92//! assert!(!bytes.is_empty());
93//! # Ok(())
94//! # }
95//! ```
96//!
97//! ## Using the Row-by-Row Encoder for Message Queues
98//!
99//! ```
100//! use std::sync::Arc;
101//! use arrow_array::{ArrayRef, Int32Array, RecordBatch};
102//! use arrow_schema::{DataType, Field, Schema};
103//! use arrow_avro::writer::{WriterBuilder, format::AvroSoeFormat};
104//! use arrow_avro::schema::FingerprintStrategy;
105//!
106//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
107//! let schema = Schema::new(vec![Field::new("x", DataType::Int32, false)]);
108//! let batch = RecordBatch::try_new(
109//!     Arc::new(schema.clone()),
110//!     vec![Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef],
111//! )?;
112//!
113//! // Build an Encoder with Confluent wire format (schema ID = 42)
114//! let mut encoder = WriterBuilder::new(schema)
115//!     .with_fingerprint_strategy(FingerprintStrategy::Id(42))
116//!     .build_encoder::<AvroSoeFormat>()?;
117//!
118//! encoder.encode(&batch)?;
119//!
120//! // Get the buffered rows (zero-copy views into a single backing buffer)
121//! let rows = encoder.flush();
122//! assert_eq!(rows.len(), 3);
123//!
124//! // Each row has Confluent wire format: magic byte + 4-byte schema ID + body
125//! for row in rows.iter() {
126//!     assert_eq!(row[0], 0x00); // Confluent magic byte
127//! }
128//! # Ok(())
129//! # }
130//! ```
131//!
132//! ---
133use crate::codec::AvroFieldBuilder;
134use crate::compression::CompressionCodec;
135use crate::errors::AvroError;
136use crate::schema::{
137    AvroSchema, Fingerprint, FingerprintAlgorithm, FingerprintStrategy, SCHEMA_METADATA_KEY,
138};
139use crate::writer::encoder::{RecordEncoder, RecordEncoderBuilder, write_long};
140use crate::writer::format::{AvroFormat, AvroOcfFormat, AvroSoeFormat};
141use arrow_array::RecordBatch;
142use arrow_schema::{Schema, SchemaRef};
143use bytes::{Bytes, BytesMut};
144use std::io::Write;
145use std::sync::Arc;
146
147/// Encodes `RecordBatch` into the Avro binary format.
148mod encoder;
149/// Logic for different Avro container file formats.
150pub mod format;
151
152/// A contiguous set of Avro encoded rows.
153///
154/// `EncodedRows` stores:
155/// - a single backing byte buffer (`bytes::Bytes`)
156/// - a `Vec<usize>` of row boundary offsets (length = `rows + 1`)
157///
158/// This lets callers get per-row payloads as zero-copy `Bytes` slices.
159///
160/// For compatibility with APIs that require owned `Vec<u8>`, use:
161/// `let vecs: Vec<Vec<u8>> = rows.iter().map(|b| b.to_vec()).collect();`
162#[derive(Debug, Clone)]
163pub struct EncodedRows {
164    data: Bytes,
165    offsets: Vec<usize>,
166}
167
168impl EncodedRows {
169    /// Create a new `EncodedRows` from a backing buffer and row boundary offsets.
170    ///
171    /// `offsets` must have length `rows + 1`, and be monotonically non-decreasing.
172    /// The last offset should equal `data.len()`.
173    pub fn new(data: Bytes, offsets: Vec<usize>) -> Self {
174        Self { data, offsets }
175    }
176
177    /// Returns the number of encoded rows stored in this container.
178    #[inline]
179    pub fn len(&self) -> usize {
180        self.offsets.len().saturating_sub(1)
181    }
182
183    /// Returns `true` if this container holds no encoded rows.
184    #[inline]
185    pub fn is_empty(&self) -> bool {
186        self.len() == 0
187    }
188
189    /// Returns a reference to the single contiguous backing buffer.
190    ///
191    /// This buffer contains the payloads of all rows concatenated together.
192    ///
193    /// # Note
194    ///
195    /// To access individual row payloads, prefer using [`Self::row`] or [`Self::iter`]
196    /// rather than slicing this buffer manually.
197    #[inline]
198    pub fn bytes(&self) -> &Bytes {
199        &self.data
200    }
201
202    /// Returns the row boundary offsets.
203    ///
204    /// The returned slice always has the length `self.len() + 1`. The `n`th row payload
205    /// corresponds to `bytes[offsets[n] ... offsets[n+1]]`.
206    #[inline]
207    pub fn offsets(&self) -> &[usize] {
208        &self.offsets
209    }
210
211    /// Return the `n`th row as a zero-copy `Bytes` slice.
212    ///
213    /// # Errors
214    ///
215    /// Returns an error if `n` is out of bounds or if the internal offsets are invalid
216    /// (e.g., offsets are not within the backing buffer).
217    ///
218    /// # Examples
219    ///
220    /// ```
221    /// use std::sync::Arc;
222    /// use arrow_array::{ArrayRef, Int32Array, RecordBatch};
223    /// use arrow_schema::{DataType, Field, Schema};
224    /// use arrow_avro::writer::WriterBuilder;
225    /// use arrow_avro::writer::format::AvroSoeFormat;
226    ///
227    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
228    /// let schema = Schema::new(vec![Field::new("x", DataType::Int32, false)]);
229    /// let batch = RecordBatch::try_new(
230    ///     Arc::new(schema.clone()),
231    ///     vec![Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef],
232    /// )?;
233    ///
234    /// let mut encoder = WriterBuilder::new(schema).build_encoder::<AvroSoeFormat>()?;
235    /// encoder.encode(&batch)?;
236    /// let rows = encoder.flush();
237    ///
238    /// assert_eq!(rows.iter().count(), 2);
239    /// # Ok(())
240    /// # }
241    /// ```
242    pub fn row(&self, n: usize) -> Result<Bytes, AvroError> {
243        if n >= self.len() {
244            return Err(AvroError::General(format!(
245                "Row index {n} out of bounds for len {}",
246                self.len()
247            )));
248        }
249        // SAFETY:
250        // self.len() is defined as self.offsets.len().saturating_sub(1).
251        // The check `n >= self.len()` above ensures that `n < self.offsets.len() - 1`.
252        // Therefore, both `n` and `n + 1` are strictly within the bounds of `self.offsets`.
253        let (start, end) = unsafe {
254            (
255                *self.offsets.get_unchecked(n),
256                *self.offsets.get_unchecked(n + 1),
257            )
258        };
259        if start > end || end > self.data.len() {
260            return Err(AvroError::General(format!(
261                "Invalid row offsets for row {n}: start={start}, end={end}, data_len={}",
262                self.data.len()
263            )));
264        }
265        Ok(self.data.slice(start..end))
266    }
267
268    /// Iterate over rows as zero-copy `Bytes` slices.
269    ///
270    /// This iterator is infallible and is intended for the common case where
271    /// `EncodedRows` is produced by [`Encoder::flush`], which guarantees valid offsets.
272    ///
273    /// # Examples
274    ///
275    /// ```
276    /// use std::sync::Arc;
277    /// use arrow_array::{ArrayRef, Int32Array, RecordBatch};
278    /// use arrow_schema::{DataType, Field, Schema};
279    /// use arrow_avro::writer::WriterBuilder;
280    /// use arrow_avro::writer::format::AvroSoeFormat;
281    ///
282    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
283    /// let schema = Schema::new(vec![Field::new("x", DataType::Int32, false)]);
284    /// let batch = RecordBatch::try_new(
285    ///     Arc::new(schema.clone()),
286    ///     vec![Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef],
287    /// )?;
288    ///
289    /// let mut encoder = WriterBuilder::new(schema).build_encoder::<AvroSoeFormat>()?;
290    /// encoder.encode(&batch)?;
291    /// let rows = encoder.flush();
292    ///
293    /// assert_eq!(rows.iter().count(), 2);
294    /// # Ok(())
295    /// # }
296    /// ```
297    #[inline]
298    pub fn iter(&self) -> impl ExactSizeIterator<Item = Bytes> + '_ {
299        self.offsets.windows(2).map(|w| self.data.slice(w[0]..w[1]))
300    }
301}
302
303/// Builder to configure and create a `Writer`.
304#[derive(Debug, Clone)]
305pub struct WriterBuilder {
306    schema: Schema,
307    codec: Option<CompressionCodec>,
308    row_capacity: Option<usize>,
309    capacity: usize,
310    fingerprint_strategy: Option<FingerprintStrategy>,
311}
312
313impl WriterBuilder {
314    /// Create a new builder with default settings.
315    ///
316    /// The Avro schema used for writing is determined as follows:
317    /// 1) If the Arrow schema metadata contains `avro::schema` (see `SCHEMA_METADATA_KEY`),
318    ///    that JSON is used verbatim.
319    /// 2) Otherwise, the Arrow schema is converted to an Avro record schema.
320    pub fn new(schema: Schema) -> Self {
321        Self {
322            schema,
323            codec: None,
324            row_capacity: None,
325            capacity: 1024,
326            fingerprint_strategy: None,
327        }
328    }
329
330    /// Set the fingerprinting strategy for the stream writer.
331    /// This determines the per-record prefix format.
332    pub fn with_fingerprint_strategy(mut self, strategy: FingerprintStrategy) -> Self {
333        self.fingerprint_strategy = Some(strategy);
334        self
335    }
336
337    /// Change the compression codec.
338    pub fn with_compression(mut self, codec: Option<CompressionCodec>) -> Self {
339        self.codec = codec;
340        self
341    }
342
343    /// Sets the expected capacity (in bytes) for internal buffers.
344    ///
345    /// This is used as a hint to pre-allocate staging buffers for writing.
346    pub fn with_capacity(mut self, capacity: usize) -> Self {
347        self.capacity = capacity;
348        self
349    }
350
351    /// Sets the expected byte size for each encoded row.
352    ///
353    /// This setting affects [`Encoder`] created via [`build_encoder`](Self::build_encoder).
354    /// It is used as a hint to reduce reallocations when the typical encoded row size is known.
355    pub fn with_row_capacity(mut self, capacity: usize) -> Self {
356        self.row_capacity = Some(capacity);
357        self
358    }
359
360    fn prepare_encoder<F: AvroFormat>(&self) -> Result<(Arc<Schema>, RecordEncoder), AvroError> {
361        let avro_schema = match self.schema.metadata.get(SCHEMA_METADATA_KEY) {
362            Some(json) => AvroSchema::new(json.clone()),
363            None => AvroSchema::try_from(&self.schema)?,
364        };
365        let maybe_fingerprint = if F::NEEDS_PREFIX {
366            match &self.fingerprint_strategy {
367                Some(FingerprintStrategy::Id(id)) => Some(Fingerprint::Id(*id)),
368                Some(FingerprintStrategy::Id64(id)) => Some(Fingerprint::Id64(*id)),
369                Some(strategy) => {
370                    Some(avro_schema.fingerprint(FingerprintAlgorithm::from(*strategy))?)
371                }
372                None => Some(
373                    avro_schema
374                        .fingerprint(FingerprintAlgorithm::from(FingerprintStrategy::Rabin))?,
375                ),
376            }
377        } else {
378            None
379        };
380        let mut md = self.schema.metadata().clone();
381        md.insert(
382            SCHEMA_METADATA_KEY.to_string(),
383            avro_schema.clone().json_string,
384        );
385        let schema = Arc::new(Schema::new_with_metadata(self.schema.fields().clone(), md));
386        let avro_root = AvroFieldBuilder::new(&avro_schema.schema()?).build()?;
387        let encoder = RecordEncoderBuilder::new(&avro_root, schema.as_ref())
388            .with_fingerprint(maybe_fingerprint)
389            .build()?;
390        Ok((schema, encoder))
391    }
392
393    /// Build a new [`Encoder`] for the given [`AvroFormat`].
394    ///
395    /// `Encoder` only supports stream formats (no OCF sync markers). Attempting to build an
396    /// encoder with an OCF format (e.g. [`AvroOcfFormat`]) will return an error.
397    pub fn build_encoder<F: AvroFormat>(self) -> Result<Encoder, AvroError> {
398        if F::default().sync_marker().is_some() {
399            return Err(AvroError::InvalidArgument(
400                "Encoder only supports stream formats (no OCF header/sync marker)".to_string(),
401            ));
402        }
403        let (schema, encoder) = self.prepare_encoder::<F>()?;
404        Ok(Encoder {
405            schema,
406            encoder,
407            row_capacity: self.row_capacity,
408            buffer: BytesMut::with_capacity(self.capacity),
409            offsets: vec![0],
410        })
411    }
412
413    /// Build a new [`Writer`] with the specified [`AvroFormat`] and builder options.
414    pub fn build<W, F>(self, mut writer: W) -> Result<Writer<W, F>, AvroError>
415    where
416        W: Write,
417        F: AvroFormat,
418    {
419        let mut format = F::default();
420        if format.sync_marker().is_none() && !F::NEEDS_PREFIX {
421            return Err(AvroError::InvalidArgument(
422                "AvroBinaryFormat is only supported with Encoder, use build_encoder instead"
423                    .to_string(),
424            ));
425        }
426        let (schema, encoder) = self.prepare_encoder::<F>()?;
427        format.start_stream(&mut writer, &schema, self.codec)?;
428        Ok(Writer {
429            writer,
430            schema,
431            format,
432            compression: self.codec,
433            capacity: self.capacity,
434            encoder,
435        })
436    }
437}
438
439/// A row-by-row encoder for Avro *stream/message* formats (SOE / registry wire formats / raw binary).
440///
441/// Unlike [`Writer`], which emits a single continuous byte stream to a [`std::io::Write`] sink,
442/// `Encoder` tracks row boundaries during encoding and returns an [`EncodedRows`] containing:
443/// - one backing buffer (`Bytes`)
444/// - row boundary offsets
445///
446/// This enables zero-copy per-row payloads (for instance, one Kafka message per Arrow row) without
447/// re-encoding or decoding the byte stream to recover record boundaries.
448///
449/// ### Example
450///
451/// ```
452/// use std::sync::Arc;
453/// use arrow_array::{ArrayRef, Int32Array, RecordBatch};
454/// use arrow_schema::{DataType, Field, Schema};
455/// use arrow_avro::writer::{WriterBuilder, format::AvroSoeFormat};
456/// use arrow_avro::schema::FingerprintStrategy;
457///
458/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
459/// let schema = Schema::new(vec![Field::new("value", DataType::Int32, false)]);
460/// let batch = RecordBatch::try_new(
461///     Arc::new(schema.clone()),
462///     vec![Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef],
463/// )?;
464///
465/// // Configure the encoder (here: Confluent Wire Format with schema ID 100)
466/// let mut encoder = WriterBuilder::new(schema)
467///     .with_fingerprint_strategy(FingerprintStrategy::Id(100))
468///     .build_encoder::<AvroSoeFormat>()?;
469///
470/// // Encode the batch
471/// encoder.encode(&batch)?;
472///
473/// // Get the encoded rows
474/// let rows = encoder.flush();
475///
476/// // Convert to owned Vec<u8> payloads (e.g., for a Kafka producer)
477/// let payloads: Vec<Vec<u8>> = rows.iter().map(|row| row.to_vec()).collect();
478///
479/// assert_eq!(payloads.len(), 3);
480/// assert_eq!(payloads[0][0], 0x00); // Magic byte
481/// # Ok(())
482/// # }
483/// ```
484#[derive(Debug)]
485pub struct Encoder {
486    schema: SchemaRef,
487    encoder: RecordEncoder,
488    row_capacity: Option<usize>,
489    buffer: BytesMut,
490    offsets: Vec<usize>,
491}
492
493impl Encoder {
494    /// Serialize one [`RecordBatch`] into the internal buffer.
495    pub fn encode(&mut self, batch: &RecordBatch) -> Result<(), AvroError> {
496        if batch.schema().fields() != self.schema.fields() {
497            return Err(AvroError::SchemaError(
498                "Schema of RecordBatch differs from Writer schema".to_string(),
499            ));
500        }
501        self.encoder.encode_rows(
502            batch,
503            self.row_capacity.unwrap_or(0),
504            &mut self.buffer,
505            &mut self.offsets,
506        )?;
507        Ok(())
508    }
509
510    /// A convenience method to write a slice of [`RecordBatch`] values.
511    pub fn encode_batches(&mut self, batches: &[RecordBatch]) -> Result<(), AvroError> {
512        for b in batches {
513            self.encode(b)?;
514        }
515        Ok(())
516    }
517
518    /// Drain and return all currently buffered encoded rows.
519    ///
520    /// The returned [`EncodedRows`] provides per-row payloads as `Bytes` slices.
521    pub fn flush(&mut self) -> EncodedRows {
522        let data = self.buffer.split().freeze();
523        let mut offsets = Vec::with_capacity(self.offsets.len());
524        offsets.append(&mut self.offsets);
525        self.offsets.push(0);
526        EncodedRows::new(data, offsets)
527    }
528
529    /// Returns the Arrow schema used by this encoder.
530    ///
531    /// The returned schema includes metadata with the Avro schema JSON under
532    /// the `avro.schema` key.
533    pub fn schema(&self) -> SchemaRef {
534        self.schema.clone()
535    }
536
537    /// Returns the number of encoded rows currently buffered.
538    pub fn buffered_len(&self) -> usize {
539        self.offsets.len().saturating_sub(1)
540    }
541}
542
543/// Generic Avro writer.
544///
545/// This type is generic over the output Write sink (`W`) and the Avro format (`F`).
546/// You’ll usually use the concrete aliases:
547///
548/// * **[`AvroWriter`]** for **OCF** (self‑describing container file)
549/// * **[`AvroStreamWriter`]** for **SOE** Avro streams
550#[derive(Debug)]
551pub struct Writer<W: Write, F: AvroFormat> {
552    writer: W,
553    schema: SchemaRef,
554    format: F,
555    compression: Option<CompressionCodec>,
556    capacity: usize,
557    encoder: RecordEncoder,
558}
559
560/// Alias for an Avro **Object Container File** writer.
561///
562/// ### Quickstart (runnable)
563///
564/// ```
565/// use std::io::Cursor;
566/// use std::sync::Arc;
567/// use arrow_array::{ArrayRef, Int64Array, StringArray, RecordBatch};
568/// use arrow_schema::{DataType, Field, Schema};
569/// use arrow_avro::writer::AvroWriter;
570/// use arrow_avro::reader::ReaderBuilder;
571///
572/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
573/// // Writer schema: { id: long, name: string }
574/// let writer_schema = Schema::new(vec![
575///     Field::new("id", DataType::Int64, false),
576///     Field::new("name", DataType::Utf8, false),
577/// ]);
578///
579/// // Build a RecordBatch with two rows
580/// let batch = RecordBatch::try_new(
581///     Arc::new(writer_schema.clone()),
582///     vec![
583///         Arc::new(Int64Array::from(vec![1, 2])) as ArrayRef,
584///         Arc::new(StringArray::from(vec!["a", "b"])) as ArrayRef,
585///     ],
586/// )?;
587///
588/// // Write an Avro **Object Container File** (OCF) to memory
589/// let mut w = AvroWriter::new(Vec::<u8>::new(), writer_schema.clone())?;
590/// w.write(&batch)?;
591/// w.finish()?;
592/// let bytes = w.into_inner();
593///
594/// // Build a Reader and decode the batch back
595/// let mut r = ReaderBuilder::new().build(Cursor::new(bytes))?;
596/// let out = r.next().unwrap()?;
597/// assert_eq!(out.num_rows(), 2);
598/// # Ok(()) }
599/// ```
600pub type AvroWriter<W> = Writer<W, AvroOcfFormat>;
601
602/// Alias for an Avro **Single Object Encoding** stream writer.
603///
604/// ### Example
605///
606/// This writer automatically adds the appropriate per-record prefix (based on the
607/// fingerprint strategy) before the Avro body of each record. The default is Single
608/// Object Encoding (SOE) with a Rabin fingerprint.
609///
610/// ```
611/// use std::sync::Arc;
612/// use arrow_array::{ArrayRef, Int64Array, RecordBatch};
613/// use arrow_schema::{DataType, Field, Schema};
614/// use arrow_avro::writer::AvroStreamWriter;
615///
616/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
617/// // One‑column Arrow batch
618/// let schema = Schema::new(vec![Field::new("x", DataType::Int64, false)]);
619/// let batch = RecordBatch::try_new(
620///     Arc::new(schema.clone()),
621///     vec![Arc::new(Int64Array::from(vec![10, 20])) as ArrayRef],
622/// )?;
623///
624/// // Write an Avro Single Object Encoding stream to a Vec<u8>
625/// let sink: Vec<u8> = Vec::new();
626/// let mut w = AvroStreamWriter::new(sink, schema)?;
627/// w.write(&batch)?;
628/// w.finish()?;
629/// let bytes = w.into_inner();
630/// assert!(!bytes.is_empty());
631/// # Ok(()) }
632/// ```
633pub type AvroStreamWriter<W> = Writer<W, AvroSoeFormat>;
634
635impl<W: Write> Writer<W, AvroOcfFormat> {
636    /// Convenience constructor – same as [`WriterBuilder::build`] with `AvroOcfFormat`.
637    ///
638    /// ### Example
639    ///
640    /// ```
641    /// use std::sync::Arc;
642    /// use arrow_array::{ArrayRef, Int32Array, RecordBatch};
643    /// use arrow_schema::{DataType, Field, Schema};
644    /// use arrow_avro::writer::AvroWriter;
645    ///
646    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
647    /// let schema = Schema::new(vec![Field::new("id", DataType::Int32, false)]);
648    /// let batch = RecordBatch::try_new(
649    ///     Arc::new(schema.clone()),
650    ///     vec![Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef],
651    /// )?;
652    ///
653    /// let buf: Vec<u8> = Vec::new();
654    /// let mut w = AvroWriter::new(buf, schema)?;
655    /// w.write(&batch)?;
656    /// w.finish()?;
657    /// let bytes = w.into_inner();
658    /// assert!(!bytes.is_empty());
659    /// # Ok(()) }
660    /// ```
661    pub fn new(writer: W, schema: Schema) -> Result<Self, AvroError> {
662        WriterBuilder::new(schema).build::<W, AvroOcfFormat>(writer)
663    }
664
665    /// Return a reference to the 16‑byte sync marker generated for this file.
666    pub fn sync_marker(&self) -> Option<&[u8; 16]> {
667        self.format.sync_marker()
668    }
669}
670
671impl<W: Write> Writer<W, AvroSoeFormat> {
672    /// Convenience constructor to create a new [`AvroStreamWriter`].
673    ///
674    /// The resulting stream contains **Single Object Encodings** (no OCF header/sync).
675    ///
676    /// ### Example
677    ///
678    /// ```
679    /// use std::sync::Arc;
680    /// use arrow_array::{ArrayRef, Int64Array, RecordBatch};
681    /// use arrow_schema::{DataType, Field, Schema};
682    /// use arrow_avro::writer::AvroStreamWriter;
683    ///
684    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
685    /// let schema = Schema::new(vec![Field::new("x", DataType::Int64, false)]);
686    /// let batch = RecordBatch::try_new(
687    ///     Arc::new(schema.clone()),
688    ///     vec![Arc::new(Int64Array::from(vec![10, 20])) as ArrayRef],
689    /// )?;
690    ///
691    /// let sink: Vec<u8> = Vec::new();
692    /// let mut w = AvroStreamWriter::new(sink, schema)?;
693    /// w.write(&batch)?;
694    /// w.finish()?;
695    /// let bytes = w.into_inner();
696    /// assert!(!bytes.is_empty());
697    /// # Ok(()) }
698    /// ```
699    pub fn new(writer: W, schema: Schema) -> Result<Self, AvroError> {
700        WriterBuilder::new(schema).build::<W, AvroSoeFormat>(writer)
701    }
702}
703
704impl<W: Write, F: AvroFormat> Writer<W, F> {
705    /// Serialize one [`RecordBatch`] to the output.
706    pub fn write(&mut self, batch: &RecordBatch) -> Result<(), AvroError> {
707        if batch.schema().fields() != self.schema.fields() {
708            return Err(AvroError::SchemaError(
709                "Schema of RecordBatch differs from Writer schema".to_string(),
710            ));
711        }
712        match self.format.sync_marker() {
713            Some(&sync) => self.write_ocf_block(batch, &sync),
714            None => self.write_stream(batch),
715        }
716    }
717
718    /// A convenience method to write a slice of [`RecordBatch`].
719    ///
720    /// This is equivalent to calling `write` for each batch in the slice.
721    pub fn write_batches(&mut self, batches: &[&RecordBatch]) -> Result<(), AvroError> {
722        for b in batches {
723            self.write(b)?;
724        }
725        Ok(())
726    }
727
728    /// Flush remaining buffered data and (for OCF) ensure the header is present.
729    pub fn finish(&mut self) -> Result<(), AvroError> {
730        self.writer
731            .flush()
732            .map_err(|e| AvroError::IoError(format!("Error flushing writer: {e}"), e))
733    }
734
735    /// Consume the writer, returning the underlying output object.
736    pub fn into_inner(self) -> W {
737        self.writer
738    }
739
740    fn write_ocf_block(&mut self, batch: &RecordBatch, sync: &[u8; 16]) -> Result<(), AvroError> {
741        let mut buf = Vec::<u8>::with_capacity(self.capacity);
742        self.encoder.encode(&mut buf, batch)?;
743        let encoded = match self.compression {
744            Some(codec) => codec.compress(&buf)?,
745            None => buf,
746        };
747        write_long(&mut self.writer, batch.num_rows() as i64)?;
748        write_long(&mut self.writer, encoded.len() as i64)?;
749        self.writer
750            .write_all(&encoded)
751            .map_err(|e| AvroError::IoError(format!("Error writing Avro block: {e}"), e))?;
752        self.writer
753            .write_all(sync)
754            .map_err(|e| AvroError::IoError(format!("Error writing Avro sync: {e}"), e))?;
755        Ok(())
756    }
757
758    fn write_stream(&mut self, batch: &RecordBatch) -> Result<(), AvroError> {
759        self.encoder.encode(&mut self.writer, batch)?;
760        Ok(())
761    }
762}
763
764#[cfg(test)]
765mod tests {
766    use super::*;
767    use crate::compression::CompressionCodec;
768    use crate::reader::ReaderBuilder;
769    use crate::schema::AVRO_NAME_METADATA_KEY;
770    use crate::schema::{AvroSchema, SchemaStore};
771    use crate::test_util::arrow_test_data;
772    use arrow::datatypes::TimeUnit;
773    use arrow::util::pretty::pretty_format_batches;
774    #[cfg(not(feature = "avro_custom_types"))]
775    use arrow_array::Float32Array;
776    #[cfg(feature = "avro_custom_types")]
777    use arrow_array::RunArray;
778    use arrow_array::builder::{Int32Builder, ListBuilder};
779    use arrow_array::cast::AsArray;
780    #[cfg(feature = "avro_custom_types")]
781    use arrow_array::types::{Int16Type, Int64Type};
782    use arrow_array::types::{
783        Int32Type, Time32MillisecondType, Time64MicrosecondType, TimestampMicrosecondType,
784        TimestampMillisecondType, TimestampNanosecondType,
785    };
786    use arrow_array::{
787        Array, ArrayRef, BinaryArray, BooleanArray, Date32Array, Date64Array, Float16Array,
788        Int8Array, Int16Array, Int32Array, Int64Array, IntervalDayTimeArray,
789        IntervalMonthDayNanoArray, IntervalYearMonthArray, PrimitiveArray, RecordBatch,
790        StringArray, StructArray, Time32MillisecondArray, Time32SecondArray,
791        Time64MicrosecondArray, Time64NanosecondArray, TimestampMillisecondArray,
792        TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, UnionArray,
793    };
794    use arrow_buffer::{IntervalDayTime, IntervalMonthDayNano};
795    #[cfg(not(feature = "avro_custom_types"))]
796    use arrow_schema::{DataType, Field, Schema};
797    #[cfg(feature = "avro_custom_types")]
798    use arrow_schema::{DataType, Field, Schema};
799    use arrow_schema::{IntervalUnit, UnionMode};
800    use bytes::BytesMut;
801    use half::f16;
802    use serde_json::{Value, json};
803    use std::collections::HashMap;
804    use std::collections::HashSet;
805    use std::fs::File;
806    use std::io::{BufReader, Cursor};
807    use std::path::PathBuf;
808    use std::sync::Arc;
809    use tempfile::NamedTempFile;
810
811    fn files() -> impl Iterator<Item = &'static str> {
812        [
813            // TODO: avoid requiring snappy for this file
814            #[cfg(feature = "snappy")]
815            "avro/alltypes_plain.avro",
816            // Compression codecs are unsupported by Miri
817            #[cfg(all(feature = "snappy", not(miri)))]
818            "avro/alltypes_plain.snappy.avro",
819            #[cfg(all(feature = "zstd", not(miri)))]
820            "avro/alltypes_plain.zstandard.avro",
821            #[cfg(all(feature = "bzip2", not(miri)))]
822            "avro/alltypes_plain.bzip2.avro",
823            #[cfg(all(feature = "xz", not(miri)))]
824            "avro/alltypes_plain.xz.avro",
825        ]
826        .into_iter()
827    }
828
829    fn make_schema() -> Schema {
830        Schema::new(vec![
831            Field::new("id", DataType::Int32, false),
832            Field::new("name", DataType::Binary, false),
833        ])
834    }
835
836    fn make_batch() -> RecordBatch {
837        let ids = Int32Array::from(vec![1, 2, 3]);
838        let names = BinaryArray::from_vec(vec![b"a".as_ref(), b"b".as_ref(), b"c".as_ref()]);
839        RecordBatch::try_new(
840            Arc::new(make_schema()),
841            vec![Arc::new(ids) as ArrayRef, Arc::new(names) as ArrayRef],
842        )
843        .expect("failed to build test RecordBatch")
844    }
845
846    #[test]
847    fn test_stream_writer_writes_prefix_per_row_rt() -> Result<(), AvroError> {
848        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
849        let batch = RecordBatch::try_new(
850            Arc::new(schema.clone()),
851            vec![Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef],
852        )?;
853        let buf: Vec<u8> = Vec::new();
854        let mut writer = AvroStreamWriter::new(buf, schema.clone())?;
855        writer.write(&batch)?;
856        let encoded = writer.into_inner();
857        let mut store = SchemaStore::new(); // Rabin by default
858        let avro_schema = AvroSchema::try_from(&schema)?;
859        let _fp = store.register(avro_schema)?;
860        let mut decoder = ReaderBuilder::new()
861            .with_writer_schema_store(store)
862            .build_decoder()?;
863        let _consumed = decoder.decode(&encoded)?;
864        let decoded = decoder
865            .flush()?
866            .expect("expected at least one batch from decoder");
867        assert_eq!(decoded.num_columns(), 1);
868        assert_eq!(decoded.num_rows(), 2);
869        let col = decoded.column(0).as_primitive::<Int32Type>();
870        assert_eq!(col, &Int32Array::from(vec![10, 20]));
871        Ok(())
872    }
873
874    #[test]
875    fn test_nullable_struct_with_nonnullable_field_sliced_encoding() {
876        use arrow_array::{ArrayRef, Int32Array, StringArray, StructArray};
877        use arrow_buffer::NullBuffer;
878        use arrow_schema::{DataType, Field, Fields, Schema};
879        use std::sync::Arc;
880        let inner_fields = Fields::from(vec![
881            Field::new("id", DataType::Int32, false), // non-nullable
882            Field::new("name", DataType::Utf8, true), // nullable
883        ]);
884        let inner_struct_type = DataType::Struct(inner_fields.clone());
885        let schema = Schema::new(vec![
886            Field::new("before", inner_struct_type.clone(), true), // nullable struct
887            Field::new("after", inner_struct_type.clone(), true),  // nullable struct
888            Field::new("op", DataType::Utf8, false),               // non-nullable
889        ]);
890        let before_ids = Int32Array::from(vec![None, None]);
891        let before_names = StringArray::from(vec![None::<&str>, None]);
892        let before_struct = StructArray::new(
893            inner_fields.clone(),
894            vec![
895                Arc::new(before_ids) as ArrayRef,
896                Arc::new(before_names) as ArrayRef,
897            ],
898            Some(NullBuffer::from(vec![false, false])),
899        );
900        let after_ids = Int32Array::from(vec![1, 2]); // non-nullable, no nulls
901        let after_names = StringArray::from(vec![Some("Alice"), Some("Bob")]);
902        let after_struct = StructArray::new(
903            inner_fields.clone(),
904            vec![
905                Arc::new(after_ids) as ArrayRef,
906                Arc::new(after_names) as ArrayRef,
907            ],
908            Some(NullBuffer::from(vec![true, true])),
909        );
910        let op_col = StringArray::from(vec!["r", "r"]);
911        let batch = RecordBatch::try_new(
912            Arc::new(schema.clone()),
913            vec![
914                Arc::new(before_struct) as ArrayRef,
915                Arc::new(after_struct) as ArrayRef,
916                Arc::new(op_col) as ArrayRef,
917            ],
918        )
919        .expect("failed to create test batch");
920        let mut sink = Vec::new();
921        let mut writer = WriterBuilder::new(schema)
922            .with_fingerprint_strategy(FingerprintStrategy::Id(1))
923            .build::<_, AvroSoeFormat>(&mut sink)
924            .expect("failed to create writer");
925        for row_idx in 0..batch.num_rows() {
926            let single_row = batch.slice(row_idx, 1);
927            let after_col = single_row.column(1);
928            assert_eq!(
929                after_col.null_count(),
930                0,
931                "after column should have no nulls in sliced row"
932            );
933            writer
934                .write(&single_row)
935                .unwrap_or_else(|e| panic!("Failed to encode row {row_idx}: {e}"));
936        }
937        writer.finish().expect("failed to finish writer");
938        assert!(!sink.is_empty(), "encoded output should not be empty");
939    }
940
941    #[test]
942    fn test_nullable_struct_with_decimal_and_timestamp_sliced() {
943        use arrow_array::{
944            ArrayRef, Decimal128Array, Int32Array, StringArray, StructArray,
945            TimestampMicrosecondArray,
946        };
947        use arrow_buffer::NullBuffer;
948        use arrow_schema::{DataType, Field, Fields, Schema};
949        use std::sync::Arc;
950        let row_fields = Fields::from(vec![
951            Field::new("id", DataType::Int32, false),
952            Field::new("name", DataType::Utf8, true),
953            Field::new("category", DataType::Utf8, true),
954            Field::new("price", DataType::Decimal128(10, 2), true),
955            Field::new("stock_quantity", DataType::Int32, true),
956            Field::new(
957                "created_at",
958                DataType::Timestamp(TimeUnit::Microsecond, None),
959                true,
960            ),
961        ]);
962        let row_struct_type = DataType::Struct(row_fields.clone());
963        let schema = Schema::new(vec![
964            Field::new("before", row_struct_type.clone(), true),
965            Field::new("after", row_struct_type.clone(), true),
966            Field::new("op", DataType::Utf8, false),
967        ]);
968        let before_struct = StructArray::new_null(row_fields.clone(), 2);
969        let ids = Int32Array::from(vec![1, 2]);
970        let names = StringArray::from(vec![Some("Widget"), Some("Gadget")]);
971        let categories = StringArray::from(vec![Some("Electronics"), Some("Electronics")]);
972        let prices = Decimal128Array::from(vec![Some(1999), Some(2999)])
973            .with_precision_and_scale(10, 2)
974            .unwrap();
975        let quantities = Int32Array::from(vec![Some(100), Some(50)]);
976        let timestamps = TimestampMicrosecondArray::from(vec![
977            Some(1700000000000000i64),
978            Some(1700000001000000i64),
979        ]);
980        let after_struct = StructArray::new(
981            row_fields.clone(),
982            vec![
983                Arc::new(ids) as ArrayRef,
984                Arc::new(names) as ArrayRef,
985                Arc::new(categories) as ArrayRef,
986                Arc::new(prices) as ArrayRef,
987                Arc::new(quantities) as ArrayRef,
988                Arc::new(timestamps) as ArrayRef,
989            ],
990            Some(NullBuffer::from(vec![true, true])),
991        );
992        let op_col = StringArray::from(vec!["r", "r"]);
993        let batch = RecordBatch::try_new(
994            Arc::new(schema.clone()),
995            vec![
996                Arc::new(before_struct) as ArrayRef,
997                Arc::new(after_struct) as ArrayRef,
998                Arc::new(op_col) as ArrayRef,
999            ],
1000        )
1001        .expect("failed to create products batch");
1002        let mut sink = Vec::new();
1003        let mut writer = WriterBuilder::new(schema)
1004            .with_fingerprint_strategy(FingerprintStrategy::Id(1))
1005            .build::<_, AvroSoeFormat>(&mut sink)
1006            .expect("failed to create writer");
1007        // Encode row by row
1008        for row_idx in 0..batch.num_rows() {
1009            let single_row = batch.slice(row_idx, 1);
1010            writer
1011                .write(&single_row)
1012                .unwrap_or_else(|e| panic!("Failed to encode product row {row_idx}: {e}"));
1013        }
1014        writer.finish().expect("failed to finish writer");
1015        assert!(!sink.is_empty());
1016    }
1017
1018    #[test]
1019    fn non_nullable_child_in_nullable_struct_should_encode_per_row() {
1020        use arrow_array::{
1021            ArrayRef, Int32Array, Int64Array, RecordBatch, StringArray, StructArray,
1022        };
1023        use arrow_schema::{DataType, Field, Fields, Schema};
1024        use std::sync::Arc;
1025        let row_fields = Fields::from(vec![
1026            Field::new("id", DataType::Int32, false),
1027            Field::new("name", DataType::Utf8, true),
1028        ]);
1029        let row_struct_dt = DataType::Struct(row_fields.clone());
1030        let before: ArrayRef = Arc::new(StructArray::new_null(row_fields.clone(), 1));
1031        let id_col: ArrayRef = Arc::new(Int32Array::from(vec![1]));
1032        let name_col: ArrayRef = Arc::new(StringArray::from(vec![None::<&str>]));
1033        let after: ArrayRef = Arc::new(StructArray::new(
1034            row_fields.clone(),
1035            vec![id_col, name_col],
1036            None,
1037        ));
1038        let schema = Arc::new(Schema::new(vec![
1039            Field::new("before", row_struct_dt.clone(), true),
1040            Field::new("after", row_struct_dt, true),
1041            Field::new("op", DataType::Utf8, false),
1042            Field::new("ts_ms", DataType::Int64, false),
1043        ]));
1044        let op = Arc::new(StringArray::from(vec!["r"])) as ArrayRef;
1045        let ts_ms = Arc::new(Int64Array::from(vec![1732900000000_i64])) as ArrayRef;
1046        let batch = RecordBatch::try_new(schema.clone(), vec![before, after, op, ts_ms]).unwrap();
1047        let mut buf = Vec::new();
1048        let mut writer = WriterBuilder::new(schema.as_ref().clone())
1049            .build::<_, AvroSoeFormat>(&mut buf)
1050            .unwrap();
1051        let single = batch.slice(0, 1);
1052        let res = writer.write(&single);
1053        assert!(
1054            res.is_ok(),
1055            "expected to encode successfully, got: {:?}",
1056            res.err()
1057        );
1058    }
1059
1060    #[test]
1061    fn test_union_nonzero_type_ids() -> Result<(), AvroError> {
1062        use arrow_array::UnionArray;
1063        use arrow_buffer::Buffer;
1064        use arrow_schema::UnionFields;
1065        let union_fields = UnionFields::try_new(
1066            vec![2, 5],
1067            vec![
1068                Field::new("v_str", DataType::Utf8, true),
1069                Field::new("v_int", DataType::Int32, true),
1070            ],
1071        )
1072        .unwrap();
1073        let strings = StringArray::from(vec!["hello", "world"]);
1074        let ints = Int32Array::from(vec![10, 20, 30]);
1075        let type_ids = Buffer::from_slice_ref([2_i8, 5, 5, 2, 5]);
1076        let offsets = Buffer::from_slice_ref([0_i32, 0, 1, 1, 2]);
1077        let union_array = UnionArray::try_new(
1078            union_fields.clone(),
1079            type_ids.into(),
1080            Some(offsets.into()),
1081            vec![Arc::new(strings) as ArrayRef, Arc::new(ints) as ArrayRef],
1082        )?;
1083        let schema = Schema::new(vec![Field::new(
1084            "union_col",
1085            DataType::Union(union_fields, UnionMode::Dense),
1086            false,
1087        )]);
1088        let batch = RecordBatch::try_new(
1089            Arc::new(schema.clone()),
1090            vec![Arc::new(union_array) as ArrayRef],
1091        )?;
1092        let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
1093        assert!(
1094            writer.write(&batch).is_ok(),
1095            "Expected no error from writing"
1096        );
1097        writer.finish()?;
1098        assert!(
1099            writer.finish().is_ok(),
1100            "Expected no error from finishing writer"
1101        );
1102        Ok(())
1103    }
1104
1105    #[test]
1106    fn test_stream_writer_with_id_fingerprint_rt() -> Result<(), AvroError> {
1107        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
1108        let batch = RecordBatch::try_new(
1109            Arc::new(schema.clone()),
1110            vec![Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef],
1111        )?;
1112        let schema_id: u32 = 42;
1113        let mut writer = WriterBuilder::new(schema.clone())
1114            .with_fingerprint_strategy(FingerprintStrategy::Id(schema_id))
1115            .build::<_, AvroSoeFormat>(Vec::new())?;
1116        writer.write(&batch)?;
1117        let encoded = writer.into_inner();
1118        let mut store = SchemaStore::new_with_type(FingerprintAlgorithm::Id);
1119        let avro_schema = AvroSchema::try_from(&schema)?;
1120        let _ = store.set(Fingerprint::Id(schema_id), avro_schema)?;
1121        let mut decoder = ReaderBuilder::new()
1122            .with_writer_schema_store(store)
1123            .build_decoder()?;
1124        let _ = decoder.decode(&encoded)?;
1125        let decoded = decoder
1126            .flush()?
1127            .expect("expected at least one batch from decoder");
1128        assert_eq!(decoded.num_columns(), 1);
1129        assert_eq!(decoded.num_rows(), 3);
1130        let col = decoded.column(0).as_primitive::<Int32Type>();
1131        assert_eq!(col, &Int32Array::from(vec![1, 2, 3]));
1132        Ok(())
1133    }
1134
1135    #[test]
1136    fn test_stream_writer_with_id64_fingerprint_rt() -> Result<(), AvroError> {
1137        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
1138        let batch = RecordBatch::try_new(
1139            Arc::new(schema.clone()),
1140            vec![Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef],
1141        )?;
1142        let schema_id: u64 = 42;
1143        let mut writer = WriterBuilder::new(schema.clone())
1144            .with_fingerprint_strategy(FingerprintStrategy::Id64(schema_id))
1145            .build::<_, AvroSoeFormat>(Vec::new())?;
1146        writer.write(&batch)?;
1147        let encoded = writer.into_inner();
1148        let mut store = SchemaStore::new_with_type(FingerprintAlgorithm::Id64);
1149        let avro_schema = AvroSchema::try_from(&schema)?;
1150        let _ = store.set(Fingerprint::Id64(schema_id), avro_schema)?;
1151        let mut decoder = ReaderBuilder::new()
1152            .with_writer_schema_store(store)
1153            .build_decoder()?;
1154        let _ = decoder.decode(&encoded)?;
1155        let decoded = decoder
1156            .flush()?
1157            .expect("expected at least one batch from decoder");
1158        assert_eq!(decoded.num_columns(), 1);
1159        assert_eq!(decoded.num_rows(), 3);
1160        let col = decoded.column(0).as_primitive::<Int32Type>();
1161        assert_eq!(col, &Int32Array::from(vec![1, 2, 3]));
1162        Ok(())
1163    }
1164
1165    #[test]
1166    fn test_ocf_writer_generates_header_and_sync() -> Result<(), AvroError> {
1167        let batch = make_batch();
1168        let buffer: Vec<u8> = Vec::new();
1169        let mut writer = AvroWriter::new(buffer, make_schema())?;
1170        writer.write(&batch)?;
1171        writer.finish()?;
1172        let out = writer.into_inner();
1173        assert_eq!(&out[..4], b"Obj\x01", "OCF magic bytes missing/incorrect");
1174        let trailer = &out[out.len() - 16..];
1175        assert_eq!(trailer.len(), 16, "expected 16‑byte sync marker");
1176        Ok(())
1177    }
1178
1179    #[test]
1180    fn test_schema_mismatch_yields_error() {
1181        let batch = make_batch();
1182        let alt_schema = Schema::new(vec![Field::new("x", DataType::Int32, false)]);
1183        let buffer = Vec::<u8>::new();
1184        let mut writer = AvroWriter::new(buffer, alt_schema).unwrap();
1185        let err = writer.write(&batch).unwrap_err();
1186        assert!(matches!(err, AvroError::SchemaError(_)));
1187    }
1188
1189    #[test]
1190    fn test_write_batches_accumulates_multiple() -> Result<(), AvroError> {
1191        let batch1 = make_batch();
1192        let batch2 = make_batch();
1193        let buffer = Vec::<u8>::new();
1194        let mut writer = AvroWriter::new(buffer, make_schema())?;
1195        writer.write_batches(&[&batch1, &batch2])?;
1196        writer.finish()?;
1197        let out = writer.into_inner();
1198        assert!(out.len() > 4, "combined batches produced tiny file");
1199        Ok(())
1200    }
1201
1202    #[test]
1203    fn test_finish_without_write_adds_header() -> Result<(), AvroError> {
1204        let buffer = Vec::<u8>::new();
1205        let mut writer = AvroWriter::new(buffer, make_schema())?;
1206        writer.finish()?;
1207        let out = writer.into_inner();
1208        assert_eq!(&out[..4], b"Obj\x01", "finish() should emit OCF header");
1209        Ok(())
1210    }
1211
1212    #[test]
1213    fn test_write_long_encodes_zigzag_varint() -> Result<(), AvroError> {
1214        let mut buf = Vec::new();
1215        write_long(&mut buf, 0)?;
1216        write_long(&mut buf, -1)?;
1217        write_long(&mut buf, 1)?;
1218        write_long(&mut buf, -2)?;
1219        write_long(&mut buf, 2147483647)?;
1220        assert!(
1221            buf.starts_with(&[0x00, 0x01, 0x02, 0x03]),
1222            "zig‑zag varint encodings incorrect: {buf:?}"
1223        );
1224        Ok(())
1225    }
1226
1227    #[test]
1228    fn test_roundtrip_alltypes_roundtrip_writer() -> Result<(), AvroError> {
1229        for rel in files() {
1230            let path = arrow_test_data(rel);
1231            let rdr_file = File::open(&path).expect("open input avro");
1232            let reader = ReaderBuilder::new()
1233                .build(BufReader::new(rdr_file))
1234                .expect("build reader");
1235            let schema = reader.schema();
1236            let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
1237            let original =
1238                arrow::compute::concat_batches(&schema, &input_batches).expect("concat input");
1239            let tmp = NamedTempFile::new().expect("create temp file");
1240            let out_path = tmp.into_temp_path();
1241            let out_file = File::create(&out_path).expect("create temp avro");
1242            let codec = if rel.contains(".snappy.") {
1243                Some(CompressionCodec::Snappy)
1244            } else if rel.contains(".zstandard.") {
1245                Some(CompressionCodec::ZStandard)
1246            } else if rel.contains(".bzip2.") {
1247                Some(CompressionCodec::Bzip2)
1248            } else if rel.contains(".xz.") {
1249                Some(CompressionCodec::Xz)
1250            } else {
1251                None
1252            };
1253            let mut writer = WriterBuilder::new(original.schema().as_ref().clone())
1254                .with_compression(codec)
1255                .build::<_, AvroOcfFormat>(out_file)?;
1256            writer.write(&original)?;
1257            writer.finish()?;
1258            drop(writer);
1259            let rt_file = File::open(&out_path).expect("open roundtrip avro");
1260            let rt_reader = ReaderBuilder::new()
1261                .build(BufReader::new(rt_file))
1262                .expect("build roundtrip reader");
1263            let rt_schema = rt_reader.schema();
1264            let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1265            let roundtrip =
1266                arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat roundtrip");
1267            assert_eq!(
1268                roundtrip, original,
1269                "Round-trip batch mismatch for file: {rel}"
1270            );
1271        }
1272        Ok(())
1273    }
1274
1275    #[test]
1276    fn test_roundtrip_nested_records_writer() -> Result<(), AvroError> {
1277        let path = arrow_test_data("avro/nested_records.avro");
1278        let rdr_file = File::open(&path).expect("open nested_records.avro");
1279        let reader = ReaderBuilder::new()
1280            .build(BufReader::new(rdr_file))
1281            .expect("build reader for nested_records.avro");
1282        let schema = reader.schema();
1283        let batches = reader.collect::<Result<Vec<_>, _>>()?;
1284        let original = arrow::compute::concat_batches(&schema, &batches).expect("concat original");
1285        let tmp = NamedTempFile::new().expect("create temp file");
1286        let out_path = tmp.into_temp_path();
1287        {
1288            let out_file = File::create(&out_path).expect("create output avro");
1289            let mut writer = AvroWriter::new(out_file, original.schema().as_ref().clone())?;
1290            writer.write(&original)?;
1291            writer.finish()?;
1292        }
1293        let rt_file = File::open(&out_path).expect("open round_trip avro");
1294        let rt_reader = ReaderBuilder::new()
1295            .build(BufReader::new(rt_file))
1296            .expect("build round_trip reader");
1297        let rt_schema = rt_reader.schema();
1298        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1299        let round_trip =
1300            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
1301        assert_eq!(
1302            round_trip, original,
1303            "Round-trip batch mismatch for nested_records.avro"
1304        );
1305        Ok(())
1306    }
1307
1308    #[test]
1309    #[cfg(feature = "snappy")]
1310    fn test_roundtrip_nested_lists_writer() -> Result<(), AvroError> {
1311        let path = arrow_test_data("avro/nested_lists.snappy.avro");
1312        let rdr_file = File::open(&path).expect("open nested_lists.snappy.avro");
1313        let reader = ReaderBuilder::new()
1314            .build(BufReader::new(rdr_file))
1315            .expect("build reader for nested_lists.snappy.avro");
1316        let schema = reader.schema();
1317        let batches = reader.collect::<Result<Vec<_>, _>>()?;
1318        let original = arrow::compute::concat_batches(&schema, &batches).expect("concat original");
1319        let tmp = NamedTempFile::new().expect("create temp file");
1320        let out_path = tmp.into_temp_path();
1321        {
1322            let out_file = File::create(&out_path).expect("create output avro");
1323            let mut writer = WriterBuilder::new(original.schema().as_ref().clone())
1324                .with_compression(Some(CompressionCodec::Snappy))
1325                .build::<_, AvroOcfFormat>(out_file)?;
1326            writer.write(&original)?;
1327            writer.finish()?;
1328        }
1329        let rt_file = File::open(&out_path).expect("open round_trip avro");
1330        let rt_reader = ReaderBuilder::new()
1331            .build(BufReader::new(rt_file))
1332            .expect("build round_trip reader");
1333        let rt_schema = rt_reader.schema();
1334        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1335        let round_trip =
1336            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
1337        assert_eq!(
1338            round_trip, original,
1339            "Round-trip batch mismatch for nested_lists.snappy.avro"
1340        );
1341        Ok(())
1342    }
1343
1344    #[test]
1345    fn test_round_trip_simple_fixed_ocf() -> Result<(), AvroError> {
1346        let path = arrow_test_data("avro/simple_fixed.avro");
1347        let rdr_file = File::open(&path).expect("open avro/simple_fixed.avro");
1348        let reader = ReaderBuilder::new()
1349            .build(BufReader::new(rdr_file))
1350            .expect("build avro reader");
1351        let schema = reader.schema();
1352        let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
1353        let original =
1354            arrow::compute::concat_batches(&schema, &input_batches).expect("concat input");
1355        let tmp = NamedTempFile::new().expect("create temp file");
1356        let out_file = File::create(tmp.path()).expect("create temp avro");
1357        let mut writer = AvroWriter::new(out_file, original.schema().as_ref().clone())?;
1358        writer.write(&original)?;
1359        writer.finish()?;
1360        drop(writer);
1361        let rt_file = File::open(tmp.path()).expect("open round_trip avro");
1362        let rt_reader = ReaderBuilder::new()
1363            .build(BufReader::new(rt_file))
1364            .expect("build round_trip reader");
1365        let rt_schema = rt_reader.schema();
1366        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1367        let round_trip =
1368            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
1369        assert_eq!(round_trip, original);
1370        Ok(())
1371    }
1372
1373    // Strict equality (schema + values) only when canonical extension types are enabled
1374    #[test]
1375    #[cfg(feature = "canonical_extension_types")]
1376    fn test_round_trip_duration_and_uuid_ocf() -> Result<(), AvroError> {
1377        use arrow_schema::{DataType, IntervalUnit};
1378        let in_file =
1379            File::open("test/data/duration_uuid.avro").expect("open test/data/duration_uuid.avro");
1380        let reader = ReaderBuilder::new()
1381            .build(BufReader::new(in_file))
1382            .expect("build reader for duration_uuid.avro");
1383        let in_schema = reader.schema();
1384        let has_mdn = in_schema.fields().iter().any(|f| {
1385            matches!(
1386                f.data_type(),
1387                DataType::Interval(IntervalUnit::MonthDayNano)
1388            )
1389        });
1390        assert!(
1391            has_mdn,
1392            "expected at least one Interval(MonthDayNano) field in duration_uuid.avro"
1393        );
1394        let has_uuid_fixed = in_schema
1395            .fields()
1396            .iter()
1397            .any(|f| matches!(f.data_type(), DataType::FixedSizeBinary(16)));
1398        assert!(
1399            has_uuid_fixed,
1400            "expected at least one FixedSizeBinary(16) (uuid) field in duration_uuid.avro"
1401        );
1402        let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
1403        let input =
1404            arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
1405        // Write to an in‑memory OCF and read back
1406        let mut writer = AvroWriter::new(Vec::<u8>::new(), in_schema.as_ref().clone())?;
1407        writer.write(&input)?;
1408        writer.finish()?;
1409        let bytes = writer.into_inner();
1410        let rt_reader = ReaderBuilder::new()
1411            .build(Cursor::new(bytes))
1412            .expect("build round_trip reader");
1413        let rt_schema = rt_reader.schema();
1414        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1415        let round_trip =
1416            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
1417        assert_eq!(round_trip, input);
1418        Ok(())
1419    }
1420
1421    // Feature OFF: only values are asserted equal; schema may legitimately differ (uuid as fixed(16))
1422    #[test]
1423    #[cfg(not(feature = "canonical_extension_types"))]
1424    fn test_duration_and_uuid_ocf_without_extensions_round_trips_values() -> Result<(), AvroError> {
1425        use arrow::datatypes::{DataType, IntervalUnit};
1426        use std::io::BufReader;
1427
1428        // Read input Avro (duration + uuid)
1429        let in_file =
1430            File::open("test/data/duration_uuid.avro").expect("open test/data/duration_uuid.avro");
1431        let reader = ReaderBuilder::new()
1432            .build(BufReader::new(in_file))
1433            .expect("build reader for duration_uuid.avro");
1434        let in_schema = reader.schema();
1435
1436        // Sanity checks: has MonthDayNano and a FixedSizeBinary(16)
1437        assert!(
1438            in_schema.fields().iter().any(|f| {
1439                matches!(
1440                    f.data_type(),
1441                    DataType::Interval(IntervalUnit::MonthDayNano)
1442                )
1443            }),
1444            "expected at least one Interval(MonthDayNano) field"
1445        );
1446        assert!(
1447            in_schema
1448                .fields()
1449                .iter()
1450                .any(|f| matches!(f.data_type(), DataType::FixedSizeBinary(16))),
1451            "expected a FixedSizeBinary(16) field (uuid)"
1452        );
1453
1454        let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
1455        let input =
1456            arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
1457
1458        // Write to a temp OCF and read back
1459        let mut writer = AvroWriter::new(Vec::<u8>::new(), in_schema.as_ref().clone())?;
1460        writer.write(&input)?;
1461        writer.finish()?;
1462        let bytes = writer.into_inner();
1463        let rt_reader = ReaderBuilder::new()
1464            .build(Cursor::new(bytes))
1465            .expect("build round_trip reader");
1466        let rt_schema = rt_reader.schema();
1467        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1468        let round_trip =
1469            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
1470
1471        // 1) Values must round-trip for both columns
1472        assert_eq!(
1473            round_trip.column(0),
1474            input.column(0),
1475            "duration column values differ"
1476        );
1477        assert_eq!(round_trip.column(1), input.column(1), "uuid bytes differ");
1478
1479        // 2) Schema expectation without extensions:
1480        //    uuid is written as named fixed(16), so reader attaches avro.name
1481        let uuid_rt = rt_schema.field_with_name("uuid_field")?;
1482        assert_eq!(uuid_rt.data_type(), &DataType::FixedSizeBinary(16));
1483        assert_eq!(
1484            uuid_rt.metadata().get("logicalType").map(|s| s.as_str()),
1485            Some("uuid"),
1486            "expected `logicalType = \"uuid\"` on round-tripped field metadata"
1487        );
1488
1489        // 3) Duration remains Interval(MonthDayNano)
1490        let dur_rt = rt_schema.field_with_name("duration_field")?;
1491        assert!(matches!(
1492            dur_rt.data_type(),
1493            DataType::Interval(IntervalUnit::MonthDayNano)
1494        ));
1495
1496        Ok(())
1497    }
1498
1499    // This test reads the same 'nonnullable.impala.avro' used by the reader tests,
1500    // writes it back out with the writer (hitting Map encoding paths), then reads it
1501    // again and asserts exact Arrow equivalence.
1502    #[test]
1503    // TODO: avoid requiring snappy for this file
1504    #[cfg(feature = "snappy")]
1505    fn test_nonnullable_impala_roundtrip_writer() -> Result<(), AvroError> {
1506        // Load source Avro with Map fields
1507        let path = arrow_test_data("avro/nonnullable.impala.avro");
1508        let rdr_file = File::open(&path).expect("open avro/nonnullable.impala.avro");
1509        let reader = ReaderBuilder::new()
1510            .build(BufReader::new(rdr_file))
1511            .expect("build reader for nonnullable.impala.avro");
1512        // Collect all input batches and concatenate to a single RecordBatch
1513        let in_schema = reader.schema();
1514        // Sanity: ensure the file actually contains at least one Map field
1515        let has_map = in_schema
1516            .fields()
1517            .iter()
1518            .any(|f| matches!(f.data_type(), DataType::Map(_, _)));
1519        assert!(
1520            has_map,
1521            "expected at least one Map field in avro/nonnullable.impala.avro"
1522        );
1523
1524        let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
1525        let original =
1526            arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
1527        // Write out using the OCF writer into an in-memory Vec<u8>
1528        let buffer = Vec::<u8>::new();
1529        let mut writer = AvroWriter::new(buffer, in_schema.as_ref().clone())?;
1530        writer.write(&original)?;
1531        writer.finish()?;
1532        let out_bytes = writer.into_inner();
1533        // Read the produced bytes back with the Reader
1534        let rt_reader = ReaderBuilder::new()
1535            .build(Cursor::new(out_bytes))
1536            .expect("build reader for round-tripped in-memory OCF");
1537        let rt_schema = rt_reader.schema();
1538        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1539        let roundtrip =
1540            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat roundtrip");
1541        // Exact value fidelity (schema + data)
1542        assert_eq!(
1543            roundtrip, original,
1544            "Round-trip Avro map data mismatch for nonnullable.impala.avro"
1545        );
1546        Ok(())
1547    }
1548
1549    #[test]
1550    // TODO: avoid requiring snappy for these files
1551    #[cfg(feature = "snappy")]
1552    fn test_roundtrip_decimals_via_writer() -> Result<(), AvroError> {
1553        // (file, resolve via ARROW_TEST_DATA?)
1554        let files: [(&str, bool); 8] = [
1555            ("avro/fixed_length_decimal.avro", true), // fixed-backed -> Decimal128(25,2)
1556            ("avro/fixed_length_decimal_legacy.avro", true), // legacy fixed[8] -> Decimal64(13,2)
1557            ("avro/int32_decimal.avro", true),        // bytes-backed -> Decimal32(4,2)
1558            ("avro/int64_decimal.avro", true),        // bytes-backed -> Decimal64(10,2)
1559            ("test/data/int256_decimal.avro", false), // bytes-backed -> Decimal256(76,2)
1560            ("test/data/fixed256_decimal.avro", false), // fixed[32]-backed -> Decimal256(76,10)
1561            ("test/data/fixed_length_decimal_legacy_32.avro", false), // legacy fixed[4] -> Decimal32(9,2)
1562            ("test/data/int128_decimal.avro", false), // bytes-backed -> Decimal128(38,2)
1563        ];
1564        for (rel, in_test_data_dir) in files {
1565            // Resolve path the same way as reader::test_decimal
1566            let path: String = if in_test_data_dir {
1567                arrow_test_data(rel)
1568            } else {
1569                PathBuf::from(env!("CARGO_MANIFEST_DIR"))
1570                    .join(rel)
1571                    .to_string_lossy()
1572                    .into_owned()
1573            };
1574            // Read original file into a single RecordBatch for comparison
1575            let f_in = File::open(&path).expect("open input avro");
1576            let rdr = ReaderBuilder::new().build(BufReader::new(f_in))?;
1577            let in_schema = rdr.schema();
1578            let in_batches = rdr.collect::<Result<Vec<_>, _>>()?;
1579            let original =
1580                arrow::compute::concat_batches(&in_schema, &in_batches).expect("concat input");
1581            // Write it out with the OCF writer (no special compression)
1582            let tmp = NamedTempFile::new().expect("create temp file");
1583            let out_path = tmp.into_temp_path();
1584            let out_file = File::create(&out_path).expect("create temp avro");
1585            let mut writer = AvroWriter::new(out_file, original.schema().as_ref().clone())?;
1586            writer.write(&original)?;
1587            writer.finish()?;
1588            // Read back the file we just wrote and compare equality (schema + data)
1589            let f_rt = File::open(&out_path).expect("open roundtrip avro");
1590            let rt_rdr = ReaderBuilder::new().build(BufReader::new(f_rt))?;
1591            let rt_schema = rt_rdr.schema();
1592            let rt_batches = rt_rdr.collect::<Result<Vec<_>, _>>()?;
1593            let roundtrip =
1594                arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat rt");
1595            assert_eq!(roundtrip, original, "decimal round-trip mismatch for {rel}");
1596        }
1597        Ok(())
1598    }
1599
1600    #[test]
1601    fn test_named_types_complex_roundtrip() -> Result<(), AvroError> {
1602        // 1. Read the new, more complex named references file.
1603        let path =
1604            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("test/data/named_types_complex.avro");
1605        let rdr_file = File::open(&path).expect("open avro/named_types_complex.avro");
1606
1607        let reader = ReaderBuilder::new()
1608            .build(BufReader::new(rdr_file))
1609            .expect("build reader for named_types_complex.avro");
1610
1611        // 2. Concatenate all batches to one RecordBatch.
1612        let in_schema = reader.schema();
1613        let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
1614        let original =
1615            arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
1616
1617        // 3. Sanity Checks: Validate that all named types were reused correctly.
1618        {
1619            let arrow_schema = original.schema();
1620
1621            // --- A. Validate 'User' record reuse ---
1622            let author_field = arrow_schema.field_with_name("author")?;
1623            let author_type = author_field.data_type();
1624            let editors_field = arrow_schema.field_with_name("editors")?;
1625            let editors_item_type = match editors_field.data_type() {
1626                DataType::List(item_field) => item_field.data_type(),
1627                other => panic!("Editors field should be a List, but was {other:?}"),
1628            };
1629            assert_eq!(
1630                author_type, editors_item_type,
1631                "The DataType for the 'author' struct and the 'editors' list items must be identical"
1632            );
1633
1634            // --- B. Validate 'PostStatus' enum reuse ---
1635            let status_field = arrow_schema.field_with_name("status")?;
1636            let status_type = status_field.data_type();
1637            assert!(
1638                matches!(status_type, DataType::Dictionary(_, _)),
1639                "Status field should be a Dictionary (Enum)"
1640            );
1641
1642            let prev_status_field = arrow_schema.field_with_name("previous_status")?;
1643            let prev_status_type = prev_status_field.data_type();
1644            assert_eq!(
1645                status_type, prev_status_type,
1646                "The DataType for 'status' and 'previous_status' enums must be identical"
1647            );
1648
1649            // --- C. Validate 'MD5' fixed reuse ---
1650            let content_hash_field = arrow_schema.field_with_name("content_hash")?;
1651            let content_hash_type = content_hash_field.data_type();
1652            assert!(
1653                matches!(content_hash_type, DataType::FixedSizeBinary(16)),
1654                "Content hash should be FixedSizeBinary(16)"
1655            );
1656
1657            let thumb_hash_field = arrow_schema.field_with_name("thumbnail_hash")?;
1658            let thumb_hash_type = thumb_hash_field.data_type();
1659            assert_eq!(
1660                content_hash_type, thumb_hash_type,
1661                "The DataType for 'content_hash' and 'thumbnail_hash' fixed types must be identical"
1662            );
1663        }
1664
1665        // 4. Write the data to an in-memory buffer.
1666        let buffer: Vec<u8> = Vec::new();
1667        let mut writer = AvroWriter::new(buffer, original.schema().as_ref().clone())?;
1668        writer.write(&original)?;
1669        writer.finish()?;
1670        let bytes = writer.into_inner();
1671
1672        // 5. Read the data back and compare for exact equality.
1673        let rt_reader = ReaderBuilder::new()
1674            .build(Cursor::new(bytes))
1675            .expect("build reader for round-trip");
1676        let rt_schema = rt_reader.schema();
1677        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1678        let roundtrip =
1679            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat roundtrip");
1680
1681        assert_eq!(
1682            roundtrip, original,
1683            "Avro complex named types round-trip mismatch"
1684        );
1685
1686        Ok(())
1687    }
1688
1689    // Union Roundtrip Test Helpers
1690
1691    // Asserts that the `actual` schema is a semantically equivalent superset of the `expected` one.
1692    // This allows the `actual` schema to contain additional metadata keys
1693    // (`arrowUnionMode`, `arrowUnionTypeIds`, `avro.name`) that are added during an Arrow-to-Avro-to-Arrow
1694    // roundtrip, while ensuring no other information was lost or changed.
1695    fn assert_schema_is_semantically_equivalent(expected: &Schema, actual: &Schema) {
1696        // Compare top-level schema metadata using the same superset logic.
1697        assert_metadata_is_superset(expected.metadata(), actual.metadata(), "Schema");
1698
1699        // Compare fields.
1700        assert_eq!(
1701            expected.fields().len(),
1702            actual.fields().len(),
1703            "Schema must have the same number of fields"
1704        );
1705
1706        for (expected_field, actual_field) in expected.fields().iter().zip(actual.fields().iter()) {
1707            assert_field_is_semantically_equivalent(expected_field, actual_field);
1708        }
1709    }
1710
1711    fn assert_field_is_semantically_equivalent(expected: &Field, actual: &Field) {
1712        let context = format!("Field '{}'", expected.name());
1713
1714        assert_eq!(
1715            expected.name(),
1716            actual.name(),
1717            "{context}: names must match"
1718        );
1719        assert_eq!(
1720            expected.is_nullable(),
1721            actual.is_nullable(),
1722            "{context}: nullability must match"
1723        );
1724
1725        // Recursively check the data types.
1726        assert_datatype_is_semantically_equivalent(
1727            expected.data_type(),
1728            actual.data_type(),
1729            &context,
1730        );
1731
1732        // Check that metadata is a valid superset.
1733        assert_metadata_is_superset(expected.metadata(), actual.metadata(), &context);
1734    }
1735
1736    fn assert_datatype_is_semantically_equivalent(
1737        expected: &DataType,
1738        actual: &DataType,
1739        context: &str,
1740    ) {
1741        match (expected, actual) {
1742            (DataType::List(expected_field), DataType::List(actual_field))
1743            | (DataType::LargeList(expected_field), DataType::LargeList(actual_field))
1744            | (DataType::Map(expected_field, _), DataType::Map(actual_field, _)) => {
1745                assert_field_is_semantically_equivalent(expected_field, actual_field);
1746            }
1747            (DataType::Struct(expected_fields), DataType::Struct(actual_fields)) => {
1748                assert_eq!(
1749                    expected_fields.len(),
1750                    actual_fields.len(),
1751                    "{context}: struct must have same number of fields"
1752                );
1753                for (ef, af) in expected_fields.iter().zip(actual_fields.iter()) {
1754                    assert_field_is_semantically_equivalent(ef, af);
1755                }
1756            }
1757            (
1758                DataType::Union(expected_fields, expected_mode),
1759                DataType::Union(actual_fields, actual_mode),
1760            ) => {
1761                assert_eq!(
1762                    expected_mode, actual_mode,
1763                    "{context}: union mode must match"
1764                );
1765                assert_eq!(
1766                    expected_fields.len(),
1767                    actual_fields.len(),
1768                    "{context}: union must have same number of variants"
1769                );
1770                for ((exp_id, exp_field), (act_id, act_field)) in
1771                    expected_fields.iter().zip(actual_fields.iter())
1772                {
1773                    assert_eq!(exp_id, act_id, "{context}: union type ids must match");
1774                    assert_field_is_semantically_equivalent(exp_field, act_field);
1775                }
1776            }
1777            _ => {
1778                assert_eq!(expected, actual, "{context}: data types must be identical");
1779            }
1780        }
1781    }
1782
1783    fn assert_batch_data_is_identical(expected: &RecordBatch, actual: &RecordBatch) {
1784        assert_eq!(
1785            expected.num_columns(),
1786            actual.num_columns(),
1787            "RecordBatches must have the same number of columns"
1788        );
1789        assert_eq!(
1790            expected.num_rows(),
1791            actual.num_rows(),
1792            "RecordBatches must have the same number of rows"
1793        );
1794
1795        for i in 0..expected.num_columns() {
1796            let context = format!("Column {i}");
1797            let expected_col = expected.column(i);
1798            let actual_col = actual.column(i);
1799            assert_array_data_is_identical(expected_col, actual_col, &context);
1800        }
1801    }
1802
1803    /// Recursively asserts that the data content of two Arrays is identical.
1804    fn assert_array_data_is_identical(expected: &dyn Array, actual: &dyn Array, context: &str) {
1805        assert_eq!(
1806            expected.nulls(),
1807            actual.nulls(),
1808            "{context}: null buffers must match"
1809        );
1810        assert_eq!(
1811            expected.len(),
1812            actual.len(),
1813            "{context}: array lengths must match"
1814        );
1815
1816        match (expected.data_type(), actual.data_type()) {
1817            (DataType::Union(expected_fields, _), DataType::Union(..)) => {
1818                let expected_union = expected.as_any().downcast_ref::<UnionArray>().unwrap();
1819                let actual_union = actual.as_any().downcast_ref::<UnionArray>().unwrap();
1820
1821                // Compare the type_ids buffer (always the first buffer).
1822                assert_eq!(
1823                    &expected.to_data().buffers()[0],
1824                    &actual.to_data().buffers()[0],
1825                    "{context}: union type_ids buffer mismatch"
1826                );
1827
1828                // For dense unions, compare the value_offsets buffer (the second buffer).
1829                if expected.to_data().buffers().len() > 1 {
1830                    assert_eq!(
1831                        &expected.to_data().buffers()[1],
1832                        &actual.to_data().buffers()[1],
1833                        "{context}: union value_offsets buffer mismatch"
1834                    );
1835                }
1836
1837                // Recursively compare children based on the fields in the DataType.
1838                for (type_id, _) in expected_fields.iter() {
1839                    let child_context = format!("{context} -> child variant {type_id}");
1840                    assert_array_data_is_identical(
1841                        expected_union.child(type_id),
1842                        actual_union.child(type_id),
1843                        &child_context,
1844                    );
1845                }
1846            }
1847            (DataType::Struct(_), DataType::Struct(_)) => {
1848                let expected_struct = expected.as_any().downcast_ref::<StructArray>().unwrap();
1849                let actual_struct = actual.as_any().downcast_ref::<StructArray>().unwrap();
1850                for i in 0..expected_struct.num_columns() {
1851                    let child_context = format!("{context} -> struct child {i}");
1852                    assert_array_data_is_identical(
1853                        expected_struct.column(i),
1854                        actual_struct.column(i),
1855                        &child_context,
1856                    );
1857                }
1858            }
1859            // Fallback for primitive types and other types where buffer comparison is sufficient.
1860            _ => {
1861                assert_eq!(
1862                    expected.to_data().buffers(),
1863                    actual.to_data().buffers(),
1864                    "{context}: data buffers must match"
1865                );
1866            }
1867        }
1868    }
1869
1870    /// Checks that `actual_meta` contains all of `expected_meta`, and any additional
1871    /// keys in `actual_meta` are from a permitted set.
1872    fn assert_metadata_is_superset(
1873        expected_meta: &arrow_schema::Metadata,
1874        actual_meta: &arrow_schema::Metadata,
1875        context: &str,
1876    ) {
1877        let allowed_additions: HashSet<&str> =
1878            vec!["arrowUnionMode", "arrowUnionTypeIds", "avro.name"]
1879                .into_iter()
1880                .collect();
1881        for (key, expected_value) in expected_meta {
1882            match actual_meta.get(key) {
1883                Some(actual_value) => assert_eq!(
1884                    expected_value, actual_value,
1885                    "{context}: preserved metadata for key '{key}' must have the same value"
1886                ),
1887                None => panic!("{context}: metadata key '{key}' was lost during roundtrip"),
1888            }
1889        }
1890        for key in actual_meta.keys() {
1891            if !expected_meta.contains_key(key) && !allowed_additions.contains(key.as_str()) {
1892                panic!("{context}: unexpected metadata key '{key}' was added during roundtrip");
1893            }
1894        }
1895    }
1896
1897    #[test]
1898    fn test_union_roundtrip() -> Result<(), AvroError> {
1899        let file_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
1900            .join("test/data/union_fields.avro")
1901            .to_string_lossy()
1902            .into_owned();
1903        let rdr_file = File::open(&file_path).expect("open avro/union_fields.avro");
1904        let reader = ReaderBuilder::new()
1905            .build(BufReader::new(rdr_file))
1906            .expect("build reader for union_fields.avro");
1907        let schema = reader.schema();
1908        let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
1909        let original =
1910            arrow::compute::concat_batches(&schema, &input_batches).expect("concat input");
1911        let mut writer = AvroWriter::new(Vec::<u8>::new(), original.schema().as_ref().clone())?;
1912        writer.write(&original)?;
1913        writer.finish()?;
1914        let bytes = writer.into_inner();
1915        let rt_reader = ReaderBuilder::new()
1916            .build(Cursor::new(bytes))
1917            .expect("build round_trip reader");
1918        let rt_schema = rt_reader.schema();
1919        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1920        let round_trip =
1921            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
1922
1923        // The nature of the crate is such that metadata gets appended during the roundtrip,
1924        // so we can't compare the schemas directly. Instead, we semantically compare the schemas and data.
1925        assert_schema_is_semantically_equivalent(&original.schema(), &round_trip.schema());
1926
1927        assert_batch_data_is_identical(&original, &round_trip);
1928        Ok(())
1929    }
1930
1931    #[test]
1932    fn test_enum_roundtrip_uses_reader_fixture() -> Result<(), AvroError> {
1933        // Read the known-good enum file (same as reader::test_simple)
1934        let path = arrow_test_data("avro/simple_enum.avro");
1935        let rdr_file = File::open(&path).expect("open avro/simple_enum.avro");
1936        let reader = ReaderBuilder::new()
1937            .build(BufReader::new(rdr_file))
1938            .expect("build reader for simple_enum.avro");
1939        // Concatenate all batches to one RecordBatch for a clean equality check
1940        let in_schema = reader.schema();
1941        let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
1942        let original =
1943            arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
1944        // Sanity: expect at least one Dictionary(Int32, Utf8) column (enum)
1945        let has_enum_dict = in_schema.fields().iter().any(|f| {
1946            matches!(
1947                f.data_type(),
1948                DataType::Dictionary(k, v) if **k == DataType::Int32 && **v == DataType::Utf8
1949            )
1950        });
1951        assert!(
1952            has_enum_dict,
1953            "Expected at least one enum-mapped Dictionary<Int32, Utf8> field"
1954        );
1955        // Write with OCF writer into memory using the reader-provided Arrow schema.
1956        // The writer will embed the Avro JSON from `avro.schema` metadata if present.
1957        let buffer: Vec<u8> = Vec::new();
1958        let mut writer = AvroWriter::new(buffer, in_schema.as_ref().clone())?;
1959        writer.write(&original)?;
1960        writer.finish()?;
1961        let bytes = writer.into_inner();
1962        // Read back and compare for exact equality (schema + data)
1963        let rt_reader = ReaderBuilder::new()
1964            .build(Cursor::new(bytes))
1965            .expect("reader for round-trip");
1966        let rt_schema = rt_reader.schema();
1967        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
1968        let roundtrip =
1969            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat roundtrip");
1970        assert_eq!(roundtrip, original, "Avro enum round-trip mismatch");
1971        Ok(())
1972    }
1973
1974    #[test]
1975    fn test_builder_propagates_capacity_to_writer() -> Result<(), AvroError> {
1976        let cap = 64 * 1024;
1977        let buffer = Vec::<u8>::new();
1978        let mut writer = WriterBuilder::new(make_schema())
1979            .with_capacity(cap)
1980            .build::<_, AvroOcfFormat>(buffer)?;
1981        assert_eq!(writer.capacity, cap, "builder capacity not propagated");
1982        let batch = make_batch();
1983        writer.write(&batch)?;
1984        writer.finish()?;
1985        let out = writer.into_inner();
1986        assert_eq!(&out[..4], b"Obj\x01", "OCF magic missing/incorrect");
1987        Ok(())
1988    }
1989
1990    #[test]
1991    fn test_stream_writer_stores_capacity_direct_writes() -> Result<(), AvroError> {
1992        use arrow_array::{ArrayRef, Int32Array};
1993        use arrow_schema::{DataType, Field, Schema};
1994        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
1995        let batch = RecordBatch::try_new(
1996            Arc::new(schema.clone()),
1997            vec![Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef],
1998        )?;
1999        let cap = 8192;
2000        let mut writer = WriterBuilder::new(schema)
2001            .with_capacity(cap)
2002            .build::<_, AvroSoeFormat>(Vec::new())?;
2003        assert_eq!(writer.capacity, cap);
2004        writer.write(&batch)?;
2005        let _bytes = writer.into_inner();
2006        Ok(())
2007    }
2008
2009    #[cfg(feature = "avro_custom_types")]
2010    #[test]
2011    fn test_roundtrip_duration_logical_types_ocf() -> Result<(), AvroError> {
2012        let file_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
2013            .join("test/data/duration_logical_types.avro")
2014            .to_string_lossy()
2015            .into_owned();
2016
2017        let in_file = File::open(&file_path)
2018            .unwrap_or_else(|_| panic!("Failed to open test file: {file_path}"));
2019
2020        let reader = ReaderBuilder::new()
2021            .build(BufReader::new(in_file))
2022            .expect("build reader for duration_logical_types.avro");
2023        let in_schema = reader.schema();
2024
2025        let expected_units: HashSet<TimeUnit> = [
2026            TimeUnit::Nanosecond,
2027            TimeUnit::Microsecond,
2028            TimeUnit::Millisecond,
2029            TimeUnit::Second,
2030        ]
2031        .into_iter()
2032        .collect();
2033
2034        let found_units: HashSet<TimeUnit> = in_schema
2035            .fields()
2036            .iter()
2037            .filter_map(|f| match f.data_type() {
2038                DataType::Duration(unit) => Some(*unit),
2039                _ => None,
2040            })
2041            .collect();
2042
2043        assert_eq!(
2044            found_units, expected_units,
2045            "Expected to find all four Duration TimeUnits in the schema from the initial read"
2046        );
2047
2048        let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
2049        let input =
2050            arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
2051
2052        let tmp = NamedTempFile::new().expect("create temp file");
2053        {
2054            let out_file = File::create(tmp.path()).expect("create temp avro");
2055            let mut writer = AvroWriter::new(out_file, in_schema.as_ref().clone())?;
2056            writer.write(&input)?;
2057            writer.finish()?;
2058        }
2059
2060        let rt_file = File::open(tmp.path()).expect("open round_trip avro");
2061        let rt_reader = ReaderBuilder::new()
2062            .build(BufReader::new(rt_file))
2063            .expect("build round_trip reader");
2064        let rt_schema = rt_reader.schema();
2065        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2066        let round_trip =
2067            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
2068
2069        assert_eq!(round_trip, input);
2070        Ok(())
2071    }
2072
2073    #[cfg(feature = "avro_custom_types")]
2074    #[test]
2075    fn test_run_end_encoded_roundtrip_writer() -> Result<(), AvroError> {
2076        let run_ends = Int32Array::from(vec![3, 5, 7, 8]);
2077        let run_values = Int32Array::from(vec![Some(1), Some(2), None, Some(3)]);
2078        let ree = RunArray::<Int32Type>::try_new(&run_ends, &run_values)?;
2079        let field = Field::new("x", ree.data_type().clone(), true);
2080        let schema = Schema::new(vec![field]);
2081        let batch = RecordBatch::try_new(
2082            Arc::new(schema.clone()),
2083            vec![Arc::new(ree.clone()) as ArrayRef],
2084        )?;
2085        let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
2086        writer.write(&batch)?;
2087        writer.finish()?;
2088        let bytes = writer.into_inner();
2089        let reader = ReaderBuilder::new().build(Cursor::new(bytes))?;
2090        let out_schema = reader.schema();
2091        let batches = reader.collect::<Result<Vec<_>, _>>()?;
2092        let out = arrow::compute::concat_batches(&out_schema, &batches).expect("concat output");
2093        assert_eq!(out.num_columns(), 1);
2094        assert_eq!(out.num_rows(), 8);
2095        match out.schema().field(0).data_type() {
2096            DataType::RunEndEncoded(run_ends_field, values_field) => {
2097                assert_eq!(
2098                    run_ends_field.name(),
2099                    Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME
2100                );
2101                assert_eq!(run_ends_field.data_type(), &DataType::Int32);
2102                assert_eq!(values_field.name(), Field::REE_VALUES_FIELD_DEFAULT_NAME);
2103                assert_eq!(values_field.data_type(), &DataType::Int32);
2104                assert!(values_field.is_nullable());
2105                let got_ree = out
2106                    .column(0)
2107                    .as_any()
2108                    .downcast_ref::<RunArray<Int32Type>>()
2109                    .expect("RunArray<Int32Type>");
2110                assert_eq!(got_ree, &ree);
2111            }
2112            other => {
2113                panic!("Unexpected DataType for round-tripped RunEndEncoded column: {other:?}")
2114            }
2115        }
2116        Ok(())
2117    }
2118
2119    #[cfg(feature = "avro_custom_types")]
2120    #[test]
2121    fn test_run_end_encoded_string_values_int16_run_ends_roundtrip_writer() -> Result<(), AvroError>
2122    {
2123        let run_ends = Int16Array::from(vec![2, 5, 7]); // end indices
2124        let run_values = StringArray::from(vec![Some("a"), None, Some("c")]);
2125        let ree = RunArray::<Int16Type>::try_new(&run_ends, &run_values)?;
2126        let field = Field::new("s", ree.data_type().clone(), true);
2127        let schema = Schema::new(vec![field]);
2128        let batch = RecordBatch::try_new(
2129            Arc::new(schema.clone()),
2130            vec![Arc::new(ree.clone()) as ArrayRef],
2131        )?;
2132        let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
2133        writer.write(&batch)?;
2134        writer.finish()?;
2135        let bytes = writer.into_inner();
2136        let reader = ReaderBuilder::new().build(Cursor::new(bytes))?;
2137        let out_schema = reader.schema();
2138        let batches = reader.collect::<Result<Vec<_>, _>>()?;
2139        let out = arrow::compute::concat_batches(&out_schema, &batches).expect("concat output");
2140        assert_eq!(out.num_columns(), 1);
2141        assert_eq!(out.num_rows(), 7);
2142        match out.schema().field(0).data_type() {
2143            DataType::RunEndEncoded(run_ends_field, values_field) => {
2144                assert_eq!(run_ends_field.data_type(), &DataType::Int16);
2145                assert_eq!(values_field.data_type(), &DataType::Utf8);
2146                assert!(
2147                    values_field.is_nullable(),
2148                    "REE 'values' child should be nullable"
2149                );
2150                let got = out
2151                    .column(0)
2152                    .as_any()
2153                    .downcast_ref::<RunArray<Int16Type>>()
2154                    .expect("RunArray<Int16Type>");
2155                assert_eq!(got, &ree);
2156            }
2157            other => panic!("Unexpected DataType: {other:?}"),
2158        }
2159        Ok(())
2160    }
2161
2162    #[cfg(feature = "avro_custom_types")]
2163    #[test]
2164    fn test_run_end_encoded_int64_run_ends_numeric_values_roundtrip_writer() -> Result<(), AvroError>
2165    {
2166        let run_ends = Int64Array::from(vec![4_i64, 8_i64]);
2167        let run_values = Int32Array::from(vec![Some(999), Some(-5)]);
2168        let ree = RunArray::<Int64Type>::try_new(&run_ends, &run_values)?;
2169        let field = Field::new("y", ree.data_type().clone(), true);
2170        let schema = Schema::new(vec![field]);
2171        let batch = RecordBatch::try_new(
2172            Arc::new(schema.clone()),
2173            vec![Arc::new(ree.clone()) as ArrayRef],
2174        )?;
2175        let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
2176        writer.write(&batch)?;
2177        writer.finish()?;
2178        let bytes = writer.into_inner();
2179        let reader = ReaderBuilder::new().build(Cursor::new(bytes))?;
2180        let out_schema = reader.schema();
2181        let batches = reader.collect::<Result<Vec<_>, _>>()?;
2182        let out = arrow::compute::concat_batches(&out_schema, &batches).expect("concat output");
2183        assert_eq!(out.num_columns(), 1);
2184        assert_eq!(out.num_rows(), 8);
2185        match out.schema().field(0).data_type() {
2186            DataType::RunEndEncoded(run_ends_field, values_field) => {
2187                assert_eq!(run_ends_field.data_type(), &DataType::Int64);
2188                assert_eq!(values_field.data_type(), &DataType::Int32);
2189                assert!(values_field.is_nullable());
2190                let got = out
2191                    .column(0)
2192                    .as_any()
2193                    .downcast_ref::<RunArray<Int64Type>>()
2194                    .expect("RunArray<Int64Type>");
2195                assert_eq!(got, &ree);
2196            }
2197            other => panic!("Unexpected DataType for REE column: {other:?}"),
2198        }
2199        Ok(())
2200    }
2201
2202    #[cfg(feature = "avro_custom_types")]
2203    #[test]
2204    fn test_run_end_encoded_sliced_roundtrip_writer() -> Result<(), AvroError> {
2205        let run_ends = Int32Array::from(vec![3, 5, 7, 8]);
2206        let run_values = Int32Array::from(vec![Some(1), Some(2), None, Some(3)]);
2207        let base = RunArray::<Int32Type>::try_new(&run_ends, &run_values)?;
2208        let offset = 1usize;
2209        let length = 6usize;
2210        let base_values = base.values().as_primitive::<Int32Type>();
2211        let mut logical_window: Vec<Option<i32>> = Vec::with_capacity(length);
2212        for i in offset..offset + length {
2213            let phys = base.get_physical_index(i);
2214            let v = if base_values.is_null(phys) {
2215                None
2216            } else {
2217                Some(base_values.value(phys))
2218            };
2219            logical_window.push(v);
2220        }
2221
2222        fn compress_run_ends_i32(vals: &[Option<i32>]) -> (Int32Array, Int32Array) {
2223            if vals.is_empty() {
2224                return (Int32Array::new_null(0), Int32Array::new_null(0));
2225            }
2226            let mut run_ends_out: Vec<i32> = Vec::new();
2227            let mut run_vals_out: Vec<Option<i32>> = Vec::new();
2228            let mut cur = vals[0];
2229            let mut len = 1i32;
2230            for v in &vals[1..] {
2231                if *v == cur {
2232                    len += 1;
2233                } else {
2234                    let last_end = run_ends_out.last().copied().unwrap_or(0);
2235                    run_ends_out.push(last_end + len);
2236                    run_vals_out.push(cur);
2237                    cur = *v;
2238                    len = 1;
2239                }
2240            }
2241            let last_end = run_ends_out.last().copied().unwrap_or(0);
2242            run_ends_out.push(last_end + len);
2243            run_vals_out.push(cur);
2244            (
2245                Int32Array::from(run_ends_out),
2246                Int32Array::from(run_vals_out),
2247            )
2248        }
2249        let (owned_run_ends, owned_run_values) = compress_run_ends_i32(&logical_window);
2250        let owned_slice = RunArray::<Int32Type>::try_new(&owned_run_ends, &owned_run_values)?;
2251        let field = Field::new("x", owned_slice.data_type().clone(), true);
2252        let schema = Schema::new(vec![field]);
2253        let batch = RecordBatch::try_new(
2254            Arc::new(schema.clone()),
2255            vec![Arc::new(owned_slice.clone()) as ArrayRef],
2256        )?;
2257        let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
2258        writer.write(&batch)?;
2259        writer.finish()?;
2260        let bytes = writer.into_inner();
2261        let reader = ReaderBuilder::new().build(Cursor::new(bytes))?;
2262        let out_schema = reader.schema();
2263        let batches = reader.collect::<Result<Vec<_>, _>>()?;
2264        let out = arrow::compute::concat_batches(&out_schema, &batches).expect("concat output");
2265        assert_eq!(out.num_columns(), 1);
2266        assert_eq!(out.num_rows(), length);
2267        match out.schema().field(0).data_type() {
2268            DataType::RunEndEncoded(run_ends_field, values_field) => {
2269                assert_eq!(run_ends_field.data_type(), &DataType::Int32);
2270                assert_eq!(values_field.data_type(), &DataType::Int32);
2271                assert!(values_field.is_nullable());
2272                let got = out
2273                    .column(0)
2274                    .as_any()
2275                    .downcast_ref::<RunArray<Int32Type>>()
2276                    .expect("RunArray<Int32Type>");
2277                fn expand_ree_to_int32(a: &RunArray<Int32Type>) -> Int32Array {
2278                    let vals = a.values().as_primitive::<Int32Type>();
2279                    let mut out: Vec<Option<i32>> = Vec::with_capacity(a.len());
2280                    for i in 0..a.len() {
2281                        let phys = a.get_physical_index(i);
2282                        out.push(if vals.is_null(phys) {
2283                            None
2284                        } else {
2285                            Some(vals.value(phys))
2286                        });
2287                    }
2288                    Int32Array::from(out)
2289                }
2290                let got_logical = expand_ree_to_int32(got);
2291                let expected_logical = Int32Array::from(logical_window);
2292                assert_eq!(
2293                    got_logical, expected_logical,
2294                    "Logical values differ after REE slice round-trip"
2295                );
2296            }
2297            other => panic!("Unexpected DataType for REE column: {other:?}"),
2298        }
2299        Ok(())
2300    }
2301
2302    #[cfg(not(feature = "avro_custom_types"))]
2303    #[test]
2304    fn test_run_end_encoded_roundtrip_writer_feature_off() -> Result<(), AvroError> {
2305        use arrow_schema::{DataType, Field, Schema};
2306        let run_ends = arrow_array::Int32Array::from(vec![3, 5, 7, 8]);
2307        let run_values = arrow_array::Int32Array::from(vec![Some(1), Some(2), None, Some(3)]);
2308        let ree = arrow_array::RunArray::<arrow_array::types::Int32Type>::try_new(
2309            &run_ends,
2310            &run_values,
2311        )?;
2312        let field = Field::new("x", ree.data_type().clone(), true);
2313        let schema = Schema::new(vec![field]);
2314        let batch =
2315            RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(ree) as ArrayRef])?;
2316        let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
2317        writer.write(&batch)?;
2318        writer.finish()?;
2319        let bytes = writer.into_inner();
2320        let reader = ReaderBuilder::new().build(Cursor::new(bytes))?;
2321        let out_schema = reader.schema();
2322        let batches = reader.collect::<Result<Vec<_>, _>>()?;
2323        let out = arrow::compute::concat_batches(&out_schema, &batches).expect("concat output");
2324        assert_eq!(out.num_columns(), 1);
2325        assert_eq!(out.num_rows(), 8);
2326        assert_eq!(out.schema().field(0).data_type(), &DataType::Int32);
2327        let got = out.column(0).as_primitive::<Int32Type>();
2328        let expected = Int32Array::from(vec![
2329            Some(1),
2330            Some(1),
2331            Some(1),
2332            Some(2),
2333            Some(2),
2334            None,
2335            None,
2336            Some(3),
2337        ]);
2338        assert_eq!(got, &expected);
2339        Ok(())
2340    }
2341
2342    #[cfg(not(feature = "avro_custom_types"))]
2343    #[test]
2344    fn test_run_end_encoded_string_values_int16_run_ends_roundtrip_writer_feature_off()
2345    -> Result<(), AvroError> {
2346        use arrow_schema::{DataType, Field, Schema};
2347        let run_ends = arrow_array::Int16Array::from(vec![2, 5, 7]);
2348        let run_values = arrow_array::StringArray::from(vec![Some("a"), None, Some("c")]);
2349        let ree = arrow_array::RunArray::<arrow_array::types::Int16Type>::try_new(
2350            &run_ends,
2351            &run_values,
2352        )?;
2353        let field = Field::new("s", ree.data_type().clone(), true);
2354        let schema = Schema::new(vec![field]);
2355        let batch =
2356            RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(ree) as ArrayRef])?;
2357        let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
2358        writer.write(&batch)?;
2359        writer.finish()?;
2360        let bytes = writer.into_inner();
2361        let reader = ReaderBuilder::new().build(Cursor::new(bytes))?;
2362        let out_schema = reader.schema();
2363        let batches = reader.collect::<Result<Vec<_>, _>>()?;
2364        let out = arrow::compute::concat_batches(&out_schema, &batches).expect("concat output");
2365        assert_eq!(out.num_columns(), 1);
2366        assert_eq!(out.num_rows(), 7);
2367        assert_eq!(out.schema().field(0).data_type(), &DataType::Utf8);
2368        let got = out
2369            .column(0)
2370            .as_any()
2371            .downcast_ref::<arrow_array::StringArray>()
2372            .expect("StringArray");
2373        let expected = arrow_array::StringArray::from(vec![
2374            Some("a"),
2375            Some("a"),
2376            None,
2377            None,
2378            None,
2379            Some("c"),
2380            Some("c"),
2381        ]);
2382        assert_eq!(got, &expected);
2383        Ok(())
2384    }
2385
2386    #[cfg(not(feature = "avro_custom_types"))]
2387    #[test]
2388    fn test_run_end_encoded_int64_run_ends_numeric_values_roundtrip_writer_feature_off()
2389    -> Result<(), AvroError> {
2390        use arrow_schema::{DataType, Field, Schema};
2391        let run_ends = arrow_array::Int64Array::from(vec![4_i64, 8_i64]);
2392        let run_values = Int32Array::from(vec![Some(999), Some(-5)]);
2393        let ree = arrow_array::RunArray::<arrow_array::types::Int64Type>::try_new(
2394            &run_ends,
2395            &run_values,
2396        )?;
2397        let field = Field::new("y", ree.data_type().clone(), true);
2398        let schema = Schema::new(vec![field]);
2399        let batch =
2400            RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(ree) as ArrayRef])?;
2401        let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
2402        writer.write(&batch)?;
2403        writer.finish()?;
2404        let bytes = writer.into_inner();
2405        let reader = ReaderBuilder::new().build(Cursor::new(bytes))?;
2406        let out_schema = reader.schema();
2407        let batches = reader.collect::<Result<Vec<_>, _>>()?;
2408        let out = arrow::compute::concat_batches(&out_schema, &batches).expect("concat output");
2409        assert_eq!(out.num_columns(), 1);
2410        assert_eq!(out.num_rows(), 8);
2411        assert_eq!(out.schema().field(0).data_type(), &DataType::Int32);
2412        let got = out.column(0).as_primitive::<Int32Type>();
2413        let expected = Int32Array::from(vec![
2414            Some(999),
2415            Some(999),
2416            Some(999),
2417            Some(999),
2418            Some(-5),
2419            Some(-5),
2420            Some(-5),
2421            Some(-5),
2422        ]);
2423        assert_eq!(got, &expected);
2424        Ok(())
2425    }
2426
2427    #[cfg(not(feature = "avro_custom_types"))]
2428    #[test]
2429    fn test_run_end_encoded_sliced_roundtrip_writer_feature_off() -> Result<(), AvroError> {
2430        use arrow_schema::{DataType, Field, Schema};
2431        let run_ends = Int32Array::from(vec![2, 4, 6]);
2432        let run_values = Int32Array::from(vec![Some(1), Some(2), None]);
2433        let ree = arrow_array::RunArray::<arrow_array::types::Int32Type>::try_new(
2434            &run_ends,
2435            &run_values,
2436        )?;
2437        let field = Field::new("x", ree.data_type().clone(), true);
2438        let schema = Schema::new(vec![field]);
2439        let batch =
2440            RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(ree) as ArrayRef])?;
2441        let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
2442        writer.write(&batch)?;
2443        writer.finish()?;
2444        let bytes = writer.into_inner();
2445        let reader = ReaderBuilder::new().build(Cursor::new(bytes))?;
2446        let out_schema = reader.schema();
2447        let batches = reader.collect::<Result<Vec<_>, _>>()?;
2448        let out = arrow::compute::concat_batches(&out_schema, &batches).expect("concat output");
2449        assert_eq!(out.num_columns(), 1);
2450        assert_eq!(out.num_rows(), 6);
2451        assert_eq!(out.schema().field(0).data_type(), &DataType::Int32);
2452        let got = out.column(0).as_primitive::<Int32Type>();
2453        let expected = Int32Array::from(vec![Some(1), Some(1), Some(2), Some(2), None, None]);
2454        assert_eq!(got, &expected);
2455        Ok(())
2456    }
2457
2458    #[test]
2459    // TODO: avoid requiring snappy for this file
2460    #[cfg(feature = "snappy")]
2461    fn test_nullable_impala_roundtrip() -> Result<(), AvroError> {
2462        let path = arrow_test_data("avro/nullable.impala.avro");
2463        let rdr_file = File::open(&path).expect("open avro/nullable.impala.avro");
2464        let reader = ReaderBuilder::new()
2465            .build(BufReader::new(rdr_file))
2466            .expect("build reader for nullable.impala.avro");
2467        let in_schema = reader.schema();
2468        assert!(
2469            in_schema.fields().iter().any(|f| f.is_nullable()),
2470            "expected at least one nullable field in avro/nullable.impala.avro"
2471        );
2472        let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
2473        let original =
2474            arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
2475        let buffer: Vec<u8> = Vec::new();
2476        let mut writer = AvroWriter::new(buffer, in_schema.as_ref().clone())?;
2477        writer.write(&original)?;
2478        writer.finish()?;
2479        let out_bytes = writer.into_inner();
2480        let rt_reader = ReaderBuilder::new()
2481            .build(Cursor::new(out_bytes))
2482            .expect("build reader for round-tripped in-memory OCF");
2483        let rt_schema = rt_reader.schema();
2484        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2485        let roundtrip =
2486            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat roundtrip");
2487        assert_eq!(
2488            roundtrip, original,
2489            "Round-trip Avro data mismatch for nullable.impala.avro"
2490        );
2491        Ok(())
2492    }
2493
2494    #[test]
2495    #[cfg(feature = "snappy")]
2496    fn test_datapage_v2_roundtrip() -> Result<(), AvroError> {
2497        let path = arrow_test_data("avro/datapage_v2.snappy.avro");
2498        let rdr_file = File::open(&path).expect("open avro/datapage_v2.snappy.avro");
2499        let reader = ReaderBuilder::new()
2500            .build(BufReader::new(rdr_file))
2501            .expect("build reader for datapage_v2.snappy.avro");
2502        let in_schema = reader.schema();
2503        let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
2504        let original =
2505            arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
2506        let mut writer = AvroWriter::new(Vec::<u8>::new(), in_schema.as_ref().clone())?;
2507        writer.write(&original)?;
2508        writer.finish()?;
2509        let bytes = writer.into_inner();
2510        let rt_reader = ReaderBuilder::new()
2511            .build(Cursor::new(bytes))
2512            .expect("build round-trip reader");
2513        let rt_schema = rt_reader.schema();
2514        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2515        let round_trip =
2516            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
2517        assert_eq!(
2518            round_trip, original,
2519            "Round-trip batch mismatch for datapage_v2.snappy.avro"
2520        );
2521        Ok(())
2522    }
2523
2524    #[test]
2525    #[cfg(feature = "snappy")]
2526    fn test_single_nan_roundtrip() -> Result<(), AvroError> {
2527        let path = arrow_test_data("avro/single_nan.avro");
2528        let in_file = File::open(&path).expect("open avro/single_nan.avro");
2529        let reader = ReaderBuilder::new()
2530            .build(BufReader::new(in_file))
2531            .expect("build reader for single_nan.avro");
2532        let in_schema = reader.schema();
2533        let in_batches = reader.collect::<Result<Vec<_>, _>>()?;
2534        let original =
2535            arrow::compute::concat_batches(&in_schema, &in_batches).expect("concat input");
2536        let mut writer = AvroWriter::new(Vec::<u8>::new(), original.schema().as_ref().clone())?;
2537        writer.write(&original)?;
2538        writer.finish()?;
2539        let bytes = writer.into_inner();
2540        let rt_reader = ReaderBuilder::new()
2541            .build(Cursor::new(bytes))
2542            .expect("build round_trip reader");
2543        let rt_schema = rt_reader.schema();
2544        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2545        let round_trip =
2546            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
2547        assert_eq!(
2548            round_trip, original,
2549            "Round-trip batch mismatch for avro/single_nan.avro"
2550        );
2551        Ok(())
2552    }
2553    #[test]
2554    // TODO: avoid requiring snappy for this file
2555    #[cfg(feature = "snappy")]
2556    fn test_dict_pages_offset_zero_roundtrip() -> Result<(), AvroError> {
2557        let path = arrow_test_data("avro/dict-page-offset-zero.avro");
2558        let rdr_file = File::open(&path).expect("open avro/dict-page-offset-zero.avro");
2559        let reader = ReaderBuilder::new()
2560            .build(BufReader::new(rdr_file))
2561            .expect("build reader for dict-page-offset-zero.avro");
2562        let in_schema = reader.schema();
2563        let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
2564        let original =
2565            arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
2566        let buffer: Vec<u8> = Vec::new();
2567        let mut writer = AvroWriter::new(buffer, original.schema().as_ref().clone())?;
2568        writer.write(&original)?;
2569        writer.finish()?;
2570        let bytes = writer.into_inner();
2571        let rt_reader = ReaderBuilder::new()
2572            .build(Cursor::new(bytes))
2573            .expect("build reader for round-trip");
2574        let rt_schema = rt_reader.schema();
2575        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2576        let roundtrip =
2577            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat roundtrip");
2578        assert_eq!(
2579            roundtrip, original,
2580            "Round-trip batch mismatch for avro/dict-page-offset-zero.avro"
2581        );
2582        Ok(())
2583    }
2584
2585    #[test]
2586    #[cfg(feature = "snappy")]
2587    fn test_repeated_no_annotation_roundtrip() -> Result<(), AvroError> {
2588        let path = arrow_test_data("avro/repeated_no_annotation.avro");
2589        let in_file = File::open(&path).expect("open avro/repeated_no_annotation.avro");
2590        let reader = ReaderBuilder::new()
2591            .build(BufReader::new(in_file))
2592            .expect("build reader for repeated_no_annotation.avro");
2593        let in_schema = reader.schema();
2594        let in_batches = reader.collect::<Result<Vec<_>, _>>()?;
2595        let original =
2596            arrow::compute::concat_batches(&in_schema, &in_batches).expect("concat input");
2597        let mut writer = AvroWriter::new(Vec::<u8>::new(), original.schema().as_ref().clone())?;
2598        writer.write(&original)?;
2599        writer.finish()?;
2600        let bytes = writer.into_inner();
2601        let rt_reader = ReaderBuilder::new()
2602            .build(Cursor::new(bytes))
2603            .expect("build reader for round-trip buffer");
2604        let rt_schema = rt_reader.schema();
2605        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2606        let round_trip =
2607            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round-trip");
2608        assert_eq!(
2609            round_trip, original,
2610            "Round-trip batch mismatch for avro/repeated_no_annotation.avro"
2611        );
2612        Ok(())
2613    }
2614
2615    #[test]
2616    fn test_nested_record_type_reuse_roundtrip() -> Result<(), AvroError> {
2617        let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
2618            .join("test/data/nested_record_reuse.avro")
2619            .to_string_lossy()
2620            .into_owned();
2621        let in_file = File::open(&path).expect("open avro/nested_record_reuse.avro");
2622        let reader = ReaderBuilder::new()
2623            .build(BufReader::new(in_file))
2624            .expect("build reader for nested_record_reuse.avro");
2625        let in_schema = reader.schema();
2626        let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
2627        let input =
2628            arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
2629        let mut writer = AvroWriter::new(Vec::<u8>::new(), in_schema.as_ref().clone())?;
2630        writer.write(&input)?;
2631        writer.finish()?;
2632        let bytes = writer.into_inner();
2633        let rt_reader = ReaderBuilder::new()
2634            .build(Cursor::new(bytes))
2635            .expect("build round_trip reader");
2636        let rt_schema = rt_reader.schema();
2637        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2638        let round_trip =
2639            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
2640        assert_eq!(
2641            round_trip, input,
2642            "Round-trip batch mismatch for nested_record_reuse.avro"
2643        );
2644        Ok(())
2645    }
2646
2647    #[test]
2648    fn test_enum_type_reuse_roundtrip() -> Result<(), AvroError> {
2649        let path =
2650            std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("test/data/enum_reuse.avro");
2651        let rdr_file = std::fs::File::open(&path).expect("open test/data/enum_reuse.avro");
2652        let reader = ReaderBuilder::new()
2653            .build(std::io::BufReader::new(rdr_file))
2654            .expect("build reader for enum_reuse.avro");
2655        let in_schema = reader.schema();
2656        let input_batches = reader.collect::<Result<Vec<_>, _>>()?;
2657        let original =
2658            arrow::compute::concat_batches(&in_schema, &input_batches).expect("concat input");
2659        let mut writer = AvroWriter::new(Vec::<u8>::new(), original.schema().as_ref().clone())?;
2660        writer.write(&original)?;
2661        writer.finish()?;
2662        let bytes = writer.into_inner();
2663        let rt_reader = ReaderBuilder::new()
2664            .build(std::io::Cursor::new(bytes))
2665            .expect("build round_trip reader");
2666        let rt_schema = rt_reader.schema();
2667        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2668        let round_trip =
2669            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat round_trip");
2670        assert_eq!(
2671            round_trip, original,
2672            "Avro enum type reuse round-trip mismatch"
2673        );
2674        Ok(())
2675    }
2676
2677    #[test]
2678    #[cfg_attr(miri, ignore)] // Takes too long
2679    fn comprehensive_e2e_test_roundtrip() -> Result<(), AvroError> {
2680        let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
2681            .join("test/data/comprehensive_e2e.avro");
2682        let rdr_file = File::open(&path).expect("open test/data/comprehensive_e2e.avro");
2683        let reader = ReaderBuilder::new()
2684            .build(BufReader::new(rdr_file))
2685            .expect("build reader for comprehensive_e2e.avro");
2686        let in_schema = reader.schema();
2687        let in_batches = reader.collect::<Result<Vec<_>, _>>()?;
2688        let original =
2689            arrow::compute::concat_batches(&in_schema, &in_batches).expect("concat input");
2690        let sink: Vec<u8> = Vec::new();
2691        let mut writer = AvroWriter::new(sink, original.schema().as_ref().clone())?;
2692        writer.write(&original)?;
2693        writer.finish()?;
2694        let bytes = writer.into_inner();
2695        let rt_reader = ReaderBuilder::new()
2696            .build(Cursor::new(bytes))
2697            .expect("build round-trip reader");
2698        let rt_schema = rt_reader.schema();
2699        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2700        let roundtrip =
2701            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat roundtrip");
2702        assert_eq!(
2703            roundtrip, original,
2704            "Round-trip batch mismatch for comprehensive_e2e.avro"
2705        );
2706        Ok(())
2707    }
2708
2709    #[test]
2710    fn test_roundtrip_new_time_encoders_writer() -> Result<(), AvroError> {
2711        let schema = Schema::new(vec![
2712            Field::new("d32", DataType::Date32, false),
2713            Field::new("t32_ms", DataType::Time32(TimeUnit::Millisecond), false),
2714            Field::new("t64_us", DataType::Time64(TimeUnit::Microsecond), false),
2715            Field::new(
2716                "ts_ms",
2717                DataType::Timestamp(TimeUnit::Millisecond, None),
2718                false,
2719            ),
2720            Field::new(
2721                "ts_us",
2722                DataType::Timestamp(TimeUnit::Microsecond, None),
2723                false,
2724            ),
2725            Field::new(
2726                "ts_ns",
2727                DataType::Timestamp(TimeUnit::Nanosecond, None),
2728                false,
2729            ),
2730        ]);
2731        let d32 = Date32Array::from(vec![0, 1, -1]);
2732        let t32_ms: PrimitiveArray<Time32MillisecondType> =
2733            vec![0_i32, 12_345_i32, 86_399_999_i32].into();
2734        let t64_us: PrimitiveArray<Time64MicrosecondType> =
2735            vec![0_i64, 1_234_567_i64, 86_399_999_999_i64].into();
2736        let ts_ms: PrimitiveArray<TimestampMillisecondType> =
2737            vec![0_i64, -1_i64, 1_700_000_000_000_i64].into();
2738        let ts_us: PrimitiveArray<TimestampMicrosecondType> = vec![0_i64, 1_i64, -1_i64].into();
2739        let ts_ns: PrimitiveArray<TimestampNanosecondType> = vec![0_i64, 1_i64, -1_i64].into();
2740        let batch = RecordBatch::try_new(
2741            Arc::new(schema.clone()),
2742            vec![
2743                Arc::new(d32) as ArrayRef,
2744                Arc::new(t32_ms) as ArrayRef,
2745                Arc::new(t64_us) as ArrayRef,
2746                Arc::new(ts_ms) as ArrayRef,
2747                Arc::new(ts_us) as ArrayRef,
2748                Arc::new(ts_ns) as ArrayRef,
2749            ],
2750        )?;
2751        let mut writer = AvroWriter::new(Vec::<u8>::new(), schema.clone())?;
2752        writer.write(&batch)?;
2753        writer.finish()?;
2754        let bytes = writer.into_inner();
2755        let rt_reader = ReaderBuilder::new()
2756            .build(std::io::Cursor::new(bytes))
2757            .expect("build reader for round-trip of new time encoders");
2758        let rt_schema = rt_reader.schema();
2759        let rt_batches = rt_reader.collect::<Result<Vec<_>, _>>()?;
2760        let roundtrip =
2761            arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat roundtrip");
2762        assert_eq!(roundtrip, batch);
2763        Ok(())
2764    }
2765
2766    fn make_encoder_schema() -> Schema {
2767        Schema::new(vec![
2768            Field::new("a", DataType::Int32, false),
2769            Field::new("b", DataType::Int32, false),
2770        ])
2771    }
2772
2773    fn make_encoder_batch(schema: &Schema) -> RecordBatch {
2774        let a = Int32Array::from(vec![1, 2, 3]);
2775        let b = Int32Array::from(vec![10, 20, 30]);
2776        RecordBatch::try_new(
2777            Arc::new(schema.clone()),
2778            vec![Arc::new(a) as ArrayRef, Arc::new(b) as ArrayRef],
2779        )
2780        .expect("failed to build test RecordBatch")
2781    }
2782
2783    fn make_real_avro_schema_and_batch() -> Result<(Schema, RecordBatch, AvroSchema), AvroError> {
2784        let avro_json = r#"
2785        {
2786          "type": "record",
2787          "name": "User",
2788          "fields": [
2789            { "name": "id",     "type": "long" },
2790            { "name": "name",   "type": "string" },
2791            { "name": "active", "type": "boolean" },
2792            { "name": "tags",   "type": { "type": "array", "items": "int" } },
2793            { "name": "opt",    "type": ["null", "string"], "default": null }
2794          ]
2795        }"#;
2796        let avro_schema = AvroSchema::new(avro_json.to_string());
2797        let mut md = HashMap::new();
2798        md.insert(
2799            SCHEMA_METADATA_KEY.to_string(),
2800            avro_schema.json_string.clone(),
2801        );
2802        let item_field = Arc::new(Field::new(
2803            Field::LIST_FIELD_DEFAULT_NAME,
2804            DataType::Int32,
2805            false,
2806        ));
2807        let schema = Schema::new_with_metadata(
2808            vec![
2809                Field::new("id", DataType::Int64, false),
2810                Field::new("name", DataType::Utf8, false),
2811                Field::new("active", DataType::Boolean, false),
2812                Field::new("tags", DataType::List(item_field.clone()), false),
2813                Field::new("opt", DataType::Utf8, true),
2814            ],
2815            md,
2816        );
2817        let id = Int64Array::from(vec![1, 2, 3]);
2818        let name = StringArray::from(vec!["alice", "bob", "carol"]);
2819        let active = BooleanArray::from(vec![true, false, true]);
2820        let mut tags_builder = ListBuilder::new(Int32Builder::new()).with_field(item_field);
2821        tags_builder.values().append_value(1);
2822        tags_builder.values().append_value(2);
2823        tags_builder.append(true);
2824        tags_builder.append(true);
2825        tags_builder.values().append_value(3);
2826        tags_builder.append(true);
2827        let tags = tags_builder.finish();
2828        let opt = StringArray::from(vec![Some("x"), None, Some("z")]);
2829        let batch = RecordBatch::try_new(
2830            Arc::new(schema.clone()),
2831            vec![
2832                Arc::new(id) as ArrayRef,
2833                Arc::new(name) as ArrayRef,
2834                Arc::new(active) as ArrayRef,
2835                Arc::new(tags) as ArrayRef,
2836                Arc::new(opt) as ArrayRef,
2837            ],
2838        )?;
2839        Ok((schema, batch, avro_schema))
2840    }
2841
2842    #[test]
2843    fn test_row_writer_matches_stream_writer_soe() -> Result<(), AvroError> {
2844        let schema = make_encoder_schema();
2845        let batch = make_encoder_batch(&schema);
2846        let mut stream = AvroStreamWriter::new(Vec::<u8>::new(), schema.clone())?;
2847        stream.write(&batch)?;
2848        stream.finish()?;
2849        let stream_bytes = stream.into_inner();
2850        let mut row_writer = WriterBuilder::new(schema).build_encoder::<AvroSoeFormat>()?;
2851        row_writer.encode(&batch)?;
2852        let rows = row_writer.flush();
2853        let row_bytes: Vec<u8> = rows.bytes().to_vec();
2854        assert_eq!(stream_bytes, row_bytes);
2855        Ok(())
2856    }
2857
2858    #[test]
2859    fn test_row_writer_flush_clears_buffer() -> Result<(), AvroError> {
2860        let schema = make_encoder_schema();
2861        let batch = make_encoder_batch(&schema);
2862        let mut row_writer = WriterBuilder::new(schema).build_encoder::<AvroSoeFormat>()?;
2863        row_writer.encode(&batch)?;
2864        assert_eq!(row_writer.buffered_len(), batch.num_rows());
2865        let out1 = row_writer.flush();
2866        assert_eq!(out1.len(), batch.num_rows());
2867        assert_eq!(row_writer.buffered_len(), 0);
2868        let out2 = row_writer.flush();
2869        assert_eq!(out2.len(), 0);
2870        Ok(())
2871    }
2872
2873    #[test]
2874    fn test_row_writer_roundtrip_decoder_soe_real_avro_data() -> Result<(), AvroError> {
2875        let (schema, batch, avro_schema) = make_real_avro_schema_and_batch()?;
2876        let mut store = SchemaStore::new();
2877        store.register(avro_schema.clone())?;
2878        let mut row_writer = WriterBuilder::new(schema).build_encoder::<AvroSoeFormat>()?;
2879        row_writer.encode(&batch)?;
2880        let rows = row_writer.flush();
2881        let mut decoder = ReaderBuilder::new()
2882            .with_writer_schema_store(store)
2883            .with_batch_size(1024)
2884            .build_decoder()?;
2885        for row in rows.iter() {
2886            let consumed = decoder.decode(row.as_ref())?;
2887            assert_eq!(
2888                consumed,
2889                row.len(),
2890                "decoder should consume the full row frame"
2891            );
2892        }
2893        let out = decoder.flush()?.expect("decoded batch");
2894        let expected = pretty_format_batches(std::slice::from_ref(&batch))?.to_string();
2895        let actual = pretty_format_batches(&[out])?.to_string();
2896        assert_eq!(expected, actual);
2897        Ok(())
2898    }
2899
2900    #[test]
2901    fn test_row_writer_roundtrip_decoder_soe_streaming_chunks() -> Result<(), AvroError> {
2902        let (schema, batch, avro_schema) = make_real_avro_schema_and_batch()?;
2903        let mut store = SchemaStore::new();
2904        store.register(avro_schema.clone())?;
2905        let mut row_writer = WriterBuilder::new(schema).build_encoder::<AvroSoeFormat>()?;
2906        row_writer.encode(&batch)?;
2907        let rows = row_writer.flush();
2908        // Build a contiguous stream and frame boundaries (prefix sums) from EncodedRows.
2909        let mut stream: Vec<u8> = Vec::new();
2910        let mut boundaries: Vec<usize> = Vec::with_capacity(rows.len() + 1);
2911        boundaries.push(0usize);
2912        for row in rows.iter() {
2913            stream.extend_from_slice(row.as_ref());
2914            boundaries.push(stream.len());
2915        }
2916        let mut decoder = ReaderBuilder::new()
2917            .with_writer_schema_store(store)
2918            .with_batch_size(1024)
2919            .build_decoder()?;
2920        let mut buffered = BytesMut::new();
2921        let chunk_rows = [1usize, 2, 3, 1, 4, 2];
2922        let mut row_idx = 0usize;
2923        let mut i = 0usize;
2924        let n_rows = rows.len();
2925        while row_idx < n_rows {
2926            let take = chunk_rows[i % chunk_rows.len()];
2927            i += 1;
2928            let end_row = (row_idx + take).min(n_rows);
2929            let byte_start = boundaries[row_idx];
2930            let byte_end = boundaries[end_row];
2931            buffered.extend_from_slice(&stream[byte_start..byte_end]);
2932            loop {
2933                let consumed = decoder.decode(&buffered)?;
2934                if consumed == 0 {
2935                    break;
2936                }
2937                let _ = buffered.split_to(consumed);
2938            }
2939            assert!(
2940                buffered.is_empty(),
2941                "expected decoder to consume the entire frame-aligned chunk"
2942            );
2943            row_idx = end_row;
2944        }
2945        let out = decoder.flush()?.expect("decoded batch");
2946        let expected = pretty_format_batches(std::slice::from_ref(&batch))?.to_string();
2947        let actual = pretty_format_batches(&[out])?.to_string();
2948        assert_eq!(expected, actual);
2949        Ok(())
2950    }
2951
2952    #[test]
2953    fn test_row_writer_roundtrip_decoder_confluent_wire_format_id() -> Result<(), AvroError> {
2954        let (schema, batch, avro_schema) = make_real_avro_schema_and_batch()?;
2955        let schema_id: u32 = 42;
2956        let mut store = SchemaStore::new_with_type(FingerprintAlgorithm::Id);
2957        store.set(Fingerprint::Id(schema_id), avro_schema.clone())?;
2958        let mut row_writer = WriterBuilder::new(schema)
2959            .with_fingerprint_strategy(FingerprintStrategy::Id(schema_id))
2960            .build_encoder::<AvroSoeFormat>()?;
2961        row_writer.encode(&batch)?;
2962        let rows = row_writer.flush();
2963        let mut decoder = ReaderBuilder::new()
2964            .with_writer_schema_store(store)
2965            .with_batch_size(1024)
2966            .build_decoder()?;
2967        for row in rows.iter() {
2968            let consumed = decoder.decode(row.as_ref())?;
2969            assert_eq!(consumed, row.len());
2970        }
2971        let out = decoder.flush()?.expect("decoded batch");
2972        let expected = pretty_format_batches(std::slice::from_ref(&batch))?.to_string();
2973        let actual = pretty_format_batches(&[out])?.to_string();
2974        assert_eq!(expected, actual);
2975        Ok(())
2976    }
2977    #[test]
2978    fn test_encoder_encode_batches_flush_and_encoded_rows_methods_with_avro_binary_format()
2979    -> Result<(), AvroError> {
2980        use crate::writer::format::AvroBinaryFormat;
2981        use arrow_array::{ArrayRef, Int32Array, RecordBatch};
2982        use arrow_schema::{DataType, Field, Schema};
2983        use std::sync::Arc;
2984        let schema = Schema::new(vec![
2985            Field::new("a", DataType::Int32, false),
2986            Field::new("b", DataType::Int32, false),
2987        ]);
2988        let schema_ref = Arc::new(schema.clone());
2989        let batch1 = RecordBatch::try_new(
2990            schema_ref.clone(),
2991            vec![
2992                Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef,
2993                Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef,
2994            ],
2995        )?;
2996        let batch2 = RecordBatch::try_new(
2997            schema_ref,
2998            vec![
2999                Arc::new(Int32Array::from(vec![4, 5])) as ArrayRef,
3000                Arc::new(Int32Array::from(vec![40, 50])) as ArrayRef,
3001            ],
3002        )?;
3003        let mut encoder = WriterBuilder::new(schema).build_encoder::<AvroBinaryFormat>()?;
3004        let empty = Encoder::flush(&mut encoder);
3005        assert_eq!(EncodedRows::len(&empty), 0);
3006        assert!(EncodedRows::is_empty(&empty));
3007        assert_eq!(EncodedRows::bytes(&empty).as_ref(), &[] as &[u8]);
3008        assert_eq!(EncodedRows::offsets(&empty), &[0usize]);
3009        assert_eq!(EncodedRows::iter(&empty).count(), 0);
3010        let empty_vecs: Vec<Vec<u8>> = empty.iter().map(|b| b.to_vec()).collect();
3011        assert!(empty_vecs.is_empty());
3012        let batches = vec![batch1, batch2];
3013        Encoder::encode_batches(&mut encoder, &batches)?;
3014        assert_eq!(encoder.buffered_len(), 5);
3015        let rows = Encoder::flush(&mut encoder);
3016        assert_eq!(
3017            encoder.buffered_len(),
3018            0,
3019            "Encoder::flush should reset the internal offsets"
3020        );
3021        assert_eq!(EncodedRows::len(&rows), 5);
3022        assert!(!EncodedRows::is_empty(&rows));
3023        let expected_offsets: &[usize] = &[0, 2, 4, 6, 8, 10];
3024        assert_eq!(EncodedRows::offsets(&rows), expected_offsets);
3025        let expected_rows: Vec<Vec<u8>> = vec![
3026            vec![2, 20],
3027            vec![4, 40],
3028            vec![6, 60],
3029            vec![8, 80],
3030            vec![10, 100],
3031        ];
3032        let expected_stream: Vec<u8> = expected_rows.concat();
3033        assert_eq!(
3034            EncodedRows::bytes(&rows).as_ref(),
3035            expected_stream.as_slice()
3036        );
3037        for (i, expected) in expected_rows.iter().enumerate() {
3038            assert_eq!(EncodedRows::row(&rows, i)?.as_ref(), expected.as_slice());
3039        }
3040        let iter_rows: Vec<Vec<u8>> = EncodedRows::iter(&rows).map(|b| b.to_vec()).collect();
3041        assert_eq!(iter_rows, expected_rows);
3042        let recreated = EncodedRows::new(
3043            EncodedRows::bytes(&rows).clone(),
3044            EncodedRows::offsets(&rows).to_vec(),
3045        );
3046        assert_eq!(EncodedRows::len(&recreated), EncodedRows::len(&rows));
3047        assert_eq!(EncodedRows::bytes(&recreated), EncodedRows::bytes(&rows));
3048        assert_eq!(
3049            EncodedRows::offsets(&recreated),
3050            EncodedRows::offsets(&rows)
3051        );
3052        let rec_vecs: Vec<Vec<u8>> = recreated.iter().map(|b| b.to_vec()).collect();
3053        assert_eq!(rec_vecs, iter_rows);
3054        let empty_again = Encoder::flush(&mut encoder);
3055        assert!(EncodedRows::is_empty(&empty_again));
3056        Ok(())
3057    }
3058
3059    #[test]
3060    fn test_writer_builder_build_rejects_avro_binary_format() {
3061        use crate::writer::format::AvroBinaryFormat;
3062        use arrow_schema::{DataType, Field, Schema};
3063        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
3064        let err = WriterBuilder::new(schema)
3065            .build::<_, AvroBinaryFormat>(Vec::<u8>::new())
3066            .unwrap_err();
3067        match err {
3068            AvroError::InvalidArgument(msg) => assert_eq!(
3069                msg,
3070                "AvroBinaryFormat is only supported with Encoder, use build_encoder instead"
3071            ),
3072            other => panic!("expected InvalidArgumentError, got {other:?}"),
3073        }
3074    }
3075    #[test]
3076    fn test_row_encoder_avro_binary_format_roundtrip_decoder_with_soe_framing()
3077    -> Result<(), AvroError> {
3078        use crate::writer::format::AvroBinaryFormat;
3079        let (schema, batch, avro_schema) = make_real_avro_schema_and_batch()?;
3080        let batches: Vec<RecordBatch> = vec![batch.clone(), batch.slice(1, 2)];
3081        let expected = arrow::compute::concat_batches(&batch.schema(), &batches)?;
3082        let mut binary_encoder =
3083            WriterBuilder::new(schema.clone()).build_encoder::<AvroBinaryFormat>()?;
3084        binary_encoder.encode_batches(&batches)?;
3085        let binary_rows = binary_encoder.flush();
3086        assert_eq!(
3087            binary_rows.len(),
3088            expected.num_rows(),
3089            "binary encoder row count mismatch"
3090        );
3091        let mut soe_encoder = WriterBuilder::new(schema).build_encoder::<AvroSoeFormat>()?;
3092        soe_encoder.encode_batches(&batches)?;
3093        let soe_rows = soe_encoder.flush();
3094        assert_eq!(
3095            soe_rows.len(),
3096            binary_rows.len(),
3097            "SOE vs binary row count mismatch"
3098        );
3099        let mut store = SchemaStore::new(); // Rabin by default
3100        let fp = store.register(avro_schema)?;
3101        let fp_le_bytes = match fp {
3102            Fingerprint::Rabin(v) => v.to_le_bytes(),
3103            other => panic!("expected Rabin fingerprint from SchemaStore::new(), got {other:?}"),
3104        };
3105        const SOE_MAGIC: [u8; 2] = [0xC3, 0x01];
3106        const SOE_PREFIX_LEN: usize = 2 + 8;
3107        for i in 0..binary_rows.len() {
3108            let body = binary_rows.row(i)?;
3109            let soe = soe_rows.row(i)?;
3110            assert!(
3111                soe.len() >= SOE_PREFIX_LEN,
3112                "expected SOE row to include prefix"
3113            );
3114            assert_eq!(&soe.as_ref()[..2], &SOE_MAGIC);
3115            assert_eq!(&soe.as_ref()[2..SOE_PREFIX_LEN], &fp_le_bytes);
3116            assert_eq!(
3117                &soe.as_ref()[SOE_PREFIX_LEN..],
3118                body.as_ref(),
3119                "SOE body bytes differ from AvroBinaryFormat body bytes (row {i})"
3120            );
3121        }
3122        let mut decoder = ReaderBuilder::new()
3123            .with_writer_schema_store(store)
3124            .with_batch_size(1024)
3125            .build_decoder()?;
3126        for body in binary_rows.iter() {
3127            let mut framed = Vec::with_capacity(SOE_PREFIX_LEN + body.len());
3128            framed.extend_from_slice(&SOE_MAGIC);
3129            framed.extend_from_slice(&fp_le_bytes);
3130            framed.extend_from_slice(body.as_ref());
3131            let consumed = decoder.decode(&framed)?;
3132            assert_eq!(
3133                consumed,
3134                framed.len(),
3135                "decoder should consume the full SOE-framed message"
3136            );
3137        }
3138        let out = decoder.flush()?.expect("expected a decoded RecordBatch");
3139        let expected_str = pretty_format_batches(&[expected])?.to_string();
3140        let actual_str = pretty_format_batches(&[out])?.to_string();
3141        assert_eq!(expected_str, actual_str);
3142        Ok(())
3143    }
3144
3145    #[test]
3146    fn test_row_encoder_avro_binary_format_roundtrip_decoder_streaming_chunks()
3147    -> Result<(), AvroError> {
3148        use crate::writer::format::AvroBinaryFormat;
3149        let (schema, batch, avro_schema) = make_real_avro_schema_and_batch()?;
3150        let mut encoder = WriterBuilder::new(schema).build_encoder::<AvroBinaryFormat>()?;
3151        encoder.encode(&batch)?;
3152        let rows = encoder.flush();
3153        let mut store = SchemaStore::new();
3154        let fp = store.register(avro_schema)?;
3155        let fp_le_bytes = match fp {
3156            Fingerprint::Rabin(v) => v.to_le_bytes(),
3157            other => panic!("expected Rabin fingerprint from SchemaStore::new(), got {other:?}"),
3158        };
3159        const SOE_MAGIC: [u8; 2] = [0xC3, 0x01];
3160        const SOE_PREFIX_LEN: usize = 2 + 8;
3161        let mut stream: Vec<u8> = Vec::new();
3162        for body in rows.iter() {
3163            let msg_len: u32 = (SOE_PREFIX_LEN + body.len())
3164                .try_into()
3165                .expect("message length must fit in u32");
3166            stream.extend_from_slice(&msg_len.to_le_bytes());
3167            stream.extend_from_slice(&SOE_MAGIC);
3168            stream.extend_from_slice(&fp_le_bytes);
3169            stream.extend_from_slice(body.as_ref());
3170        }
3171        let mut decoder = ReaderBuilder::new()
3172            .with_writer_schema_store(store)
3173            .with_batch_size(1024)
3174            .build_decoder()?;
3175        let chunk_sizes = [1usize, 2, 3, 5, 8, 13, 21, 34];
3176        let mut pos = 0usize;
3177        let mut i = 0usize;
3178        let mut buffered = BytesMut::new();
3179        let mut decoded_frames = 0usize;
3180        while pos < stream.len() {
3181            let take = chunk_sizes[i % chunk_sizes.len()];
3182            i += 1;
3183            let end = (pos + take).min(stream.len());
3184            buffered.extend_from_slice(&stream[pos..end]);
3185            pos = end;
3186            loop {
3187                if buffered.len() < 4 {
3188                    break;
3189                }
3190                let msg_len =
3191                    u32::from_le_bytes([buffered[0], buffered[1], buffered[2], buffered[3]])
3192                        as usize;
3193                if buffered.len() < 4 + msg_len {
3194                    break;
3195                }
3196                let frame = buffered.split_to(4 + msg_len);
3197                let payload = &frame[4..];
3198                let consumed = decoder.decode(payload)?;
3199                assert_eq!(
3200                    consumed,
3201                    payload.len(),
3202                    "decoder should consume the full SOE-framed message"
3203                );
3204
3205                decoded_frames += 1;
3206            }
3207        }
3208        assert!(
3209            buffered.is_empty(),
3210            "expected transport framer to consume all bytes; leftover = {}",
3211            buffered.len()
3212        );
3213        assert_eq!(
3214            decoded_frames,
3215            rows.len(),
3216            "expected to decode exactly one frame per encoded row"
3217        );
3218        let out = decoder.flush()?.expect("expected decoded RecordBatch");
3219        let expected_str = pretty_format_batches(std::slice::from_ref(&batch))?.to_string();
3220        let actual_str = pretty_format_batches(&[out])?.to_string();
3221        assert_eq!(expected_str, actual_str);
3222        Ok(())
3223    }
3224
3225    /// Helper to roundtrip a RecordBatch through OCF writer/reader
3226    fn roundtrip_ocf(batch: &RecordBatch) -> Result<RecordBatch, AvroError> {
3227        let schema = batch.schema();
3228        let mut buffer = Vec::<u8>::new();
3229        let mut writer = AvroWriter::new(&mut buffer, schema.as_ref().clone())?;
3230        writer.write(batch)?;
3231        writer.finish()?;
3232        drop(writer);
3233        let reader = ReaderBuilder::new()
3234            .build(Cursor::new(buffer))
3235            .expect("build reader for roundtrip OCF");
3236        // Get the Avro schema JSON from the OCF header
3237        let avro_schema_json = reader
3238            .avro_header()
3239            .get(SCHEMA_METADATA_KEY)
3240            .map(|raw| std::str::from_utf8(raw).expect("valid UTF-8").to_string());
3241        // Get the Arrow schema and add the Avro schema metadata
3242        let arrow_schema = reader.schema();
3243        let rt_schema = if let Some(json) = avro_schema_json {
3244            let mut metadata = arrow_schema.metadata().clone();
3245            metadata.insert(SCHEMA_METADATA_KEY.to_string(), json);
3246            Arc::new(Schema::new_with_metadata(
3247                arrow_schema.fields().clone(),
3248                metadata,
3249            ))
3250        } else {
3251            arrow_schema
3252        };
3253        let rt_batches: Vec<RecordBatch> = reader.collect::<Result<Vec<_>, _>>()?;
3254        Ok(arrow::compute::concat_batches(&rt_schema, &rt_batches).expect("concat roundtrip"))
3255    }
3256
3257    /// Assert that an array roundtrips through Avro OCF and comes back identical.
3258    #[cfg(feature = "avro_custom_types")]
3259    fn assert_round_trip(array: ArrayRef) {
3260        assert_round_trip_widened(array.clone(), array);
3261    }
3262
3263    /// Assert that an input array roundtrips through Avro OCF and produces the expected output.
3264    fn assert_round_trip_widened(input: ArrayRef, expected: ArrayRef) {
3265        let schema = Schema::new(vec![Field::new("val", input.data_type().clone(), true)]);
3266        let batch =
3267            RecordBatch::try_new(Arc::new(schema), vec![input]).expect("failed to create batch");
3268        let roundtrip = roundtrip_ocf(&batch).expect("roundtrip failed");
3269        assert_eq!(
3270            roundtrip.column(0).data_type(),
3271            expected.data_type(),
3272            "output data type mismatch"
3273        );
3274        assert_eq!(
3275            roundtrip.column(0).to_data(),
3276            expected.to_data(),
3277            "output data mismatch"
3278        );
3279    }
3280
3281    #[cfg(feature = "avro_custom_types")]
3282    #[test]
3283    fn test_roundtrip_int8_custom_types() {
3284        assert_round_trip(Arc::new(Int8Array::from(vec![
3285            Some(i8::MIN),
3286            Some(-1),
3287            Some(0),
3288            None,
3289            Some(1),
3290            Some(i8::MAX),
3291        ])));
3292    }
3293
3294    #[cfg(not(feature = "avro_custom_types"))]
3295    #[test]
3296    fn test_roundtrip_int8_no_custom_widens_to_int32() {
3297        assert_round_trip_widened(
3298            Arc::new(Int8Array::from(vec![
3299                Some(i8::MIN),
3300                Some(-1),
3301                Some(0),
3302                None,
3303                Some(1),
3304                Some(i8::MAX),
3305            ])),
3306            Arc::new(Int32Array::from(vec![
3307                Some(i8::MIN as i32),
3308                Some(-1),
3309                Some(0),
3310                None,
3311                Some(1),
3312                Some(i8::MAX as i32),
3313            ])),
3314        );
3315    }
3316
3317    #[cfg(feature = "avro_custom_types")]
3318    #[test]
3319    fn test_roundtrip_int16_custom_types() {
3320        assert_round_trip(Arc::new(Int16Array::from(vec![
3321            Some(i16::MIN),
3322            Some(-1),
3323            Some(0),
3324            None,
3325            Some(1),
3326            Some(i16::MAX),
3327        ])));
3328    }
3329
3330    #[cfg(not(feature = "avro_custom_types"))]
3331    #[test]
3332    fn test_roundtrip_int16_no_custom_widens_to_int32() {
3333        assert_round_trip_widened(
3334            Arc::new(Int16Array::from(vec![
3335                Some(i16::MIN),
3336                Some(-1),
3337                Some(0),
3338                None,
3339                Some(1),
3340                Some(i16::MAX),
3341            ])),
3342            Arc::new(Int32Array::from(vec![
3343                Some(i16::MIN as i32),
3344                Some(-1),
3345                Some(0),
3346                None,
3347                Some(1),
3348                Some(i16::MAX as i32),
3349            ])),
3350        );
3351    }
3352
3353    #[cfg(feature = "avro_custom_types")]
3354    #[test]
3355    fn test_roundtrip_uint8_custom_types() {
3356        assert_round_trip(Arc::new(UInt8Array::from(vec![
3357            Some(0u8),
3358            Some(1),
3359            None,
3360            Some(127),
3361            Some(u8::MAX),
3362        ])));
3363    }
3364
3365    #[cfg(not(feature = "avro_custom_types"))]
3366    #[test]
3367    fn test_roundtrip_uint8_no_custom_widens_to_int32() {
3368        assert_round_trip_widened(
3369            Arc::new(UInt8Array::from(vec![
3370                Some(0u8),
3371                Some(1),
3372                None,
3373                Some(127),
3374                Some(u8::MAX),
3375            ])),
3376            Arc::new(Int32Array::from(vec![
3377                Some(0i32),
3378                Some(1),
3379                None,
3380                Some(127),
3381                Some(u8::MAX as i32),
3382            ])),
3383        );
3384    }
3385
3386    #[cfg(feature = "avro_custom_types")]
3387    #[test]
3388    fn test_roundtrip_uint16_custom_types() {
3389        assert_round_trip(Arc::new(UInt16Array::from(vec![
3390            Some(0u16),
3391            Some(1),
3392            None,
3393            Some(32767),
3394            Some(u16::MAX),
3395        ])));
3396    }
3397
3398    #[cfg(not(feature = "avro_custom_types"))]
3399    #[test]
3400    fn test_roundtrip_uint16_no_custom_widens_to_int32() {
3401        assert_round_trip_widened(
3402            Arc::new(UInt16Array::from(vec![
3403                Some(0u16),
3404                Some(1),
3405                None,
3406                Some(32767),
3407                Some(u16::MAX),
3408            ])),
3409            Arc::new(Int32Array::from(vec![
3410                Some(0i32),
3411                Some(1),
3412                None,
3413                Some(32767),
3414                Some(u16::MAX as i32),
3415            ])),
3416        );
3417    }
3418
3419    #[cfg(feature = "avro_custom_types")]
3420    #[test]
3421    fn test_roundtrip_uint32_custom_types() {
3422        assert_round_trip(Arc::new(UInt32Array::from(vec![
3423            Some(0u32),
3424            Some(1),
3425            None,
3426            Some(i32::MAX as u32),
3427            Some(u32::MAX),
3428        ])));
3429    }
3430
3431    #[cfg(not(feature = "avro_custom_types"))]
3432    #[test]
3433    fn test_roundtrip_uint32_no_custom_widens_to_int64() {
3434        assert_round_trip_widened(
3435            Arc::new(UInt32Array::from(vec![
3436                Some(0u32),
3437                Some(1),
3438                None,
3439                Some(i32::MAX as u32),
3440                Some(u32::MAX),
3441            ])),
3442            Arc::new(Int64Array::from(vec![
3443                Some(0i64),
3444                Some(1),
3445                None,
3446                Some(i32::MAX as i64),
3447                Some(u32::MAX as i64),
3448            ])),
3449        );
3450    }
3451
3452    #[cfg(feature = "avro_custom_types")]
3453    #[test]
3454    fn test_roundtrip_uint64_custom_types() {
3455        assert_round_trip(Arc::new(UInt64Array::from(vec![
3456            Some(0u64),
3457            Some(1),
3458            None,
3459            Some(i64::MAX as u64),
3460            Some(u64::MAX),
3461        ])));
3462    }
3463
3464    #[cfg(not(feature = "avro_custom_types"))]
3465    #[test]
3466    fn test_roundtrip_uint64_no_custom_widens_to_int64() {
3467        assert_round_trip_widened(
3468            Arc::new(UInt64Array::from(vec![
3469                Some(0u64),
3470                Some(1),
3471                None,
3472                Some(i64::MAX as u64),
3473            ])),
3474            Arc::new(Int64Array::from(vec![
3475                Some(0i64),
3476                Some(1),
3477                None,
3478                Some(i64::MAX),
3479            ])),
3480        );
3481    }
3482
3483    #[cfg(not(feature = "avro_custom_types"))]
3484    #[test]
3485    fn test_roundtrip_uint64_overflow_errors_without_custom() {
3486        use arrow_array::UInt64Array;
3487        let schema = Schema::new(vec![Field::new("val", DataType::UInt64, false)]);
3488        let values: Vec<u64> = vec![u64::MAX];
3489        let array = UInt64Array::from(values);
3490        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array) as ArrayRef])
3491            .expect("create batch");
3492        let result = roundtrip_ocf(&batch);
3493        assert!(
3494            result.is_err(),
3495            "Expected error when encoding UInt64 > i64::MAX without avro_custom_types"
3496        );
3497    }
3498
3499    #[cfg(feature = "avro_custom_types")]
3500    #[test]
3501    fn test_roundtrip_float16_custom_types() {
3502        assert_round_trip(Arc::new(Float16Array::from(vec![
3503            Some(f16::ZERO),
3504            Some(f16::ONE),
3505            None,
3506            Some(f16::NEG_ONE),
3507            Some(f16::MAX),
3508            Some(f16::MIN),
3509        ])));
3510    }
3511
3512    #[cfg(not(feature = "avro_custom_types"))]
3513    #[test]
3514    #[cfg_attr(miri, ignore)] // Unsupported inline assembly
3515    fn test_roundtrip_float16_no_custom_widens_to_float32() {
3516        assert_round_trip_widened(
3517            Arc::new(Float16Array::from(vec![
3518                Some(f16::ZERO),
3519                Some(f16::ONE),
3520                None,
3521                Some(f16::NEG_ONE),
3522            ])),
3523            Arc::new(Float32Array::from(vec![
3524                Some(0.0f32),
3525                Some(1.0),
3526                None,
3527                Some(-1.0),
3528            ])),
3529        );
3530    }
3531
3532    #[cfg(feature = "avro_custom_types")]
3533    #[test]
3534    fn test_roundtrip_date64_custom_types() {
3535        assert_round_trip(Arc::new(Date64Array::from(vec![
3536            Some(0i64),
3537            Some(86_400_000),
3538            None,
3539            Some(1_609_459_200_000),
3540        ])));
3541    }
3542
3543    #[cfg(not(feature = "avro_custom_types"))]
3544    #[test]
3545    fn test_roundtrip_date64_no_custom_as_timestamp_millis() {
3546        assert_round_trip_widened(
3547            Arc::new(Date64Array::from(vec![
3548                Some(0i64),
3549                Some(86_400_000),
3550                None,
3551                Some(1_609_459_200_000),
3552            ])),
3553            Arc::new(TimestampMillisecondArray::from(vec![
3554                Some(0i64),
3555                Some(86_400_000),
3556                None,
3557                Some(1_609_459_200_000),
3558            ])),
3559        );
3560    }
3561
3562    #[cfg(feature = "avro_custom_types")]
3563    #[test]
3564    fn test_roundtrip_time64_nanosecond_custom_types() {
3565        assert_round_trip(Arc::new(Time64NanosecondArray::from(vec![
3566            Some(0i64),
3567            Some(1_000_000_000),
3568            None,
3569            Some(86_399_999_999_999),
3570        ])));
3571    }
3572
3573    #[cfg(not(feature = "avro_custom_types"))]
3574    #[test]
3575    fn test_roundtrip_time64_nanos_no_custom_truncates_to_micros() {
3576        // Use values evenly divisible by 1000 to avoid truncation issues
3577        assert_round_trip_widened(
3578            Arc::new(Time64NanosecondArray::from(vec![
3579                Some(0i64),
3580                Some(1_000_000_000),
3581                None,
3582                Some(86_399_999_000_000),
3583            ])),
3584            Arc::new(Time64MicrosecondArray::from(vec![
3585                Some(0i64),
3586                Some(1_000_000),
3587                None,
3588                Some(86_399_999_000),
3589            ])),
3590        );
3591    }
3592
3593    #[cfg(feature = "avro_custom_types")]
3594    #[test]
3595    fn test_roundtrip_time32_second_custom_types() {
3596        assert_round_trip(Arc::new(Time32SecondArray::from(vec![
3597            Some(0i32),
3598            Some(3600),
3599            None,
3600            Some(86399),
3601        ])));
3602    }
3603
3604    #[cfg(not(feature = "avro_custom_types"))]
3605    #[test]
3606    fn test_roundtrip_time32_second_no_custom_scales_to_millis() {
3607        assert_round_trip_widened(
3608            Arc::new(Time32SecondArray::from(vec![
3609                Some(0i32),
3610                Some(3600),
3611                None,
3612                Some(86399),
3613            ])),
3614            Arc::new(Time32MillisecondArray::from(vec![
3615                Some(0i32),
3616                Some(3_600_000),
3617                None,
3618                Some(86_399_000),
3619            ])),
3620        );
3621    }
3622
3623    #[cfg(feature = "avro_custom_types")]
3624    #[test]
3625    fn test_roundtrip_timestamp_second_custom_types() {
3626        assert_round_trip(Arc::new(
3627            TimestampSecondArray::from(vec![Some(0i64), Some(1609459200), None, Some(1735689600)])
3628                .with_timezone("+00:00"),
3629        ));
3630    }
3631
3632    #[cfg(not(feature = "avro_custom_types"))]
3633    #[test]
3634    fn test_roundtrip_timestamp_second_no_custom_scales_to_millis() {
3635        assert_round_trip_widened(
3636            Arc::new(
3637                TimestampSecondArray::from(vec![
3638                    Some(0i64),
3639                    Some(1609459200),
3640                    None,
3641                    Some(1735689600),
3642                ])
3643                .with_timezone("+00:00"),
3644            ),
3645            Arc::new(
3646                TimestampMillisecondArray::from(vec![
3647                    Some(0i64),
3648                    Some(1_609_459_200_000),
3649                    None,
3650                    Some(1_735_689_600_000),
3651                ])
3652                .with_timezone("+00:00"),
3653            ),
3654        );
3655    }
3656
3657    #[cfg(feature = "avro_custom_types")]
3658    #[test]
3659    fn test_roundtrip_interval_year_month_custom_types() {
3660        assert_round_trip(Arc::new(IntervalYearMonthArray::from(vec![
3661            Some(0i32),
3662            Some(12),
3663            None,
3664            Some(-6),
3665            Some(25),
3666        ])));
3667    }
3668
3669    #[cfg(not(feature = "avro_custom_types"))]
3670    #[test]
3671    fn test_roundtrip_interval_year_month_no_custom() {
3672        // Only non-negative values for standard Avro duration
3673        assert_round_trip_widened(
3674            Arc::new(IntervalYearMonthArray::from(vec![
3675                Some(0i32),
3676                Some(12),
3677                None,
3678                Some(25),
3679            ])),
3680            Arc::new(IntervalMonthDayNanoArray::from(vec![
3681                Some(IntervalMonthDayNano::new(0, 0, 0)),
3682                Some(IntervalMonthDayNano::new(12, 0, 0)),
3683                None,
3684                Some(IntervalMonthDayNano::new(25, 0, 0)),
3685            ])),
3686        );
3687    }
3688
3689    #[cfg(feature = "avro_custom_types")]
3690    #[test]
3691    fn test_roundtrip_interval_day_time_custom_types() {
3692        assert_round_trip(Arc::new(IntervalDayTimeArray::from(vec![
3693            Some(IntervalDayTime::new(0, 0)),
3694            Some(IntervalDayTime::new(1, 1000)),
3695            None,
3696            Some(IntervalDayTime::new(30, 3600000)),
3697        ])));
3698    }
3699
3700    #[cfg(not(feature = "avro_custom_types"))]
3701    #[test]
3702    fn test_roundtrip_interval_day_time_no_custom() {
3703        assert_round_trip_widened(
3704            Arc::new(IntervalDayTimeArray::from(vec![
3705                Some(IntervalDayTime::new(0, 0)),
3706                Some(IntervalDayTime::new(1, 1000)),
3707                None,
3708                Some(IntervalDayTime::new(30, 3600000)),
3709            ])),
3710            Arc::new(IntervalMonthDayNanoArray::from(vec![
3711                Some(IntervalMonthDayNano::new(0, 0, 0)),
3712                Some(IntervalMonthDayNano::new(0, 1, 1_000_000_000)),
3713                None,
3714                Some(IntervalMonthDayNano::new(0, 30, 3_600_000_000_000)),
3715            ])),
3716        );
3717    }
3718
3719    #[cfg(feature = "avro_custom_types")]
3720    #[test]
3721    fn test_roundtrip_interval_month_day_nano_custom_types() {
3722        assert_round_trip(Arc::new(IntervalMonthDayNanoArray::from(vec![
3723            Some(IntervalMonthDayNano::new(0, 0, 0)),
3724            Some(IntervalMonthDayNano::new(1, 2, 3)),
3725            None,
3726            Some(IntervalMonthDayNano::new(-4, -5, -6)),
3727        ])));
3728    }
3729
3730    #[cfg(not(feature = "avro_custom_types"))]
3731    #[test]
3732    fn test_roundtrip_interval_month_day_nano_no_custom() {
3733        // Only representable values for Avro duration: non-negative and whole milliseconds
3734        assert_round_trip_widened(
3735            Arc::new(IntervalMonthDayNanoArray::from(vec![
3736                Some(IntervalMonthDayNano::new(0, 0, 0)),
3737                Some(IntervalMonthDayNano::new(1, 2, 3_000_000)),
3738                None,
3739                Some(IntervalMonthDayNano::new(4, 5, 6_000_000)),
3740            ])),
3741            Arc::new(IntervalMonthDayNanoArray::from(vec![
3742                Some(IntervalMonthDayNano::new(0, 0, 0)),
3743                Some(IntervalMonthDayNano::new(1, 2, 3_000_000)),
3744                None,
3745                Some(IntervalMonthDayNano::new(4, 5, 6_000_000)),
3746            ])),
3747        );
3748    }
3749
3750    fn schemas_equal_ignoring_metadata(left: &Schema, right: &Schema) -> bool {
3751        if left.fields().len() != right.fields().len() {
3752            return false;
3753        }
3754        for (l, r) in left.fields().iter().zip(right.fields().iter()) {
3755            if l.name() != r.name()
3756                || l.data_type() != r.data_type()
3757                || l.is_nullable() != r.is_nullable()
3758            {
3759                return false;
3760            }
3761        }
3762        true
3763    }
3764
3765    fn avro_field_type<'a>(avro_schema: &'a Value, name: &str) -> &'a Value {
3766        let fields = avro_schema
3767            .get("fields")
3768            .and_then(|v| v.as_array())
3769            .expect("avro schema has 'fields' array");
3770        fields
3771            .iter()
3772            .find(|f| f.get("name").and_then(|n| n.as_str()) == Some(name))
3773            .unwrap_or_else(|| panic!("avro schema missing field '{name}'"))
3774            .get("type")
3775            .expect("field has 'type'")
3776    }
3777
3778    #[test]
3779    #[cfg_attr(miri, ignore)] // Unsupported inline assembly
3780    fn e2e_types_and_schema_alignment() -> Result<(), AvroError> {
3781        // Values are chosen to:
3782        // - exercise full UInt64 range when `avro_custom_types` is enabled
3783        // - exercise negative / sub-millisecond intervals when `avro_custom_types` is enabled
3784        // - remain representable under standard Avro logical types when `avro_custom_types` is disabled
3785        let i8_values: Vec<Option<i8>> = vec![Some(i8::MIN), Some(-1), Some(i8::MAX)];
3786        let i16_values: Vec<Option<i16>> = vec![Some(i16::MIN), Some(-1), Some(i16::MAX)];
3787        let u8_values: Vec<Option<u8>> = vec![Some(0), Some(1), Some(u8::MAX)];
3788        let u16_values: Vec<Option<u16>> = vec![Some(0), Some(1), Some(u16::MAX)];
3789        let u32_values: Vec<Option<u32>> = vec![Some(0), Some(1), Some(u32::MAX)];
3790        let u64_values: Vec<Option<u64>> = if cfg!(feature = "avro_custom_types") {
3791            vec![Some(0), Some(i64::MAX as u64), Some((i64::MAX as u64) + 1)]
3792        } else {
3793            // Must remain <= i64::MAX when `avro_custom_types` is disabled
3794            vec![Some(0), Some((i64::MAX as u64) - 1), Some(i64::MAX as u64)]
3795        };
3796        let f16_values: Vec<Option<f16>> = vec![
3797            Some(f16::from_f32(1.5)),
3798            Some(f16::from_f32(-2.0)),
3799            Some(f16::from_f32(0.0)),
3800        ];
3801        let date64_values: Vec<Option<i64>> = vec![Some(-86_400_000), Some(0), Some(86_400_000)];
3802        let time32s_values: Vec<Option<i32>> = vec![Some(0), Some(1), Some(86_399)];
3803        let time64ns_values: Vec<Option<i64>> = vec![
3804            Some(0),
3805            Some(1_234_567_890), // truncation case for no-custom (nanos -> micros)
3806            Some(86_399_000_000_123_i64), // near end-of-day, also truncation
3807        ];
3808        let ts_s_local_values: Vec<Option<i64>> = vec![Some(-1), Some(0), Some(1)];
3809        let ts_s_utc_values: Vec<Option<i64>> = vec![Some(1), Some(2), Some(3)];
3810        let iv_ym_values: Vec<Option<i32>> = if cfg!(feature = "avro_custom_types") {
3811            vec![Some(0), Some(-6), Some(25)]
3812        } else {
3813            // Avro duration cannot represent negative months without custom types
3814            vec![Some(0), Some(12), Some(25)]
3815        };
3816        let iv_dt_values: Vec<Option<IntervalDayTime>> = if cfg!(feature = "avro_custom_types") {
3817            vec![
3818                Some(IntervalDayTime::new(0, 0)),
3819                Some(IntervalDayTime::new(1, 1000)),
3820                Some(IntervalDayTime::new(-1, -1000)),
3821            ]
3822        } else {
3823            // Avro duration cannot represent negative day-time without custom types
3824            vec![
3825                Some(IntervalDayTime::new(0, 0)),
3826                Some(IntervalDayTime::new(1, 1000)),
3827                Some(IntervalDayTime::new(30, 3_600_000)),
3828            ]
3829        };
3830        let iv_mdn_values: Vec<Option<IntervalMonthDayNano>> =
3831            if cfg!(feature = "avro_custom_types") {
3832                vec![
3833                    Some(IntervalMonthDayNano::new(0, 0, 0)),
3834                    Some(IntervalMonthDayNano::new(1, 2, 3)), // sub-millisecond
3835                    Some(IntervalMonthDayNano::new(-1, -2, -3)), // negative
3836                ]
3837            } else {
3838                // Avro duration requires non-negative and whole milliseconds
3839                vec![
3840                    Some(IntervalMonthDayNano::new(0, 0, 0)),
3841                    Some(IntervalMonthDayNano::new(1, 2, 3_000_000)), // 3ms
3842                    Some(IntervalMonthDayNano::new(10, 20, 30_000_000_000)), // 30s
3843                ]
3844            };
3845        // Build a batch containing all impacted types from issue #9290
3846        let schema = Schema::new(vec![
3847            Field::new("i8", DataType::Int8, false),
3848            Field::new("i16", DataType::Int16, false),
3849            Field::new("u8", DataType::UInt8, false),
3850            Field::new("u16", DataType::UInt16, false),
3851            Field::new("u32", DataType::UInt32, false),
3852            Field::new("u64", DataType::UInt64, false),
3853            Field::new("f16", DataType::Float16, false),
3854            Field::new("date64", DataType::Date64, false),
3855            Field::new("time32s", DataType::Time32(TimeUnit::Second), false),
3856            Field::new("time64ns", DataType::Time64(TimeUnit::Nanosecond), false),
3857            Field::new(
3858                "ts_s_local",
3859                DataType::Timestamp(TimeUnit::Second, None),
3860                false,
3861            ),
3862            Field::new(
3863                "ts_s_utc",
3864                DataType::Timestamp(TimeUnit::Second, Some("+00:00".into())),
3865                false,
3866            ),
3867            Field::new("iv_ym", DataType::Interval(IntervalUnit::YearMonth), false),
3868            Field::new("iv_dt", DataType::Interval(IntervalUnit::DayTime), false),
3869            Field::new(
3870                "iv_mdn",
3871                DataType::Interval(IntervalUnit::MonthDayNano),
3872                false,
3873            ),
3874        ]);
3875        let batch = RecordBatch::try_new(
3876            Arc::new(schema.clone()),
3877            vec![
3878                Arc::new(Int8Array::from(i8_values.clone())) as ArrayRef,
3879                Arc::new(Int16Array::from(i16_values.clone())) as ArrayRef,
3880                Arc::new(UInt8Array::from(u8_values.clone())) as ArrayRef,
3881                Arc::new(UInt16Array::from(u16_values.clone())) as ArrayRef,
3882                Arc::new(UInt32Array::from(u32_values.clone())) as ArrayRef,
3883                Arc::new(UInt64Array::from(u64_values.clone())) as ArrayRef,
3884                Arc::new(Float16Array::from(f16_values.clone())) as ArrayRef,
3885                Arc::new(Date64Array::from(date64_values.clone())) as ArrayRef,
3886                Arc::new(Time32SecondArray::from(time32s_values.clone())) as ArrayRef,
3887                Arc::new(Time64NanosecondArray::from(time64ns_values.clone())) as ArrayRef,
3888                Arc::new(TimestampSecondArray::from(ts_s_local_values.clone())) as ArrayRef,
3889                Arc::new(
3890                    TimestampSecondArray::from(ts_s_utc_values.clone()).with_timezone("+00:00"),
3891                ) as ArrayRef,
3892                Arc::new(IntervalYearMonthArray::from(iv_ym_values.clone())) as ArrayRef,
3893                Arc::new(IntervalDayTimeArray::from(iv_dt_values.clone())) as ArrayRef,
3894                Arc::new(IntervalMonthDayNanoArray::from(iv_mdn_values.clone())) as ArrayRef,
3895            ],
3896        )?;
3897        let rt = roundtrip_ocf(&batch)?;
3898        let rt_schema = rt.schema();
3899        let avro_schema_json = rt_schema
3900            .metadata()
3901            .get(SCHEMA_METADATA_KEY)
3902            .expect("avro.schema missing in round-tripped batch metadata");
3903        let avro_schema: Value =
3904            serde_json::from_str(avro_schema_json).expect("valid avro schema json");
3905        let rt_arrow_schema = rt.schema();
3906        if cfg!(feature = "avro_custom_types") {
3907            assert!(
3908                schemas_equal_ignoring_metadata(rt_arrow_schema.as_ref(), &schema),
3909                "Schema fields mismatch.\nExpected: {schema:?}\nGot: {rt_arrow_schema:?}"
3910            );
3911            for field_name in ["u64", "f16", "iv_ym", "iv_dt", "iv_mdn"] {
3912                let field = rt_arrow_schema
3913                    .field_with_name(field_name)
3914                    .expect("field exists");
3915                assert!(
3916                    field.metadata().get(AVRO_NAME_METADATA_KEY).is_some(),
3917                    "Field '{field_name}' should have avro.name metadata"
3918                );
3919            }
3920        } else {
3921            // Without avro_custom_types, Avro's type system is narrower than Arrow's.
3922            // Each field below shows the expected type AFTER round-tripping through Avro,
3923            // which differs from the original `schema` above:
3924            let exp_schema = Schema::new(vec![
3925                Field::new("i8", DataType::Int32, false),
3926                Field::new("i16", DataType::Int32, false),
3927                Field::new("u8", DataType::Int32, false),
3928                Field::new("u16", DataType::Int32, false),
3929                Field::new("u32", DataType::Int64, false),
3930                Field::new("u64", DataType::Int64, false),
3931                Field::new("f16", DataType::Float32, false),
3932                Field::new(
3933                    "date64",
3934                    DataType::Timestamp(TimeUnit::Millisecond, None),
3935                    false,
3936                ),
3937                Field::new("time32s", DataType::Time32(TimeUnit::Millisecond), false),
3938                Field::new("time64ns", DataType::Time64(TimeUnit::Microsecond), false),
3939                Field::new(
3940                    "ts_s_local",
3941                    DataType::Timestamp(TimeUnit::Millisecond, None),
3942                    false,
3943                ),
3944                Field::new(
3945                    "ts_s_utc",
3946                    DataType::Timestamp(TimeUnit::Millisecond, Some("+00:00".into())),
3947                    false,
3948                ),
3949                Field::new(
3950                    "iv_ym",
3951                    DataType::Interval(IntervalUnit::MonthDayNano),
3952                    false,
3953                ),
3954                Field::new(
3955                    "iv_dt",
3956                    DataType::Interval(IntervalUnit::MonthDayNano),
3957                    false,
3958                ),
3959                Field::new(
3960                    "iv_mdn",
3961                    DataType::Interval(IntervalUnit::MonthDayNano),
3962                    false,
3963                ),
3964            ]);
3965            assert!(
3966                schemas_equal_ignoring_metadata(rt_arrow_schema.as_ref(), &exp_schema),
3967                "Schema fields mismatch.\nExpected: {exp_schema:?}\nGot: {rt_arrow_schema:?}"
3968            );
3969            for field_name in ["iv_ym", "iv_dt", "iv_mdn"] {
3970                let field = rt_arrow_schema
3971                    .field_with_name(field_name)
3972                    .expect("field exists");
3973                assert!(
3974                    field.metadata().get(AVRO_NAME_METADATA_KEY).is_some(),
3975                    "Field '{field_name}' should have avro.name metadata"
3976                );
3977            }
3978        }
3979        if cfg!(feature = "avro_custom_types") {
3980            assert_eq!(
3981                avro_field_type(&avro_schema, "i8"),
3982                &json!({"type":"int","logicalType":"arrow.int8"})
3983            );
3984            assert_eq!(
3985                avro_field_type(&avro_schema, "i16"),
3986                &json!({"type":"int","logicalType":"arrow.int16"})
3987            );
3988            assert_eq!(
3989                avro_field_type(&avro_schema, "u8"),
3990                &json!({"type":"int","logicalType":"arrow.uint8"})
3991            );
3992            assert_eq!(
3993                avro_field_type(&avro_schema, "u16"),
3994                &json!({"type":"int","logicalType":"arrow.uint16"})
3995            );
3996            assert_eq!(
3997                avro_field_type(&avro_schema, "u32"),
3998                &json!({"type":"long","logicalType":"arrow.uint32"})
3999            );
4000            assert_eq!(
4001                avro_field_type(&avro_schema, "u64"),
4002                &json!({"type":"fixed","name":"u64","size":8,"logicalType":"arrow.uint64"})
4003            );
4004            assert_eq!(
4005                avro_field_type(&avro_schema, "f16"),
4006                &json!({"type":"fixed","name":"f16","size":2,"logicalType":"arrow.float16"})
4007            );
4008            assert_eq!(
4009                avro_field_type(&avro_schema, "date64"),
4010                &json!({"type":"long","logicalType":"arrow.date64"})
4011            );
4012            assert_eq!(
4013                avro_field_type(&avro_schema, "time32s"),
4014                &json!({"type":"int","logicalType":"arrow.time32-second"})
4015            );
4016            assert_eq!(
4017                avro_field_type(&avro_schema, "time64ns"),
4018                &json!({"type":"long","logicalType":"arrow.time64-nanosecond"})
4019            );
4020            assert_eq!(
4021                avro_field_type(&avro_schema, "ts_s_local"),
4022                &json!({"type":"long","logicalType":"arrow.local-timestamp-second"})
4023            );
4024            assert_eq!(
4025                avro_field_type(&avro_schema, "ts_s_utc"),
4026                &json!({"type":"long","logicalType":"arrow.timestamp-second"})
4027            );
4028            assert_eq!(
4029                avro_field_type(&avro_schema, "iv_ym"),
4030                &json!({"type":"fixed","name":"iv_ym","size":4,"logicalType":"arrow.interval-year-month"})
4031            );
4032            assert_eq!(
4033                avro_field_type(&avro_schema, "iv_dt"),
4034                &json!({"type":"fixed","name":"iv_dt","size":8,"logicalType":"arrow.interval-day-time"})
4035            );
4036            assert_eq!(
4037                avro_field_type(&avro_schema, "iv_mdn"),
4038                &json!({"type":"fixed","name":"iv_mdn","size":16,"logicalType":"arrow.interval-month-day-nano"})
4039            );
4040        } else {
4041            // Without custom types:
4042            // - small ints widen to int
4043            // - UInt32/UInt64 widen to long
4044            // - Float16 widens to float
4045            // - Date64 coerces to local-timestamp-millis
4046            // - Time32(Second) coerces to time-millis and scales seconds->millis
4047            // - Time64(Nanosecond) coerces to time-micros and truncates nanos->micros
4048            // - Timestamp(Second) coerces to timestamp-millis / local-timestamp-millis and scales seconds->millis
4049            // - Intervals YearMonth/DayTime encode as Avro duration (fixed 12) with arrowIntervalUnit annotation
4050            assert_eq!(avro_field_type(&avro_schema, "i8"), &json!("int"));
4051            assert_eq!(avro_field_type(&avro_schema, "i16"), &json!("int"));
4052            assert_eq!(avro_field_type(&avro_schema, "u8"), &json!("int"));
4053            assert_eq!(avro_field_type(&avro_schema, "u16"), &json!("int"));
4054            assert_eq!(avro_field_type(&avro_schema, "u32"), &json!("long"));
4055            assert_eq!(avro_field_type(&avro_schema, "u64"), &json!("long"));
4056            assert_eq!(avro_field_type(&avro_schema, "f16"), &json!("float"));
4057            assert_eq!(
4058                avro_field_type(&avro_schema, "date64"),
4059                &json!({"type":"long","logicalType":"local-timestamp-millis"})
4060            );
4061            assert_eq!(
4062                avro_field_type(&avro_schema, "time32s"),
4063                &json!({"type":"int","logicalType":"time-millis"})
4064            );
4065            assert_eq!(
4066                avro_field_type(&avro_schema, "time64ns"),
4067                &json!({"type":"long","logicalType":"time-micros"})
4068            );
4069            assert_eq!(
4070                avro_field_type(&avro_schema, "ts_s_local"),
4071                &json!({"type":"long","logicalType":"local-timestamp-millis"})
4072            );
4073            assert_eq!(
4074                avro_field_type(&avro_schema, "ts_s_utc"),
4075                &json!({"type":"long","logicalType":"timestamp-millis"})
4076            );
4077            assert_eq!(
4078                avro_field_type(&avro_schema, "iv_ym"),
4079                &json!({"type":"fixed","name":"iv_ym","size":12,"logicalType":"duration"})
4080            );
4081            assert_eq!(
4082                avro_field_type(&avro_schema, "iv_dt"),
4083                &json!({"type":"fixed","name":"iv_dt","size":12,"logicalType":"duration"})
4084            );
4085            assert_eq!(
4086                avro_field_type(&avro_schema, "iv_mdn"),
4087                &json!({"type":"fixed","name":"iv_mdn","size":12,"logicalType":"duration"})
4088            );
4089        }
4090        if cfg!(feature = "avro_custom_types") {
4091            assert_eq!(
4092                rt.column(0).as_ref(),
4093                &Int8Array::from(i8_values) as &dyn Array
4094            );
4095            assert_eq!(
4096                rt.column(1).as_ref(),
4097                &Int16Array::from(i16_values) as &dyn Array
4098            );
4099            assert_eq!(
4100                rt.column(2).as_ref(),
4101                &UInt8Array::from(u8_values) as &dyn Array
4102            );
4103            assert_eq!(
4104                rt.column(3).as_ref(),
4105                &UInt16Array::from(u16_values) as &dyn Array
4106            );
4107            assert_eq!(
4108                rt.column(4).as_ref(),
4109                &UInt32Array::from(u32_values) as &dyn Array
4110            );
4111            assert_eq!(
4112                rt.column(5).as_ref(),
4113                &UInt64Array::from(u64_values) as &dyn Array
4114            );
4115            assert_eq!(
4116                rt.column(6).as_ref(),
4117                &Float16Array::from(f16_values) as &dyn Array
4118            );
4119            assert_eq!(
4120                rt.column(7).as_ref(),
4121                &Date64Array::from(date64_values) as &dyn Array
4122            );
4123            assert_eq!(
4124                rt.column(8).as_ref(),
4125                &Time32SecondArray::from(time32s_values) as &dyn Array
4126            );
4127            assert_eq!(
4128                rt.column(9).as_ref(),
4129                &Time64NanosecondArray::from(time64ns_values) as &dyn Array
4130            );
4131            assert_eq!(
4132                rt.column(10).as_ref(),
4133                &TimestampSecondArray::from(ts_s_local_values) as &dyn Array
4134            );
4135            assert_eq!(
4136                rt.column(11).as_ref(),
4137                &TimestampSecondArray::from(ts_s_utc_values).with_timezone("+00:00") as &dyn Array
4138            );
4139            assert_eq!(
4140                rt.column(12).as_ref(),
4141                &IntervalYearMonthArray::from(iv_ym_values) as &dyn Array
4142            );
4143            assert_eq!(
4144                rt.column(13).as_ref(),
4145                &IntervalDayTimeArray::from(iv_dt_values) as &dyn Array
4146            );
4147            assert_eq!(
4148                rt.column(14).as_ref(),
4149                &IntervalMonthDayNanoArray::from(iv_mdn_values) as &dyn Array
4150            );
4151        } else {
4152            let exp_i8: Vec<Option<i32>> = i8_values.iter().map(|v| v.map(|x| x as i32)).collect();
4153            let exp_i16: Vec<Option<i32>> =
4154                i16_values.iter().map(|v| v.map(|x| x as i32)).collect();
4155            let exp_u8: Vec<Option<i32>> = u8_values.iter().map(|v| v.map(|x| x as i32)).collect();
4156            let exp_u16: Vec<Option<i32>> =
4157                u16_values.iter().map(|v| v.map(|x| x as i32)).collect();
4158            let exp_u32: Vec<Option<i64>> =
4159                u32_values.iter().map(|v| v.map(|x| x as i64)).collect();
4160            let exp_u64: Vec<Option<i64>> =
4161                u64_values.iter().map(|v| v.map(|x| x as i64)).collect();
4162            let exp_f16: Vec<Option<f32>> =
4163                f16_values.iter().map(|v| v.map(|x| x.to_f32())).collect();
4164            let exp_time32_ms: Vec<Option<i32>> = time32s_values
4165                .iter()
4166                .map(|v| v.map(|x| x.saturating_mul(1000)))
4167                .collect();
4168            let exp_time64_us: Vec<Option<i64>> = time64ns_values
4169                .iter()
4170                .map(|v| v.map(|x| x / 1000))
4171                .collect();
4172            let exp_ts_local_ms: Vec<Option<i64>> = ts_s_local_values
4173                .iter()
4174                .map(|v| v.map(|x| x * 1000))
4175                .collect();
4176            let exp_ts_utc_ms: Vec<Option<i64>> = ts_s_utc_values
4177                .iter()
4178                .map(|v| v.map(|x| x * 1000))
4179                .collect();
4180            // Interval conversions to MonthDayNano via Avro duration
4181            let exp_iv_ym: Vec<Option<IntervalMonthDayNano>> = iv_ym_values
4182                .iter()
4183                .map(|v| v.map(|months| IntervalMonthDayNano::new(months, 0, 0)))
4184                .collect();
4185            let exp_iv_dt: Vec<Option<IntervalMonthDayNano>> = iv_dt_values
4186                .iter()
4187                .map(|v| {
4188                    v.map(|dt| {
4189                        IntervalMonthDayNano::new(0, dt.days, (dt.milliseconds as i64) * 1_000_000)
4190                    })
4191                })
4192                .collect();
4193            assert_eq!(
4194                rt.column(0).as_ref(),
4195                &Int32Array::from(exp_i8) as &dyn Array
4196            );
4197            assert_eq!(
4198                rt.column(1).as_ref(),
4199                &Int32Array::from(exp_i16) as &dyn Array
4200            );
4201            assert_eq!(
4202                rt.column(2).as_ref(),
4203                &Int32Array::from(exp_u8) as &dyn Array
4204            );
4205            assert_eq!(
4206                rt.column(3).as_ref(),
4207                &Int32Array::from(exp_u16) as &dyn Array
4208            );
4209            assert_eq!(
4210                rt.column(4).as_ref(),
4211                &arrow_array::Int64Array::from(exp_u32) as &dyn Array
4212            );
4213            assert_eq!(
4214                rt.column(5).as_ref(),
4215                &arrow_array::Int64Array::from(exp_u64) as &dyn Array
4216            );
4217            assert_eq!(
4218                rt.column(6).as_ref(),
4219                &arrow_array::Float32Array::from(exp_f16) as &dyn Array
4220            );
4221            assert_eq!(
4222                rt.column(7).as_ref(),
4223                &TimestampMillisecondArray::from(date64_values) as &dyn Array
4224            );
4225            assert_eq!(
4226                rt.column(8).as_ref(),
4227                &Time32MillisecondArray::from(exp_time32_ms) as &dyn Array
4228            );
4229            assert_eq!(
4230                rt.column(9).as_ref(),
4231                &Time64MicrosecondArray::from(exp_time64_us) as &dyn Array
4232            );
4233            assert_eq!(
4234                rt.column(10).as_ref(),
4235                &TimestampMillisecondArray::from(exp_ts_local_ms) as &dyn Array
4236            );
4237            assert_eq!(
4238                rt.column(11).as_ref(),
4239                &TimestampMillisecondArray::from(exp_ts_utc_ms).with_timezone("+00:00")
4240                    as &dyn Array
4241            );
4242            assert_eq!(
4243                rt.column(12).as_ref(),
4244                &IntervalMonthDayNanoArray::from(exp_iv_ym) as &dyn Array
4245            );
4246            assert_eq!(
4247                rt.column(13).as_ref(),
4248                &IntervalMonthDayNanoArray::from(exp_iv_dt) as &dyn Array
4249            );
4250            assert_eq!(
4251                rt.column(14).as_ref(),
4252                &IntervalMonthDayNanoArray::from(iv_mdn_values) as &dyn Array
4253            );
4254        }
4255        Ok(())
4256    }
4257
4258    #[cfg(not(feature = "avro_custom_types"))]
4259    #[test]
4260    fn non_custom_uint64_overflow_errors() -> Result<(), AvroError> {
4261        let schema = Schema::new(vec![Field::new("u64", DataType::UInt64, false)]);
4262        let values: Vec<Option<u64>> = vec![Some((i64::MAX as u64) + 1)];
4263        let batch = RecordBatch::try_new(
4264            Arc::new(schema.clone()),
4265            vec![Arc::new(UInt64Array::from(values)) as ArrayRef],
4266        )?;
4267        let mut w = AvroWriter::new(Vec::<u8>::new(), schema)?;
4268        let err = w
4269            .write(&batch)
4270            .expect_err("expected UInt64 overflow error when avro_custom_types is disabled");
4271        match err {
4272            AvroError::InvalidArgument(msg) => {
4273                assert_eq!(
4274                    msg,
4275                    "UInt64 value 9223372036854775808 exceeds i64::MAX; enable avro_custom_types feature for full UInt64 support"
4276                );
4277            }
4278            other => panic!("expected InvalidArgument, got {other:?}"),
4279        }
4280        Ok(())
4281    }
4282
4283    #[cfg(not(feature = "avro_custom_types"))]
4284    #[test]
4285    fn non_custom_interval_year_month_negative_errors() -> Result<(), AvroError> {
4286        let schema = Schema::new(vec![Field::new(
4287            "iv_ym",
4288            DataType::Interval(IntervalUnit::YearMonth),
4289            false,
4290        )]);
4291        let values: Vec<Option<i32>> = vec![Some(-1)];
4292        let batch = RecordBatch::try_new(
4293            Arc::new(schema.clone()),
4294            vec![Arc::new(IntervalYearMonthArray::from(values)) as ArrayRef],
4295        )?;
4296
4297        let mut w = AvroWriter::new(Vec::<u8>::new(), schema)?;
4298        let err = w
4299            .write(&batch)
4300            .expect_err("expected negative Interval(YearMonth) error");
4301        match err {
4302            AvroError::InvalidArgument(msg) => {
4303                assert_eq!(
4304                    msg,
4305                    "Avro 'duration' cannot encode negative months; enable `avro_custom_types` to round-trip signed Arrow Interval(YearMonth)"
4306                );
4307            }
4308            other => panic!("expected InvalidArgument, got {other:?}"),
4309        }
4310        Ok(())
4311    }
4312
4313    #[cfg(not(feature = "avro_custom_types"))]
4314    #[test]
4315    fn non_custom_interval_day_time_negative_errors() -> Result<(), AvroError> {
4316        let schema = Schema::new(vec![Field::new(
4317            "iv_dt",
4318            DataType::Interval(IntervalUnit::DayTime),
4319            false,
4320        )]);
4321        let values: Vec<Option<IntervalDayTime>> = vec![Some(IntervalDayTime::new(-1, 0))];
4322        let batch = RecordBatch::try_new(
4323            Arc::new(schema.clone()),
4324            vec![Arc::new(IntervalDayTimeArray::from(values)) as ArrayRef],
4325        )?;
4326        let mut w = AvroWriter::new(Vec::<u8>::new(), schema)?;
4327        let err = w
4328            .write(&batch)
4329            .expect_err("expected negative Interval(DayTime) error");
4330        match err {
4331            AvroError::InvalidArgument(msg) => {
4332                assert_eq!(
4333                    msg,
4334                    "Avro 'duration' cannot encode negative days or milliseconds; enable `avro_custom_types` to round-trip signed Arrow Interval(DayTime)"
4335                );
4336            }
4337            other => panic!("expected InvalidArgument, got {other:?}"),
4338        }
4339        Ok(())
4340    }
4341
4342    #[cfg(not(feature = "avro_custom_types"))]
4343    #[test]
4344    fn non_custom_interval_month_day_nano_negative_errors() -> Result<(), AvroError> {
4345        let schema = Schema::new(vec![Field::new(
4346            "iv_mdn",
4347            DataType::Interval(IntervalUnit::MonthDayNano),
4348            false,
4349        )]);
4350        let values: Vec<Option<IntervalMonthDayNano>> =
4351            vec![Some(IntervalMonthDayNano::new(-1, 0, 0))];
4352        let batch = RecordBatch::try_new(
4353            Arc::new(schema.clone()),
4354            vec![Arc::new(IntervalMonthDayNanoArray::from(values)) as ArrayRef],
4355        )?;
4356        let mut w = AvroWriter::new(Vec::<u8>::new(), schema)?;
4357        let err = w
4358            .write(&batch)
4359            .expect_err("expected negative Interval(MonthDayNano) error");
4360        match err {
4361            AvroError::InvalidArgument(msg) => {
4362                assert_eq!(
4363                    msg,
4364                    "Avro 'duration' cannot encode negative months/days/nanoseconds; enable `avro_custom_types` to round-trip signed Arrow intervals"
4365                );
4366            }
4367            other => panic!("expected InvalidArgument, got {other:?}"),
4368        }
4369        Ok(())
4370    }
4371
4372    #[cfg(not(feature = "avro_custom_types"))]
4373    #[test]
4374    fn non_custom_interval_month_day_nano_sub_millis_errors() -> Result<(), AvroError> {
4375        let schema = Schema::new(vec![Field::new(
4376            "iv_mdn",
4377            DataType::Interval(IntervalUnit::MonthDayNano),
4378            false,
4379        )]);
4380        let values: Vec<Option<IntervalMonthDayNano>> =
4381            vec![Some(IntervalMonthDayNano::new(0, 0, 1))];
4382        let batch = RecordBatch::try_new(
4383            Arc::new(schema.clone()),
4384            vec![Arc::new(IntervalMonthDayNanoArray::from(values)) as ArrayRef],
4385        )?;
4386        let mut w = AvroWriter::new(Vec::<u8>::new(), schema)?;
4387        let err = w
4388            .write(&batch)
4389            .expect_err("expected sub-millisecond Interval(MonthDayNano) error");
4390        match err {
4391            AvroError::InvalidArgument(msg) => {
4392                assert_eq!(
4393                    msg,
4394                    "Avro 'duration' requires whole milliseconds; nanoseconds must be divisible by 1_000_000 (enable `avro_custom_types` to preserve nanosecond intervals)"
4395                );
4396            }
4397            other => panic!("expected InvalidArgument, got {other:?}"),
4398        }
4399        Ok(())
4400    }
4401
4402    #[cfg(not(feature = "avro_custom_types"))]
4403    #[test]
4404    fn non_custom_time32_second_scaling_overflow_errors() -> Result<(), AvroError> {
4405        let schema = Schema::new(vec![Field::new(
4406            "time32s",
4407            DataType::Time32(TimeUnit::Second),
4408            false,
4409        )]);
4410        let values: Vec<Option<i32>> = vec![Some((i32::MAX / 1000) + 1)];
4411        let batch = RecordBatch::try_new(
4412            Arc::new(schema.clone()),
4413            vec![Arc::new(Time32SecondArray::from(values)) as ArrayRef],
4414        )?;
4415        let mut w = AvroWriter::new(Vec::<u8>::new(), schema)?;
4416        let err = w
4417            .write(&batch)
4418            .expect_err("expected time32 seconds->millis overflow error");
4419        match err {
4420            AvroError::InvalidArgument(msg) => {
4421                assert_eq!(msg, "time32(secs) * 1000 overflowed");
4422            }
4423            other => panic!("expected InvalidArgument, got {other:?}"),
4424        }
4425        Ok(())
4426    }
4427
4428    #[cfg(not(feature = "avro_custom_types"))]
4429    #[test]
4430    fn non_custom_timestamp_second_scaling_overflow_errors() -> Result<(), AvroError> {
4431        let schema = Schema::new(vec![Field::new(
4432            "ts_s_local",
4433            DataType::Timestamp(TimeUnit::Second, None),
4434            false,
4435        )]);
4436        // i64::MAX / 1000 + 1 will overflow when multiplied by 1000
4437        let values: Vec<Option<i64>> = vec![Some((i64::MAX / 1000) + 1)];
4438        let batch = RecordBatch::try_new(
4439            Arc::new(schema.clone()),
4440            vec![Arc::new(TimestampSecondArray::from(values)) as ArrayRef],
4441        )?;
4442        let mut w = AvroWriter::new(Vec::<u8>::new(), schema)?;
4443        let err = w
4444            .write(&batch)
4445            .expect_err("expected timestamp seconds->millis overflow error");
4446        match err {
4447            AvroError::InvalidArgument(msg) => {
4448                assert_eq!(msg, "timestamp(secs) * 1000 overflowed");
4449            }
4450            other => panic!("expected InvalidArgument, got {other:?}"),
4451        }
4452        Ok(())
4453    }
4454}