Skip to main content

arrow_avro/reader/async_reader/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Asynchronous implementation of Avro file reader.
6//!
7//! This module provides [`AsyncAvroFileReader`], which supports reading and decoding
8//! the Avro OCF format from any source that implements [`AsyncFileReader`].
9
10use crate::compression::CompressionCodec;
11use crate::reader::Decoder;
12use crate::reader::block::{BlockDecoder, BlockDecoderState};
13use arrow_array::RecordBatch;
14use arrow_schema::{ArrowError, SchemaRef};
15use bytes::Bytes;
16use futures::future::BoxFuture;
17use futures::{FutureExt, Stream};
18use std::mem;
19use std::ops::Range;
20use std::pin::Pin;
21use std::task::{Context, Poll};
22
23mod async_file_reader;
24mod builder;
25mod spawn;
26
27pub use async_file_reader::AsyncFileReader;
28pub use builder::{ReaderBuilder, read_header_info};
29pub use spawn::SpawnedReader;
30
31#[cfg(feature = "object_store")]
32mod store;
33
34use crate::errors::AvroError;
35#[expect(deprecated)]
36#[cfg(feature = "object_store")]
37pub use store::AvroObjectReader;
38
39enum FetchNextBehaviour {
40    /// Initial read: scan for sync marker, then move to decoding blocks
41    ReadSyncMarker,
42    /// Parse VLQ header bytes one at a time until Data state, then continue decoding
43    DecodeVLQHeader,
44    /// Continue decoding the current block with the fetched data
45    ContinueDecoding,
46}
47
48enum ReaderState<R> {
49    /// Intermediate state to fix ownership issues
50    InvalidState,
51    /// Initial state, fetch initial range
52    Idle { reader: R },
53    /// Fetching data from the reader
54    FetchingData {
55        future: BoxFuture<'static, Result<(R, Bytes), AvroError>>,
56        next_behaviour: FetchNextBehaviour,
57    },
58    /// Decode a block in a loop until completion
59    DecodingBlock { data: Bytes, reader: R },
60    /// Output batches from a decoded block
61    ReadingBatches {
62        data: Bytes,
63        block_data: Bytes,
64        remaining_in_block: usize,
65        reader: R,
66    },
67    /// Successfully finished reading file contents; drain any remaining buffered records
68    /// from the decoder into (possibly partial) output batches.
69    Flushing,
70    /// Done, flush decoder and return
71    Finished,
72}
73
74/// An asynchronous Avro file reader that implements `Stream<Item = Result<RecordBatch, ArrowError>>`.
75/// This uses an [`AsyncFileReader`] to fetch data ranges as needed, starting with fetching the header,
76/// then reading all the blocks in the provided range where:
77/// 1. Reads and decodes data until the header is fully decoded.
78/// 2. Searching from `range.start` for the first sync marker, and starting with the following block.
79///    (If `range.start` is less than the header length, we start at the header length minus the sync marker bytes)
80/// 3. Reading blocks sequentially, decoding them into RecordBatches.
81/// 4. If a block is incomplete (due to range ending mid-block), fetching the remaining bytes from the [`AsyncFileReader`].
82/// 5. If no range was originally provided, reads the full file.
83/// 6. If the range is 0, file_size is 0, or `range.end` is less than the header length, finish immediately.
84///
85/// # Example
86///
87/// ```
88/// #[tokio::main(flavor = "current_thread")]
89/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
90/// use std::io::Cursor;
91/// use std::sync::Arc;
92/// use arrow_array::{ArrayRef, Int32Array, RecordBatch};
93/// use arrow_schema::{DataType, Field, Schema};
94/// use arrow_avro::reader::AsyncAvroFileReader;
95/// use arrow_avro::writer::AvroWriter;
96/// use futures::TryStreamExt;
97///
98/// // Build a minimal Arrow schema and batch
99/// let schema = Schema::new(vec![Field::new("id", DataType::Int32, false)]);
100/// let batch = RecordBatch::try_new(
101///     Arc::new(schema.clone()),
102///     vec![Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef],
103/// )?;
104///
105/// // Write an Avro OCF to memory
106/// let buffer: Vec<u8> = Vec::new();
107/// let mut writer = AvroWriter::new(buffer, schema)?;
108/// writer.write(&batch)?;
109/// writer.finish()?;
110/// let bytes = writer.into_inner();
111///
112/// // Create an async reader from the in-memory bytes
113/// // `tokio::fs::File` also implements `AsyncFileReader` for reading from disk
114/// let file_size = bytes.len();
115/// let cursor = Cursor::new(bytes);
116/// let reader = AsyncAvroFileReader::builder(cursor, file_size as u64, 1024)
117///     .try_build()
118///     .await?;
119///
120/// // Consume the stream of RecordBatches
121/// let batches: Vec<RecordBatch> = reader.try_collect().await?;
122/// assert_eq!(batches.len(), 1);
123/// assert_eq!(batches[0].num_rows(), 3);
124/// Ok(())
125/// }
126/// ```
127pub struct AsyncAvroFileReader<R> {
128    // Members required to fetch data
129    range: Range<u64>,
130    file_size: u64,
131
132    // Members required to actually decode and read data
133    decoder: Decoder,
134    block_decoder: BlockDecoder,
135    codec: Option<CompressionCodec>,
136    sync_marker: [u8; 16],
137
138    // Members keeping the current state of the reader
139    reader_state: ReaderState<R>,
140    finishing_partial_block: bool,
141}
142
143impl<R> AsyncAvroFileReader<R> {
144    /// Returns a builder for a new [`Self`], allowing some optional parameters.
145    pub fn builder(reader: R, file_size: u64, batch_size: usize) -> ReaderBuilder<R> {
146        ReaderBuilder::new(reader, file_size, batch_size)
147    }
148
149    fn new(
150        range: Range<u64>,
151        file_size: u64,
152        decoder: Decoder,
153        codec: Option<CompressionCodec>,
154        sync_marker: [u8; 16],
155        reader_state: ReaderState<R>,
156    ) -> Self {
157        Self {
158            range,
159            file_size,
160
161            decoder,
162            block_decoder: Default::default(),
163            codec,
164            sync_marker,
165
166            reader_state,
167            finishing_partial_block: false,
168        }
169    }
170
171    /// Returns the Arrow schema for batches produced by this reader.
172    ///
173    /// The schema is determined by the writer schema in the file and the reader schema provided to the builder.
174    pub fn schema(&self) -> SchemaRef {
175        self.decoder.schema()
176    }
177
178    /// Calculate the byte range needed to complete the current block.
179    /// Only valid when block_decoder is in Data or Sync state.
180    /// Returns the range to fetch, or an error if EOF would be reached.
181    fn remaining_block_range(&self) -> Result<Range<u64>, AvroError> {
182        let remaining = self.block_decoder.bytes_remaining() as u64
183            + match self.block_decoder.state() {
184                BlockDecoderState::Data => 16, // Include sync marker
185                BlockDecoderState::Sync => 0,
186                state => {
187                    return Err(AvroError::General(format!(
188                        "remaining_block_range called in unexpected state: {state:?}"
189                    )));
190                }
191            };
192
193        let fetch_end = self.range.end + remaining;
194        if fetch_end > self.file_size {
195            return Err(AvroError::EOF(
196                "Avro block requires more bytes than what exists in the file".into(),
197            ));
198        }
199
200        Ok(self.range.end..fetch_end)
201    }
202
203    /// Terminate the stream after returning this error once.
204    #[inline]
205    fn finish_with_error(
206        &mut self,
207        error: AvroError,
208    ) -> Poll<Option<Result<RecordBatch, AvroError>>> {
209        self.reader_state = ReaderState::Finished;
210        Poll::Ready(Some(Err(error)))
211    }
212
213    #[inline]
214    fn start_flushing(&mut self) {
215        self.reader_state = ReaderState::Flushing;
216    }
217
218    /// Drain any remaining buffered records from the decoder.
219    #[inline]
220    fn poll_flush(&mut self) -> Poll<Option<Result<RecordBatch, AvroError>>> {
221        match self.decoder.flush_block() {
222            Ok(Some(batch)) => {
223                self.reader_state = ReaderState::Flushing;
224                Poll::Ready(Some(Ok(batch)))
225            }
226            Ok(None) => {
227                self.reader_state = ReaderState::Finished;
228                Poll::Ready(None)
229            }
230            Err(e) => self.finish_with_error(e),
231        }
232    }
233}
234
235impl<R: AsyncFileReader + Unpin + 'static> AsyncAvroFileReader<R> {
236    // The forbid question mark thing shouldn't apply here, as it is within the future,
237    // so exported this to a separate function.
238    async fn fetch_bytes(mut reader: R, range: Range<u64>) -> Result<(R, Bytes), AvroError> {
239        let data = reader.get_bytes(range).await?;
240        Ok((reader, data))
241    }
242
243    #[forbid(clippy::question_mark_used)]
244    fn read_next(&mut self, cx: &mut Context<'_>) -> Poll<Option<Result<RecordBatch, AvroError>>> {
245        loop {
246            match mem::replace(&mut self.reader_state, ReaderState::InvalidState) {
247                ReaderState::Idle { reader } => {
248                    let range = self.range.clone();
249                    if range.start >= range.end {
250                        return self.finish_with_error(AvroError::InvalidArgument(format!(
251                            "Invalid range specified for Avro file: start {} >= end {}, file_size: {}",
252                            range.start, range.end, self.file_size
253                        )));
254                    }
255
256                    let future = Self::fetch_bytes(reader, range).boxed();
257                    self.reader_state = ReaderState::FetchingData {
258                        future,
259                        next_behaviour: FetchNextBehaviour::ReadSyncMarker,
260                    };
261                }
262                ReaderState::FetchingData {
263                    mut future,
264                    next_behaviour,
265                } => {
266                    let (reader, data_chunk) = match future.poll_unpin(cx) {
267                        Poll::Ready(Ok(data)) => data,
268                        Poll::Ready(Err(e)) => return self.finish_with_error(e),
269                        Poll::Pending => {
270                            self.reader_state = ReaderState::FetchingData {
271                                future,
272                                next_behaviour,
273                            };
274                            return Poll::Pending;
275                        }
276                    };
277
278                    match next_behaviour {
279                        FetchNextBehaviour::ReadSyncMarker => {
280                            let sync_marker_pos = data_chunk
281                                .windows(16)
282                                .position(|slice| slice == self.sync_marker);
283                            let block_start = match sync_marker_pos {
284                                Some(pos) => pos + 16, // Move past the sync marker
285                                None => {
286                                    // Sync marker not found, valid if we arbitrarily split the file at its end.
287                                    self.reader_state = ReaderState::Finished;
288                                    return Poll::Ready(None);
289                                }
290                            };
291
292                            self.reader_state = ReaderState::DecodingBlock {
293                                reader,
294                                data: data_chunk.slice(block_start..),
295                            };
296                        }
297                        FetchNextBehaviour::DecodeVLQHeader => {
298                            let mut data = data_chunk;
299
300                            // Feed bytes one at a time until we reach Data state (VLQ header complete)
301                            while !matches!(self.block_decoder.state(), BlockDecoderState::Data) {
302                                if data.is_empty() {
303                                    return self.finish_with_error(AvroError::EOF(
304                                        "Unexpected EOF while reading Avro block header".into(),
305                                    ));
306                                }
307                                let consumed = match self.block_decoder.decode(&data[..1]) {
308                                    Ok(consumed) => consumed,
309                                    Err(e) => return self.finish_with_error(e),
310                                };
311                                if consumed == 0 {
312                                    return self.finish_with_error(AvroError::General(
313                                        "BlockDecoder failed to consume byte during VLQ header parsing"
314                                            .into(),
315                                    ));
316                                }
317                                data = data.slice(consumed..);
318                            }
319
320                            // Now we know the block size. Slice remaining data to what we need.
321                            let bytes_remaining = self.block_decoder.bytes_remaining();
322                            let data_to_use = data.slice(..data.len().min(bytes_remaining));
323                            let consumed = match self.block_decoder.decode(&data_to_use) {
324                                Ok(consumed) => consumed,
325                                Err(e) => return self.finish_with_error(e),
326                            };
327                            if consumed != data_to_use.len() {
328                                return self.finish_with_error(AvroError::General(
329                                    "BlockDecoder failed to consume all bytes after VLQ header parsing"
330                                        .into(),
331                                ));
332                            }
333
334                            // May need more data to finish the block.
335                            let range_to_fetch = match self.remaining_block_range() {
336                                Ok(range) if range.is_empty() => {
337                                    // All bytes fetched, move to decoding block directly
338                                    self.reader_state = ReaderState::DecodingBlock {
339                                        reader,
340                                        data: Bytes::new(),
341                                    };
342                                    continue;
343                                }
344                                Ok(range) => range,
345                                Err(e) => return self.finish_with_error(e),
346                            };
347
348                            let future = Self::fetch_bytes(reader, range_to_fetch).boxed();
349                            self.reader_state = ReaderState::FetchingData {
350                                future,
351                                next_behaviour: FetchNextBehaviour::ContinueDecoding,
352                            };
353                        }
354                        FetchNextBehaviour::ContinueDecoding => {
355                            self.reader_state = ReaderState::DecodingBlock {
356                                reader,
357                                data: data_chunk,
358                            };
359                        }
360                    }
361                }
362                ReaderState::InvalidState => {
363                    return self.finish_with_error(AvroError::General(
364                        "AsyncAvroFileReader in invalid state".into(),
365                    ));
366                }
367                ReaderState::DecodingBlock { reader, mut data } => {
368                    // Try to decode another block from the buffered reader.
369                    let consumed = match self.block_decoder.decode(&data) {
370                        Ok(consumed) => consumed,
371                        Err(e) => return self.finish_with_error(e),
372                    };
373                    data = data.slice(consumed..);
374
375                    // If we reached the end of the block, flush it, and move to read batches.
376                    if let Some(block) = self.block_decoder.flush() {
377                        // Successfully decoded a block.
378                        if block.sync != self.sync_marker {
379                            return self.finish_with_error(AvroError::ParseError(
380                                "Avro block sync marker does not match file header".to_string(),
381                            ));
382                        }
383                        let block_count = block.count;
384                        let block_data = Bytes::from_owner(if let Some(ref codec) = self.codec {
385                            match codec.decompress(&block.data) {
386                                Ok(decompressed) => decompressed,
387                                Err(e) => return self.finish_with_error(e),
388                            }
389                        } else {
390                            block.data
391                        });
392
393                        // Since we have an active block, move to reading batches
394                        self.reader_state = ReaderState::ReadingBatches {
395                            reader,
396                            data,
397                            block_data,
398                            remaining_in_block: block_count,
399                        };
400                        continue;
401                    }
402
403                    // data should always be consumed unless Finished, if it wasn't, something went wrong
404                    if !data.is_empty() {
405                        return self.finish_with_error(AvroError::General(
406                            "BlockDecoder failed to make progress decoding Avro block".into(),
407                        ));
408                    }
409
410                    if matches!(self.block_decoder.state(), BlockDecoderState::Finished) {
411                        // We've already flushed, so if no batch was produced, we are simply done.
412                        self.finishing_partial_block = false;
413                        self.start_flushing();
414                        continue;
415                    }
416
417                    // If we've tried the following stage before, and still can't decode,
418                    // this means the file is truncated or corrupted.
419                    if self.finishing_partial_block {
420                        return self.finish_with_error(AvroError::EOF(
421                            "Unexpected EOF while reading last Avro block".into(),
422                        ));
423                    }
424
425                    // Avro splitting case: block is incomplete, we need to:
426                    // 1. Parse the length so we know how much to read
427                    // 2. Fetch more data from the reader
428                    // 3. Create a new block data from the remaining slice and the newly fetched data
429                    // 4. Continue decoding until end of block
430                    self.finishing_partial_block = true;
431
432                    // Mid-block, but we don't know how many bytes are missing yet
433                    if matches!(
434                        self.block_decoder.state(),
435                        BlockDecoderState::Count | BlockDecoderState::Size
436                    ) {
437                        // Max VLQ header is 20 bytes (10 bytes each for count and size).
438                        // Fetch just enough to complete it.
439                        const MAX_VLQ_HEADER_SIZE: u64 = 20;
440                        let fetch_end = (self.range.end + MAX_VLQ_HEADER_SIZE).min(self.file_size);
441
442                        // If there is nothing more to fetch, error out
443                        if fetch_end == self.range.end {
444                            return self.finish_with_error(AvroError::EOF(
445                                "Unexpected EOF while reading Avro block header".into(),
446                            ));
447                        }
448
449                        let range_to_fetch = self.range.end..fetch_end;
450                        self.range.end = fetch_end; // Track that we've fetched these bytes
451
452                        let future = Self::fetch_bytes(reader, range_to_fetch).boxed();
453                        self.reader_state = ReaderState::FetchingData {
454                            future,
455                            next_behaviour: FetchNextBehaviour::DecodeVLQHeader,
456                        };
457                        continue;
458                    }
459
460                    // Otherwise, we're mid-block but know how many bytes are remaining to fetch.
461                    let range_to_fetch = match self.remaining_block_range() {
462                        Ok(range) => range,
463                        Err(e) => return self.finish_with_error(e),
464                    };
465
466                    let future = Self::fetch_bytes(reader, range_to_fetch).boxed();
467                    self.reader_state = ReaderState::FetchingData {
468                        future,
469                        next_behaviour: FetchNextBehaviour::ContinueDecoding,
470                    };
471                }
472                ReaderState::ReadingBatches {
473                    reader,
474                    data,
475                    mut block_data,
476                    mut remaining_in_block,
477                } => {
478                    let (consumed, records_decoded) =
479                        match self.decoder.decode_block(&block_data, remaining_in_block) {
480                            Ok((consumed, records_decoded)) => (consumed, records_decoded),
481                            Err(e) => return self.finish_with_error(e),
482                        };
483
484                    remaining_in_block -= records_decoded;
485
486                    if remaining_in_block == 0 {
487                        if data.is_empty() {
488                            // No more data to read, drain remaining buffered records
489                            self.start_flushing();
490                        } else {
491                            // Finished this block, move to decode next block in the next iteration
492                            self.reader_state = ReaderState::DecodingBlock { reader, data };
493                        }
494                    } else {
495                        // Still more records to decode in this block, slice the already-read data and stay in this state
496                        block_data = block_data.slice(consumed..);
497                        self.reader_state = ReaderState::ReadingBatches {
498                            reader,
499                            data,
500                            block_data,
501                            remaining_in_block,
502                        };
503                    }
504
505                    // We have a full batch ready, emit it
506                    // (This is not mutually exclusive with the block being finished, so the state change is valid)
507                    if self.decoder.batch_is_full() {
508                        return match self.decoder.flush_block() {
509                            Ok(Some(batch)) => Poll::Ready(Some(Ok(batch))),
510                            Ok(None) => self.finish_with_error(AvroError::General(
511                                "Decoder reported a full batch, but flush returned None".into(),
512                            )),
513                            Err(e) => self.finish_with_error(e),
514                        };
515                    }
516                }
517                ReaderState::Flushing => {
518                    return self.poll_flush();
519                }
520                ReaderState::Finished => {
521                    // Terminal: once finished (including after an error), always yield None
522                    self.reader_state = ReaderState::Finished;
523                    return Poll::Ready(None);
524                }
525            }
526        }
527    }
528}
529
530// To maintain compatibility with the expected stream results in the ecosystem, this returns ArrowError.
531impl<R: AsyncFileReader + Unpin + 'static> Stream for AsyncAvroFileReader<R> {
532    type Item = Result<RecordBatch, ArrowError>;
533
534    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
535        self.read_next(cx).map_err(Into::into)
536    }
537}
538
539#[cfg(test)]
540mod tests {
541    use super::*;
542    use crate::codec::Tz;
543    use crate::schema::{
544        AVRO_NAME_METADATA_KEY, AVRO_NAMESPACE_METADATA_KEY, AvroSchema, SCHEMA_METADATA_KEY,
545    };
546    use arrow_array::cast::AsArray;
547    use arrow_array::types::{Int32Type, Int64Type};
548    use arrow_array::*;
549    use arrow_schema::{DataType, Field, Metadata, Schema, SchemaRef, TimeUnit};
550    use futures::{StreamExt, TryStreamExt};
551    use object_store::local::LocalFileSystem;
552    use object_store::path::Path;
553    use object_store::{ObjectStore, ObjectStoreExt};
554    use std::collections::HashMap;
555    use std::sync::Arc;
556
557    /// An [`AsyncFileReader`] reading via an [`ObjectStore`], mirroring the
558    /// example on the [`AsyncFileReader`] trait documentation
559    #[derive(Clone, Debug)]
560    struct ObjectStoreReader {
561        store: Arc<dyn ObjectStore>,
562        path: Path,
563    }
564
565    impl ObjectStoreReader {
566        fn new(store: Arc<dyn ObjectStore>, path: Path) -> Self {
567            Self { store, path }
568        }
569    }
570
571    impl AsyncFileReader for ObjectStoreReader {
572        fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes, AvroError>> {
573            async move {
574                self.store
575                    .get_range(&self.path, range)
576                    .await
577                    .map_err(|e| AvroError::General(e.to_string()))
578            }
579            .boxed()
580        }
581
582        fn get_byte_ranges(
583            &mut self,
584            ranges: Vec<Range<u64>>,
585        ) -> BoxFuture<'_, Result<Vec<Bytes>, AvroError>> {
586            async move {
587                self.store
588                    .get_ranges(&self.path, &ranges)
589                    .await
590                    .map_err(|e| AvroError::General(e.to_string()))
591            }
592            .boxed()
593        }
594    }
595
596    fn arrow_test_data(file: &str) -> String {
597        let base =
598            std::env::var("ARROW_TEST_DATA").unwrap_or_else(|_| "../testing/data".to_string());
599        format!("{base}/{file}")
600    }
601
602    fn get_alltypes_schema() -> SchemaRef {
603        get_alltypes_schema_with_tz("+00:00")
604    }
605
606    fn get_alltypes_schema_with_tz(tz_id: &str) -> SchemaRef {
607        let schema = Schema::new(vec![
608            Field::new("id", DataType::Int32, true),
609            Field::new("bool_col", DataType::Boolean, true),
610            Field::new("tinyint_col", DataType::Int32, true),
611            Field::new("smallint_col", DataType::Int32, true),
612            Field::new("int_col", DataType::Int32, true),
613            Field::new("bigint_col", DataType::Int64, true),
614            Field::new("float_col", DataType::Float32, true),
615            Field::new("double_col", DataType::Float64, true),
616            Field::new("date_string_col", DataType::Binary, true),
617            Field::new("string_col", DataType::Binary, true),
618            Field::new(
619                "timestamp_col",
620                DataType::Timestamp(TimeUnit::Microsecond, Some(tz_id.into())),
621                true,
622            ),
623        ])
624        .with_metadata(HashMap::from([(
625            SCHEMA_METADATA_KEY.into(),
626            r#"{
627    "type": "record",
628    "name": "topLevelRecord",
629    "fields": [
630        {
631            "name": "id",
632            "type": [
633                "int",
634                "null"
635            ]
636        },
637        {
638            "name": "bool_col",
639            "type": [
640                "boolean",
641                "null"
642            ]
643        },
644        {
645            "name": "tinyint_col",
646            "type": [
647                "int",
648                "null"
649            ]
650        },
651        {
652            "name": "smallint_col",
653            "type": [
654                "int",
655                "null"
656            ]
657        },
658        {
659            "name": "int_col",
660            "type": [
661                "int",
662                "null"
663            ]
664        },
665        {
666            "name": "bigint_col",
667            "type": [
668                "long",
669                "null"
670            ]
671        },
672        {
673            "name": "float_col",
674            "type": [
675                "float",
676                "null"
677            ]
678        },
679        {
680            "name": "double_col",
681            "type": [
682                "double",
683                "null"
684            ]
685        },
686        {
687            "name": "date_string_col",
688            "type": [
689                "bytes",
690                "null"
691            ]
692        },
693        {
694            "name": "string_col",
695            "type": [
696                "bytes",
697                "null"
698            ]
699        },
700        {
701            "name": "timestamp_col",
702            "type": [
703                {
704                    "type": "long",
705                    "logicalType": "timestamp-micros"
706                },
707                "null"
708            ]
709        }
710    ]
711}
712"#
713            .into(),
714        )]));
715        Arc::new(schema)
716    }
717
718    fn get_alltypes_with_nulls_schema() -> SchemaRef {
719        let schema = Schema::new(vec![
720            Field::new("string_col", DataType::Binary, true),
721            Field::new("int_col", DataType::Int32, true),
722            Field::new("bool_col", DataType::Boolean, true),
723            Field::new("bigint_col", DataType::Int64, true),
724            Field::new("float_col", DataType::Float32, true),
725            Field::new("double_col", DataType::Float64, true),
726            Field::new("bytes_col", DataType::Binary, true),
727        ])
728        .with_metadata(HashMap::from([(
729            SCHEMA_METADATA_KEY.into(),
730            r#"{
731    "type": "record",
732    "name": "topLevelRecord",
733    "fields": [
734        {
735            "name": "string_col",
736            "type": [
737                "null",
738                "string"
739            ],
740            "default": null
741        },
742        {
743            "name": "int_col",
744            "type": [
745                "null",
746                "int"
747            ],
748            "default": null
749        },
750        {
751            "name": "bool_col",
752            "type": [
753                "null",
754                "boolean"
755            ],
756            "default": null
757        },
758        {
759            "name": "bigint_col",
760            "type": [
761                "null",
762                "long"
763            ],
764            "default": null
765        },
766        {
767            "name": "float_col",
768            "type": [
769                "null",
770                "float"
771            ],
772            "default": null
773        },
774        {
775            "name": "double_col",
776            "type": [
777                "null",
778                "double"
779            ],
780            "default": null
781        },
782        {
783            "name": "bytes_col",
784            "type": [
785                "null",
786                "bytes"
787            ],
788            "default": null
789        }
790    ]
791}"#
792            .into(),
793        )]));
794
795        Arc::new(schema)
796    }
797
798    fn get_nested_records_schema() -> SchemaRef {
799        let schema = Schema::new(vec![
800            Field::new(
801                "f1",
802                DataType::Struct(
803                    vec![
804                        Field::new("f1_1", DataType::Utf8, false),
805                        Field::new("f1_2", DataType::Int32, false),
806                        Field::new(
807                            "f1_3",
808                            DataType::Struct(
809                                vec![Field::new("f1_3_1", DataType::Float64, false)].into(),
810                            ),
811                            false,
812                        )
813                        .with_metadata(HashMap::from([
814                            (AVRO_NAMESPACE_METADATA_KEY.to_owned(), "ns3".to_owned()),
815                            (AVRO_NAME_METADATA_KEY.to_owned(), "record3".to_owned()),
816                        ])),
817                    ]
818                    .into(),
819                ),
820                false,
821            )
822            .with_metadata(HashMap::from([
823                (AVRO_NAMESPACE_METADATA_KEY.to_owned(), "ns2".to_owned()),
824                (AVRO_NAME_METADATA_KEY.to_owned(), "record2".to_owned()),
825            ])),
826            Field::new(
827                "f2",
828                DataType::List(Arc::new(
829                    Field::new(
830                        "item",
831                        DataType::Struct(
832                            vec![
833                                Field::new("f2_1", DataType::Boolean, false),
834                                Field::new("f2_2", DataType::Float32, false),
835                            ]
836                            .into(),
837                        ),
838                        false,
839                    )
840                    .with_metadata(HashMap::from([
841                        (AVRO_NAMESPACE_METADATA_KEY.to_owned(), "ns4".to_owned()),
842                        (AVRO_NAME_METADATA_KEY.to_owned(), "record4".to_owned()),
843                    ])),
844                )),
845                false,
846            ),
847            Field::new(
848                "f3",
849                DataType::Struct(vec![Field::new("f3_1", DataType::Utf8, false)].into()),
850                true,
851            )
852            .with_metadata(HashMap::from([
853                (AVRO_NAMESPACE_METADATA_KEY.to_owned(), "ns5".to_owned()),
854                (AVRO_NAME_METADATA_KEY.to_owned(), "record5".to_owned()),
855            ])),
856            Field::new(
857                "f4",
858                DataType::List(Arc::new(
859                    Field::new(
860                        "item",
861                        DataType::Struct(vec![Field::new("f4_1", DataType::Int64, false)].into()),
862                        true,
863                    )
864                    .with_metadata(HashMap::from([
865                        (AVRO_NAMESPACE_METADATA_KEY.to_owned(), "ns6".to_owned()),
866                        (AVRO_NAME_METADATA_KEY.to_owned(), "record6".to_owned()),
867                    ])),
868                )),
869                false,
870            ),
871        ])
872        .with_metadata(HashMap::from([(
873            SCHEMA_METADATA_KEY.into(),
874            r#"{
875    "type": "record",
876    "namespace": "ns1",
877    "name": "record1",
878    "fields": [
879        {
880            "name": "f1",
881            "type": {
882                "type": "record",
883                "namespace": "ns2",
884                "name": "record2",
885                "fields": [
886                    {
887                        "name": "f1_1",
888                        "type": "string"
889                    },
890                    {
891                        "name": "f1_2",
892                        "type": "int"
893                    },
894                    {
895                        "name": "f1_3",
896                        "type": {
897                            "type": "record",
898                            "namespace": "ns3",
899                            "name": "record3",
900                            "fields": [
901                                {
902                                    "name": "f1_3_1",
903                                    "type": "double"
904                                }
905                            ]
906                        }
907                    }
908                ]
909            }
910        },
911        {
912            "name": "f2",
913            "type": {
914                "type": "array",
915                "items": {
916                    "type": "record",
917                    "namespace": "ns4",
918                    "name": "record4",
919                    "fields": [
920                        {
921                            "name": "f2_1",
922                            "type": "boolean"
923                        },
924                        {
925                            "name": "f2_2",
926                            "type": "float"
927                        }
928                    ]
929                }
930            }
931        },
932        {
933            "name": "f3",
934            "type": [
935                "null",
936                {
937                    "type": "record",
938                    "namespace": "ns5",
939                    "name": "record5",
940                    "fields": [
941                        {
942                            "name": "f3_1",
943                            "type": "string"
944                        }
945                    ]
946                }
947            ],
948            "default": null
949        },
950        {
951            "name": "f4",
952            "type": {
953                "type": "array",
954                "items": [
955                    "null",
956                    {
957                        "type": "record",
958                        "namespace": "ns6",
959                        "name": "record6",
960                        "fields": [
961                            {
962                                "name": "f4_1",
963                                "type": "long"
964                            }
965                        ]
966                    }
967                ]
968            }
969        }
970    ]
971}
972"#
973            .into(),
974        )]));
975
976        Arc::new(schema)
977    }
978
979    async fn read_async_file(
980        path: &str,
981        batch_size: usize,
982        range: Option<Range<u64>>,
983        schema: Option<SchemaRef>,
984        projection: Option<Vec<usize>>,
985    ) -> Result<Vec<RecordBatch>, ArrowError> {
986        let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
987        let location = Path::from_filesystem_path(path).unwrap();
988
989        let file_size = store.head(&location).await.unwrap().size;
990
991        let file_reader = ObjectStoreReader::new(store, location);
992        let mut builder = AsyncAvroFileReader::builder(file_reader, file_size, batch_size);
993
994        if let Some(s) = schema {
995            let reader_schema = AvroSchema::try_from(s.as_ref())?;
996            builder = builder.with_reader_schema(reader_schema);
997        }
998
999        if let Some(proj) = projection {
1000            builder = builder.with_projection(proj);
1001        }
1002
1003        if let Some(range) = range {
1004            builder = builder.with_range(range);
1005        }
1006
1007        let reader = builder.try_build().await?;
1008        reader.try_collect().await
1009    }
1010
1011    #[tokio::test]
1012    async fn test_full_file_read() {
1013        let file = arrow_test_data("avro/alltypes_plain.avro");
1014        let schema = get_alltypes_schema();
1015        let batches = read_async_file(&file, 1024, None, Some(schema), None)
1016            .await
1017            .unwrap();
1018        let batch = &batches[0];
1019
1020        assert_eq!(batch.num_rows(), 8);
1021        assert_eq!(batch.num_columns(), 11);
1022
1023        let id_array = batch
1024            .column(0)
1025            .as_any()
1026            .downcast_ref::<Int32Array>()
1027            .unwrap();
1028        assert_eq!(id_array.value(0), 4);
1029        assert_eq!(id_array.value(7), 1);
1030    }
1031
1032    #[tokio::test]
1033    async fn test_small_batch_size() {
1034        let file = arrow_test_data("avro/alltypes_plain.avro");
1035        let schema = get_alltypes_schema();
1036        let batches = read_async_file(&file, 2, None, Some(schema), None)
1037            .await
1038            .unwrap();
1039        assert_eq!(batches.len(), 4);
1040
1041        let batch = &batches[0];
1042
1043        assert_eq!(batch.num_rows(), 2);
1044        assert_eq!(batch.num_columns(), 11);
1045    }
1046
1047    #[tokio::test]
1048    async fn test_batch_size_one() {
1049        let file = arrow_test_data("avro/alltypes_plain.avro");
1050        let schema = get_alltypes_schema();
1051        let batches = read_async_file(&file, 1, None, Some(schema), None)
1052            .await
1053            .unwrap();
1054        let batch = &batches[0];
1055
1056        assert_eq!(batches.len(), 8);
1057        assert_eq!(batch.num_rows(), 1);
1058    }
1059
1060    #[tokio::test]
1061    async fn test_batch_size_larger_than_file() {
1062        let file = arrow_test_data("avro/alltypes_plain.avro");
1063        let schema = get_alltypes_schema();
1064        let batches = read_async_file(&file, 10000, None, Some(schema), None)
1065            .await
1066            .unwrap();
1067        let batch = &batches[0];
1068
1069        assert_eq!(batch.num_rows(), 8);
1070    }
1071
1072    #[tokio::test]
1073    async fn test_empty_range() {
1074        let file = arrow_test_data("avro/alltypes_plain.avro");
1075        let range = 100..100;
1076        let schema = get_alltypes_schema();
1077        let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1078            .await
1079            .unwrap();
1080        assert_eq!(batches.len(), 0);
1081    }
1082
1083    #[tokio::test]
1084    async fn test_range_starting_at_zero() {
1085        // Tests that range starting at 0 correctly skips header
1086        let file = arrow_test_data("avro/alltypes_plain.avro");
1087        let store = Arc::new(LocalFileSystem::new());
1088        let location = Path::from_filesystem_path(&file).unwrap();
1089        let meta = store.head(&location).await.unwrap();
1090
1091        let range = 0..meta.size;
1092        let schema = get_alltypes_schema();
1093        let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1094            .await
1095            .unwrap();
1096        let batch = &batches[0];
1097
1098        assert_eq!(batch.num_rows(), 8);
1099    }
1100
1101    #[tokio::test]
1102    async fn test_range_after_header() {
1103        let file = arrow_test_data("avro/alltypes_plain.avro");
1104        let store = Arc::new(LocalFileSystem::new());
1105        let location = Path::from_filesystem_path(&file).unwrap();
1106        let meta = store.head(&location).await.unwrap();
1107
1108        let range = 100..meta.size;
1109        let schema = get_alltypes_schema();
1110        let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1111            .await
1112            .unwrap();
1113        let batch = &batches[0];
1114
1115        assert!(batch.num_rows() > 0);
1116    }
1117
1118    #[tokio::test]
1119    async fn test_block_sync_marker_mismatch_errors() {
1120        use tempfile::tempdir;
1121        let file = arrow_test_data("avro/alltypes_plain.avro");
1122        let mut bytes = std::fs::read(&file).unwrap();
1123        // The file ends with the final block's 16-byte sync marker.
1124        let last = bytes.len() - 1;
1125        bytes[last] ^= 0xFF;
1126        let dir = tempdir().unwrap();
1127        let path = dir.path().join("corrupt_sync.avro");
1128        std::fs::write(&path, bytes).unwrap();
1129        let schema = get_alltypes_schema();
1130        let err = read_async_file(path.to_str().unwrap(), 1024, None, Some(schema), None)
1131            .await
1132            .expect_err("corrupted block sync marker should fail the read");
1133        assert!(err.to_string().contains("sync marker"), "{err}");
1134    }
1135
1136    #[tokio::test]
1137    async fn test_range_no_sync_marker() {
1138        // Small range unlikely to contain sync marker
1139        let file = arrow_test_data("avro/alltypes_plain.avro");
1140        let range = 50..150;
1141        let schema = get_alltypes_schema();
1142        let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1143            .await
1144            .unwrap();
1145        assert_eq!(batches.len(), 0);
1146    }
1147
1148    #[tokio::test]
1149    async fn test_range_starting_mid_file() {
1150        let file = arrow_test_data("avro/alltypes_plain.avro");
1151
1152        let range = 700..768; // Header ends at 675, so this should be mid-block
1153        let schema = get_alltypes_schema();
1154        let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1155            .await
1156            .unwrap();
1157        assert_eq!(batches.len(), 0);
1158    }
1159
1160    #[tokio::test]
1161    async fn test_range_ending_at_file_size() {
1162        let file = arrow_test_data("avro/alltypes_plain.avro");
1163        let store = Arc::new(LocalFileSystem::new());
1164        let location = Path::from_filesystem_path(&file).unwrap();
1165        let meta = store.head(&location).await.unwrap();
1166
1167        let range = 200..meta.size;
1168        let schema = get_alltypes_schema();
1169        let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1170            .await
1171            .unwrap();
1172        let batch = &batches[0];
1173
1174        assert_eq!(batch.num_rows(), 8);
1175    }
1176
1177    #[tokio::test]
1178    async fn test_incomplete_block_requires_fetch() {
1179        // Range ends mid-block, should trigger fetching_rem_block logic
1180        let file = arrow_test_data("avro/alltypes_plain.avro");
1181        let range = 0..1200;
1182        let schema = get_alltypes_schema();
1183        let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1184            .await
1185            .unwrap();
1186        let batch = &batches[0];
1187
1188        assert_eq!(batch.num_rows(), 8)
1189    }
1190
1191    #[tokio::test]
1192    async fn test_partial_vlq_header_requires_fetch() {
1193        // Range ends mid-VLQ header, triggering the Count|Size partial fetch logic.
1194        let file = arrow_test_data("avro/alltypes_plain.avro");
1195        let range = 16..676; // Header should end at 675
1196        let schema = get_alltypes_schema();
1197        let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1198            .await
1199            .unwrap();
1200        let batch = &batches[0];
1201
1202        assert_eq!(batch.num_rows(), 8)
1203    }
1204
1205    #[cfg(feature = "snappy")]
1206    #[tokio::test]
1207    async fn test_snappy_compressed_with_range() {
1208        {
1209            let file = arrow_test_data("avro/alltypes_plain.snappy.avro");
1210            let store = Arc::new(LocalFileSystem::new());
1211            let location = Path::from_filesystem_path(&file).unwrap();
1212            let meta = store.head(&location).await.unwrap();
1213
1214            let range = 200..meta.size;
1215            let schema = get_alltypes_schema();
1216            let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1217                .await
1218                .unwrap();
1219            let batch = &batches[0];
1220
1221            assert!(batch.num_rows() > 0);
1222        }
1223    }
1224
1225    #[tokio::test]
1226    async fn test_nulls() {
1227        let file = arrow_test_data("avro/alltypes_nulls_plain.avro");
1228        let schema = get_alltypes_with_nulls_schema();
1229        let batches = read_async_file(&file, 1024, None, Some(schema), None)
1230            .await
1231            .unwrap();
1232        let batch = &batches[0];
1233
1234        assert_eq!(batch.num_rows(), 1);
1235        for col in batch.columns() {
1236            assert!(col.is_null(0));
1237        }
1238    }
1239
1240    #[tokio::test]
1241    async fn test_nested_records() {
1242        let file = arrow_test_data("avro/nested_records.avro");
1243        let schema = get_nested_records_schema();
1244        let batches = read_async_file(&file, 1024, None, Some(schema), None)
1245            .await
1246            .unwrap();
1247        let batch = &batches[0];
1248
1249        assert_eq!(batch.num_rows(), 2);
1250        assert!(batch.num_columns() > 0);
1251    }
1252
1253    #[tokio::test]
1254    async fn test_stream_produces_multiple_batches() {
1255        let file = arrow_test_data("avro/alltypes_plain.avro");
1256        let store = Arc::new(LocalFileSystem::new());
1257        let location = Path::from_filesystem_path(&file).unwrap();
1258
1259        let file_size = store.head(&location).await.unwrap().size;
1260
1261        let file_reader = ObjectStoreReader::new(store, location);
1262        let schema = get_alltypes_schema();
1263        let reader_schema = AvroSchema::try_from(schema.as_ref()).unwrap();
1264        let reader = AsyncAvroFileReader::builder(
1265            file_reader,
1266            file_size,
1267            2, // Small batch size to force multiple batches
1268        )
1269        .with_reader_schema(reader_schema)
1270        .try_build()
1271        .await
1272        .unwrap();
1273
1274        let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1275
1276        assert!(batches.len() > 1);
1277        let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
1278        assert_eq!(total_rows, 8);
1279    }
1280
1281    #[tokio::test]
1282    async fn test_stream_early_termination() {
1283        let file = arrow_test_data("avro/alltypes_plain.avro");
1284        let store = Arc::new(LocalFileSystem::new());
1285        let location = Path::from_filesystem_path(&file).unwrap();
1286
1287        let file_size = store.head(&location).await.unwrap().size;
1288
1289        let file_reader = ObjectStoreReader::new(store, location);
1290        let schema = get_alltypes_schema();
1291        let reader_schema = AvroSchema::try_from(schema.as_ref()).unwrap();
1292        let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1)
1293            .with_reader_schema(reader_schema)
1294            .try_build()
1295            .await
1296            .unwrap();
1297
1298        let first_batch = reader.take(1).try_collect::<Vec<_>>().await.unwrap();
1299
1300        assert_eq!(first_batch.len(), 1);
1301        assert!(first_batch[0].num_rows() > 0);
1302    }
1303
1304    #[tokio::test]
1305    async fn test_various_batch_sizes() {
1306        let file = arrow_test_data("avro/alltypes_plain.avro");
1307
1308        for batch_size in [1, 2, 3, 5, 7, 11, 100] {
1309            let schema = get_alltypes_schema();
1310            let batches = read_async_file(&file, batch_size, None, Some(schema), None)
1311                .await
1312                .unwrap();
1313            let batch = &batches[0];
1314
1315            // Size should be what was provided, to the limit of the batch in the file
1316            assert_eq!(
1317                batch.num_rows(),
1318                batch_size.min(8),
1319                "Failed with batch_size={batch_size}"
1320            );
1321        }
1322    }
1323
1324    #[tokio::test]
1325    async fn test_range_larger_than_file() {
1326        let file = arrow_test_data("avro/alltypes_plain.avro");
1327        let store = Arc::new(LocalFileSystem::new());
1328        let location = Path::from_filesystem_path(&file).unwrap();
1329        let meta = store.head(&location).await.unwrap();
1330
1331        // Range extends beyond file size
1332        let range = 100..(meta.size + 1000);
1333        let schema = get_alltypes_schema();
1334        let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1335            .await
1336            .unwrap();
1337        let batch = &batches[0];
1338
1339        // Should clamp to file size
1340        assert_eq!(batch.num_rows(), 8);
1341    }
1342
1343    #[tokio::test]
1344    async fn test_builder_with_header_info() {
1345        let file = arrow_test_data("avro/alltypes_plain.avro");
1346        let store = Arc::new(LocalFileSystem::new());
1347        let location = Path::from_filesystem_path(&file).unwrap();
1348
1349        let file_size = store.head(&location).await.unwrap().size;
1350
1351        let mut file_reader = ObjectStoreReader::new(store, location);
1352
1353        let header_info = read_header_info(&mut file_reader, file_size, None)
1354            .await
1355            .unwrap();
1356
1357        assert_eq!(header_info.header_len(), 675);
1358
1359        let writer_schema = header_info.writer_schema().unwrap();
1360        let expected_avro_json: serde_json::Value = serde_json::from_str(
1361            get_alltypes_schema()
1362                .metadata()
1363                .get(SCHEMA_METADATA_KEY)
1364                .unwrap(),
1365        )
1366        .unwrap();
1367        let actual_avro_json: serde_json::Value =
1368            serde_json::from_str(&writer_schema.json_string).unwrap();
1369        assert_eq!(actual_avro_json, expected_avro_json);
1370
1371        let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1372            .build_with_header(header_info)
1373            .unwrap();
1374
1375        let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1376
1377        let batch = &batches[0];
1378        assert_eq!(batch.num_rows(), 8)
1379    }
1380
1381    #[tokio::test]
1382    async fn test_roundtrip_write_then_async_read() {
1383        use crate::writer::AvroWriter;
1384        use arrow_array::{Float64Array, StringArray};
1385        use std::fs::File;
1386        use std::io::BufWriter;
1387        use tempfile::tempdir;
1388
1389        // Schema with nullable and non-nullable fields of various types
1390        let schema = Arc::new(Schema::new(vec![
1391            Field::new("id", DataType::Int32, false),
1392            Field::new("name", DataType::Utf8, true),
1393            Field::new("score", DataType::Float64, true),
1394            Field::new("count", DataType::Int64, false),
1395        ]));
1396
1397        let dir = tempdir().unwrap();
1398        let file_path = dir.path().join("roundtrip_test.avro");
1399
1400        // Write multiple batches with nulls
1401        {
1402            let file = File::create(&file_path).unwrap();
1403            let writer = BufWriter::new(file);
1404            let mut avro_writer = AvroWriter::new(writer, schema.as_ref().clone()).unwrap();
1405
1406            // First batch: 3 rows with some nulls
1407            let batch1 = RecordBatch::try_new(
1408                schema.clone(),
1409                vec![
1410                    Arc::new(Int32Array::from(vec![1, 2, 3])),
1411                    Arc::new(StringArray::from(vec![
1412                        Some("alice"),
1413                        None,
1414                        Some("charlie"),
1415                    ])),
1416                    Arc::new(Float64Array::from(vec![Some(95.5), Some(87.3), None])),
1417                    Arc::new(Int64Array::from(vec![10, 20, 30])),
1418                ],
1419            )
1420            .unwrap();
1421            avro_writer.write(&batch1).unwrap();
1422
1423            // Second batch: 2 rows
1424            let batch2 = RecordBatch::try_new(
1425                schema.clone(),
1426                vec![
1427                    Arc::new(Int32Array::from(vec![4, 5])),
1428                    Arc::new(StringArray::from(vec![Some("diana"), Some("eve")])),
1429                    Arc::new(Float64Array::from(vec![None, Some(88.0)])),
1430                    Arc::new(Int64Array::from(vec![40, 50])),
1431                ],
1432            )
1433            .unwrap();
1434            avro_writer.write(&batch2).unwrap();
1435
1436            avro_writer.finish().unwrap();
1437        }
1438
1439        // Read back with small batch size to produce multiple output batches
1440        let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1441        let location = Path::from_filesystem_path(&file_path).unwrap();
1442        let file_size = store.head(&location).await.unwrap().size;
1443
1444        let file_reader = ObjectStoreReader::new(store, location);
1445        let reader = AsyncAvroFileReader::builder(file_reader, file_size, 2)
1446            .try_build()
1447            .await
1448            .unwrap();
1449
1450        let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1451
1452        // Verify we got multiple output batches due to small batch_size
1453        assert!(
1454            batches.len() > 1,
1455            "Expected multiple batches with batch_size=2"
1456        );
1457
1458        // Verify total row count
1459        let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
1460        assert_eq!(total_rows, 5);
1461
1462        // Concatenate all batches to verify data
1463        let combined = arrow::compute::concat_batches(&batches[0].schema(), &batches).unwrap();
1464        assert_eq!(combined.num_rows(), 5);
1465        assert_eq!(combined.num_columns(), 4);
1466
1467        // Check id column (non-nullable)
1468        let id_array = combined
1469            .column(0)
1470            .as_any()
1471            .downcast_ref::<Int32Array>()
1472            .unwrap();
1473        assert_eq!(id_array.values(), &[1, 2, 3, 4, 5]);
1474
1475        // Check name column (nullable) - verify nulls are preserved
1476        // Avro strings are read as Binary by default
1477        let name_col = combined.column(1);
1478        let name_array = name_col.as_string::<i32>();
1479        assert_eq!(name_array.value(0), "alice");
1480        assert!(name_col.is_null(1)); // second row has null name
1481        assert_eq!(name_array.value(2), "charlie");
1482
1483        // Check score column (nullable) - verify nulls are preserved
1484        let score_array = combined
1485            .column(2)
1486            .as_any()
1487            .downcast_ref::<Float64Array>()
1488            .unwrap();
1489        assert!(!score_array.is_null(0));
1490        assert!((score_array.value(0) - 95.5).abs() < f64::EPSILON);
1491        assert!(score_array.is_null(2)); // third row has null score
1492        assert!(score_array.is_null(3)); // fourth row has null score
1493        assert!(!score_array.is_null(4));
1494        assert!((score_array.value(4) - 88.0).abs() < f64::EPSILON);
1495
1496        // Check count column (non-nullable)
1497        let count_array = combined
1498            .column(3)
1499            .as_any()
1500            .downcast_ref::<Int64Array>()
1501            .unwrap();
1502        assert_eq!(count_array.values(), &[10, 20, 30, 40, 50]);
1503    }
1504
1505    #[tokio::test]
1506    async fn test_alltypes_no_schema_no_projection() {
1507        // No reader schema, no projection - uses writer schema from file
1508        let file = arrow_test_data("avro/alltypes_plain.avro");
1509        let batches = read_async_file(&file, 1024, None, None, None)
1510            .await
1511            .unwrap();
1512        let batch = &batches[0];
1513
1514        assert_eq!(batch.num_rows(), 8);
1515        assert_eq!(batch.num_columns(), 11);
1516        assert_eq!(batch.schema().field(0).name(), "id");
1517    }
1518
1519    #[tokio::test]
1520    async fn test_alltypes_no_schema_with_projection() {
1521        // No reader schema, with projection - project writer schema
1522        let file = arrow_test_data("avro/alltypes_plain.avro");
1523        // Project [tinyint_col, id, bigint_col] = indices [2, 0, 5]
1524        let batches = read_async_file(&file, 1024, None, None, Some(vec![2, 0, 5]))
1525            .await
1526            .unwrap();
1527        let batch = &batches[0];
1528
1529        assert_eq!(batch.num_rows(), 8);
1530        assert_eq!(batch.num_columns(), 3);
1531        assert_eq!(batch.schema().field(0).name(), "tinyint_col");
1532        assert_eq!(batch.schema().field(1).name(), "id");
1533        assert_eq!(batch.schema().field(2).name(), "bigint_col");
1534
1535        // Verify data values
1536        let tinyint_col = batch.column(0).as_primitive::<Int32Type>();
1537        assert_eq!(tinyint_col.values(), &[0, 1, 0, 1, 0, 1, 0, 1]);
1538
1539        let id = batch.column(1).as_primitive::<Int32Type>();
1540        assert_eq!(id.values(), &[4, 5, 6, 7, 2, 3, 0, 1]);
1541
1542        let bigint_col = batch.column(2).as_primitive::<Int64Type>();
1543        assert_eq!(bigint_col.values(), &[0, 10, 0, 10, 0, 10, 0, 10]);
1544    }
1545
1546    #[tokio::test]
1547    async fn test_alltypes_with_schema_no_projection() {
1548        // With reader schema, no projection
1549        let file = arrow_test_data("avro/alltypes_plain.avro");
1550        let schema = get_alltypes_schema();
1551        let batches = read_async_file(&file, 1024, None, Some(schema), None)
1552            .await
1553            .unwrap();
1554        let batch = &batches[0];
1555
1556        assert_eq!(batch.num_rows(), 8);
1557        assert_eq!(batch.num_columns(), 11);
1558    }
1559
1560    #[tokio::test]
1561    async fn test_alltypes_with_schema_with_projection() {
1562        // With reader schema, with projection
1563        let file = arrow_test_data("avro/alltypes_plain.avro");
1564        let schema = get_alltypes_schema();
1565        // Project [bool_col, id] = indices [1, 0]
1566        let batches = read_async_file(&file, 1024, None, Some(schema), Some(vec![1, 0]))
1567            .await
1568            .unwrap();
1569        let batch = &batches[0];
1570
1571        assert_eq!(batch.num_rows(), 8);
1572        assert_eq!(batch.num_columns(), 2);
1573        assert_eq!(batch.schema().field(0).name(), "bool_col");
1574        assert_eq!(batch.schema().field(1).name(), "id");
1575
1576        let bool_col = batch.column(0).as_boolean();
1577        assert!(bool_col.value(0));
1578        assert!(!bool_col.value(1));
1579
1580        let id = batch.column(1).as_primitive::<Int32Type>();
1581        assert_eq!(id.values(), &[4, 5, 6, 7, 2, 3, 0, 1]);
1582    }
1583
1584    #[tokio::test]
1585    async fn test_alltypes_with_empty_schema_large_batch() {
1586        // With an empty reader schema -- should count rows but produce no columns
1587        let file = arrow_test_data("avro/alltypes_plain.avro");
1588        let schema = Arc::new(Schema::new(Vec::<Field>::new()));
1589        let batches = read_async_file(&file, 1024, None, Some(schema), None)
1590            .await
1591            .unwrap();
1592        assert_eq!(batches.len(), 1);
1593        let batch = &batches[0];
1594
1595        assert_eq!(batch.num_rows(), 8);
1596        assert_eq!(batch.num_columns(), 0);
1597    }
1598
1599    #[tokio::test]
1600    async fn test_alltypes_with_empty_schema_small_batch() {
1601        // With an empty reader schema -- should count rows but produce no columns
1602        let file = arrow_test_data("avro/alltypes_plain.avro");
1603        let schema = Arc::new(Schema::new(Vec::<Field>::new()));
1604        let batches = read_async_file(&file, 5, None, Some(schema), None)
1605            .await
1606            .unwrap();
1607
1608        assert_eq!(batches.len(), 2);
1609
1610        assert_eq!(batches[0].num_rows(), 5);
1611        assert_eq!(batches[0].num_columns(), 0);
1612        assert_eq!(batches[1].num_rows(), 3);
1613        assert_eq!(batches[1].num_columns(), 0);
1614    }
1615
1616    #[tokio::test]
1617    async fn test_nested_no_schema_no_projection() {
1618        // No reader schema, no projection
1619        let file = arrow_test_data("avro/nested_records.avro");
1620        let batches = read_async_file(&file, 1024, None, None, None)
1621            .await
1622            .unwrap();
1623        let batch = &batches[0];
1624
1625        assert_eq!(batch.num_rows(), 2);
1626        assert_eq!(batch.num_columns(), 4);
1627        assert_eq!(batch.schema().field(0).name(), "f1");
1628        assert_eq!(batch.schema().field(1).name(), "f2");
1629        assert_eq!(batch.schema().field(2).name(), "f3");
1630        assert_eq!(batch.schema().field(3).name(), "f4");
1631    }
1632
1633    #[tokio::test]
1634    async fn test_nested_no_schema_with_projection() {
1635        // No reader schema, with projection - reorder nested fields
1636        let file = arrow_test_data("avro/nested_records.avro");
1637        // Project [f3, f1] = indices [2, 0]
1638        let batches = read_async_file(&file, 1024, None, None, Some(vec![2, 0]))
1639            .await
1640            .unwrap();
1641        let batch = &batches[0];
1642
1643        assert_eq!(batch.num_rows(), 2);
1644        assert_eq!(batch.num_columns(), 2);
1645        assert_eq!(batch.schema().field(0).name(), "f3");
1646        assert_eq!(batch.schema().field(1).name(), "f1");
1647    }
1648
1649    #[tokio::test]
1650    async fn test_nested_with_schema_no_projection() {
1651        // With reader schema, no projection
1652        let file = arrow_test_data("avro/nested_records.avro");
1653        let schema = get_nested_records_schema();
1654        let batches = read_async_file(&file, 1024, None, Some(schema), None)
1655            .await
1656            .unwrap();
1657        let batch = &batches[0];
1658
1659        assert_eq!(batch.num_rows(), 2);
1660        assert_eq!(batch.num_columns(), 4);
1661    }
1662
1663    #[tokio::test]
1664    async fn test_nested_with_schema_with_projection() {
1665        // With reader schema, with projection
1666        let file = arrow_test_data("avro/nested_records.avro");
1667        let schema = get_nested_records_schema();
1668        // Project [f4, f2, f1] = indices [3, 1, 0]
1669        let batches = read_async_file(&file, 1024, None, Some(schema), Some(vec![3, 1, 0]))
1670            .await
1671            .unwrap();
1672        let batch = &batches[0];
1673
1674        assert_eq!(batch.num_rows(), 2);
1675        assert_eq!(batch.num_columns(), 3);
1676        assert_eq!(batch.schema().field(0).name(), "f4");
1677        assert_eq!(batch.schema().field(1).name(), "f2");
1678        assert_eq!(batch.schema().field(2).name(), "f1");
1679    }
1680
1681    #[tokio::test]
1682    async fn test_nested_with_empty_schema() {
1683        // With an empty reader schema -- should count rows but produce no columns
1684        let file = arrow_test_data("avro/nested_records.avro");
1685        let schema = Arc::new(
1686            Schema::new(Vec::<Field>::new()).with_metadata(HashMap::from([(
1687                SCHEMA_METADATA_KEY.into(),
1688                r#"{
1689                    "type": "record",
1690                    "namespace": "ns1",
1691                    "name": "record1",
1692                    "fields": []
1693                }"#
1694                .to_owned(),
1695            )])),
1696        );
1697        let batches = read_async_file(&file, 1024, None, Some(schema), None)
1698            .await
1699            .unwrap();
1700        let batch = &batches[0];
1701
1702        assert_eq!(batch.num_rows(), 2);
1703        assert_eq!(batch.num_columns(), 0);
1704    }
1705
1706    #[tokio::test]
1707    async fn test_projection_error_out_of_bounds() {
1708        let file = arrow_test_data("avro/alltypes_plain.avro");
1709        // Index 100 is out of bounds for the 11-field schema
1710        let err = read_async_file(&file, 1024, None, None, Some(vec![100]))
1711            .await
1712            .unwrap_err();
1713        assert!(matches!(err, ArrowError::AvroError(_)));
1714        assert!(err.to_string().contains("out of bounds"));
1715    }
1716
1717    #[tokio::test]
1718    async fn test_projection_error_duplicate_index() {
1719        let file = arrow_test_data("avro/alltypes_plain.avro");
1720        // Duplicate index 0
1721        let err = read_async_file(&file, 1024, None, None, Some(vec![0, 0]))
1722            .await
1723            .unwrap_err();
1724        assert!(matches!(err, ArrowError::AvroError(_)));
1725        assert!(err.to_string().contains("Duplicate projection index"));
1726    }
1727
1728    #[tokio::test]
1729    async fn test_arrow_schema_from_reader_no_reader_schema() {
1730        let file = arrow_test_data("avro/alltypes_plain.avro");
1731        let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1732        let location = Path::from_filesystem_path(&file).unwrap();
1733        let file_size = store.head(&location).await.unwrap().size;
1734
1735        let file_reader = ObjectStoreReader::new(store, location);
1736        let expected_schema = get_alltypes_schema()
1737            .as_ref()
1738            .clone()
1739            .with_metadata(Metadata::default());
1740
1741        // Build reader without providing reader schema - should use writer schema from file
1742        let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1743            .try_build()
1744            .await
1745            .unwrap();
1746
1747        assert_eq!(reader.schema().as_ref(), &expected_schema);
1748
1749        let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1750        let batch = &batches[0];
1751
1752        assert_eq!(batch.schema().as_ref(), &expected_schema);
1753    }
1754
1755    #[tokio::test]
1756    async fn test_arrow_schema_from_reader_with_reader_schema() {
1757        let file = arrow_test_data("avro/alltypes_plain.avro");
1758        let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1759        let location = Path::from_filesystem_path(&file).unwrap();
1760        let file_size = store.head(&location).await.unwrap().size;
1761
1762        let file_reader = ObjectStoreReader::new(store, location);
1763        let schema = get_alltypes_schema()
1764            .project(&[0, 1, 7])
1765            .unwrap()
1766            .with_metadata(Metadata::default());
1767        let reader_schema = AvroSchema::try_from(&schema).unwrap();
1768        let expected_schema = schema.clone();
1769
1770        // Build reader with provided reader schema - must apply the projection
1771        let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1772            .with_reader_schema(reader_schema)
1773            .try_build()
1774            .await
1775            .unwrap();
1776
1777        assert_eq!(reader.schema().as_ref(), &expected_schema);
1778
1779        let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1780        let batch = &batches[0];
1781
1782        assert_eq!(batch.schema().as_ref(), &expected_schema);
1783    }
1784
1785    #[tokio::test]
1786    async fn test_arrow_schema_from_reader_nested_records() {
1787        let file = arrow_test_data("avro/nested_records.avro");
1788        let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1789        let location = Path::from_filesystem_path(&file).unwrap();
1790        let file_size = store.head(&location).await.unwrap().size;
1791
1792        let file_reader = ObjectStoreReader::new(store, location);
1793
1794        // The schema produced by the reader should match the expected schema,
1795        // attaching Avro type name metadata to fields of record and list types.
1796        let expected_schema = get_nested_records_schema()
1797            .as_ref()
1798            .clone()
1799            .with_metadata(Metadata::default());
1800
1801        let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1802            .try_build()
1803            .await
1804            .unwrap();
1805
1806        assert_eq!(reader.schema().as_ref(), &expected_schema);
1807
1808        let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1809        let batch = &batches[0];
1810
1811        assert_eq!(batch.schema().as_ref(), &expected_schema);
1812    }
1813
1814    #[tokio::test]
1815    async fn test_with_header_size_hint_small() {
1816        // Use a very small header size hint to force multiple fetches
1817        let file = arrow_test_data("avro/alltypes_plain.avro");
1818        let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1819        let location = Path::from_filesystem_path(&file).unwrap();
1820        let file_size = store.head(&location).await.unwrap().size;
1821
1822        let file_reader = ObjectStoreReader::new(store, location);
1823        let schema = get_alltypes_schema();
1824        let reader_schema = AvroSchema::try_from(schema.as_ref()).unwrap();
1825
1826        // Use a tiny header hint (64 bytes) - header is much larger
1827        let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1828            .with_reader_schema(reader_schema)
1829            .with_header_size_hint(64)
1830            .try_build()
1831            .await
1832            .unwrap();
1833
1834        let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1835        let batch = &batches[0];
1836
1837        assert_eq!(batch.num_rows(), 8);
1838        assert_eq!(batch.num_columns(), 11);
1839    }
1840
1841    #[tokio::test]
1842    async fn test_with_header_size_hint_large() {
1843        // Use a larger header size hint than needed
1844        let file = arrow_test_data("avro/alltypes_plain.avro");
1845        let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1846        let location = Path::from_filesystem_path(&file).unwrap();
1847        let file_size = store.head(&location).await.unwrap().size;
1848
1849        let file_reader = ObjectStoreReader::new(store, location);
1850        let schema = get_alltypes_schema();
1851        let reader_schema = AvroSchema::try_from(schema.as_ref()).unwrap();
1852
1853        // Use a large header hint (64KB)
1854        let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1855            .with_reader_schema(reader_schema)
1856            .with_header_size_hint(64 * 1024)
1857            .try_build()
1858            .await
1859            .unwrap();
1860
1861        let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1862        let batch = &batches[0];
1863
1864        assert_eq!(batch.num_rows(), 8);
1865        assert_eq!(batch.num_columns(), 11);
1866    }
1867
1868    #[tokio::test]
1869    async fn test_with_tz_utc() {
1870        let file = arrow_test_data("avro/alltypes_plain.avro");
1871        let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1872        let location = Path::from_filesystem_path(&file).unwrap();
1873        let file_size = store.head(&location).await.unwrap().size;
1874
1875        let file_reader = ObjectStoreReader::new(store, location);
1876        let schema = get_alltypes_schema_with_tz("UTC");
1877        let reader_schema = AvroSchema::try_from(schema.as_ref()).unwrap();
1878
1879        // Specify the time zone ID of "UTC" for timestamp fields with time zone.
1880        let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1881            .with_reader_schema(reader_schema)
1882            .with_tz(Tz::Utc)
1883            .try_build()
1884            .await
1885            .unwrap();
1886
1887        let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1888        let batch = &batches[0];
1889
1890        assert_eq!(batch.num_columns(), 11);
1891
1892        let schema = batch.schema();
1893        let ts_field = schema.field_with_name("timestamp_col").unwrap();
1894        assert!(
1895            matches!(
1896                ts_field.data_type(),
1897                DataType::Timestamp(TimeUnit::Microsecond, Some(tz)) if tz.as_ref() == "UTC"
1898            ),
1899            "expected Timestamp(Microsecond, Some(\"UTC\")), got {:?}",
1900            ts_field.data_type()
1901        );
1902    }
1903
1904    #[tokio::test]
1905    async fn test_with_utf8_view_enabled() {
1906        // Test that utf8_view produces StringViewArray instead of StringArray
1907        let file = arrow_test_data("avro/nested_records.avro");
1908        let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1909        let location = Path::from_filesystem_path(&file).unwrap();
1910        let file_size = store.head(&location).await.unwrap().size;
1911
1912        let file_reader = ObjectStoreReader::new(store, location);
1913
1914        let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1915            .with_utf8_view(true)
1916            .try_build()
1917            .await
1918            .unwrap();
1919
1920        let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1921        let batch = &batches[0];
1922
1923        assert_eq!(batch.num_rows(), 2);
1924
1925        // The f1 struct contains f1_1 which is a string field
1926        // With utf8_view enabled, it should be Utf8View type
1927        let f1_col = batch.column(0);
1928        let f1_struct = f1_col.as_struct();
1929        let f1_1_field = f1_struct.column_by_name("f1_1").unwrap();
1930
1931        // Check that the data type is Utf8View
1932        assert_eq!(f1_1_field.data_type(), &DataType::Utf8View);
1933    }
1934
1935    #[tokio::test]
1936    async fn test_with_utf8_view_disabled() {
1937        // Test that without utf8_view, we get regular Utf8
1938        let file = arrow_test_data("avro/nested_records.avro");
1939        let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1940        let location = Path::from_filesystem_path(&file).unwrap();
1941        let file_size = store.head(&location).await.unwrap().size;
1942
1943        let file_reader = ObjectStoreReader::new(store, location);
1944
1945        let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1946            .with_utf8_view(false)
1947            .try_build()
1948            .await
1949            .unwrap();
1950
1951        let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1952        let batch = &batches[0];
1953
1954        assert_eq!(batch.num_rows(), 2);
1955
1956        // The f1 struct contains f1_1 which is a string field
1957        // Without utf8_view, it should be regular Utf8
1958        let f1_col = batch.column(0);
1959        let f1_struct = f1_col.as_struct();
1960        let f1_1_field = f1_struct.column_by_name("f1_1").unwrap();
1961
1962        assert_eq!(f1_1_field.data_type(), &DataType::Utf8);
1963    }
1964
1965    #[tokio::test]
1966    async fn test_with_strict_mode_disabled_allows_null_second() {
1967        // Test that with strict_mode disabled, unions of ['T', 'null'] are allowed
1968        // The alltypes_nulls_plain.avro file has unions with null second
1969        let file = arrow_test_data("avro/alltypes_nulls_plain.avro");
1970        let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1971        let location = Path::from_filesystem_path(&file).unwrap();
1972        let file_size = store.head(&location).await.unwrap().size;
1973
1974        let file_reader = ObjectStoreReader::new(store, location);
1975
1976        // Without strict mode, this should succeed
1977        let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1978            .with_strict_mode(false)
1979            .try_build()
1980            .await
1981            .unwrap();
1982
1983        let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1984        assert_eq!(batches.len(), 1);
1985        assert_eq!(batches[0].num_rows(), 1);
1986    }
1987
1988    #[tokio::test]
1989    async fn test_with_strict_mode_enabled_rejects_null_second() {
1990        // Test that with strict_mode enabled, unions of ['T', 'null'] are rejected
1991        // The alltypes_plain.avro file has unions like ["int", "null"] (null second)
1992        let file = arrow_test_data("avro/alltypes_plain.avro");
1993        let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1994        let location = Path::from_filesystem_path(&file).unwrap();
1995        let file_size = store.head(&location).await.unwrap().size;
1996
1997        let file_reader = ObjectStoreReader::new(store, location);
1998
1999        // With strict mode, this should fail because of ['T', 'null'] unions
2000        let result = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
2001            .with_strict_mode(true)
2002            .try_build()
2003            .await;
2004
2005        match result {
2006            Ok(_) => panic!("Expected error for strict_mode with ['T', 'null'] union"),
2007            Err(err) => {
2008                assert!(
2009                    err.to_string().contains("disallowed in strict_mode"),
2010                    "Expected strict_mode error, got: {err}"
2011                );
2012            }
2013        }
2014    }
2015
2016    #[tokio::test]
2017    async fn test_with_strict_mode_enabled_valid_schema() {
2018        // Test that strict_mode works with schemas that have proper ['null', 'T'] unions
2019        // The nested_records.avro file has properly ordered unions
2020        let file = arrow_test_data("avro/nested_records.avro");
2021        let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
2022        let location = Path::from_filesystem_path(&file).unwrap();
2023        let file_size = store.head(&location).await.unwrap().size;
2024
2025        let file_reader = ObjectStoreReader::new(store, location);
2026
2027        // With strict mode, properly ordered unions should still work
2028        let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
2029            .with_strict_mode(true)
2030            .try_build()
2031            .await
2032            .unwrap();
2033
2034        let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
2035        assert_eq!(batches.len(), 1);
2036        assert_eq!(batches[0].num_rows(), 2);
2037    }
2038
2039    #[tokio::test]
2040    async fn test_builder_options_combined() {
2041        // Test combining multiple builder options
2042        let file = arrow_test_data("avro/nested_records.avro");
2043        let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
2044        let location = Path::from_filesystem_path(&file).unwrap();
2045        let file_size = store.head(&location).await.unwrap().size;
2046
2047        let file_reader = ObjectStoreReader::new(store, location);
2048
2049        let reader = AsyncAvroFileReader::builder(file_reader, file_size, 2)
2050            .with_header_size_hint(128)
2051            .with_utf8_view(true)
2052            .with_strict_mode(true)
2053            .with_projection(vec![0, 2]) // f1 and f3
2054            .try_build()
2055            .await
2056            .unwrap();
2057
2058        let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
2059        let batch = &batches[0];
2060
2061        // Should have 2 columns (f1 and f3) due to projection
2062        assert_eq!(batch.num_columns(), 2);
2063        assert_eq!(batch.schema().field(0).name(), "f1");
2064        assert_eq!(batch.schema().field(1).name(), "f3");
2065
2066        // Verify utf8_view is applied
2067        let f1_col = batch.column(0);
2068        let f1_struct = f1_col.as_struct();
2069        let f1_1_field = f1_struct.column_by_name("f1_1").unwrap();
2070        assert_eq!(f1_1_field.data_type(), &DataType::Utf8View);
2071    }
2072}