1use crate::compression::CompressionCodec;
11use crate::reader::Decoder;
12use crate::reader::block::{BlockDecoder, BlockDecoderState};
13use arrow_array::RecordBatch;
14use arrow_schema::{ArrowError, SchemaRef};
15use bytes::Bytes;
16use futures::future::BoxFuture;
17use futures::{FutureExt, Stream};
18use std::mem;
19use std::ops::Range;
20use std::pin::Pin;
21use std::task::{Context, Poll};
22
23mod async_file_reader;
24mod builder;
25mod spawn;
26
27pub use async_file_reader::AsyncFileReader;
28pub use builder::{ReaderBuilder, read_header_info};
29pub use spawn::SpawnedReader;
30
31#[cfg(feature = "object_store")]
32mod store;
33
34use crate::errors::AvroError;
35#[expect(deprecated)]
36#[cfg(feature = "object_store")]
37pub use store::AvroObjectReader;
38
39enum FetchNextBehaviour {
40 ReadSyncMarker,
42 DecodeVLQHeader,
44 ContinueDecoding,
46}
47
48enum ReaderState<R> {
49 InvalidState,
51 Idle { reader: R },
53 FetchingData {
55 future: BoxFuture<'static, Result<(R, Bytes), AvroError>>,
56 next_behaviour: FetchNextBehaviour,
57 },
58 DecodingBlock { data: Bytes, reader: R },
60 ReadingBatches {
62 data: Bytes,
63 block_data: Bytes,
64 remaining_in_block: usize,
65 reader: R,
66 },
67 Flushing,
70 Finished,
72}
73
74pub struct AsyncAvroFileReader<R> {
128 range: Range<u64>,
130 file_size: u64,
131
132 decoder: Decoder,
134 block_decoder: BlockDecoder,
135 codec: Option<CompressionCodec>,
136 sync_marker: [u8; 16],
137
138 reader_state: ReaderState<R>,
140 finishing_partial_block: bool,
141}
142
143impl<R> AsyncAvroFileReader<R> {
144 pub fn builder(reader: R, file_size: u64, batch_size: usize) -> ReaderBuilder<R> {
146 ReaderBuilder::new(reader, file_size, batch_size)
147 }
148
149 fn new(
150 range: Range<u64>,
151 file_size: u64,
152 decoder: Decoder,
153 codec: Option<CompressionCodec>,
154 sync_marker: [u8; 16],
155 reader_state: ReaderState<R>,
156 ) -> Self {
157 Self {
158 range,
159 file_size,
160
161 decoder,
162 block_decoder: Default::default(),
163 codec,
164 sync_marker,
165
166 reader_state,
167 finishing_partial_block: false,
168 }
169 }
170
171 pub fn schema(&self) -> SchemaRef {
175 self.decoder.schema()
176 }
177
178 fn remaining_block_range(&self) -> Result<Range<u64>, AvroError> {
182 let remaining = self.block_decoder.bytes_remaining() as u64
183 + match self.block_decoder.state() {
184 BlockDecoderState::Data => 16, BlockDecoderState::Sync => 0,
186 state => {
187 return Err(AvroError::General(format!(
188 "remaining_block_range called in unexpected state: {state:?}"
189 )));
190 }
191 };
192
193 let fetch_end = self.range.end + remaining;
194 if fetch_end > self.file_size {
195 return Err(AvroError::EOF(
196 "Avro block requires more bytes than what exists in the file".into(),
197 ));
198 }
199
200 Ok(self.range.end..fetch_end)
201 }
202
203 #[inline]
205 fn finish_with_error(
206 &mut self,
207 error: AvroError,
208 ) -> Poll<Option<Result<RecordBatch, AvroError>>> {
209 self.reader_state = ReaderState::Finished;
210 Poll::Ready(Some(Err(error)))
211 }
212
213 #[inline]
214 fn start_flushing(&mut self) {
215 self.reader_state = ReaderState::Flushing;
216 }
217
218 #[inline]
220 fn poll_flush(&mut self) -> Poll<Option<Result<RecordBatch, AvroError>>> {
221 match self.decoder.flush_block() {
222 Ok(Some(batch)) => {
223 self.reader_state = ReaderState::Flushing;
224 Poll::Ready(Some(Ok(batch)))
225 }
226 Ok(None) => {
227 self.reader_state = ReaderState::Finished;
228 Poll::Ready(None)
229 }
230 Err(e) => self.finish_with_error(e),
231 }
232 }
233}
234
235impl<R: AsyncFileReader + Unpin + 'static> AsyncAvroFileReader<R> {
236 async fn fetch_bytes(mut reader: R, range: Range<u64>) -> Result<(R, Bytes), AvroError> {
239 let data = reader.get_bytes(range).await?;
240 Ok((reader, data))
241 }
242
243 #[forbid(clippy::question_mark_used)]
244 fn read_next(&mut self, cx: &mut Context<'_>) -> Poll<Option<Result<RecordBatch, AvroError>>> {
245 loop {
246 match mem::replace(&mut self.reader_state, ReaderState::InvalidState) {
247 ReaderState::Idle { reader } => {
248 let range = self.range.clone();
249 if range.start >= range.end {
250 return self.finish_with_error(AvroError::InvalidArgument(format!(
251 "Invalid range specified for Avro file: start {} >= end {}, file_size: {}",
252 range.start, range.end, self.file_size
253 )));
254 }
255
256 let future = Self::fetch_bytes(reader, range).boxed();
257 self.reader_state = ReaderState::FetchingData {
258 future,
259 next_behaviour: FetchNextBehaviour::ReadSyncMarker,
260 };
261 }
262 ReaderState::FetchingData {
263 mut future,
264 next_behaviour,
265 } => {
266 let (reader, data_chunk) = match future.poll_unpin(cx) {
267 Poll::Ready(Ok(data)) => data,
268 Poll::Ready(Err(e)) => return self.finish_with_error(e),
269 Poll::Pending => {
270 self.reader_state = ReaderState::FetchingData {
271 future,
272 next_behaviour,
273 };
274 return Poll::Pending;
275 }
276 };
277
278 match next_behaviour {
279 FetchNextBehaviour::ReadSyncMarker => {
280 let sync_marker_pos = data_chunk
281 .windows(16)
282 .position(|slice| slice == self.sync_marker);
283 let block_start = match sync_marker_pos {
284 Some(pos) => pos + 16, None => {
286 self.reader_state = ReaderState::Finished;
288 return Poll::Ready(None);
289 }
290 };
291
292 self.reader_state = ReaderState::DecodingBlock {
293 reader,
294 data: data_chunk.slice(block_start..),
295 };
296 }
297 FetchNextBehaviour::DecodeVLQHeader => {
298 let mut data = data_chunk;
299
300 while !matches!(self.block_decoder.state(), BlockDecoderState::Data) {
302 if data.is_empty() {
303 return self.finish_with_error(AvroError::EOF(
304 "Unexpected EOF while reading Avro block header".into(),
305 ));
306 }
307 let consumed = match self.block_decoder.decode(&data[..1]) {
308 Ok(consumed) => consumed,
309 Err(e) => return self.finish_with_error(e),
310 };
311 if consumed == 0 {
312 return self.finish_with_error(AvroError::General(
313 "BlockDecoder failed to consume byte during VLQ header parsing"
314 .into(),
315 ));
316 }
317 data = data.slice(consumed..);
318 }
319
320 let bytes_remaining = self.block_decoder.bytes_remaining();
322 let data_to_use = data.slice(..data.len().min(bytes_remaining));
323 let consumed = match self.block_decoder.decode(&data_to_use) {
324 Ok(consumed) => consumed,
325 Err(e) => return self.finish_with_error(e),
326 };
327 if consumed != data_to_use.len() {
328 return self.finish_with_error(AvroError::General(
329 "BlockDecoder failed to consume all bytes after VLQ header parsing"
330 .into(),
331 ));
332 }
333
334 let range_to_fetch = match self.remaining_block_range() {
336 Ok(range) if range.is_empty() => {
337 self.reader_state = ReaderState::DecodingBlock {
339 reader,
340 data: Bytes::new(),
341 };
342 continue;
343 }
344 Ok(range) => range,
345 Err(e) => return self.finish_with_error(e),
346 };
347
348 let future = Self::fetch_bytes(reader, range_to_fetch).boxed();
349 self.reader_state = ReaderState::FetchingData {
350 future,
351 next_behaviour: FetchNextBehaviour::ContinueDecoding,
352 };
353 }
354 FetchNextBehaviour::ContinueDecoding => {
355 self.reader_state = ReaderState::DecodingBlock {
356 reader,
357 data: data_chunk,
358 };
359 }
360 }
361 }
362 ReaderState::InvalidState => {
363 return self.finish_with_error(AvroError::General(
364 "AsyncAvroFileReader in invalid state".into(),
365 ));
366 }
367 ReaderState::DecodingBlock { reader, mut data } => {
368 let consumed = match self.block_decoder.decode(&data) {
370 Ok(consumed) => consumed,
371 Err(e) => return self.finish_with_error(e),
372 };
373 data = data.slice(consumed..);
374
375 if let Some(block) = self.block_decoder.flush() {
377 if block.sync != self.sync_marker {
379 return self.finish_with_error(AvroError::ParseError(
380 "Avro block sync marker does not match file header".to_string(),
381 ));
382 }
383 let block_count = block.count;
384 let block_data = Bytes::from_owner(if let Some(ref codec) = self.codec {
385 match codec.decompress(&block.data) {
386 Ok(decompressed) => decompressed,
387 Err(e) => return self.finish_with_error(e),
388 }
389 } else {
390 block.data
391 });
392
393 self.reader_state = ReaderState::ReadingBatches {
395 reader,
396 data,
397 block_data,
398 remaining_in_block: block_count,
399 };
400 continue;
401 }
402
403 if !data.is_empty() {
405 return self.finish_with_error(AvroError::General(
406 "BlockDecoder failed to make progress decoding Avro block".into(),
407 ));
408 }
409
410 if matches!(self.block_decoder.state(), BlockDecoderState::Finished) {
411 self.finishing_partial_block = false;
413 self.start_flushing();
414 continue;
415 }
416
417 if self.finishing_partial_block {
420 return self.finish_with_error(AvroError::EOF(
421 "Unexpected EOF while reading last Avro block".into(),
422 ));
423 }
424
425 self.finishing_partial_block = true;
431
432 if matches!(
434 self.block_decoder.state(),
435 BlockDecoderState::Count | BlockDecoderState::Size
436 ) {
437 const MAX_VLQ_HEADER_SIZE: u64 = 20;
440 let fetch_end = (self.range.end + MAX_VLQ_HEADER_SIZE).min(self.file_size);
441
442 if fetch_end == self.range.end {
444 return self.finish_with_error(AvroError::EOF(
445 "Unexpected EOF while reading Avro block header".into(),
446 ));
447 }
448
449 let range_to_fetch = self.range.end..fetch_end;
450 self.range.end = fetch_end; let future = Self::fetch_bytes(reader, range_to_fetch).boxed();
453 self.reader_state = ReaderState::FetchingData {
454 future,
455 next_behaviour: FetchNextBehaviour::DecodeVLQHeader,
456 };
457 continue;
458 }
459
460 let range_to_fetch = match self.remaining_block_range() {
462 Ok(range) => range,
463 Err(e) => return self.finish_with_error(e),
464 };
465
466 let future = Self::fetch_bytes(reader, range_to_fetch).boxed();
467 self.reader_state = ReaderState::FetchingData {
468 future,
469 next_behaviour: FetchNextBehaviour::ContinueDecoding,
470 };
471 }
472 ReaderState::ReadingBatches {
473 reader,
474 data,
475 mut block_data,
476 mut remaining_in_block,
477 } => {
478 let (consumed, records_decoded) =
479 match self.decoder.decode_block(&block_data, remaining_in_block) {
480 Ok((consumed, records_decoded)) => (consumed, records_decoded),
481 Err(e) => return self.finish_with_error(e),
482 };
483
484 remaining_in_block -= records_decoded;
485
486 if remaining_in_block == 0 {
487 if data.is_empty() {
488 self.start_flushing();
490 } else {
491 self.reader_state = ReaderState::DecodingBlock { reader, data };
493 }
494 } else {
495 block_data = block_data.slice(consumed..);
497 self.reader_state = ReaderState::ReadingBatches {
498 reader,
499 data,
500 block_data,
501 remaining_in_block,
502 };
503 }
504
505 if self.decoder.batch_is_full() {
508 return match self.decoder.flush_block() {
509 Ok(Some(batch)) => Poll::Ready(Some(Ok(batch))),
510 Ok(None) => self.finish_with_error(AvroError::General(
511 "Decoder reported a full batch, but flush returned None".into(),
512 )),
513 Err(e) => self.finish_with_error(e),
514 };
515 }
516 }
517 ReaderState::Flushing => {
518 return self.poll_flush();
519 }
520 ReaderState::Finished => {
521 self.reader_state = ReaderState::Finished;
523 return Poll::Ready(None);
524 }
525 }
526 }
527 }
528}
529
530impl<R: AsyncFileReader + Unpin + 'static> Stream for AsyncAvroFileReader<R> {
532 type Item = Result<RecordBatch, ArrowError>;
533
534 fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
535 self.read_next(cx).map_err(Into::into)
536 }
537}
538
539#[cfg(test)]
540mod tests {
541 use super::*;
542 use crate::codec::Tz;
543 use crate::schema::{
544 AVRO_NAME_METADATA_KEY, AVRO_NAMESPACE_METADATA_KEY, AvroSchema, SCHEMA_METADATA_KEY,
545 };
546 use arrow_array::cast::AsArray;
547 use arrow_array::types::{Int32Type, Int64Type};
548 use arrow_array::*;
549 use arrow_schema::{DataType, Field, Metadata, Schema, SchemaRef, TimeUnit};
550 use futures::{StreamExt, TryStreamExt};
551 use object_store::local::LocalFileSystem;
552 use object_store::path::Path;
553 use object_store::{ObjectStore, ObjectStoreExt};
554 use std::collections::HashMap;
555 use std::sync::Arc;
556
557 #[derive(Clone, Debug)]
560 struct ObjectStoreReader {
561 store: Arc<dyn ObjectStore>,
562 path: Path,
563 }
564
565 impl ObjectStoreReader {
566 fn new(store: Arc<dyn ObjectStore>, path: Path) -> Self {
567 Self { store, path }
568 }
569 }
570
571 impl AsyncFileReader for ObjectStoreReader {
572 fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes, AvroError>> {
573 async move {
574 self.store
575 .get_range(&self.path, range)
576 .await
577 .map_err(|e| AvroError::General(e.to_string()))
578 }
579 .boxed()
580 }
581
582 fn get_byte_ranges(
583 &mut self,
584 ranges: Vec<Range<u64>>,
585 ) -> BoxFuture<'_, Result<Vec<Bytes>, AvroError>> {
586 async move {
587 self.store
588 .get_ranges(&self.path, &ranges)
589 .await
590 .map_err(|e| AvroError::General(e.to_string()))
591 }
592 .boxed()
593 }
594 }
595
596 fn arrow_test_data(file: &str) -> String {
597 let base =
598 std::env::var("ARROW_TEST_DATA").unwrap_or_else(|_| "../testing/data".to_string());
599 format!("{base}/{file}")
600 }
601
602 fn get_alltypes_schema() -> SchemaRef {
603 get_alltypes_schema_with_tz("+00:00")
604 }
605
606 fn get_alltypes_schema_with_tz(tz_id: &str) -> SchemaRef {
607 let schema = Schema::new(vec![
608 Field::new("id", DataType::Int32, true),
609 Field::new("bool_col", DataType::Boolean, true),
610 Field::new("tinyint_col", DataType::Int32, true),
611 Field::new("smallint_col", DataType::Int32, true),
612 Field::new("int_col", DataType::Int32, true),
613 Field::new("bigint_col", DataType::Int64, true),
614 Field::new("float_col", DataType::Float32, true),
615 Field::new("double_col", DataType::Float64, true),
616 Field::new("date_string_col", DataType::Binary, true),
617 Field::new("string_col", DataType::Binary, true),
618 Field::new(
619 "timestamp_col",
620 DataType::Timestamp(TimeUnit::Microsecond, Some(tz_id.into())),
621 true,
622 ),
623 ])
624 .with_metadata(HashMap::from([(
625 SCHEMA_METADATA_KEY.into(),
626 r#"{
627 "type": "record",
628 "name": "topLevelRecord",
629 "fields": [
630 {
631 "name": "id",
632 "type": [
633 "int",
634 "null"
635 ]
636 },
637 {
638 "name": "bool_col",
639 "type": [
640 "boolean",
641 "null"
642 ]
643 },
644 {
645 "name": "tinyint_col",
646 "type": [
647 "int",
648 "null"
649 ]
650 },
651 {
652 "name": "smallint_col",
653 "type": [
654 "int",
655 "null"
656 ]
657 },
658 {
659 "name": "int_col",
660 "type": [
661 "int",
662 "null"
663 ]
664 },
665 {
666 "name": "bigint_col",
667 "type": [
668 "long",
669 "null"
670 ]
671 },
672 {
673 "name": "float_col",
674 "type": [
675 "float",
676 "null"
677 ]
678 },
679 {
680 "name": "double_col",
681 "type": [
682 "double",
683 "null"
684 ]
685 },
686 {
687 "name": "date_string_col",
688 "type": [
689 "bytes",
690 "null"
691 ]
692 },
693 {
694 "name": "string_col",
695 "type": [
696 "bytes",
697 "null"
698 ]
699 },
700 {
701 "name": "timestamp_col",
702 "type": [
703 {
704 "type": "long",
705 "logicalType": "timestamp-micros"
706 },
707 "null"
708 ]
709 }
710 ]
711}
712"#
713 .into(),
714 )]));
715 Arc::new(schema)
716 }
717
718 fn get_alltypes_with_nulls_schema() -> SchemaRef {
719 let schema = Schema::new(vec![
720 Field::new("string_col", DataType::Binary, true),
721 Field::new("int_col", DataType::Int32, true),
722 Field::new("bool_col", DataType::Boolean, true),
723 Field::new("bigint_col", DataType::Int64, true),
724 Field::new("float_col", DataType::Float32, true),
725 Field::new("double_col", DataType::Float64, true),
726 Field::new("bytes_col", DataType::Binary, true),
727 ])
728 .with_metadata(HashMap::from([(
729 SCHEMA_METADATA_KEY.into(),
730 r#"{
731 "type": "record",
732 "name": "topLevelRecord",
733 "fields": [
734 {
735 "name": "string_col",
736 "type": [
737 "null",
738 "string"
739 ],
740 "default": null
741 },
742 {
743 "name": "int_col",
744 "type": [
745 "null",
746 "int"
747 ],
748 "default": null
749 },
750 {
751 "name": "bool_col",
752 "type": [
753 "null",
754 "boolean"
755 ],
756 "default": null
757 },
758 {
759 "name": "bigint_col",
760 "type": [
761 "null",
762 "long"
763 ],
764 "default": null
765 },
766 {
767 "name": "float_col",
768 "type": [
769 "null",
770 "float"
771 ],
772 "default": null
773 },
774 {
775 "name": "double_col",
776 "type": [
777 "null",
778 "double"
779 ],
780 "default": null
781 },
782 {
783 "name": "bytes_col",
784 "type": [
785 "null",
786 "bytes"
787 ],
788 "default": null
789 }
790 ]
791}"#
792 .into(),
793 )]));
794
795 Arc::new(schema)
796 }
797
798 fn get_nested_records_schema() -> SchemaRef {
799 let schema = Schema::new(vec![
800 Field::new(
801 "f1",
802 DataType::Struct(
803 vec![
804 Field::new("f1_1", DataType::Utf8, false),
805 Field::new("f1_2", DataType::Int32, false),
806 Field::new(
807 "f1_3",
808 DataType::Struct(
809 vec![Field::new("f1_3_1", DataType::Float64, false)].into(),
810 ),
811 false,
812 )
813 .with_metadata(HashMap::from([
814 (AVRO_NAMESPACE_METADATA_KEY.to_owned(), "ns3".to_owned()),
815 (AVRO_NAME_METADATA_KEY.to_owned(), "record3".to_owned()),
816 ])),
817 ]
818 .into(),
819 ),
820 false,
821 )
822 .with_metadata(HashMap::from([
823 (AVRO_NAMESPACE_METADATA_KEY.to_owned(), "ns2".to_owned()),
824 (AVRO_NAME_METADATA_KEY.to_owned(), "record2".to_owned()),
825 ])),
826 Field::new(
827 "f2",
828 DataType::List(Arc::new(
829 Field::new(
830 "item",
831 DataType::Struct(
832 vec![
833 Field::new("f2_1", DataType::Boolean, false),
834 Field::new("f2_2", DataType::Float32, false),
835 ]
836 .into(),
837 ),
838 false,
839 )
840 .with_metadata(HashMap::from([
841 (AVRO_NAMESPACE_METADATA_KEY.to_owned(), "ns4".to_owned()),
842 (AVRO_NAME_METADATA_KEY.to_owned(), "record4".to_owned()),
843 ])),
844 )),
845 false,
846 ),
847 Field::new(
848 "f3",
849 DataType::Struct(vec![Field::new("f3_1", DataType::Utf8, false)].into()),
850 true,
851 )
852 .with_metadata(HashMap::from([
853 (AVRO_NAMESPACE_METADATA_KEY.to_owned(), "ns5".to_owned()),
854 (AVRO_NAME_METADATA_KEY.to_owned(), "record5".to_owned()),
855 ])),
856 Field::new(
857 "f4",
858 DataType::List(Arc::new(
859 Field::new(
860 "item",
861 DataType::Struct(vec![Field::new("f4_1", DataType::Int64, false)].into()),
862 true,
863 )
864 .with_metadata(HashMap::from([
865 (AVRO_NAMESPACE_METADATA_KEY.to_owned(), "ns6".to_owned()),
866 (AVRO_NAME_METADATA_KEY.to_owned(), "record6".to_owned()),
867 ])),
868 )),
869 false,
870 ),
871 ])
872 .with_metadata(HashMap::from([(
873 SCHEMA_METADATA_KEY.into(),
874 r#"{
875 "type": "record",
876 "namespace": "ns1",
877 "name": "record1",
878 "fields": [
879 {
880 "name": "f1",
881 "type": {
882 "type": "record",
883 "namespace": "ns2",
884 "name": "record2",
885 "fields": [
886 {
887 "name": "f1_1",
888 "type": "string"
889 },
890 {
891 "name": "f1_2",
892 "type": "int"
893 },
894 {
895 "name": "f1_3",
896 "type": {
897 "type": "record",
898 "namespace": "ns3",
899 "name": "record3",
900 "fields": [
901 {
902 "name": "f1_3_1",
903 "type": "double"
904 }
905 ]
906 }
907 }
908 ]
909 }
910 },
911 {
912 "name": "f2",
913 "type": {
914 "type": "array",
915 "items": {
916 "type": "record",
917 "namespace": "ns4",
918 "name": "record4",
919 "fields": [
920 {
921 "name": "f2_1",
922 "type": "boolean"
923 },
924 {
925 "name": "f2_2",
926 "type": "float"
927 }
928 ]
929 }
930 }
931 },
932 {
933 "name": "f3",
934 "type": [
935 "null",
936 {
937 "type": "record",
938 "namespace": "ns5",
939 "name": "record5",
940 "fields": [
941 {
942 "name": "f3_1",
943 "type": "string"
944 }
945 ]
946 }
947 ],
948 "default": null
949 },
950 {
951 "name": "f4",
952 "type": {
953 "type": "array",
954 "items": [
955 "null",
956 {
957 "type": "record",
958 "namespace": "ns6",
959 "name": "record6",
960 "fields": [
961 {
962 "name": "f4_1",
963 "type": "long"
964 }
965 ]
966 }
967 ]
968 }
969 }
970 ]
971}
972"#
973 .into(),
974 )]));
975
976 Arc::new(schema)
977 }
978
979 async fn read_async_file(
980 path: &str,
981 batch_size: usize,
982 range: Option<Range<u64>>,
983 schema: Option<SchemaRef>,
984 projection: Option<Vec<usize>>,
985 ) -> Result<Vec<RecordBatch>, ArrowError> {
986 let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
987 let location = Path::from_filesystem_path(path).unwrap();
988
989 let file_size = store.head(&location).await.unwrap().size;
990
991 let file_reader = ObjectStoreReader::new(store, location);
992 let mut builder = AsyncAvroFileReader::builder(file_reader, file_size, batch_size);
993
994 if let Some(s) = schema {
995 let reader_schema = AvroSchema::try_from(s.as_ref())?;
996 builder = builder.with_reader_schema(reader_schema);
997 }
998
999 if let Some(proj) = projection {
1000 builder = builder.with_projection(proj);
1001 }
1002
1003 if let Some(range) = range {
1004 builder = builder.with_range(range);
1005 }
1006
1007 let reader = builder.try_build().await?;
1008 reader.try_collect().await
1009 }
1010
1011 #[tokio::test]
1012 async fn test_full_file_read() {
1013 let file = arrow_test_data("avro/alltypes_plain.avro");
1014 let schema = get_alltypes_schema();
1015 let batches = read_async_file(&file, 1024, None, Some(schema), None)
1016 .await
1017 .unwrap();
1018 let batch = &batches[0];
1019
1020 assert_eq!(batch.num_rows(), 8);
1021 assert_eq!(batch.num_columns(), 11);
1022
1023 let id_array = batch
1024 .column(0)
1025 .as_any()
1026 .downcast_ref::<Int32Array>()
1027 .unwrap();
1028 assert_eq!(id_array.value(0), 4);
1029 assert_eq!(id_array.value(7), 1);
1030 }
1031
1032 #[tokio::test]
1033 async fn test_small_batch_size() {
1034 let file = arrow_test_data("avro/alltypes_plain.avro");
1035 let schema = get_alltypes_schema();
1036 let batches = read_async_file(&file, 2, None, Some(schema), None)
1037 .await
1038 .unwrap();
1039 assert_eq!(batches.len(), 4);
1040
1041 let batch = &batches[0];
1042
1043 assert_eq!(batch.num_rows(), 2);
1044 assert_eq!(batch.num_columns(), 11);
1045 }
1046
1047 #[tokio::test]
1048 async fn test_batch_size_one() {
1049 let file = arrow_test_data("avro/alltypes_plain.avro");
1050 let schema = get_alltypes_schema();
1051 let batches = read_async_file(&file, 1, None, Some(schema), None)
1052 .await
1053 .unwrap();
1054 let batch = &batches[0];
1055
1056 assert_eq!(batches.len(), 8);
1057 assert_eq!(batch.num_rows(), 1);
1058 }
1059
1060 #[tokio::test]
1061 async fn test_batch_size_larger_than_file() {
1062 let file = arrow_test_data("avro/alltypes_plain.avro");
1063 let schema = get_alltypes_schema();
1064 let batches = read_async_file(&file, 10000, None, Some(schema), None)
1065 .await
1066 .unwrap();
1067 let batch = &batches[0];
1068
1069 assert_eq!(batch.num_rows(), 8);
1070 }
1071
1072 #[tokio::test]
1073 async fn test_empty_range() {
1074 let file = arrow_test_data("avro/alltypes_plain.avro");
1075 let range = 100..100;
1076 let schema = get_alltypes_schema();
1077 let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1078 .await
1079 .unwrap();
1080 assert_eq!(batches.len(), 0);
1081 }
1082
1083 #[tokio::test]
1084 async fn test_range_starting_at_zero() {
1085 let file = arrow_test_data("avro/alltypes_plain.avro");
1087 let store = Arc::new(LocalFileSystem::new());
1088 let location = Path::from_filesystem_path(&file).unwrap();
1089 let meta = store.head(&location).await.unwrap();
1090
1091 let range = 0..meta.size;
1092 let schema = get_alltypes_schema();
1093 let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1094 .await
1095 .unwrap();
1096 let batch = &batches[0];
1097
1098 assert_eq!(batch.num_rows(), 8);
1099 }
1100
1101 #[tokio::test]
1102 async fn test_range_after_header() {
1103 let file = arrow_test_data("avro/alltypes_plain.avro");
1104 let store = Arc::new(LocalFileSystem::new());
1105 let location = Path::from_filesystem_path(&file).unwrap();
1106 let meta = store.head(&location).await.unwrap();
1107
1108 let range = 100..meta.size;
1109 let schema = get_alltypes_schema();
1110 let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1111 .await
1112 .unwrap();
1113 let batch = &batches[0];
1114
1115 assert!(batch.num_rows() > 0);
1116 }
1117
1118 #[tokio::test]
1119 async fn test_block_sync_marker_mismatch_errors() {
1120 use tempfile::tempdir;
1121 let file = arrow_test_data("avro/alltypes_plain.avro");
1122 let mut bytes = std::fs::read(&file).unwrap();
1123 let last = bytes.len() - 1;
1125 bytes[last] ^= 0xFF;
1126 let dir = tempdir().unwrap();
1127 let path = dir.path().join("corrupt_sync.avro");
1128 std::fs::write(&path, bytes).unwrap();
1129 let schema = get_alltypes_schema();
1130 let err = read_async_file(path.to_str().unwrap(), 1024, None, Some(schema), None)
1131 .await
1132 .expect_err("corrupted block sync marker should fail the read");
1133 assert!(err.to_string().contains("sync marker"), "{err}");
1134 }
1135
1136 #[tokio::test]
1137 async fn test_range_no_sync_marker() {
1138 let file = arrow_test_data("avro/alltypes_plain.avro");
1140 let range = 50..150;
1141 let schema = get_alltypes_schema();
1142 let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1143 .await
1144 .unwrap();
1145 assert_eq!(batches.len(), 0);
1146 }
1147
1148 #[tokio::test]
1149 async fn test_range_starting_mid_file() {
1150 let file = arrow_test_data("avro/alltypes_plain.avro");
1151
1152 let range = 700..768; let schema = get_alltypes_schema();
1154 let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1155 .await
1156 .unwrap();
1157 assert_eq!(batches.len(), 0);
1158 }
1159
1160 #[tokio::test]
1161 async fn test_range_ending_at_file_size() {
1162 let file = arrow_test_data("avro/alltypes_plain.avro");
1163 let store = Arc::new(LocalFileSystem::new());
1164 let location = Path::from_filesystem_path(&file).unwrap();
1165 let meta = store.head(&location).await.unwrap();
1166
1167 let range = 200..meta.size;
1168 let schema = get_alltypes_schema();
1169 let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1170 .await
1171 .unwrap();
1172 let batch = &batches[0];
1173
1174 assert_eq!(batch.num_rows(), 8);
1175 }
1176
1177 #[tokio::test]
1178 async fn test_incomplete_block_requires_fetch() {
1179 let file = arrow_test_data("avro/alltypes_plain.avro");
1181 let range = 0..1200;
1182 let schema = get_alltypes_schema();
1183 let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1184 .await
1185 .unwrap();
1186 let batch = &batches[0];
1187
1188 assert_eq!(batch.num_rows(), 8)
1189 }
1190
1191 #[tokio::test]
1192 async fn test_partial_vlq_header_requires_fetch() {
1193 let file = arrow_test_data("avro/alltypes_plain.avro");
1195 let range = 16..676; let schema = get_alltypes_schema();
1197 let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1198 .await
1199 .unwrap();
1200 let batch = &batches[0];
1201
1202 assert_eq!(batch.num_rows(), 8)
1203 }
1204
1205 #[cfg(feature = "snappy")]
1206 #[tokio::test]
1207 async fn test_snappy_compressed_with_range() {
1208 {
1209 let file = arrow_test_data("avro/alltypes_plain.snappy.avro");
1210 let store = Arc::new(LocalFileSystem::new());
1211 let location = Path::from_filesystem_path(&file).unwrap();
1212 let meta = store.head(&location).await.unwrap();
1213
1214 let range = 200..meta.size;
1215 let schema = get_alltypes_schema();
1216 let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1217 .await
1218 .unwrap();
1219 let batch = &batches[0];
1220
1221 assert!(batch.num_rows() > 0);
1222 }
1223 }
1224
1225 #[tokio::test]
1226 async fn test_nulls() {
1227 let file = arrow_test_data("avro/alltypes_nulls_plain.avro");
1228 let schema = get_alltypes_with_nulls_schema();
1229 let batches = read_async_file(&file, 1024, None, Some(schema), None)
1230 .await
1231 .unwrap();
1232 let batch = &batches[0];
1233
1234 assert_eq!(batch.num_rows(), 1);
1235 for col in batch.columns() {
1236 assert!(col.is_null(0));
1237 }
1238 }
1239
1240 #[tokio::test]
1241 async fn test_nested_records() {
1242 let file = arrow_test_data("avro/nested_records.avro");
1243 let schema = get_nested_records_schema();
1244 let batches = read_async_file(&file, 1024, None, Some(schema), None)
1245 .await
1246 .unwrap();
1247 let batch = &batches[0];
1248
1249 assert_eq!(batch.num_rows(), 2);
1250 assert!(batch.num_columns() > 0);
1251 }
1252
1253 #[tokio::test]
1254 async fn test_stream_produces_multiple_batches() {
1255 let file = arrow_test_data("avro/alltypes_plain.avro");
1256 let store = Arc::new(LocalFileSystem::new());
1257 let location = Path::from_filesystem_path(&file).unwrap();
1258
1259 let file_size = store.head(&location).await.unwrap().size;
1260
1261 let file_reader = ObjectStoreReader::new(store, location);
1262 let schema = get_alltypes_schema();
1263 let reader_schema = AvroSchema::try_from(schema.as_ref()).unwrap();
1264 let reader = AsyncAvroFileReader::builder(
1265 file_reader,
1266 file_size,
1267 2, )
1269 .with_reader_schema(reader_schema)
1270 .try_build()
1271 .await
1272 .unwrap();
1273
1274 let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1275
1276 assert!(batches.len() > 1);
1277 let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
1278 assert_eq!(total_rows, 8);
1279 }
1280
1281 #[tokio::test]
1282 async fn test_stream_early_termination() {
1283 let file = arrow_test_data("avro/alltypes_plain.avro");
1284 let store = Arc::new(LocalFileSystem::new());
1285 let location = Path::from_filesystem_path(&file).unwrap();
1286
1287 let file_size = store.head(&location).await.unwrap().size;
1288
1289 let file_reader = ObjectStoreReader::new(store, location);
1290 let schema = get_alltypes_schema();
1291 let reader_schema = AvroSchema::try_from(schema.as_ref()).unwrap();
1292 let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1)
1293 .with_reader_schema(reader_schema)
1294 .try_build()
1295 .await
1296 .unwrap();
1297
1298 let first_batch = reader.take(1).try_collect::<Vec<_>>().await.unwrap();
1299
1300 assert_eq!(first_batch.len(), 1);
1301 assert!(first_batch[0].num_rows() > 0);
1302 }
1303
1304 #[tokio::test]
1305 async fn test_various_batch_sizes() {
1306 let file = arrow_test_data("avro/alltypes_plain.avro");
1307
1308 for batch_size in [1, 2, 3, 5, 7, 11, 100] {
1309 let schema = get_alltypes_schema();
1310 let batches = read_async_file(&file, batch_size, None, Some(schema), None)
1311 .await
1312 .unwrap();
1313 let batch = &batches[0];
1314
1315 assert_eq!(
1317 batch.num_rows(),
1318 batch_size.min(8),
1319 "Failed with batch_size={batch_size}"
1320 );
1321 }
1322 }
1323
1324 #[tokio::test]
1325 async fn test_range_larger_than_file() {
1326 let file = arrow_test_data("avro/alltypes_plain.avro");
1327 let store = Arc::new(LocalFileSystem::new());
1328 let location = Path::from_filesystem_path(&file).unwrap();
1329 let meta = store.head(&location).await.unwrap();
1330
1331 let range = 100..(meta.size + 1000);
1333 let schema = get_alltypes_schema();
1334 let batches = read_async_file(&file, 1024, Some(range), Some(schema), None)
1335 .await
1336 .unwrap();
1337 let batch = &batches[0];
1338
1339 assert_eq!(batch.num_rows(), 8);
1341 }
1342
1343 #[tokio::test]
1344 async fn test_builder_with_header_info() {
1345 let file = arrow_test_data("avro/alltypes_plain.avro");
1346 let store = Arc::new(LocalFileSystem::new());
1347 let location = Path::from_filesystem_path(&file).unwrap();
1348
1349 let file_size = store.head(&location).await.unwrap().size;
1350
1351 let mut file_reader = ObjectStoreReader::new(store, location);
1352
1353 let header_info = read_header_info(&mut file_reader, file_size, None)
1354 .await
1355 .unwrap();
1356
1357 assert_eq!(header_info.header_len(), 675);
1358
1359 let writer_schema = header_info.writer_schema().unwrap();
1360 let expected_avro_json: serde_json::Value = serde_json::from_str(
1361 get_alltypes_schema()
1362 .metadata()
1363 .get(SCHEMA_METADATA_KEY)
1364 .unwrap(),
1365 )
1366 .unwrap();
1367 let actual_avro_json: serde_json::Value =
1368 serde_json::from_str(&writer_schema.json_string).unwrap();
1369 assert_eq!(actual_avro_json, expected_avro_json);
1370
1371 let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1372 .build_with_header(header_info)
1373 .unwrap();
1374
1375 let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1376
1377 let batch = &batches[0];
1378 assert_eq!(batch.num_rows(), 8)
1379 }
1380
1381 #[tokio::test]
1382 async fn test_roundtrip_write_then_async_read() {
1383 use crate::writer::AvroWriter;
1384 use arrow_array::{Float64Array, StringArray};
1385 use std::fs::File;
1386 use std::io::BufWriter;
1387 use tempfile::tempdir;
1388
1389 let schema = Arc::new(Schema::new(vec![
1391 Field::new("id", DataType::Int32, false),
1392 Field::new("name", DataType::Utf8, true),
1393 Field::new("score", DataType::Float64, true),
1394 Field::new("count", DataType::Int64, false),
1395 ]));
1396
1397 let dir = tempdir().unwrap();
1398 let file_path = dir.path().join("roundtrip_test.avro");
1399
1400 {
1402 let file = File::create(&file_path).unwrap();
1403 let writer = BufWriter::new(file);
1404 let mut avro_writer = AvroWriter::new(writer, schema.as_ref().clone()).unwrap();
1405
1406 let batch1 = RecordBatch::try_new(
1408 schema.clone(),
1409 vec![
1410 Arc::new(Int32Array::from(vec![1, 2, 3])),
1411 Arc::new(StringArray::from(vec![
1412 Some("alice"),
1413 None,
1414 Some("charlie"),
1415 ])),
1416 Arc::new(Float64Array::from(vec![Some(95.5), Some(87.3), None])),
1417 Arc::new(Int64Array::from(vec![10, 20, 30])),
1418 ],
1419 )
1420 .unwrap();
1421 avro_writer.write(&batch1).unwrap();
1422
1423 let batch2 = RecordBatch::try_new(
1425 schema.clone(),
1426 vec![
1427 Arc::new(Int32Array::from(vec![4, 5])),
1428 Arc::new(StringArray::from(vec![Some("diana"), Some("eve")])),
1429 Arc::new(Float64Array::from(vec![None, Some(88.0)])),
1430 Arc::new(Int64Array::from(vec![40, 50])),
1431 ],
1432 )
1433 .unwrap();
1434 avro_writer.write(&batch2).unwrap();
1435
1436 avro_writer.finish().unwrap();
1437 }
1438
1439 let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1441 let location = Path::from_filesystem_path(&file_path).unwrap();
1442 let file_size = store.head(&location).await.unwrap().size;
1443
1444 let file_reader = ObjectStoreReader::new(store, location);
1445 let reader = AsyncAvroFileReader::builder(file_reader, file_size, 2)
1446 .try_build()
1447 .await
1448 .unwrap();
1449
1450 let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1451
1452 assert!(
1454 batches.len() > 1,
1455 "Expected multiple batches with batch_size=2"
1456 );
1457
1458 let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
1460 assert_eq!(total_rows, 5);
1461
1462 let combined = arrow::compute::concat_batches(&batches[0].schema(), &batches).unwrap();
1464 assert_eq!(combined.num_rows(), 5);
1465 assert_eq!(combined.num_columns(), 4);
1466
1467 let id_array = combined
1469 .column(0)
1470 .as_any()
1471 .downcast_ref::<Int32Array>()
1472 .unwrap();
1473 assert_eq!(id_array.values(), &[1, 2, 3, 4, 5]);
1474
1475 let name_col = combined.column(1);
1478 let name_array = name_col.as_string::<i32>();
1479 assert_eq!(name_array.value(0), "alice");
1480 assert!(name_col.is_null(1)); assert_eq!(name_array.value(2), "charlie");
1482
1483 let score_array = combined
1485 .column(2)
1486 .as_any()
1487 .downcast_ref::<Float64Array>()
1488 .unwrap();
1489 assert!(!score_array.is_null(0));
1490 assert!((score_array.value(0) - 95.5).abs() < f64::EPSILON);
1491 assert!(score_array.is_null(2)); assert!(score_array.is_null(3)); assert!(!score_array.is_null(4));
1494 assert!((score_array.value(4) - 88.0).abs() < f64::EPSILON);
1495
1496 let count_array = combined
1498 .column(3)
1499 .as_any()
1500 .downcast_ref::<Int64Array>()
1501 .unwrap();
1502 assert_eq!(count_array.values(), &[10, 20, 30, 40, 50]);
1503 }
1504
1505 #[tokio::test]
1506 async fn test_alltypes_no_schema_no_projection() {
1507 let file = arrow_test_data("avro/alltypes_plain.avro");
1509 let batches = read_async_file(&file, 1024, None, None, None)
1510 .await
1511 .unwrap();
1512 let batch = &batches[0];
1513
1514 assert_eq!(batch.num_rows(), 8);
1515 assert_eq!(batch.num_columns(), 11);
1516 assert_eq!(batch.schema().field(0).name(), "id");
1517 }
1518
1519 #[tokio::test]
1520 async fn test_alltypes_no_schema_with_projection() {
1521 let file = arrow_test_data("avro/alltypes_plain.avro");
1523 let batches = read_async_file(&file, 1024, None, None, Some(vec![2, 0, 5]))
1525 .await
1526 .unwrap();
1527 let batch = &batches[0];
1528
1529 assert_eq!(batch.num_rows(), 8);
1530 assert_eq!(batch.num_columns(), 3);
1531 assert_eq!(batch.schema().field(0).name(), "tinyint_col");
1532 assert_eq!(batch.schema().field(1).name(), "id");
1533 assert_eq!(batch.schema().field(2).name(), "bigint_col");
1534
1535 let tinyint_col = batch.column(0).as_primitive::<Int32Type>();
1537 assert_eq!(tinyint_col.values(), &[0, 1, 0, 1, 0, 1, 0, 1]);
1538
1539 let id = batch.column(1).as_primitive::<Int32Type>();
1540 assert_eq!(id.values(), &[4, 5, 6, 7, 2, 3, 0, 1]);
1541
1542 let bigint_col = batch.column(2).as_primitive::<Int64Type>();
1543 assert_eq!(bigint_col.values(), &[0, 10, 0, 10, 0, 10, 0, 10]);
1544 }
1545
1546 #[tokio::test]
1547 async fn test_alltypes_with_schema_no_projection() {
1548 let file = arrow_test_data("avro/alltypes_plain.avro");
1550 let schema = get_alltypes_schema();
1551 let batches = read_async_file(&file, 1024, None, Some(schema), None)
1552 .await
1553 .unwrap();
1554 let batch = &batches[0];
1555
1556 assert_eq!(batch.num_rows(), 8);
1557 assert_eq!(batch.num_columns(), 11);
1558 }
1559
1560 #[tokio::test]
1561 async fn test_alltypes_with_schema_with_projection() {
1562 let file = arrow_test_data("avro/alltypes_plain.avro");
1564 let schema = get_alltypes_schema();
1565 let batches = read_async_file(&file, 1024, None, Some(schema), Some(vec![1, 0]))
1567 .await
1568 .unwrap();
1569 let batch = &batches[0];
1570
1571 assert_eq!(batch.num_rows(), 8);
1572 assert_eq!(batch.num_columns(), 2);
1573 assert_eq!(batch.schema().field(0).name(), "bool_col");
1574 assert_eq!(batch.schema().field(1).name(), "id");
1575
1576 let bool_col = batch.column(0).as_boolean();
1577 assert!(bool_col.value(0));
1578 assert!(!bool_col.value(1));
1579
1580 let id = batch.column(1).as_primitive::<Int32Type>();
1581 assert_eq!(id.values(), &[4, 5, 6, 7, 2, 3, 0, 1]);
1582 }
1583
1584 #[tokio::test]
1585 async fn test_alltypes_with_empty_schema_large_batch() {
1586 let file = arrow_test_data("avro/alltypes_plain.avro");
1588 let schema = Arc::new(Schema::new(Vec::<Field>::new()));
1589 let batches = read_async_file(&file, 1024, None, Some(schema), None)
1590 .await
1591 .unwrap();
1592 assert_eq!(batches.len(), 1);
1593 let batch = &batches[0];
1594
1595 assert_eq!(batch.num_rows(), 8);
1596 assert_eq!(batch.num_columns(), 0);
1597 }
1598
1599 #[tokio::test]
1600 async fn test_alltypes_with_empty_schema_small_batch() {
1601 let file = arrow_test_data("avro/alltypes_plain.avro");
1603 let schema = Arc::new(Schema::new(Vec::<Field>::new()));
1604 let batches = read_async_file(&file, 5, None, Some(schema), None)
1605 .await
1606 .unwrap();
1607
1608 assert_eq!(batches.len(), 2);
1609
1610 assert_eq!(batches[0].num_rows(), 5);
1611 assert_eq!(batches[0].num_columns(), 0);
1612 assert_eq!(batches[1].num_rows(), 3);
1613 assert_eq!(batches[1].num_columns(), 0);
1614 }
1615
1616 #[tokio::test]
1617 async fn test_nested_no_schema_no_projection() {
1618 let file = arrow_test_data("avro/nested_records.avro");
1620 let batches = read_async_file(&file, 1024, None, None, None)
1621 .await
1622 .unwrap();
1623 let batch = &batches[0];
1624
1625 assert_eq!(batch.num_rows(), 2);
1626 assert_eq!(batch.num_columns(), 4);
1627 assert_eq!(batch.schema().field(0).name(), "f1");
1628 assert_eq!(batch.schema().field(1).name(), "f2");
1629 assert_eq!(batch.schema().field(2).name(), "f3");
1630 assert_eq!(batch.schema().field(3).name(), "f4");
1631 }
1632
1633 #[tokio::test]
1634 async fn test_nested_no_schema_with_projection() {
1635 let file = arrow_test_data("avro/nested_records.avro");
1637 let batches = read_async_file(&file, 1024, None, None, Some(vec![2, 0]))
1639 .await
1640 .unwrap();
1641 let batch = &batches[0];
1642
1643 assert_eq!(batch.num_rows(), 2);
1644 assert_eq!(batch.num_columns(), 2);
1645 assert_eq!(batch.schema().field(0).name(), "f3");
1646 assert_eq!(batch.schema().field(1).name(), "f1");
1647 }
1648
1649 #[tokio::test]
1650 async fn test_nested_with_schema_no_projection() {
1651 let file = arrow_test_data("avro/nested_records.avro");
1653 let schema = get_nested_records_schema();
1654 let batches = read_async_file(&file, 1024, None, Some(schema), None)
1655 .await
1656 .unwrap();
1657 let batch = &batches[0];
1658
1659 assert_eq!(batch.num_rows(), 2);
1660 assert_eq!(batch.num_columns(), 4);
1661 }
1662
1663 #[tokio::test]
1664 async fn test_nested_with_schema_with_projection() {
1665 let file = arrow_test_data("avro/nested_records.avro");
1667 let schema = get_nested_records_schema();
1668 let batches = read_async_file(&file, 1024, None, Some(schema), Some(vec![3, 1, 0]))
1670 .await
1671 .unwrap();
1672 let batch = &batches[0];
1673
1674 assert_eq!(batch.num_rows(), 2);
1675 assert_eq!(batch.num_columns(), 3);
1676 assert_eq!(batch.schema().field(0).name(), "f4");
1677 assert_eq!(batch.schema().field(1).name(), "f2");
1678 assert_eq!(batch.schema().field(2).name(), "f1");
1679 }
1680
1681 #[tokio::test]
1682 async fn test_nested_with_empty_schema() {
1683 let file = arrow_test_data("avro/nested_records.avro");
1685 let schema = Arc::new(
1686 Schema::new(Vec::<Field>::new()).with_metadata(HashMap::from([(
1687 SCHEMA_METADATA_KEY.into(),
1688 r#"{
1689 "type": "record",
1690 "namespace": "ns1",
1691 "name": "record1",
1692 "fields": []
1693 }"#
1694 .to_owned(),
1695 )])),
1696 );
1697 let batches = read_async_file(&file, 1024, None, Some(schema), None)
1698 .await
1699 .unwrap();
1700 let batch = &batches[0];
1701
1702 assert_eq!(batch.num_rows(), 2);
1703 assert_eq!(batch.num_columns(), 0);
1704 }
1705
1706 #[tokio::test]
1707 async fn test_projection_error_out_of_bounds() {
1708 let file = arrow_test_data("avro/alltypes_plain.avro");
1709 let err = read_async_file(&file, 1024, None, None, Some(vec![100]))
1711 .await
1712 .unwrap_err();
1713 assert!(matches!(err, ArrowError::AvroError(_)));
1714 assert!(err.to_string().contains("out of bounds"));
1715 }
1716
1717 #[tokio::test]
1718 async fn test_projection_error_duplicate_index() {
1719 let file = arrow_test_data("avro/alltypes_plain.avro");
1720 let err = read_async_file(&file, 1024, None, None, Some(vec![0, 0]))
1722 .await
1723 .unwrap_err();
1724 assert!(matches!(err, ArrowError::AvroError(_)));
1725 assert!(err.to_string().contains("Duplicate projection index"));
1726 }
1727
1728 #[tokio::test]
1729 async fn test_arrow_schema_from_reader_no_reader_schema() {
1730 let file = arrow_test_data("avro/alltypes_plain.avro");
1731 let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1732 let location = Path::from_filesystem_path(&file).unwrap();
1733 let file_size = store.head(&location).await.unwrap().size;
1734
1735 let file_reader = ObjectStoreReader::new(store, location);
1736 let expected_schema = get_alltypes_schema()
1737 .as_ref()
1738 .clone()
1739 .with_metadata(Metadata::default());
1740
1741 let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1743 .try_build()
1744 .await
1745 .unwrap();
1746
1747 assert_eq!(reader.schema().as_ref(), &expected_schema);
1748
1749 let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1750 let batch = &batches[0];
1751
1752 assert_eq!(batch.schema().as_ref(), &expected_schema);
1753 }
1754
1755 #[tokio::test]
1756 async fn test_arrow_schema_from_reader_with_reader_schema() {
1757 let file = arrow_test_data("avro/alltypes_plain.avro");
1758 let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1759 let location = Path::from_filesystem_path(&file).unwrap();
1760 let file_size = store.head(&location).await.unwrap().size;
1761
1762 let file_reader = ObjectStoreReader::new(store, location);
1763 let schema = get_alltypes_schema()
1764 .project(&[0, 1, 7])
1765 .unwrap()
1766 .with_metadata(Metadata::default());
1767 let reader_schema = AvroSchema::try_from(&schema).unwrap();
1768 let expected_schema = schema.clone();
1769
1770 let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1772 .with_reader_schema(reader_schema)
1773 .try_build()
1774 .await
1775 .unwrap();
1776
1777 assert_eq!(reader.schema().as_ref(), &expected_schema);
1778
1779 let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1780 let batch = &batches[0];
1781
1782 assert_eq!(batch.schema().as_ref(), &expected_schema);
1783 }
1784
1785 #[tokio::test]
1786 async fn test_arrow_schema_from_reader_nested_records() {
1787 let file = arrow_test_data("avro/nested_records.avro");
1788 let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1789 let location = Path::from_filesystem_path(&file).unwrap();
1790 let file_size = store.head(&location).await.unwrap().size;
1791
1792 let file_reader = ObjectStoreReader::new(store, location);
1793
1794 let expected_schema = get_nested_records_schema()
1797 .as_ref()
1798 .clone()
1799 .with_metadata(Metadata::default());
1800
1801 let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1802 .try_build()
1803 .await
1804 .unwrap();
1805
1806 assert_eq!(reader.schema().as_ref(), &expected_schema);
1807
1808 let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1809 let batch = &batches[0];
1810
1811 assert_eq!(batch.schema().as_ref(), &expected_schema);
1812 }
1813
1814 #[tokio::test]
1815 async fn test_with_header_size_hint_small() {
1816 let file = arrow_test_data("avro/alltypes_plain.avro");
1818 let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1819 let location = Path::from_filesystem_path(&file).unwrap();
1820 let file_size = store.head(&location).await.unwrap().size;
1821
1822 let file_reader = ObjectStoreReader::new(store, location);
1823 let schema = get_alltypes_schema();
1824 let reader_schema = AvroSchema::try_from(schema.as_ref()).unwrap();
1825
1826 let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1828 .with_reader_schema(reader_schema)
1829 .with_header_size_hint(64)
1830 .try_build()
1831 .await
1832 .unwrap();
1833
1834 let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1835 let batch = &batches[0];
1836
1837 assert_eq!(batch.num_rows(), 8);
1838 assert_eq!(batch.num_columns(), 11);
1839 }
1840
1841 #[tokio::test]
1842 async fn test_with_header_size_hint_large() {
1843 let file = arrow_test_data("avro/alltypes_plain.avro");
1845 let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1846 let location = Path::from_filesystem_path(&file).unwrap();
1847 let file_size = store.head(&location).await.unwrap().size;
1848
1849 let file_reader = ObjectStoreReader::new(store, location);
1850 let schema = get_alltypes_schema();
1851 let reader_schema = AvroSchema::try_from(schema.as_ref()).unwrap();
1852
1853 let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1855 .with_reader_schema(reader_schema)
1856 .with_header_size_hint(64 * 1024)
1857 .try_build()
1858 .await
1859 .unwrap();
1860
1861 let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1862 let batch = &batches[0];
1863
1864 assert_eq!(batch.num_rows(), 8);
1865 assert_eq!(batch.num_columns(), 11);
1866 }
1867
1868 #[tokio::test]
1869 async fn test_with_tz_utc() {
1870 let file = arrow_test_data("avro/alltypes_plain.avro");
1871 let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1872 let location = Path::from_filesystem_path(&file).unwrap();
1873 let file_size = store.head(&location).await.unwrap().size;
1874
1875 let file_reader = ObjectStoreReader::new(store, location);
1876 let schema = get_alltypes_schema_with_tz("UTC");
1877 let reader_schema = AvroSchema::try_from(schema.as_ref()).unwrap();
1878
1879 let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1881 .with_reader_schema(reader_schema)
1882 .with_tz(Tz::Utc)
1883 .try_build()
1884 .await
1885 .unwrap();
1886
1887 let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1888 let batch = &batches[0];
1889
1890 assert_eq!(batch.num_columns(), 11);
1891
1892 let schema = batch.schema();
1893 let ts_field = schema.field_with_name("timestamp_col").unwrap();
1894 assert!(
1895 matches!(
1896 ts_field.data_type(),
1897 DataType::Timestamp(TimeUnit::Microsecond, Some(tz)) if tz.as_ref() == "UTC"
1898 ),
1899 "expected Timestamp(Microsecond, Some(\"UTC\")), got {:?}",
1900 ts_field.data_type()
1901 );
1902 }
1903
1904 #[tokio::test]
1905 async fn test_with_utf8_view_enabled() {
1906 let file = arrow_test_data("avro/nested_records.avro");
1908 let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1909 let location = Path::from_filesystem_path(&file).unwrap();
1910 let file_size = store.head(&location).await.unwrap().size;
1911
1912 let file_reader = ObjectStoreReader::new(store, location);
1913
1914 let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1915 .with_utf8_view(true)
1916 .try_build()
1917 .await
1918 .unwrap();
1919
1920 let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1921 let batch = &batches[0];
1922
1923 assert_eq!(batch.num_rows(), 2);
1924
1925 let f1_col = batch.column(0);
1928 let f1_struct = f1_col.as_struct();
1929 let f1_1_field = f1_struct.column_by_name("f1_1").unwrap();
1930
1931 assert_eq!(f1_1_field.data_type(), &DataType::Utf8View);
1933 }
1934
1935 #[tokio::test]
1936 async fn test_with_utf8_view_disabled() {
1937 let file = arrow_test_data("avro/nested_records.avro");
1939 let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1940 let location = Path::from_filesystem_path(&file).unwrap();
1941 let file_size = store.head(&location).await.unwrap().size;
1942
1943 let file_reader = ObjectStoreReader::new(store, location);
1944
1945 let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1946 .with_utf8_view(false)
1947 .try_build()
1948 .await
1949 .unwrap();
1950
1951 let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1952 let batch = &batches[0];
1953
1954 assert_eq!(batch.num_rows(), 2);
1955
1956 let f1_col = batch.column(0);
1959 let f1_struct = f1_col.as_struct();
1960 let f1_1_field = f1_struct.column_by_name("f1_1").unwrap();
1961
1962 assert_eq!(f1_1_field.data_type(), &DataType::Utf8);
1963 }
1964
1965 #[tokio::test]
1966 async fn test_with_strict_mode_disabled_allows_null_second() {
1967 let file = arrow_test_data("avro/alltypes_nulls_plain.avro");
1970 let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1971 let location = Path::from_filesystem_path(&file).unwrap();
1972 let file_size = store.head(&location).await.unwrap().size;
1973
1974 let file_reader = ObjectStoreReader::new(store, location);
1975
1976 let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
1978 .with_strict_mode(false)
1979 .try_build()
1980 .await
1981 .unwrap();
1982
1983 let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
1984 assert_eq!(batches.len(), 1);
1985 assert_eq!(batches[0].num_rows(), 1);
1986 }
1987
1988 #[tokio::test]
1989 async fn test_with_strict_mode_enabled_rejects_null_second() {
1990 let file = arrow_test_data("avro/alltypes_plain.avro");
1993 let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
1994 let location = Path::from_filesystem_path(&file).unwrap();
1995 let file_size = store.head(&location).await.unwrap().size;
1996
1997 let file_reader = ObjectStoreReader::new(store, location);
1998
1999 let result = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
2001 .with_strict_mode(true)
2002 .try_build()
2003 .await;
2004
2005 match result {
2006 Ok(_) => panic!("Expected error for strict_mode with ['T', 'null'] union"),
2007 Err(err) => {
2008 assert!(
2009 err.to_string().contains("disallowed in strict_mode"),
2010 "Expected strict_mode error, got: {err}"
2011 );
2012 }
2013 }
2014 }
2015
2016 #[tokio::test]
2017 async fn test_with_strict_mode_enabled_valid_schema() {
2018 let file = arrow_test_data("avro/nested_records.avro");
2021 let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
2022 let location = Path::from_filesystem_path(&file).unwrap();
2023 let file_size = store.head(&location).await.unwrap().size;
2024
2025 let file_reader = ObjectStoreReader::new(store, location);
2026
2027 let reader = AsyncAvroFileReader::builder(file_reader, file_size, 1024)
2029 .with_strict_mode(true)
2030 .try_build()
2031 .await
2032 .unwrap();
2033
2034 let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
2035 assert_eq!(batches.len(), 1);
2036 assert_eq!(batches[0].num_rows(), 2);
2037 }
2038
2039 #[tokio::test]
2040 async fn test_builder_options_combined() {
2041 let file = arrow_test_data("avro/nested_records.avro");
2043 let store: Arc<dyn ObjectStore> = Arc::new(LocalFileSystem::new());
2044 let location = Path::from_filesystem_path(&file).unwrap();
2045 let file_size = store.head(&location).await.unwrap().size;
2046
2047 let file_reader = ObjectStoreReader::new(store, location);
2048
2049 let reader = AsyncAvroFileReader::builder(file_reader, file_size, 2)
2050 .with_header_size_hint(128)
2051 .with_utf8_view(true)
2052 .with_strict_mode(true)
2053 .with_projection(vec![0, 2]) .try_build()
2055 .await
2056 .unwrap();
2057
2058 let batches: Vec<RecordBatch> = reader.try_collect().await.unwrap();
2059 let batch = &batches[0];
2060
2061 assert_eq!(batch.num_columns(), 2);
2063 assert_eq!(batch.schema().field(0).name(), "f1");
2064 assert_eq!(batch.schema().field(1).name(), "f3");
2065
2066 let f1_col = batch.column(0);
2068 let f1_struct = f1_col.as_struct();
2069 let f1_1_field = f1_struct.column_by_name("f1_1").unwrap();
2070 assert_eq!(f1_1_field.data_type(), &DataType::Utf8View);
2071 }
2072}