Skip to main content

arrow_avro/reader/async_reader/
builder.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use crate::codec::{AvroFieldBuilder, Tz};
6use crate::errors::AvroError;
7use crate::reader::async_reader::ReaderState;
8use crate::reader::header::{Header, HeaderDecoder, HeaderInfo};
9use crate::reader::record::RecordDecoder;
10use crate::reader::{AsyncAvroFileReader, AsyncFileReader, Decoder};
11use crate::schema::{AvroSchema, FingerprintAlgorithm};
12use indexmap::IndexMap;
13use std::ops::Range;
14
15const DEFAULT_HEADER_SIZE_HINT: u64 = 16 * 1024; // 16 KB
16
17/// Builder for an asynchronous Avro file reader.
18pub struct ReaderBuilder<R> {
19    reader: R,
20    file_size: u64,
21    batch_size: usize,
22    range: Option<Range<u64>>,
23    reader_schema: Option<AvroSchema>,
24    projection: Option<Vec<usize>>,
25    header_size_hint: Option<u64>,
26    utf8_view: bool,
27    strict_mode: bool,
28    tz: Tz,
29}
30
31impl<R> ReaderBuilder<R> {
32    pub(super) fn new(reader: R, file_size: u64, batch_size: usize) -> Self {
33        Self {
34            reader,
35            file_size,
36            batch_size,
37            range: None,
38            reader_schema: None,
39            projection: None,
40            header_size_hint: None,
41            utf8_view: false,
42            strict_mode: false,
43            tz: Default::default(),
44        }
45    }
46
47    /// Specify a byte range to read from the Avro file.
48    /// If this is provided, the reader will read all the blocks within the specified range,
49    /// if the range ends mid-block, it will attempt to fetch the remaining bytes to complete the block,
50    /// but no further blocks will be read.
51    /// If this is omitted, the full file will be read.
52    pub fn with_range(self, range: Range<u64>) -> Self {
53        Self {
54            range: Some(range),
55            ..self
56        }
57    }
58
59    /// Specify a reader schema to use when reading the Avro file.
60    /// This can be useful to project specific columns or handle schema evolution.
61    /// If this is not provided, the schema will be derived from the Arrow schema provided.
62    pub fn with_reader_schema(self, reader_schema: AvroSchema) -> Self {
63        Self {
64            reader_schema: Some(reader_schema),
65            ..self
66        }
67    }
68
69    /// Specify a projection of column indices to read from the Avro file.
70    /// This can help optimize reading by only fetching the necessary columns.
71    pub fn with_projection(self, projection: Vec<usize>) -> Self {
72        Self {
73            projection: Some(projection),
74            ..self
75        }
76    }
77
78    /// Provide a hint for the expected size of the Avro header in bytes.
79    /// This can help optimize the initial read operation when fetching the header.
80    pub fn with_header_size_hint(self, hint: u64) -> Self {
81        Self {
82            header_size_hint: Some(hint),
83            ..self
84        }
85    }
86
87    /// Enable usage of Utf8View types when reading string data.
88    pub fn with_utf8_view(self, utf8_view: bool) -> Self {
89        Self { utf8_view, ..self }
90    }
91
92    /// Enable strict mode for schema validation and data reading.
93    pub fn with_strict_mode(self, strict_mode: bool) -> Self {
94        Self {
95            strict_mode,
96            ..self
97        }
98    }
99
100    /// Sets the timezone representation for Avro timestamp fields.
101    ///
102    /// The default is `Tz::OffsetZero`, meaning the "+00:00" time zone ID.
103    pub fn with_tz(mut self, tz: Tz) -> Self {
104        self.tz = tz;
105        self
106    }
107}
108
109/// Reads the Avro file header (magic, metadata, sync marker) asynchronously from `reader`.
110///
111/// On success, returns the parsed [`HeaderInfo`] containing the header and its length in bytes.
112pub async fn read_header_info<R>(
113    reader: &mut R,
114    file_size: u64,
115    header_size_hint: Option<u64>,
116) -> Result<HeaderInfo, AvroError>
117where
118    R: AsyncFileReader,
119{
120    read_header(reader, file_size, header_size_hint)
121        .await
122        .map(|(header, header_len)| HeaderInfo::new(header, header_len))
123}
124
125async fn read_header<R>(
126    reader: &mut R,
127    file_size: u64,
128    header_size_hint: Option<u64>,
129) -> Result<(Header, u64), AvroError>
130where
131    R: AsyncFileReader,
132{
133    let mut decoder = HeaderDecoder::default();
134    let mut position = 0;
135    loop {
136        let range_to_fetch = position
137            ..(position + header_size_hint.unwrap_or(DEFAULT_HEADER_SIZE_HINT)).min(file_size);
138
139        // Maybe EOF after the header, no actual data
140        if range_to_fetch.is_empty() {
141            break;
142        }
143
144        let current_data = reader.get_bytes(range_to_fetch.clone()).await?;
145        if current_data.is_empty() {
146            return Err(AvroError::EOF(
147                "Unexpected EOF while fetching header data".into(),
148            ));
149        }
150
151        let read = current_data.len();
152        let decoded = decoder.decode(&current_data)?;
153        if decoded != read {
154            position += decoded as u64;
155            break;
156        }
157        position += read as u64;
158    }
159
160    decoder
161        .flush()
162        .map(|header| (header, position))
163        .ok_or_else(|| AvroError::EOF("Unexpected EOF while reading Avro header".into()))
164}
165
166impl<R: AsyncFileReader> ReaderBuilder<R> {
167    /// Build the asynchronous Avro reader with the provided parameters.
168    /// This reads the header first to initialize the reader state.
169    pub async fn try_build(mut self) -> Result<AsyncAvroFileReader<R>, AvroError> {
170        if self.file_size == 0 {
171            return Err(AvroError::InvalidArgument("File size cannot be 0".into()));
172        }
173
174        // Start by reading the header from the beginning of the avro file
175        // take the writer schema from the header
176        let header_info =
177            read_header_info(&mut self.reader, self.file_size, self.header_size_hint).await?;
178
179        self.build_with_header(header_info)
180    }
181
182    /// Build the asynchronous Avro reader with the provided header.
183    ///
184    /// This allows initializing the reader with pre-parsed header information.
185    /// Note that this method is not async because it does not need to perform any I/O operations.
186    ///
187    /// Note: Any `header_size_hint` set via [`Self::with_header_size_hint`] is not used
188    /// when building with a pre-parsed header, since no header fetching occurs.
189    pub fn build_with_header(
190        self,
191        header_info: HeaderInfo,
192    ) -> Result<AsyncAvroFileReader<R>, AvroError> {
193        let writer_schema = header_info.writer_schema()?;
194
195        // If projection exists, project the reader schema,
196        // if no reader schema is provided, parse it from the header(get the raw writer schema), and project that
197        // this projected schema will be the schema used for reading.
198        let projected_reader_schema = self
199            .projection
200            .as_deref()
201            .map(|projection| {
202                let base_schema = if let Some(reader_schema) = &self.reader_schema {
203                    reader_schema
204                } else {
205                    &writer_schema
206                };
207                base_schema.project(projection)
208            })
209            .transpose()?;
210
211        // Use either the projected reader schema or the original reader schema(if no projection)
212        // (both optional, at worst no reader schema is provided, in which case we read with the writer schema)
213        let effective_reader_schema = projected_reader_schema
214            .as_ref()
215            .or(self.reader_schema.as_ref())
216            .map(|s| s.schema())
217            .transpose()?;
218
219        let root = {
220            let writer_schema = writer_schema.schema()?;
221            let mut builder = AvroFieldBuilder::new(&writer_schema);
222            if let Some(reader_schema) = &effective_reader_schema {
223                builder = builder.with_reader_schema(reader_schema);
224            }
225            builder
226                .with_utf8view(self.utf8_view)
227                .with_strict_mode(self.strict_mode)
228                .with_tz(self.tz)
229                .build()
230        }?;
231
232        let record_decoder = RecordDecoder::try_new_with_options(root.data_type())?;
233        let decoder = Decoder::from_parts(
234            self.batch_size,
235            record_decoder,
236            None,
237            IndexMap::new(),
238            FingerprintAlgorithm::Rabin,
239        );
240        let header_len = header_info.header_len();
241        let range = match self.range {
242            Some(r) => {
243                // If this PartitionedFile's range starts at 0, we need to skip the header bytes.
244                // But then we need to seek back 16 bytes to include the sync marker for the first block,
245                // as the logic in this reader searches the data for the first sync marker(after which a block starts),
246                // then reads blocks from the count, size etc.
247                let start = r.start.max(header_len.checked_sub(16).ok_or(AvroError::ParseError("Avro header length overflow, header was not long enough to contain avro bytes".to_string()))?);
248                let end = r.end.max(start).min(self.file_size); // Ensure end is not less than start, worst case range is empty
249                start..end
250            }
251            None => 0..self.file_size,
252        };
253
254        // Determine if there is actually data to fetch, note that we subtract the header len from range.start,
255        // so we need to check if range.end == header_len to see if there's no data after the header
256        let reader_state = if range.start == range.end || header_len == range.end {
257            ReaderState::Finished
258        } else {
259            ReaderState::Idle {
260                reader: self.reader,
261            }
262        };
263
264        let codec = header_info.compression()?;
265        let sync_marker = header_info.sync();
266
267        Ok(AsyncAvroFileReader::new(
268            range,
269            self.file_size,
270            decoder,
271            codec,
272            sync_marker,
273            reader_state,
274        ))
275    }
276}