arrow_avro/reader/async_reader/
builder.rs1use crate::codec::{AvroFieldBuilder, Tz};
6use crate::errors::AvroError;
7use crate::reader::async_reader::ReaderState;
8use crate::reader::header::{Header, HeaderDecoder, HeaderInfo};
9use crate::reader::record::RecordDecoder;
10use crate::reader::{AsyncAvroFileReader, AsyncFileReader, Decoder};
11use crate::schema::{AvroSchema, FingerprintAlgorithm};
12use indexmap::IndexMap;
13use std::ops::Range;
14
15const DEFAULT_HEADER_SIZE_HINT: u64 = 16 * 1024; pub struct ReaderBuilder<R> {
19 reader: R,
20 file_size: u64,
21 batch_size: usize,
22 range: Option<Range<u64>>,
23 reader_schema: Option<AvroSchema>,
24 projection: Option<Vec<usize>>,
25 header_size_hint: Option<u64>,
26 utf8_view: bool,
27 strict_mode: bool,
28 tz: Tz,
29}
30
31impl<R> ReaderBuilder<R> {
32 pub(super) fn new(reader: R, file_size: u64, batch_size: usize) -> Self {
33 Self {
34 reader,
35 file_size,
36 batch_size,
37 range: None,
38 reader_schema: None,
39 projection: None,
40 header_size_hint: None,
41 utf8_view: false,
42 strict_mode: false,
43 tz: Default::default(),
44 }
45 }
46
47 pub fn with_range(self, range: Range<u64>) -> Self {
53 Self {
54 range: Some(range),
55 ..self
56 }
57 }
58
59 pub fn with_reader_schema(self, reader_schema: AvroSchema) -> Self {
63 Self {
64 reader_schema: Some(reader_schema),
65 ..self
66 }
67 }
68
69 pub fn with_projection(self, projection: Vec<usize>) -> Self {
72 Self {
73 projection: Some(projection),
74 ..self
75 }
76 }
77
78 pub fn with_header_size_hint(self, hint: u64) -> Self {
81 Self {
82 header_size_hint: Some(hint),
83 ..self
84 }
85 }
86
87 pub fn with_utf8_view(self, utf8_view: bool) -> Self {
89 Self { utf8_view, ..self }
90 }
91
92 pub fn with_strict_mode(self, strict_mode: bool) -> Self {
94 Self {
95 strict_mode,
96 ..self
97 }
98 }
99
100 pub fn with_tz(mut self, tz: Tz) -> Self {
104 self.tz = tz;
105 self
106 }
107}
108
109pub async fn read_header_info<R>(
113 reader: &mut R,
114 file_size: u64,
115 header_size_hint: Option<u64>,
116) -> Result<HeaderInfo, AvroError>
117where
118 R: AsyncFileReader,
119{
120 read_header(reader, file_size, header_size_hint)
121 .await
122 .map(|(header, header_len)| HeaderInfo::new(header, header_len))
123}
124
125async fn read_header<R>(
126 reader: &mut R,
127 file_size: u64,
128 header_size_hint: Option<u64>,
129) -> Result<(Header, u64), AvroError>
130where
131 R: AsyncFileReader,
132{
133 let mut decoder = HeaderDecoder::default();
134 let mut position = 0;
135 loop {
136 let range_to_fetch = position
137 ..(position + header_size_hint.unwrap_or(DEFAULT_HEADER_SIZE_HINT)).min(file_size);
138
139 if range_to_fetch.is_empty() {
141 break;
142 }
143
144 let current_data = reader.get_bytes(range_to_fetch.clone()).await?;
145 if current_data.is_empty() {
146 return Err(AvroError::EOF(
147 "Unexpected EOF while fetching header data".into(),
148 ));
149 }
150
151 let read = current_data.len();
152 let decoded = decoder.decode(¤t_data)?;
153 if decoded != read {
154 position += decoded as u64;
155 break;
156 }
157 position += read as u64;
158 }
159
160 decoder
161 .flush()
162 .map(|header| (header, position))
163 .ok_or_else(|| AvroError::EOF("Unexpected EOF while reading Avro header".into()))
164}
165
166impl<R: AsyncFileReader> ReaderBuilder<R> {
167 pub async fn try_build(mut self) -> Result<AsyncAvroFileReader<R>, AvroError> {
170 if self.file_size == 0 {
171 return Err(AvroError::InvalidArgument("File size cannot be 0".into()));
172 }
173
174 let header_info =
177 read_header_info(&mut self.reader, self.file_size, self.header_size_hint).await?;
178
179 self.build_with_header(header_info)
180 }
181
182 pub fn build_with_header(
190 self,
191 header_info: HeaderInfo,
192 ) -> Result<AsyncAvroFileReader<R>, AvroError> {
193 let writer_schema = header_info.writer_schema()?;
194
195 let projected_reader_schema = self
199 .projection
200 .as_deref()
201 .map(|projection| {
202 let base_schema = if let Some(reader_schema) = &self.reader_schema {
203 reader_schema
204 } else {
205 &writer_schema
206 };
207 base_schema.project(projection)
208 })
209 .transpose()?;
210
211 let effective_reader_schema = projected_reader_schema
214 .as_ref()
215 .or(self.reader_schema.as_ref())
216 .map(|s| s.schema())
217 .transpose()?;
218
219 let root = {
220 let writer_schema = writer_schema.schema()?;
221 let mut builder = AvroFieldBuilder::new(&writer_schema);
222 if let Some(reader_schema) = &effective_reader_schema {
223 builder = builder.with_reader_schema(reader_schema);
224 }
225 builder
226 .with_utf8view(self.utf8_view)
227 .with_strict_mode(self.strict_mode)
228 .with_tz(self.tz)
229 .build()
230 }?;
231
232 let record_decoder = RecordDecoder::try_new_with_options(root.data_type())?;
233 let decoder = Decoder::from_parts(
234 self.batch_size,
235 record_decoder,
236 None,
237 IndexMap::new(),
238 FingerprintAlgorithm::Rabin,
239 );
240 let header_len = header_info.header_len();
241 let range = match self.range {
242 Some(r) => {
243 let start = r.start.max(header_len.checked_sub(16).ok_or(AvroError::ParseError("Avro header length overflow, header was not long enough to contain avro bytes".to_string()))?);
248 let end = r.end.max(start).min(self.file_size); start..end
250 }
251 None => 0..self.file_size,
252 };
253
254 let reader_state = if range.start == range.end || header_len == range.end {
257 ReaderState::Finished
258 } else {
259 ReaderState::Idle {
260 reader: self.reader,
261 }
262 };
263
264 let codec = header_info.compression()?;
265 let sync_marker = header_info.sync();
266
267 Ok(AsyncAvroFileReader::new(
268 range,
269 self.file_size,
270 decoder,
271 codec,
272 sync_marker,
273 reader_state,
274 ))
275 }
276}