1use crate::compression::{CODEC_METADATA_KEY, CompressionCodec};
8use crate::errors::AvroError;
9use crate::reader::vlq::VLQDecoder;
10use crate::schema::{AvroSchema, SCHEMA_METADATA_KEY, Schema};
11use std::io::BufRead;
12use std::str;
13use std::sync::Arc;
14
15pub(crate) fn read_header<R: BufRead>(mut reader: R) -> Result<(Header, u64), AvroError> {
19 let mut decoder = HeaderDecoder::default();
20 let mut position = 0;
21 loop {
22 let buf = reader.fill_buf()?;
23 if buf.is_empty() {
24 break;
25 }
26 let read = buf.len();
27 let decoded = decoder.decode(buf)?;
28 reader.consume(decoded);
29 position += decoded as u64;
30 if decoded != read {
31 break;
32 }
33 }
34 decoder
35 .flush()
36 .map(|header| (header, position))
37 .ok_or_else(|| AvroError::EOF("Unexpected EOF while reading Avro header".to_string()))
38}
39
40#[derive(Debug)]
41enum HeaderDecoderState {
42 Magic,
44 BlockCount,
46 BlockLen,
48 KeyLen,
50 Key,
52 ValueLen,
54 Value,
56 Sync,
58 Finished,
60}
61
62#[derive(Debug, Clone)]
64pub struct Header {
65 meta_offsets: Vec<usize>,
66 meta_buf: Vec<u8>,
67 sync: [u8; 16],
68}
69
70impl Header {
71 pub fn metadata(&self) -> impl Iterator<Item = (&[u8], &[u8])> {
73 let mut last = 0;
74 self.meta_offsets.as_chunks::<2>().0.iter().map(move |w| {
75 let start = last;
76 last = w[1];
77 (&self.meta_buf[start..w[0]], &self.meta_buf[w[0]..w[1]])
78 })
79 }
80
81 pub fn get(&self, key: impl AsRef<[u8]>) -> Option<&[u8]> {
83 self.metadata()
84 .find_map(|(k, v)| (k == key.as_ref()).then_some(v))
85 }
86
87 pub fn sync(&self) -> [u8; 16] {
89 self.sync
90 }
91
92 pub fn compression(&self) -> Result<Option<CompressionCodec>, AvroError> {
94 let v = self.get(CODEC_METADATA_KEY);
95 match v {
96 None | Some(b"null") => Ok(None),
97 Some(b"deflate") => Ok(Some(CompressionCodec::Deflate)),
98 Some(b"snappy") => Ok(Some(CompressionCodec::Snappy)),
99 Some(b"zstandard") => Ok(Some(CompressionCodec::ZStandard)),
100 Some(b"bzip2") => Ok(Some(CompressionCodec::Bzip2)),
101 Some(b"xz") => Ok(Some(CompressionCodec::Xz)),
102 Some(v) => Err(AvroError::ParseError(format!(
103 "Unrecognized compression codec \'{}\'",
104 String::from_utf8_lossy(v)
105 ))),
106 }
107 }
108
109 pub(crate) fn schema(&self) -> Result<Option<Schema<'_>>, AvroError> {
111 self.get(SCHEMA_METADATA_KEY)
112 .map(|x| {
113 serde_json::from_slice(x).map_err(|e| {
114 AvroError::ParseError(format!("Failed to parse Avro schema JSON: {e}"))
115 })
116 })
117 .transpose()
118 }
119}
120
121#[derive(Clone)]
126pub struct HeaderInfo(Arc<HeaderInfoInner>);
127
128struct HeaderInfoInner {
129 header: Header,
130 header_len: u64,
131}
132
133pub fn read_header_info<R: BufRead>(reader: R) -> Result<HeaderInfo, AvroError> {
137 let (header, header_len) = read_header(reader)?;
138 Ok(HeaderInfo::new(header, header_len))
139}
140
141impl HeaderInfo {
142 pub(crate) fn new(header: Header, header_len: u64) -> Self {
143 Self(Arc::new(HeaderInfoInner { header, header_len }))
144 }
145
146 pub fn writer_schema(&self) -> Result<AvroSchema, AvroError> {
148 let raw = self.0.header.get(SCHEMA_METADATA_KEY).ok_or_else(|| {
149 AvroError::ParseError("No Avro schema present in file header".to_string())
150 })?;
151 let json_string = str::from_utf8(raw)
152 .map_err(|e| {
153 AvroError::ParseError(format!("Invalid UTF-8 in Avro schema header: {e}"))
154 })?
155 .to_string();
156 Ok(AvroSchema::new(json_string))
157 }
158
159 pub fn compression(&self) -> Result<Option<CompressionCodec>, AvroError> {
161 self.0.header.compression()
162 }
163
164 pub fn header_len(&self) -> u64 {
166 self.0.header_len
167 }
168
169 pub fn sync(&self) -> [u8; 16] {
171 self.0.header.sync()
172 }
173}
174
175#[derive(Debug)]
180pub struct HeaderDecoder {
181 state: HeaderDecoderState,
182 vlq_decoder: VLQDecoder,
183
184 meta_offsets: Vec<usize>,
186 meta_buf: Vec<u8>,
188
189 sync_marker: [u8; 16],
191
192 tuples_remaining: usize,
194 bytes_remaining: usize,
196}
197
198impl Default for HeaderDecoder {
199 fn default() -> Self {
200 Self {
201 state: HeaderDecoderState::Magic,
202 meta_offsets: vec![],
203 meta_buf: vec![],
204 sync_marker: [0; 16],
205 vlq_decoder: Default::default(),
206 tuples_remaining: 0,
207 bytes_remaining: MAGIC.len(),
208 }
209 }
210}
211
212const MAGIC: &[u8; 4] = b"Obj\x01";
213
214impl HeaderDecoder {
215 pub fn decode(&mut self, mut buf: &[u8]) -> Result<usize, AvroError> {
227 let max_read = buf.len();
228 while !buf.is_empty() {
229 match self.state {
230 HeaderDecoderState::Magic => {
231 let remaining = &MAGIC[MAGIC.len() - self.bytes_remaining..];
232 let to_decode = buf.len().min(remaining.len());
233 if !buf.starts_with(&remaining[..to_decode]) {
234 return Err(AvroError::ParseError("Incorrect avro magic".to_string()));
235 }
236 self.bytes_remaining -= to_decode;
237 buf = &buf[to_decode..];
238 if self.bytes_remaining == 0 {
239 self.state = HeaderDecoderState::BlockCount;
240 }
241 }
242 HeaderDecoderState::BlockCount => {
243 if let Some(block_count) = self.vlq_decoder.long(&mut buf)? {
244 match block_count.try_into() {
245 Ok(0) => {
246 self.state = HeaderDecoderState::Sync;
247 self.bytes_remaining = 16;
248 }
249 Ok(remaining) => {
250 self.tuples_remaining = remaining;
251 self.state = HeaderDecoderState::KeyLen;
252 }
253 Err(_) => {
254 self.tuples_remaining = block_count.unsigned_abs() as _;
255 self.state = HeaderDecoderState::BlockLen;
256 }
257 }
258 }
259 }
260 HeaderDecoderState::BlockLen => {
261 if self.vlq_decoder.long(&mut buf)?.is_some() {
262 self.state = HeaderDecoderState::KeyLen
263 }
264 }
265 HeaderDecoderState::Key => {
266 let to_read = self.bytes_remaining.min(buf.len());
267 self.meta_buf.extend_from_slice(&buf[..to_read]);
268 self.bytes_remaining -= to_read;
269 buf = &buf[to_read..];
270 if self.bytes_remaining == 0 {
271 self.meta_offsets.push(self.meta_buf.len());
272 self.state = HeaderDecoderState::ValueLen;
273 }
274 }
275 HeaderDecoderState::Value => {
276 let to_read = self.bytes_remaining.min(buf.len());
277 self.meta_buf.extend_from_slice(&buf[..to_read]);
278 self.bytes_remaining -= to_read;
279 buf = &buf[to_read..];
280 if self.bytes_remaining == 0 {
281 self.meta_offsets.push(self.meta_buf.len());
282
283 self.tuples_remaining -= 1;
284 match self.tuples_remaining {
285 0 => self.state = HeaderDecoderState::BlockCount,
286 _ => self.state = HeaderDecoderState::KeyLen,
287 }
288 }
289 }
290 HeaderDecoderState::KeyLen => {
291 if let Some(len) = self.vlq_decoder.long(&mut buf)? {
292 self.bytes_remaining = len as _;
293 self.state = HeaderDecoderState::Key;
294 }
295 }
296 HeaderDecoderState::ValueLen => {
297 if let Some(len) = self.vlq_decoder.long(&mut buf)? {
298 self.bytes_remaining = len as _;
299 self.state = HeaderDecoderState::Value;
300 }
301 }
302 HeaderDecoderState::Sync => {
303 let to_decode = buf.len().min(self.bytes_remaining);
304 let offset = 16 - self.bytes_remaining;
306 self.sync_marker[offset..offset + to_decode].copy_from_slice(&buf[..to_decode]);
307 self.bytes_remaining -= to_decode;
308 buf = &buf[to_decode..];
309 if self.bytes_remaining == 0 {
310 self.state = HeaderDecoderState::Finished;
311 }
312 }
313 HeaderDecoderState::Finished => return Ok(max_read - buf.len()),
314 }
315 }
316 Ok(max_read)
317 }
318
319 pub fn flush(&mut self) -> Option<Header> {
321 match self.state {
322 HeaderDecoderState::Finished => {
323 self.state = HeaderDecoderState::Magic;
324 Some(Header {
325 meta_offsets: std::mem::take(&mut self.meta_offsets),
326 meta_buf: std::mem::take(&mut self.meta_buf),
327 sync: self.sync_marker,
328 })
329 }
330 _ => None,
331 }
332 }
333}
334
335#[cfg(test)]
336mod test {
337 use super::*;
338 use crate::codec::AvroField;
339 use crate::reader::read_header;
340 use crate::schema::{
341 AVRO_NAME_METADATA_KEY, AVRO_ROOT_RECORD_DEFAULT_NAME, SCHEMA_METADATA_KEY,
342 };
343 use crate::test_util::arrow_test_data;
344 use arrow_schema::{DataType, Field, Fields, TimeUnit};
345 use std::collections::HashMap;
346 use std::fs::File;
347 use std::io::BufReader;
348
349 #[test]
350 fn test_header_decode() {
351 let mut decoder = HeaderDecoder::default();
352 for m in MAGIC {
353 decoder.decode(std::slice::from_ref(m)).unwrap();
354 }
355
356 let mut decoder = HeaderDecoder::default();
357 assert_eq!(decoder.decode(MAGIC).unwrap(), 4);
358
359 let mut decoder = HeaderDecoder::default();
360 decoder.decode(b"Ob").unwrap();
361 let err = decoder.decode(b"s").unwrap_err().to_string();
362 assert_eq!(err, "Parser error: Incorrect avro magic");
363 }
364
365 fn decode_file(file: &str) -> Header {
366 let file = File::open(file).unwrap();
367 read_header(BufReader::with_capacity(1000, file)).unwrap().0
368 }
369
370 #[test]
371 fn test_header() {
372 let header = decode_file(&arrow_test_data("avro/alltypes_plain.avro"));
373 let schema_json = header.get(SCHEMA_METADATA_KEY).unwrap();
374 let expected = br#"{"type":"record","name":"topLevelRecord","fields":[{"name":"id","type":["int","null"]},{"name":"bool_col","type":["boolean","null"]},{"name":"tinyint_col","type":["int","null"]},{"name":"smallint_col","type":["int","null"]},{"name":"int_col","type":["int","null"]},{"name":"bigint_col","type":["long","null"]},{"name":"float_col","type":["float","null"]},{"name":"double_col","type":["double","null"]},{"name":"date_string_col","type":["bytes","null"]},{"name":"string_col","type":["bytes","null"]},{"name":"timestamp_col","type":[{"type":"long","logicalType":"timestamp-micros"},"null"]}]}"#;
375 assert_eq!(schema_json, expected);
376 let schema: Schema<'_> = serde_json::from_slice(schema_json).unwrap();
377 let field = AvroField::try_from(&schema).unwrap();
378
379 assert_eq!(
380 field.field(),
381 Field::new(
382 "topLevelRecord",
383 DataType::Struct(Fields::from(vec![
384 Field::new("id", DataType::Int32, true),
385 Field::new("bool_col", DataType::Boolean, true),
386 Field::new("tinyint_col", DataType::Int32, true),
387 Field::new("smallint_col", DataType::Int32, true),
388 Field::new("int_col", DataType::Int32, true),
389 Field::new("bigint_col", DataType::Int64, true),
390 Field::new("float_col", DataType::Float32, true),
391 Field::new("double_col", DataType::Float64, true),
392 Field::new("date_string_col", DataType::Binary, true),
393 Field::new("string_col", DataType::Binary, true),
394 Field::new(
395 "timestamp_col",
396 DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
397 true
398 ),
399 ])),
400 false
401 )
402 .with_metadata(HashMap::from([(
403 AVRO_NAME_METADATA_KEY.to_string(),
404 AVRO_ROOT_RECORD_DEFAULT_NAME.to_string()
405 )]))
406 );
407
408 assert_eq!(
409 u128::from_le_bytes(header.sync()),
410 226966037233754408753420635932530907102
411 );
412
413 let header = decode_file(&arrow_test_data("avro/fixed_length_decimal.avro"));
414
415 let meta: Vec<_> = header
416 .metadata()
417 .map(|(k, _)| std::str::from_utf8(k).unwrap())
418 .collect();
419
420 assert_eq!(
421 meta,
422 &["avro.schema", "org.apache.spark.version", "avro.codec"]
423 );
424
425 let schema_json = header.get(SCHEMA_METADATA_KEY).unwrap();
426 let expected = br#"{"type":"record","name":"topLevelRecord","fields":[{"name":"value","type":[{"type":"fixed","name":"fixed","namespace":"topLevelRecord.value","size":11,"logicalType":"decimal","precision":25,"scale":2},"null"]}]}"#;
427 assert_eq!(schema_json, expected);
428 let _schema: Schema<'_> = serde_json::from_slice(schema_json).unwrap();
429 assert_eq!(
430 u128::from_le_bytes(header.sync()),
431 325166208089902833952788552656412487328
432 );
433 }
434}