Skip to main content

arrow_avro/reader/
header.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Decoder for [`Header`]
6
7use crate::compression::{CODEC_METADATA_KEY, CompressionCodec};
8use crate::errors::AvroError;
9use crate::reader::vlq::VLQDecoder;
10use crate::schema::{AvroSchema, SCHEMA_METADATA_KEY, Schema};
11use std::io::BufRead;
12use std::str;
13use std::sync::Arc;
14
15/// Read the Avro file header (magic, metadata, sync marker) from `reader`.
16///
17/// On success, returns the parsed [`Header`] and the number of bytes read from `reader`.
18pub(crate) fn read_header<R: BufRead>(mut reader: R) -> Result<(Header, u64), AvroError> {
19    let mut decoder = HeaderDecoder::default();
20    let mut position = 0;
21    loop {
22        let buf = reader.fill_buf()?;
23        if buf.is_empty() {
24            break;
25        }
26        let read = buf.len();
27        let decoded = decoder.decode(buf)?;
28        reader.consume(decoded);
29        position += decoded as u64;
30        if decoded != read {
31            break;
32        }
33    }
34    decoder
35        .flush()
36        .map(|header| (header, position))
37        .ok_or_else(|| AvroError::EOF("Unexpected EOF while reading Avro header".to_string()))
38}
39
40#[derive(Debug)]
41enum HeaderDecoderState {
42    /// Decoding the [`MAGIC`] prefix
43    Magic,
44    /// Decoding a block count
45    BlockCount,
46    /// Decoding a block byte length
47    BlockLen,
48    /// Decoding a key length
49    KeyLen,
50    /// Decoding a key string
51    Key,
52    /// Decoding a value length
53    ValueLen,
54    /// Decoding a value payload
55    Value,
56    /// Decoding sync marker
57    Sync,
58    /// Finished decoding
59    Finished,
60}
61
62/// A decoded header for an [Object Container File](https://avro.apache.org/docs/1.11.1/specification/#object-container-files)
63#[derive(Debug, Clone)]
64pub struct Header {
65    meta_offsets: Vec<usize>,
66    meta_buf: Vec<u8>,
67    sync: [u8; 16],
68}
69
70impl Header {
71    /// Returns an iterator over the meta keys in this header
72    pub fn metadata(&self) -> impl Iterator<Item = (&[u8], &[u8])> {
73        let mut last = 0;
74        self.meta_offsets.as_chunks::<2>().0.iter().map(move |w| {
75            let start = last;
76            last = w[1];
77            (&self.meta_buf[start..w[0]], &self.meta_buf[w[0]..w[1]])
78        })
79    }
80
81    /// Returns the value for a given metadata key if present
82    pub fn get(&self, key: impl AsRef<[u8]>) -> Option<&[u8]> {
83        self.metadata()
84            .find_map(|(k, v)| (k == key.as_ref()).then_some(v))
85    }
86
87    /// Returns the sync token for this file
88    pub fn sync(&self) -> [u8; 16] {
89        self.sync
90    }
91
92    /// Returns the [`CompressionCodec`] if any
93    pub fn compression(&self) -> Result<Option<CompressionCodec>, AvroError> {
94        let v = self.get(CODEC_METADATA_KEY);
95        match v {
96            None | Some(b"null") => Ok(None),
97            Some(b"deflate") => Ok(Some(CompressionCodec::Deflate)),
98            Some(b"snappy") => Ok(Some(CompressionCodec::Snappy)),
99            Some(b"zstandard") => Ok(Some(CompressionCodec::ZStandard)),
100            Some(b"bzip2") => Ok(Some(CompressionCodec::Bzip2)),
101            Some(b"xz") => Ok(Some(CompressionCodec::Xz)),
102            Some(v) => Err(AvroError::ParseError(format!(
103                "Unrecognized compression codec \'{}\'",
104                String::from_utf8_lossy(v)
105            ))),
106        }
107    }
108
109    /// Returns the `Schema` if any
110    pub(crate) fn schema(&self) -> Result<Option<Schema<'_>>, AvroError> {
111        self.get(SCHEMA_METADATA_KEY)
112            .map(|x| {
113                serde_json::from_slice(x).map_err(|e| {
114                    AvroError::ParseError(format!("Failed to parse Avro schema JSON: {e}"))
115                })
116            })
117            .transpose()
118    }
119}
120
121/// Header information for an Avro OCF file.
122///
123/// The header can be parsed once and shared to construct multiple readers
124/// for the same file, and so this struct is designed to be cheaply cloneable.
125#[derive(Clone)]
126pub struct HeaderInfo(Arc<HeaderInfoInner>);
127
128struct HeaderInfoInner {
129    header: Header,
130    header_len: u64,
131}
132
133/// Reads the Avro file header (magic, metadata, sync marker) from `reader`.
134///
135/// On success, returns the parsed [`HeaderInfo`] containing the header and its length in bytes.
136pub fn read_header_info<R: BufRead>(reader: R) -> Result<HeaderInfo, AvroError> {
137    let (header, header_len) = read_header(reader)?;
138    Ok(HeaderInfo::new(header, header_len))
139}
140
141impl HeaderInfo {
142    pub(crate) fn new(header: Header, header_len: u64) -> Self {
143        Self(Arc::new(HeaderInfoInner { header, header_len }))
144    }
145
146    /// Returns the writer schema for this file.
147    pub fn writer_schema(&self) -> Result<AvroSchema, AvroError> {
148        let raw = self.0.header.get(SCHEMA_METADATA_KEY).ok_or_else(|| {
149            AvroError::ParseError("No Avro schema present in file header".to_string())
150        })?;
151        let json_string = str::from_utf8(raw)
152            .map_err(|e| {
153                AvroError::ParseError(format!("Invalid UTF-8 in Avro schema header: {e}"))
154            })?
155            .to_string();
156        Ok(AvroSchema::new(json_string))
157    }
158
159    /// Returns the [`CompressionCodec`] if any
160    pub fn compression(&self) -> Result<Option<CompressionCodec>, AvroError> {
161        self.0.header.compression()
162    }
163
164    /// Returns the length of the header in bytes.
165    pub fn header_len(&self) -> u64 {
166        self.0.header_len
167    }
168
169    /// Returns the sync token for this file.
170    pub fn sync(&self) -> [u8; 16] {
171        self.0.header.sync()
172    }
173}
174
175/// A decoder for [`Header`]
176///
177/// The avro file format does not encode the length of the header, and so it
178/// is necessary to provide a push-based decoder that can be used with streams
179#[derive(Debug)]
180pub struct HeaderDecoder {
181    state: HeaderDecoderState,
182    vlq_decoder: VLQDecoder,
183
184    /// The end offsets of strings in `meta_buf`
185    meta_offsets: Vec<usize>,
186    /// The raw binary data of the metadata map
187    meta_buf: Vec<u8>,
188
189    /// The decoded sync marker
190    sync_marker: [u8; 16],
191
192    /// The number of remaining tuples in the current block
193    tuples_remaining: usize,
194    /// The number of bytes remaining in the current string/bytes payload
195    bytes_remaining: usize,
196}
197
198impl Default for HeaderDecoder {
199    fn default() -> Self {
200        Self {
201            state: HeaderDecoderState::Magic,
202            meta_offsets: vec![],
203            meta_buf: vec![],
204            sync_marker: [0; 16],
205            vlq_decoder: Default::default(),
206            tuples_remaining: 0,
207            bytes_remaining: MAGIC.len(),
208        }
209    }
210}
211
212const MAGIC: &[u8; 4] = b"Obj\x01";
213
214impl HeaderDecoder {
215    /// Parse [`Header`] from `buf`, returning the number of bytes read
216    ///
217    /// This method can be called multiple times with consecutive chunks of data, allowing
218    /// integration with chunked IO systems like [`BufRead::fill_buf`]
219    ///
220    /// All errors should be considered fatal, and decoding aborted
221    ///
222    /// Once the entire [`Header`] has been decoded this method will not read any further
223    /// input bytes, and the header can be obtained with [`Self::flush`]
224    ///
225    /// [`BufRead::fill_buf`]: std::io::BufRead::fill_buf
226    pub fn decode(&mut self, mut buf: &[u8]) -> Result<usize, AvroError> {
227        let max_read = buf.len();
228        while !buf.is_empty() {
229            match self.state {
230                HeaderDecoderState::Magic => {
231                    let remaining = &MAGIC[MAGIC.len() - self.bytes_remaining..];
232                    let to_decode = buf.len().min(remaining.len());
233                    if !buf.starts_with(&remaining[..to_decode]) {
234                        return Err(AvroError::ParseError("Incorrect avro magic".to_string()));
235                    }
236                    self.bytes_remaining -= to_decode;
237                    buf = &buf[to_decode..];
238                    if self.bytes_remaining == 0 {
239                        self.state = HeaderDecoderState::BlockCount;
240                    }
241                }
242                HeaderDecoderState::BlockCount => {
243                    if let Some(block_count) = self.vlq_decoder.long(&mut buf)? {
244                        match block_count.try_into() {
245                            Ok(0) => {
246                                self.state = HeaderDecoderState::Sync;
247                                self.bytes_remaining = 16;
248                            }
249                            Ok(remaining) => {
250                                self.tuples_remaining = remaining;
251                                self.state = HeaderDecoderState::KeyLen;
252                            }
253                            Err(_) => {
254                                self.tuples_remaining = block_count.unsigned_abs() as _;
255                                self.state = HeaderDecoderState::BlockLen;
256                            }
257                        }
258                    }
259                }
260                HeaderDecoderState::BlockLen => {
261                    if self.vlq_decoder.long(&mut buf)?.is_some() {
262                        self.state = HeaderDecoderState::KeyLen
263                    }
264                }
265                HeaderDecoderState::Key => {
266                    let to_read = self.bytes_remaining.min(buf.len());
267                    self.meta_buf.extend_from_slice(&buf[..to_read]);
268                    self.bytes_remaining -= to_read;
269                    buf = &buf[to_read..];
270                    if self.bytes_remaining == 0 {
271                        self.meta_offsets.push(self.meta_buf.len());
272                        self.state = HeaderDecoderState::ValueLen;
273                    }
274                }
275                HeaderDecoderState::Value => {
276                    let to_read = self.bytes_remaining.min(buf.len());
277                    self.meta_buf.extend_from_slice(&buf[..to_read]);
278                    self.bytes_remaining -= to_read;
279                    buf = &buf[to_read..];
280                    if self.bytes_remaining == 0 {
281                        self.meta_offsets.push(self.meta_buf.len());
282
283                        self.tuples_remaining -= 1;
284                        match self.tuples_remaining {
285                            0 => self.state = HeaderDecoderState::BlockCount,
286                            _ => self.state = HeaderDecoderState::KeyLen,
287                        }
288                    }
289                }
290                HeaderDecoderState::KeyLen => {
291                    if let Some(len) = self.vlq_decoder.long(&mut buf)? {
292                        self.bytes_remaining = len as _;
293                        self.state = HeaderDecoderState::Key;
294                    }
295                }
296                HeaderDecoderState::ValueLen => {
297                    if let Some(len) = self.vlq_decoder.long(&mut buf)? {
298                        self.bytes_remaining = len as _;
299                        self.state = HeaderDecoderState::Value;
300                    }
301                }
302                HeaderDecoderState::Sync => {
303                    let to_decode = buf.len().min(self.bytes_remaining);
304                    // Fill from the front: the marker may arrive split across decode() calls.
305                    let offset = 16 - self.bytes_remaining;
306                    self.sync_marker[offset..offset + to_decode].copy_from_slice(&buf[..to_decode]);
307                    self.bytes_remaining -= to_decode;
308                    buf = &buf[to_decode..];
309                    if self.bytes_remaining == 0 {
310                        self.state = HeaderDecoderState::Finished;
311                    }
312                }
313                HeaderDecoderState::Finished => return Ok(max_read - buf.len()),
314            }
315        }
316        Ok(max_read)
317    }
318
319    /// Flush this decoder returning the parsed [`Header`] if any
320    pub fn flush(&mut self) -> Option<Header> {
321        match self.state {
322            HeaderDecoderState::Finished => {
323                self.state = HeaderDecoderState::Magic;
324                Some(Header {
325                    meta_offsets: std::mem::take(&mut self.meta_offsets),
326                    meta_buf: std::mem::take(&mut self.meta_buf),
327                    sync: self.sync_marker,
328                })
329            }
330            _ => None,
331        }
332    }
333}
334
335#[cfg(test)]
336mod test {
337    use super::*;
338    use crate::codec::AvroField;
339    use crate::reader::read_header;
340    use crate::schema::{
341        AVRO_NAME_METADATA_KEY, AVRO_ROOT_RECORD_DEFAULT_NAME, SCHEMA_METADATA_KEY,
342    };
343    use crate::test_util::arrow_test_data;
344    use arrow_schema::{DataType, Field, Fields, TimeUnit};
345    use std::collections::HashMap;
346    use std::fs::File;
347    use std::io::BufReader;
348
349    #[test]
350    fn test_header_decode() {
351        let mut decoder = HeaderDecoder::default();
352        for m in MAGIC {
353            decoder.decode(std::slice::from_ref(m)).unwrap();
354        }
355
356        let mut decoder = HeaderDecoder::default();
357        assert_eq!(decoder.decode(MAGIC).unwrap(), 4);
358
359        let mut decoder = HeaderDecoder::default();
360        decoder.decode(b"Ob").unwrap();
361        let err = decoder.decode(b"s").unwrap_err().to_string();
362        assert_eq!(err, "Parser error: Incorrect avro magic");
363    }
364
365    fn decode_file(file: &str) -> Header {
366        let file = File::open(file).unwrap();
367        read_header(BufReader::with_capacity(1000, file)).unwrap().0
368    }
369
370    #[test]
371    fn test_header() {
372        let header = decode_file(&arrow_test_data("avro/alltypes_plain.avro"));
373        let schema_json = header.get(SCHEMA_METADATA_KEY).unwrap();
374        let expected = br#"{"type":"record","name":"topLevelRecord","fields":[{"name":"id","type":["int","null"]},{"name":"bool_col","type":["boolean","null"]},{"name":"tinyint_col","type":["int","null"]},{"name":"smallint_col","type":["int","null"]},{"name":"int_col","type":["int","null"]},{"name":"bigint_col","type":["long","null"]},{"name":"float_col","type":["float","null"]},{"name":"double_col","type":["double","null"]},{"name":"date_string_col","type":["bytes","null"]},{"name":"string_col","type":["bytes","null"]},{"name":"timestamp_col","type":[{"type":"long","logicalType":"timestamp-micros"},"null"]}]}"#;
375        assert_eq!(schema_json, expected);
376        let schema: Schema<'_> = serde_json::from_slice(schema_json).unwrap();
377        let field = AvroField::try_from(&schema).unwrap();
378
379        assert_eq!(
380            field.field(),
381            Field::new(
382                "topLevelRecord",
383                DataType::Struct(Fields::from(vec![
384                    Field::new("id", DataType::Int32, true),
385                    Field::new("bool_col", DataType::Boolean, true),
386                    Field::new("tinyint_col", DataType::Int32, true),
387                    Field::new("smallint_col", DataType::Int32, true),
388                    Field::new("int_col", DataType::Int32, true),
389                    Field::new("bigint_col", DataType::Int64, true),
390                    Field::new("float_col", DataType::Float32, true),
391                    Field::new("double_col", DataType::Float64, true),
392                    Field::new("date_string_col", DataType::Binary, true),
393                    Field::new("string_col", DataType::Binary, true),
394                    Field::new(
395                        "timestamp_col",
396                        DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
397                        true
398                    ),
399                ])),
400                false
401            )
402            .with_metadata(HashMap::from([(
403                AVRO_NAME_METADATA_KEY.to_string(),
404                AVRO_ROOT_RECORD_DEFAULT_NAME.to_string()
405            )]))
406        );
407
408        assert_eq!(
409            u128::from_le_bytes(header.sync()),
410            226966037233754408753420635932530907102
411        );
412
413        let header = decode_file(&arrow_test_data("avro/fixed_length_decimal.avro"));
414
415        let meta: Vec<_> = header
416            .metadata()
417            .map(|(k, _)| std::str::from_utf8(k).unwrap())
418            .collect();
419
420        assert_eq!(
421            meta,
422            &["avro.schema", "org.apache.spark.version", "avro.codec"]
423        );
424
425        let schema_json = header.get(SCHEMA_METADATA_KEY).unwrap();
426        let expected = br#"{"type":"record","name":"topLevelRecord","fields":[{"name":"value","type":[{"type":"fixed","name":"fixed","namespace":"topLevelRecord.value","size":11,"logicalType":"decimal","precision":25,"scale":2},"null"]}]}"#;
427        assert_eq!(schema_json, expected);
428        let _schema: Schema<'_> = serde_json::from_slice(schema_json).unwrap();
429        assert_eq!(
430            u128::from_le_bytes(header.sync()),
431            325166208089902833952788552656412487328
432        );
433    }
434}