Skip to main content

arrow_avro/reader/
block.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Decoder for [`Block`]
6
7use crate::errors::AvroError;
8use crate::reader::vlq::VLQDecoder;
9
10/// A file data block
11///
12/// <https://avro.apache.org/docs/1.11.1/specification/#object-container-files>
13#[derive(Debug, Default)]
14pub struct Block {
15    /// The number of objects in this block
16    pub count: usize,
17    /// The serialized objects within this block
18    pub data: Vec<u8>,
19    /// The sync marker
20    pub sync: [u8; 16],
21}
22
23/// A decoder for [`Block`]
24#[derive(Debug)]
25pub struct BlockDecoder {
26    state: BlockDecoderState,
27    in_progress: Block,
28    vlq_decoder: VLQDecoder,
29    bytes_remaining: usize,
30}
31
32#[derive(Debug)]
33pub(crate) enum BlockDecoderState {
34    Count,
35    Size,
36    Data,
37    Sync,
38    Finished,
39}
40
41impl Default for BlockDecoder {
42    fn default() -> Self {
43        Self {
44            state: BlockDecoderState::Count,
45            in_progress: Default::default(),
46            vlq_decoder: Default::default(),
47            bytes_remaining: 0,
48        }
49    }
50}
51
52impl BlockDecoder {
53    /// Parse [`Block`] from `buf`, returning the number of bytes read
54    ///
55    /// This method can be called multiple times with consecutive chunks of data, allowing
56    /// integration with chunked IO systems like [`BufRead::fill_buf`]
57    ///
58    /// All errors should be considered fatal, and decoding aborted
59    ///
60    /// Once an entire [`Block`] has been decoded this method will not read any further
61    /// input bytes, until [`Self::flush`] is called. Afterwards [`Self::decode`]
62    /// can then be used again to read the next block, if any
63    ///
64    /// [`BufRead::fill_buf`]: std::io::BufRead::fill_buf
65    pub fn decode(&mut self, mut buf: &[u8]) -> Result<usize, AvroError> {
66        let max_read = buf.len();
67        while !buf.is_empty() {
68            match self.state {
69                BlockDecoderState::Count => {
70                    if let Some(c) = self.vlq_decoder.long(&mut buf)? {
71                        self.in_progress.count = c.try_into().map_err(|_| {
72                            AvroError::ParseError(format!(
73                                "Block count cannot be negative, got {c}"
74                            ))
75                        })?;
76
77                        self.state = BlockDecoderState::Size;
78                    }
79                }
80                BlockDecoderState::Size => {
81                    if let Some(c) = self.vlq_decoder.long(&mut buf)? {
82                        self.bytes_remaining = c.try_into().map_err(|_| {
83                            AvroError::ParseError(format!("Block size cannot be negative, got {c}"))
84                        })?;
85
86                        // Only reserve what the current input backs: the block size is
87                        // input specified so could be an extreme value (e.g. i64::MAX)
88                        // in case of corrupted/malicious input. The rest is reserved
89                        // lazily by `extend_from_slice` below as data arrives.
90                        self.in_progress
91                            .data
92                            .reserve(self.bytes_remaining.min(buf.len()));
93                        self.state = BlockDecoderState::Data;
94                    }
95                }
96                BlockDecoderState::Data => {
97                    let to_read = self.bytes_remaining.min(buf.len());
98                    self.in_progress.data.extend_from_slice(&buf[..to_read]);
99                    buf = &buf[to_read..];
100                    self.bytes_remaining -= to_read;
101                    if self.bytes_remaining == 0 {
102                        self.bytes_remaining = 16;
103                        self.state = BlockDecoderState::Sync;
104                    }
105                }
106                BlockDecoderState::Sync => {
107                    let to_decode = buf.len().min(self.bytes_remaining);
108                    // Fill from the front: the marker may arrive split across decode() calls.
109                    let offset = 16 - self.bytes_remaining;
110                    self.in_progress.sync[offset..offset + to_decode]
111                        .copy_from_slice(&buf[..to_decode]);
112                    self.bytes_remaining -= to_decode;
113                    buf = &buf[to_decode..];
114                    if self.bytes_remaining == 0 {
115                        self.state = BlockDecoderState::Finished;
116                    }
117                }
118                BlockDecoderState::Finished => return Ok(max_read - buf.len()),
119            }
120        }
121        Ok(max_read)
122    }
123
124    /// Flush this decoder returning the parsed [`Block`] if any
125    pub fn flush(&mut self) -> Option<Block> {
126        match self.state {
127            BlockDecoderState::Finished => {
128                self.state = BlockDecoderState::Count;
129                Some(std::mem::take(&mut self.in_progress))
130            }
131            _ => None,
132        }
133    }
134}
135
136#[cfg(feature = "async")]
137impl BlockDecoder {
138    pub(crate) fn state(&self) -> &BlockDecoderState {
139        &self.state
140    }
141
142    pub(crate) fn bytes_remaining(&self) -> usize {
143        self.bytes_remaining
144    }
145}
146
147#[cfg(test)]
148mod tests {
149    use super::*;
150
151    /// Zig-zag encode `value` as an Avro `long` (variable-length integer).
152    fn encode_long(value: i64, out: &mut Vec<u8>) {
153        let mut n = ((value << 1) ^ (value >> 63)) as u64;
154        while n >= 0x80 {
155            out.push((n as u8) | 0x80);
156            n >>= 7;
157        }
158        out.push(n as u8);
159    }
160
161    #[test]
162    fn test_oversized_block_size_bounds_reserve() {
163        // A block advertising `i64::MAX` bytes must not reserve that up front when only
164        // a few payload bytes are present, or a crafted OCF aborts on a huge alloc (#10234).
165        let mut buf = Vec::new();
166        encode_long(1, &mut buf); // object count
167        encode_long(i64::MAX, &mut buf); // attacker-controlled block size
168        buf.extend_from_slice(&[0u8; 8]); // a handful of real bytes
169
170        let mut decoder = BlockDecoder::default();
171        let read = decoder.decode(&buf).unwrap();
172
173        assert_eq!(read, buf.len(), "all available input should be consumed");
174        assert!(
175            decoder.in_progress.data.capacity() <= buf.len(),
176            "capacity {} must stay bounded by available input {}, not the advertised i64::MAX",
177            decoder.in_progress.data.capacity(),
178            buf.len(),
179        );
180    }
181
182    #[test]
183    fn test_negative_block_size_errors() {
184        let mut buf = Vec::new();
185        encode_long(1, &mut buf); // object count
186        encode_long(-1, &mut buf); // invalid (negative) block size
187
188        let mut decoder = BlockDecoder::default();
189        let err = decoder.decode(&buf).unwrap_err();
190        assert!(
191            err.to_string().contains("Block size cannot be negative"),
192            "unexpected error: {err}",
193        );
194    }
195
196    #[test]
197    fn test_well_formed_block_round_trips() {
198        // The capped reserve must not change decoding of a normal block.
199        let payload = [1u8, 2, 3, 4];
200        let sync = [7u8; 16];
201        let mut buf = Vec::new();
202        encode_long(2, &mut buf); // object count
203        encode_long(payload.len() as i64, &mut buf); // block size
204        buf.extend_from_slice(&payload);
205        buf.extend_from_slice(&sync);
206
207        let mut decoder = BlockDecoder::default();
208        assert_eq!(decoder.decode(&buf).unwrap(), buf.len());
209        let block = decoder.flush().expect("a complete block");
210        assert_eq!(block.count, 2);
211        assert_eq!(block.data, payload);
212        assert_eq!(block.sync, sync);
213    }
214
215    #[test]
216    fn test_sync_marker_split_across_decode_calls() {
217        // count=1 (zig-zag 0x02), size=1 (0x02), one data byte, 16-byte sync marker
218        let sync: [u8; 16] = core::array::from_fn(|i| i as u8);
219        let mut block_bytes = vec![0x02, 0x02, 0xAA];
220        block_bytes.extend_from_slice(&sync);
221
222        for chunk_size in 1..block_bytes.len() {
223            let mut decoder = BlockDecoder::default();
224            for chunk in block_bytes.chunks(chunk_size) {
225                decoder.decode(chunk).unwrap();
226            }
227            let block = decoder.flush().expect("complete block");
228            assert_eq!(block.count, 1);
229            assert_eq!(block.data, vec![0xAA]);
230            assert_eq!(block.sync, sync, "chunk_size {chunk_size}");
231        }
232    }
233}