arrow_avro/reader/
block.rs1use crate::errors::AvroError;
8use crate::reader::vlq::VLQDecoder;
9
10#[derive(Debug, Default)]
14pub struct Block {
15 pub count: usize,
17 pub data: Vec<u8>,
19 pub sync: [u8; 16],
21}
22
23#[derive(Debug)]
25pub struct BlockDecoder {
26 state: BlockDecoderState,
27 in_progress: Block,
28 vlq_decoder: VLQDecoder,
29 bytes_remaining: usize,
30}
31
32#[derive(Debug)]
33pub(crate) enum BlockDecoderState {
34 Count,
35 Size,
36 Data,
37 Sync,
38 Finished,
39}
40
41impl Default for BlockDecoder {
42 fn default() -> Self {
43 Self {
44 state: BlockDecoderState::Count,
45 in_progress: Default::default(),
46 vlq_decoder: Default::default(),
47 bytes_remaining: 0,
48 }
49 }
50}
51
52impl BlockDecoder {
53 pub fn decode(&mut self, mut buf: &[u8]) -> Result<usize, AvroError> {
66 let max_read = buf.len();
67 while !buf.is_empty() {
68 match self.state {
69 BlockDecoderState::Count => {
70 if let Some(c) = self.vlq_decoder.long(&mut buf)? {
71 self.in_progress.count = c.try_into().map_err(|_| {
72 AvroError::ParseError(format!(
73 "Block count cannot be negative, got {c}"
74 ))
75 })?;
76
77 self.state = BlockDecoderState::Size;
78 }
79 }
80 BlockDecoderState::Size => {
81 if let Some(c) = self.vlq_decoder.long(&mut buf)? {
82 self.bytes_remaining = c.try_into().map_err(|_| {
83 AvroError::ParseError(format!("Block size cannot be negative, got {c}"))
84 })?;
85
86 self.in_progress
91 .data
92 .reserve(self.bytes_remaining.min(buf.len()));
93 self.state = BlockDecoderState::Data;
94 }
95 }
96 BlockDecoderState::Data => {
97 let to_read = self.bytes_remaining.min(buf.len());
98 self.in_progress.data.extend_from_slice(&buf[..to_read]);
99 buf = &buf[to_read..];
100 self.bytes_remaining -= to_read;
101 if self.bytes_remaining == 0 {
102 self.bytes_remaining = 16;
103 self.state = BlockDecoderState::Sync;
104 }
105 }
106 BlockDecoderState::Sync => {
107 let to_decode = buf.len().min(self.bytes_remaining);
108 let offset = 16 - self.bytes_remaining;
110 self.in_progress.sync[offset..offset + to_decode]
111 .copy_from_slice(&buf[..to_decode]);
112 self.bytes_remaining -= to_decode;
113 buf = &buf[to_decode..];
114 if self.bytes_remaining == 0 {
115 self.state = BlockDecoderState::Finished;
116 }
117 }
118 BlockDecoderState::Finished => return Ok(max_read - buf.len()),
119 }
120 }
121 Ok(max_read)
122 }
123
124 pub fn flush(&mut self) -> Option<Block> {
126 match self.state {
127 BlockDecoderState::Finished => {
128 self.state = BlockDecoderState::Count;
129 Some(std::mem::take(&mut self.in_progress))
130 }
131 _ => None,
132 }
133 }
134}
135
136#[cfg(feature = "async")]
137impl BlockDecoder {
138 pub(crate) fn state(&self) -> &BlockDecoderState {
139 &self.state
140 }
141
142 pub(crate) fn bytes_remaining(&self) -> usize {
143 self.bytes_remaining
144 }
145}
146
147#[cfg(test)]
148mod tests {
149 use super::*;
150
151 fn encode_long(value: i64, out: &mut Vec<u8>) {
153 let mut n = ((value << 1) ^ (value >> 63)) as u64;
154 while n >= 0x80 {
155 out.push((n as u8) | 0x80);
156 n >>= 7;
157 }
158 out.push(n as u8);
159 }
160
161 #[test]
162 fn test_oversized_block_size_bounds_reserve() {
163 let mut buf = Vec::new();
166 encode_long(1, &mut buf); encode_long(i64::MAX, &mut buf); buf.extend_from_slice(&[0u8; 8]); let mut decoder = BlockDecoder::default();
171 let read = decoder.decode(&buf).unwrap();
172
173 assert_eq!(read, buf.len(), "all available input should be consumed");
174 assert!(
175 decoder.in_progress.data.capacity() <= buf.len(),
176 "capacity {} must stay bounded by available input {}, not the advertised i64::MAX",
177 decoder.in_progress.data.capacity(),
178 buf.len(),
179 );
180 }
181
182 #[test]
183 fn test_negative_block_size_errors() {
184 let mut buf = Vec::new();
185 encode_long(1, &mut buf); encode_long(-1, &mut buf); let mut decoder = BlockDecoder::default();
189 let err = decoder.decode(&buf).unwrap_err();
190 assert!(
191 err.to_string().contains("Block size cannot be negative"),
192 "unexpected error: {err}",
193 );
194 }
195
196 #[test]
197 fn test_well_formed_block_round_trips() {
198 let payload = [1u8, 2, 3, 4];
200 let sync = [7u8; 16];
201 let mut buf = Vec::new();
202 encode_long(2, &mut buf); encode_long(payload.len() as i64, &mut buf); buf.extend_from_slice(&payload);
205 buf.extend_from_slice(&sync);
206
207 let mut decoder = BlockDecoder::default();
208 assert_eq!(decoder.decode(&buf).unwrap(), buf.len());
209 let block = decoder.flush().expect("a complete block");
210 assert_eq!(block.count, 2);
211 assert_eq!(block.data, payload);
212 assert_eq!(block.sync, sync);
213 }
214
215 #[test]
216 fn test_sync_marker_split_across_decode_calls() {
217 let sync: [u8; 16] = core::array::from_fn(|i| i as u8);
219 let mut block_bytes = vec![0x02, 0x02, 0xAA];
220 block_bytes.extend_from_slice(&sync);
221
222 for chunk_size in 1..block_bytes.len() {
223 let mut decoder = BlockDecoder::default();
224 for chunk in block_bytes.chunks(chunk_size) {
225 decoder.decode(chunk).unwrap();
226 }
227 let block = decoder.flush().expect("complete block");
228 assert_eq!(block.count, 1);
229 assert_eq!(block.data, vec![0xAA]);
230 assert_eq!(block.sync, sync, "chunk_size {chunk_size}");
231 }
232 }
233}