parquet/arrow/decoder/
dictionary_index.rs1use bytes::Bytes;
6
7use crate::encodings::rle::{MAX_RLE_DICTIONARY_BIT_WIDTH, RleDecoder};
8use crate::errors::{ParquetError, Result};
9
10pub struct DictIndexDecoder {
12 decoder: RleDecoder,
14
15 index_buf: Box<[i32; 1024]>,
18 index_buf_len: usize,
20 index_offset: usize,
23
24 max_remaining_values: usize,
27}
28
29impl DictIndexDecoder {
30 pub fn new(data: Bytes, num_levels: usize, num_values: Option<usize>) -> Result<Self> {
33 let bit_width = *data
34 .first()
35 .ok_or_else(|| general_err!("dictionary index page is empty"))?;
36 if bit_width > MAX_RLE_DICTIONARY_BIT_WIDTH {
37 return Err(general_err!(
38 "Invalid or corrupted RLE bit width {bit_width}. Max allowed is {MAX_RLE_DICTIONARY_BIT_WIDTH}"
39 ));
40 }
41 let mut decoder = RleDecoder::new(bit_width);
42 decoder.set_data(data.slice(1..))?;
43
44 Ok(Self {
45 decoder,
46 index_buf: Box::new([0; 1024]),
47 index_buf_len: 0,
48 index_offset: 0,
49 max_remaining_values: num_values.unwrap_or(num_levels),
50 })
51 }
52
53 #[inline(always)]
58 pub fn read<F: FnMut(&[i32]) -> Result<()>>(&mut self, len: usize, mut f: F) -> Result<usize> {
59 let total_to_read = len.min(self.max_remaining_values);
60
61 let mut values_read = 0;
62
63 let index_buf = self.index_buf.as_mut();
64 while values_read < total_to_read {
65 if self.index_offset == self.index_buf_len {
66 let read = self.decoder.get_batch(index_buf)?;
68 if read == 0 {
69 break;
70 }
71 self.index_buf_len = read;
72 self.index_offset = 0;
73 }
74
75 let available = self.index_buf_len - self.index_offset;
76 let n = available.min(total_to_read - values_read);
77
78 f(&index_buf[self.index_offset..self.index_offset + n])?;
79
80 self.index_offset += n;
81 values_read += n;
82 }
83 self.max_remaining_values -= values_read;
84
85 Ok(values_read)
86 }
87
88 pub fn skip(&mut self, to_skip: usize) -> Result<usize> {
90 let to_skip = to_skip.min(self.max_remaining_values);
91
92 let mut values_skip = 0;
93 while values_skip < to_skip {
94 if self.index_offset == self.index_buf_len {
95 let skip = self.decoder.skip(to_skip - values_skip)?;
97
98 if skip == 0 {
99 break;
100 }
101
102 self.max_remaining_values -= skip;
103 values_skip += skip;
104 } else {
105 let skip = (to_skip - values_skip).min(self.index_buf_len - self.index_offset);
107
108 self.index_offset += skip;
109 self.max_remaining_values -= skip;
110 values_skip += skip;
111 }
112 }
113 Ok(values_skip)
114 }
115}
116
117#[cfg(test)]
118mod tests {
119 use super::*;
120
121 #[test]
122 fn test_dict_index_decoder_empty_data() {
123 let Err(err) = DictIndexDecoder::new(Bytes::new(), 0, None) else {
124 panic!("expected error");
125 };
126 assert_eq!(
127 err.to_string(),
128 "Parquet error: dictionary index page is empty"
129 );
130 }
131
132 #[test]
133 fn test_dict_index_decoder_invalid_bit_width() {
134 let data = Bytes::from_static(&[33, 0, 0, 0]);
135 let Err(err) = DictIndexDecoder::new(data, 1, None) else {
136 panic!("expected error");
137 };
138 assert_eq!(
139 err.to_string(),
140 "Parquet error: Invalid or corrupted RLE bit width 33. Max allowed is 32"
141 );
142 }
143}