Skip to main content

parquet/arrow/decoder/
dictionary_index.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use bytes::Bytes;
6
7use crate::encodings::rle::{MAX_RLE_DICTIONARY_BIT_WIDTH, RleDecoder};
8use crate::errors::{ParquetError, Result};
9
10/// Decoder for `Encoding::RLE_DICTIONARY` indices
11pub struct DictIndexDecoder {
12    /// Decoder for the dictionary offsets array
13    decoder: RleDecoder,
14
15    /// We want to decode the offsets in chunks so we will maintain an internal buffer of decoded
16    /// offsets
17    index_buf: Box<[i32; 1024]>,
18    /// Current length of `index_buf`
19    index_buf_len: usize,
20    /// Current offset into `index_buf`. If `index_buf_offset` == `index_buf_len` then we've consumed
21    /// the entire buffer and need to decode another chunk of offsets.
22    index_offset: usize,
23
24    /// This is a maximum as the null count is not always known, e.g. value data from
25    /// a v1 data page
26    max_remaining_values: usize,
27}
28
29impl DictIndexDecoder {
30    /// Create a new [`DictIndexDecoder`] with the provided data page, the number of levels
31    /// associated with this data page, and the number of non-null values (if known)
32    pub fn new(data: Bytes, num_levels: usize, num_values: Option<usize>) -> Result<Self> {
33        let bit_width = *data
34            .first()
35            .ok_or_else(|| general_err!("dictionary index page is empty"))?;
36        if bit_width > MAX_RLE_DICTIONARY_BIT_WIDTH {
37            return Err(general_err!(
38                "Invalid or corrupted RLE bit width {bit_width}. Max allowed is {MAX_RLE_DICTIONARY_BIT_WIDTH}"
39            ));
40        }
41        let mut decoder = RleDecoder::new(bit_width);
42        decoder.set_data(data.slice(1..))?;
43
44        Ok(Self {
45            decoder,
46            index_buf: Box::new([0; 1024]),
47            index_buf_len: 0,
48            index_offset: 0,
49            max_remaining_values: num_values.unwrap_or(num_levels),
50        })
51    }
52
53    /// Read up to `len` values, returning the number of values read
54    /// and calling `f` with each decoded dictionary index
55    ///
56    /// Will short-circuit and return on error
57    #[inline(always)]
58    pub fn read<F: FnMut(&[i32]) -> Result<()>>(&mut self, len: usize, mut f: F) -> Result<usize> {
59        let total_to_read = len.min(self.max_remaining_values);
60
61        let mut values_read = 0;
62
63        let index_buf = self.index_buf.as_mut();
64        while values_read < total_to_read {
65            if self.index_offset == self.index_buf_len {
66                // We've consumed the entire index buffer so we need to reload it before proceeding
67                let read = self.decoder.get_batch(index_buf)?;
68                if read == 0 {
69                    break;
70                }
71                self.index_buf_len = read;
72                self.index_offset = 0;
73            }
74
75            let available = self.index_buf_len - self.index_offset;
76            let n = available.min(total_to_read - values_read);
77
78            f(&index_buf[self.index_offset..self.index_offset + n])?;
79
80            self.index_offset += n;
81            values_read += n;
82        }
83        self.max_remaining_values -= values_read;
84
85        Ok(values_read)
86    }
87
88    /// Skip up to `to_skip` values, returning the number of values skipped
89    pub fn skip(&mut self, to_skip: usize) -> Result<usize> {
90        let to_skip = to_skip.min(self.max_remaining_values);
91
92        let mut values_skip = 0;
93        while values_skip < to_skip {
94            if self.index_offset == self.index_buf_len {
95                // Instead of reloading the buffer, just skip in the decoder
96                let skip = self.decoder.skip(to_skip - values_skip)?;
97
98                if skip == 0 {
99                    break;
100                }
101
102                self.max_remaining_values -= skip;
103                values_skip += skip;
104            } else {
105                // We still have indices buffered, so skip within the buffer
106                let skip = (to_skip - values_skip).min(self.index_buf_len - self.index_offset);
107
108                self.index_offset += skip;
109                self.max_remaining_values -= skip;
110                values_skip += skip;
111            }
112        }
113        Ok(values_skip)
114    }
115}
116
117#[cfg(test)]
118mod tests {
119    use super::*;
120
121    #[test]
122    fn test_dict_index_decoder_empty_data() {
123        let Err(err) = DictIndexDecoder::new(Bytes::new(), 0, None) else {
124            panic!("expected error");
125        };
126        assert_eq!(
127            err.to_string(),
128            "Parquet error: dictionary index page is empty"
129        );
130    }
131
132    #[test]
133    fn test_dict_index_decoder_invalid_bit_width() {
134        let data = Bytes::from_static(&[33, 0, 0, 0]);
135        let Err(err) = DictIndexDecoder::new(data, 1, None) else {
136            panic!("expected error");
137        };
138        assert_eq!(
139            err.to_string(),
140            "Parquet error: Invalid or corrupted RLE bit width 33. Max allowed is 32"
141        );
142    }
143}