Skip to main content

parquet/arrow/record_reader/
buffer.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use crate::arrow::buffer::bit_util::iter_set_bits_rev;
6use crate::errors::Result;
7
8/// A buffer that supports padding with nulls
9pub trait ValuesBuffer {
10    /// Create a new buffer with capacity for at least `capacity` logical values
11    fn with_capacity(capacity: usize) -> Self;
12
13    /// Reserve capacity for `additional` more logical values.
14    ///
15    /// Callers should use this once the level decoders have determined how
16    /// many output slots are actually needed for the next read.
17    fn reserve_exact(&mut self, additional: usize);
18
19    /// If a column contains nulls, more level data may be read than value data, as null
20    /// values are not encoded. Therefore, first the levels data is read, the null count
21    /// determined, and then the corresponding number of values read to a [`ValuesBuffer`].
22    ///
23    /// It is then necessary to move this values data into positions that correspond to
24    /// the non-null level positions. This is what this method does.
25    ///
26    /// It is provided with:
27    ///
28    /// - `read_offset` - the offset in [`ValuesBuffer`] to start null padding from
29    /// - `values_read` - the number of values read
30    /// - `levels_read` - the number of levels read
31    /// - `valid_mask` - a packed mask of valid levels
32    ///
33    /// Returns an error if the inputs are inconsistent, for example because the
34    /// decoded data was corrupt. This must not panic on such input.
35    fn pad_nulls(
36        &mut self,
37        read_offset: usize,
38        values_read: usize,
39        levels_read: usize,
40        valid_mask: &[u8],
41    ) -> Result<()>;
42}
43
44impl<T: Copy + Default> ValuesBuffer for Vec<T> {
45    fn with_capacity(capacity: usize) -> Self {
46        Vec::with_capacity(capacity)
47    }
48
49    fn reserve_exact(&mut self, additional: usize) {
50        Vec::reserve_exact(self, additional)
51    }
52
53    fn pad_nulls(
54        &mut self,
55        read_offset: usize,
56        values_read: usize,
57        levels_read: usize,
58        valid_mask: &[u8],
59    ) -> Result<()> {
60        self.resize(read_offset + levels_read, T::default());
61
62        let values_range = read_offset..read_offset + values_read;
63        for (value_pos, level_pos) in values_range.rev().zip(iter_set_bits_rev(valid_mask)) {
64            debug_assert!(level_pos >= value_pos);
65            if level_pos <= value_pos {
66                break;
67            }
68            self[level_pos] = self[value_pos];
69        }
70        Ok(())
71    }
72}