Skip to main content

parquet/file/page_index/
offset_index.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! [`OffsetIndexMetaData`] structure holding decoded [`OffsetIndex`] information
6//!
7//! [`OffsetIndex`]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
8
9use std::io::Write;
10
11use crate::parquet_thrift::{
12    ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol, ThriftCompactOutputProtocol,
13    WriteThrift, WriteThriftField, check_list_size, read_thrift_vec, validate_list_type,
14};
15use crate::{
16    errors::{ParquetError, Result},
17    thrift_struct,
18};
19
20thrift_struct!(
21/// Page location information for [`OffsetIndexMetaData`]
22pub struct PageLocation {
23  /// Offset of the page in the file
24  1: required i64 offset
25  /// Size of the page, including header. Sum of compressed_page_size and header
26  2: required i32 compressed_page_size
27  /// Index within the RowGroup of the first row of the page. When an
28  /// OffsetIndex is present, pages must begin on row boundaries
29  /// (repetition_level = 0).
30  3: required i64 first_row_index
31}
32);
33
34thrift_struct!(
35/// [`OffsetIndex`] information for a column chunk. Contains offsets and sizes for each page
36/// in the chunk. Optionally stores fully decoded page sizes for BYTE_ARRAY columns.
37///
38/// See [`PageIndex`] for more information.
39///
40/// [`PageIndex`]: crate::file::metadata::page_index::PageIndex
41/// [`OffsetIndex`]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
42pub struct OffsetIndexMetaData {
43  /// Vector of [`PageLocation`] objects, one per page in the chunk.
44  1: required list<PageLocation> page_locations
45  /// Optional vector of unencoded page sizes, one per page in the chunk.
46  /// Only defined for BYTE_ARRAY columns.
47  2: optional list<i64> unencoded_byte_array_data_bytes
48}
49);
50
51impl OffsetIndexMetaData {
52    /// Vector of [`PageLocation`] objects, one per page in the chunk.
53    pub fn page_locations(&self) -> &Vec<PageLocation> {
54        &self.page_locations
55    }
56
57    /// Optional vector of unencoded page sizes, one per page in the chunk. Only defined
58    /// for BYTE_ARRAY columns.
59    pub fn unencoded_byte_array_data_bytes(&self) -> Option<&Vec<i64>> {
60        self.unencoded_byte_array_data_bytes.as_ref()
61    }
62
63    // Fast-path read of offset index. This works because we expect all field deltas to be 1,
64    // and there's no nesting beyond PageLocation, so no need to save the last field id. Like
65    // read_page_locations(), this will fail if absolute field id's are used.
66    pub(super) fn try_from_fast<'a, R: ThriftCompactInputProtocol<'a>>(
67        prot: &mut R,
68    ) -> Result<Self> {
69        // Offset index is a struct with 2 fields. First field is an array of PageLocations,
70        // the second an optional array of i64.
71
72        // read field 1 header, then list header, then vec of PageLocations
73        let (field_type, delta) = prot.read_field_header()?;
74        if delta != 1 || field_type != FieldType::List as u8 {
75            return Err(general_err!("error reading OffsetIndex::page_locations"));
76        }
77
78        // we have to do this manually because we want to use the fast PageLocation decoder
79        let list_ident = prot.read_list_begin()?;
80        validate_list_type(ElementType::Struct, &list_ident)?;
81        // list_ident.size is already guaranteed to fit in i32 by read_list_begin()
82        let size = list_ident.size as usize;
83        check_list_size(prot.remaining_bytes(), size)?;
84        let mut page_locations = Vec::with_capacity(size);
85        for _ in 0..list_ident.size {
86            page_locations.push(read_page_location(prot)?);
87        }
88
89        let mut unencoded_byte_array_data_bytes: Option<Vec<i64>> = None;
90
91        // read second field...if it's Stop we're done
92        let (mut field_type, delta) = prot.read_field_header()?;
93        if field_type == FieldType::List as u8 {
94            if delta != 1 {
95                return Err(general_err!(
96                    "encountered unknown field while reading OffsetIndex"
97                ));
98            }
99            let vec = read_thrift_vec::<i64, R>(&mut *prot)?;
100            unencoded_byte_array_data_bytes = Some(vec);
101
102            // this one should be Stop
103            (field_type, _) = prot.read_field_header()?;
104        }
105
106        if field_type != FieldType::Stop as u8 {
107            return Err(general_err!(
108                "encountered unknown field while reading OffsetIndex"
109            ));
110        }
111
112        Ok(Self {
113            page_locations,
114            unencoded_byte_array_data_bytes,
115        })
116    }
117}
118
119// hand coding this one because it is very time critical
120
121// Note: this will fail if the fields are either out of order, or if a suboptimal
122// encoder doesn't use field deltas.
123fn read_page_location<'a, R: ThriftCompactInputProtocol<'a>>(prot: &mut R) -> Result<PageLocation> {
124    // there are 3 fields, all mandatory, so all field deltas should be 1
125    let (field_type, delta) = prot.read_field_header()?;
126    if delta != 1 || field_type != FieldType::I64 as u8 {
127        return Err(general_err!("error reading PageLocation::offset"));
128    }
129    let offset = prot.read_i64()?;
130
131    let (field_type, delta) = prot.read_field_header()?;
132    if delta != 1 || field_type != FieldType::I32 as u8 {
133        return Err(general_err!(
134            "error reading PageLocation::compressed_page_size"
135        ));
136    }
137    let compressed_page_size = prot.read_i32()?;
138
139    let (field_type, delta) = prot.read_field_header()?;
140    if delta != 1 || field_type != FieldType::I64 as u8 {
141        return Err(general_err!("error reading PageLocation::first_row_index"));
142    }
143    let first_row_index = prot.read_i64()?;
144
145    // read end of struct...return error if there are unknown fields present
146    let (field_type, _) = prot.read_field_header()?;
147    if field_type != FieldType::Stop as u8 {
148        return Err(general_err!("unexpected field in PageLocation"));
149    }
150
151    Ok(PageLocation {
152        offset,
153        compressed_page_size,
154        first_row_index,
155    })
156}
157
158#[cfg(test)]
159mod tests {
160    use super::*;
161    use crate::file::page_index::index_reader::decode_offset_index;
162    use crate::parquet_thrift::{ThriftSliceInputProtocol, tests::test_roundtrip};
163
164    #[test]
165    fn test_offset_idx_rejects_count_exceeding_remaining_input() {
166        // Field 1: list<struct> declaring 4096 entries, with no payload.
167        let data = [0x19, 0xfc, 0x80, 0x20];
168        let mut prot = ThriftSliceInputProtocol::new(&data);
169        let err = OffsetIndexMetaData::try_from_fast(&mut prot).unwrap_err();
170        assert!(
171            err.to_string()
172                .contains("Thrift list size 4096 exceeds remaining input length 0"),
173            "{err}"
174        );
175        assert!(decode_offset_index(&data).is_err());
176    }
177
178    #[test]
179    fn test_offset_idx_fast_path_roundtrip() {
180        for page_locations in [
181            Vec::new(),
182            vec![PageLocation {
183                offset: 0,
184                compressed_page_size: 10,
185                first_row_index: 0,
186            }],
187        ] {
188            let index = OffsetIndexMetaData {
189                page_locations,
190                unencoded_byte_array_data_bytes: None,
191            };
192            let mut data = Vec::new();
193            index
194                .write_thrift(&mut ThriftCompactOutputProtocol::new(&mut data))
195                .unwrap();
196            let mut prot = ThriftSliceInputProtocol::new(&data);
197            assert_eq!(
198                OffsetIndexMetaData::try_from_fast(&mut prot).unwrap(),
199                index
200            );
201            assert_eq!(decode_offset_index(&data).unwrap(), index);
202        }
203    }
204
205    #[test]
206    fn test_offset_idx_roundtrip() {
207        let page_locations = [
208            PageLocation {
209                offset: 0,
210                compressed_page_size: 10,
211                first_row_index: 0,
212            },
213            PageLocation {
214                offset: 10,
215                compressed_page_size: 20,
216                first_row_index: 100,
217            },
218        ]
219        .to_vec();
220        let unenc = [0i64, 100i64].to_vec();
221
222        test_roundtrip(OffsetIndexMetaData {
223            page_locations: page_locations.clone(),
224            unencoded_byte_array_data_bytes: Some(unenc),
225        });
226        test_roundtrip(OffsetIndexMetaData {
227            page_locations,
228            unencoded_byte_array_data_bytes: None,
229        });
230    }
231}