Skip to main content

parquet/file/page_index/
index_reader.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Support for reading [`ColumnIndexMetaData`] and [`OffsetIndexMetaData`] from parquet metadata.
6
7use crate::basic::{BoundaryOrder, Type};
8use crate::data_type::Int96;
9use crate::errors::{ParquetError, Result};
10use crate::file::page_index::column_index::{
11    ByteArrayColumnIndex, ColumnIndexMetaData, PrimitiveColumnIndex,
12};
13use crate::file::page_index::offset_index::OffsetIndexMetaData;
14use crate::parquet_thrift::{
15    ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol, ThriftCompactOutputProtocol,
16    ThriftSliceInputProtocol, WriteThrift, WriteThriftField, read_thrift_vec,
17};
18use crate::thrift_struct;
19use std::io::Write;
20
21/// Decode a Thrift [`OffsetIndex`] from the provided bytes.
22///
23/// The passed in bytes contain a serialized Thrift `OffsetIndex` struct as
24/// read from a Parquet file.
25///
26/// Returns an [`OffsetIndexMetaData`] containing page location information.
27///
28/// # Example
29///
30/// ```
31/// # use parquet::file::reader::{FileReader, SerializedFileReader};
32/// # use parquet::file::page_index::index_reader::decode_offset_index;
33/// # use std::fs::File;
34/// # use std::io::{Read, Seek};
35/// # use parquet::errors::Result;
36/// #
37/// # fn read_offset_index() -> Result<()> {
38/// // Open the Parquet file
39/// let mut file = File::open("data.parquet")?;
40/// let reader = SerializedFileReader::new(file.try_clone()?)?;
41/// let metadata = reader.metadata();
42///
43/// // Select a row group and column to read
44/// let row_group_idx = 0;
45/// let column_idx = 0;
46///
47/// // Get the column chunk metadata
48/// let row_group = metadata.row_group(row_group_idx);
49/// let column_chunk = row_group.column(column_idx);
50///
51/// // Get the offset index byte range from the column metadata
52/// if let Some(range) = column_chunk.offset_index_range() {
53///     // Read the offset index bytes from the file
54///     let mut buffer = vec![0u8; (range.end - range.start) as usize];
55///     file.seek(std::io::SeekFrom::Start(range.start))?;
56///     file.read_exact(&mut buffer)?;
57///
58///     // Decode the offset index
59///     let offset_index = decode_offset_index(&buffer)?;
60///
61///     // Access page location information
62///     for (i, page_location) in offset_index.page_locations().iter().enumerate() {
63///         println!("Page {}: offset={}, size={}, first_row={}",
64///             i,
65///             page_location.offset,
66///             page_location.compressed_page_size,
67///             page_location.first_row_index
68///         );
69///     }
70/// }
71/// # Ok(())
72/// # }
73/// ```
74///
75/// [`OffsetIndex`]: https://github.com/apache/parquet-format/blob/e94a5d090b324a0c0ee1adbb8ea6b099852dc3cc/src/main/thrift/parquet.thrift#L1253-L1273
76pub fn decode_offset_index(data: &[u8]) -> Result<OffsetIndexMetaData, ParquetError> {
77    let mut prot = ThriftSliceInputProtocol::new(data);
78
79    // Try to read fast-path first. If that fails, fall back to slower but more robust
80    // decoder.
81    match OffsetIndexMetaData::try_from_fast(&mut prot) {
82        Ok(offset_index) => Ok(offset_index),
83        Err(_) => {
84            prot = ThriftSliceInputProtocol::new(data);
85            OffsetIndexMetaData::read_thrift(&mut prot)
86        }
87    }
88}
89
90// private struct only used for decoding then discarded
91thrift_struct!(
92pub(super) struct ThriftColumnIndex<'a> {
93  1: required list<bool> null_pages
94  2: required list<'a><binary> min_values
95  3: required list<'a><binary> max_values
96  4: required BoundaryOrder boundary_order
97  5: optional list<i64> null_counts
98  6: optional list<i64> repetition_level_histograms;
99  7: optional list<i64> definition_level_histograms;
100  8: optional list<i64> nan_counts
101}
102);
103
104/// Decode a Thrift [`ColumnIndex`] from the provided bytes.
105///
106/// The passed in bytes contain a serialized Thrift `OffsetIndex` struct as
107/// read from a Parquet file. The `column_type` can be obtained via
108/// [`ColumnChunkMetaData::column_type`].
109///
110/// Returns a [`ColumnIndexMetaData`] containing per-page statistics.
111///
112/// # Example
113///
114/// ```
115/// # use parquet::file::reader::{FileReader, SerializedFileReader};
116/// # use parquet::file::page_index::index_reader::decode_column_index;
117/// # use std::fs::File;
118/// # use std::io::{Read, Seek};
119/// # use parquet::errors::Result;
120/// #
121/// # fn read_column_index() -> Result<()> {
122/// // Open the Parquet file
123/// let mut file = File::open("data.parquet")?;
124/// let reader = SerializedFileReader::new(file.try_clone()?)?;
125/// let metadata = reader.metadata();
126///
127/// // Select a row group and column to read
128/// let row_group_idx = 0;
129/// let column_idx = 0;
130///
131/// // Get the column chunk metadata
132/// let row_group = metadata.row_group(row_group_idx);
133/// let column_chunk = row_group.column(column_idx);
134///
135/// // Get the column index byte range from the column metadata
136/// if let Some(range) = column_chunk.column_index_range() {
137///     // Get the column type for proper deserialization
138///     let column_type = column_chunk.column_type();
139///
140///     // Read the column index bytes from the file
141///     let mut buffer = vec![0u8; (range.end - range.start) as usize];
142///     file.seek(std::io::SeekFrom::Start(range.start))?;
143///     file.read_exact(&mut buffer)?;
144///
145///     // Decode the column index
146///     let column_index = decode_column_index(&buffer, column_type)?;
147///
148///     // Access per-page statistics (example for INT32 column)
149///     use parquet::file::page_index::column_index::ColumnIndexMetaData;
150///     match column_index {
151///         ColumnIndexMetaData::INT32(index) => {
152///             for (i, (min, max)) in index.min_values().iter()
153///                 .zip(index.max_values().iter())
154///                 .enumerate() {
155///                 println!("Page {}: min={}, max={}", i, min, max);
156///             }
157///         }
158///         _ => println!("Column is not INT32 type"),
159///     }
160/// }
161/// # Ok(())
162/// # }
163/// ```
164///
165/// [`ColumnChunkMetaData::column_type`]: crate::file::metadata::ColumnChunkMetaData::column_type
166/// [`ColumnIndex`]: https://github.com/apache/parquet-format/blob/e94a5d090b324a0c0ee1adbb8ea6b099852dc3cc/src/main/thrift/parquet.thrift#L1275-1373
167pub fn decode_column_index(
168    data: &[u8],
169    column_type: Type,
170) -> Result<ColumnIndexMetaData, ParquetError> {
171    let mut prot = ThriftSliceInputProtocol::new(data);
172    let index = ThriftColumnIndex::read_thrift(&mut prot)?;
173
174    let index = match column_type {
175        Type::BOOLEAN => {
176            ColumnIndexMetaData::BOOLEAN(PrimitiveColumnIndex::<bool>::try_from_thrift(index)?)
177        }
178        Type::INT32 => {
179            ColumnIndexMetaData::INT32(PrimitiveColumnIndex::<i32>::try_from_thrift(index)?)
180        }
181        Type::INT64 => {
182            ColumnIndexMetaData::INT64(PrimitiveColumnIndex::<i64>::try_from_thrift(index)?)
183        }
184        Type::INT96 => {
185            ColumnIndexMetaData::INT96(PrimitiveColumnIndex::<Int96>::try_from_thrift(index)?)
186        }
187        Type::FLOAT => {
188            ColumnIndexMetaData::FLOAT(PrimitiveColumnIndex::<f32>::try_from_thrift(index)?)
189        }
190        Type::DOUBLE => {
191            ColumnIndexMetaData::DOUBLE(PrimitiveColumnIndex::<f64>::try_from_thrift(index)?)
192        }
193        Type::BYTE_ARRAY => {
194            ColumnIndexMetaData::BYTE_ARRAY(ByteArrayColumnIndex::try_from_thrift(index)?)
195        }
196        Type::FIXED_LEN_BYTE_ARRAY => {
197            ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(ByteArrayColumnIndex::try_from_thrift(index)?)
198        }
199    };
200
201    Ok(index)
202}