parquet/file/page_index/index_reader.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Support for reading [`ColumnIndexMetaData`] and [`OffsetIndexMetaData`] from parquet metadata.
6
7use crate::basic::{BoundaryOrder, Type};
8use crate::data_type::Int96;
9use crate::errors::{ParquetError, Result};
10use crate::file::page_index::column_index::{
11 ByteArrayColumnIndex, ColumnIndexMetaData, PrimitiveColumnIndex,
12};
13use crate::file::page_index::offset_index::OffsetIndexMetaData;
14use crate::parquet_thrift::{
15 ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol, ThriftCompactOutputProtocol,
16 ThriftSliceInputProtocol, WriteThrift, WriteThriftField, read_thrift_vec,
17};
18use crate::thrift_struct;
19use std::io::Write;
20
21/// Decode a Thrift [`OffsetIndex`] from the provided bytes.
22///
23/// The passed in bytes contain a serialized Thrift `OffsetIndex` struct as
24/// read from a Parquet file.
25///
26/// Returns an [`OffsetIndexMetaData`] containing page location information.
27///
28/// # Example
29///
30/// ```
31/// # use parquet::file::reader::{FileReader, SerializedFileReader};
32/// # use parquet::file::page_index::index_reader::decode_offset_index;
33/// # use std::fs::File;
34/// # use std::io::{Read, Seek};
35/// # use parquet::errors::Result;
36/// #
37/// # fn read_offset_index() -> Result<()> {
38/// // Open the Parquet file
39/// let mut file = File::open("data.parquet")?;
40/// let reader = SerializedFileReader::new(file.try_clone()?)?;
41/// let metadata = reader.metadata();
42///
43/// // Select a row group and column to read
44/// let row_group_idx = 0;
45/// let column_idx = 0;
46///
47/// // Get the column chunk metadata
48/// let row_group = metadata.row_group(row_group_idx);
49/// let column_chunk = row_group.column(column_idx);
50///
51/// // Get the offset index byte range from the column metadata
52/// if let Some(range) = column_chunk.offset_index_range() {
53/// // Read the offset index bytes from the file
54/// let mut buffer = vec![0u8; (range.end - range.start) as usize];
55/// file.seek(std::io::SeekFrom::Start(range.start))?;
56/// file.read_exact(&mut buffer)?;
57///
58/// // Decode the offset index
59/// let offset_index = decode_offset_index(&buffer)?;
60///
61/// // Access page location information
62/// for (i, page_location) in offset_index.page_locations().iter().enumerate() {
63/// println!("Page {}: offset={}, size={}, first_row={}",
64/// i,
65/// page_location.offset,
66/// page_location.compressed_page_size,
67/// page_location.first_row_index
68/// );
69/// }
70/// }
71/// # Ok(())
72/// # }
73/// ```
74///
75/// [`OffsetIndex`]: https://github.com/apache/parquet-format/blob/e94a5d090b324a0c0ee1adbb8ea6b099852dc3cc/src/main/thrift/parquet.thrift#L1253-L1273
76pub fn decode_offset_index(data: &[u8]) -> Result<OffsetIndexMetaData, ParquetError> {
77 let mut prot = ThriftSliceInputProtocol::new(data);
78
79 // Try to read fast-path first. If that fails, fall back to slower but more robust
80 // decoder.
81 match OffsetIndexMetaData::try_from_fast(&mut prot) {
82 Ok(offset_index) => Ok(offset_index),
83 Err(_) => {
84 prot = ThriftSliceInputProtocol::new(data);
85 OffsetIndexMetaData::read_thrift(&mut prot)
86 }
87 }
88}
89
90// private struct only used for decoding then discarded
91thrift_struct!(
92pub(super) struct ThriftColumnIndex<'a> {
93 1: required list<bool> null_pages
94 2: required list<'a><binary> min_values
95 3: required list<'a><binary> max_values
96 4: required BoundaryOrder boundary_order
97 5: optional list<i64> null_counts
98 6: optional list<i64> repetition_level_histograms;
99 7: optional list<i64> definition_level_histograms;
100 8: optional list<i64> nan_counts
101}
102);
103
104/// Decode a Thrift [`ColumnIndex`] from the provided bytes.
105///
106/// The passed in bytes contain a serialized Thrift `OffsetIndex` struct as
107/// read from a Parquet file. The `column_type` can be obtained via
108/// [`ColumnChunkMetaData::column_type`].
109///
110/// Returns a [`ColumnIndexMetaData`] containing per-page statistics.
111///
112/// # Example
113///
114/// ```
115/// # use parquet::file::reader::{FileReader, SerializedFileReader};
116/// # use parquet::file::page_index::index_reader::decode_column_index;
117/// # use std::fs::File;
118/// # use std::io::{Read, Seek};
119/// # use parquet::errors::Result;
120/// #
121/// # fn read_column_index() -> Result<()> {
122/// // Open the Parquet file
123/// let mut file = File::open("data.parquet")?;
124/// let reader = SerializedFileReader::new(file.try_clone()?)?;
125/// let metadata = reader.metadata();
126///
127/// // Select a row group and column to read
128/// let row_group_idx = 0;
129/// let column_idx = 0;
130///
131/// // Get the column chunk metadata
132/// let row_group = metadata.row_group(row_group_idx);
133/// let column_chunk = row_group.column(column_idx);
134///
135/// // Get the column index byte range from the column metadata
136/// if let Some(range) = column_chunk.column_index_range() {
137/// // Get the column type for proper deserialization
138/// let column_type = column_chunk.column_type();
139///
140/// // Read the column index bytes from the file
141/// let mut buffer = vec![0u8; (range.end - range.start) as usize];
142/// file.seek(std::io::SeekFrom::Start(range.start))?;
143/// file.read_exact(&mut buffer)?;
144///
145/// // Decode the column index
146/// let column_index = decode_column_index(&buffer, column_type)?;
147///
148/// // Access per-page statistics (example for INT32 column)
149/// use parquet::file::page_index::column_index::ColumnIndexMetaData;
150/// match column_index {
151/// ColumnIndexMetaData::INT32(index) => {
152/// for (i, (min, max)) in index.min_values().iter()
153/// .zip(index.max_values().iter())
154/// .enumerate() {
155/// println!("Page {}: min={}, max={}", i, min, max);
156/// }
157/// }
158/// _ => println!("Column is not INT32 type"),
159/// }
160/// }
161/// # Ok(())
162/// # }
163/// ```
164///
165/// [`ColumnChunkMetaData::column_type`]: crate::file::metadata::ColumnChunkMetaData::column_type
166/// [`ColumnIndex`]: https://github.com/apache/parquet-format/blob/e94a5d090b324a0c0ee1adbb8ea6b099852dc3cc/src/main/thrift/parquet.thrift#L1275-1373
167pub fn decode_column_index(
168 data: &[u8],
169 column_type: Type,
170) -> Result<ColumnIndexMetaData, ParquetError> {
171 let mut prot = ThriftSliceInputProtocol::new(data);
172 let index = ThriftColumnIndex::read_thrift(&mut prot)?;
173
174 let index = match column_type {
175 Type::BOOLEAN => {
176 ColumnIndexMetaData::BOOLEAN(PrimitiveColumnIndex::<bool>::try_from_thrift(index)?)
177 }
178 Type::INT32 => {
179 ColumnIndexMetaData::INT32(PrimitiveColumnIndex::<i32>::try_from_thrift(index)?)
180 }
181 Type::INT64 => {
182 ColumnIndexMetaData::INT64(PrimitiveColumnIndex::<i64>::try_from_thrift(index)?)
183 }
184 Type::INT96 => {
185 ColumnIndexMetaData::INT96(PrimitiveColumnIndex::<Int96>::try_from_thrift(index)?)
186 }
187 Type::FLOAT => {
188 ColumnIndexMetaData::FLOAT(PrimitiveColumnIndex::<f32>::try_from_thrift(index)?)
189 }
190 Type::DOUBLE => {
191 ColumnIndexMetaData::DOUBLE(PrimitiveColumnIndex::<f64>::try_from_thrift(index)?)
192 }
193 Type::BYTE_ARRAY => {
194 ColumnIndexMetaData::BYTE_ARRAY(ByteArrayColumnIndex::try_from_thrift(index)?)
195 }
196 Type::FIXED_LEN_BYTE_ARRAY => {
197 ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(ByteArrayColumnIndex::try_from_thrift(index)?)
198 }
199 };
200
201 Ok(index)
202}