Skip to main content

parquet_index/
parquet-index.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Binary that prints the [page index] of a parquet file
6//!
7//! # Install
8//!
9//! `parquet-layout` can be installed using `cargo`:
10//! ```
11//! cargo install parquet --features=cli
12//! ```
13//! After this `parquet-index` should be available:
14//! ```
15//! parquet-index XYZ.parquet COLUMN_PATH
16//! ```
17//!
18//! The binary can also be built from the source code and run as follows:
19//! ```
20//! cargo run --features=cli --bin parquet-index XYZ.parquet COLUMN_PATH
21//! ```
22//!
23//! [page index]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
24
25mod cli;
26
27use clap::Parser;
28use parquet::data_type::ByteArray;
29use parquet::errors::{ParquetError, Result};
30use parquet::file::page_index::column_index::{
31    ByteArrayColumnIndex, ColumnIndexMetaData, PrimitiveColumnIndex,
32};
33use parquet::file::page_index::offset_index::{OffsetIndexMetaData, PageLocation};
34use parquet::file::reader::{FileReader, SerializedFileReader};
35use parquet::file::serialized_reader::ReadOptionsBuilder;
36use std::fs::File;
37
38#[derive(Debug, Parser)]
39#[clap(author, version, about("Prints the page index of a parquet file"), long_about = None)]
40struct Args {
41    #[clap(help("Path to a parquet file"))]
42    file: String,
43
44    #[clap(help(
45        "Dot-separated column path to get index for. Literal dots can be escaped as `\\.`"
46    ))]
47    column: String,
48}
49
50impl Args {
51    fn run(&self) -> Result<()> {
52        let file = File::open(&self.file)?;
53        let options = ReadOptionsBuilder::new().with_page_index().build();
54        let reader = SerializedFileReader::new_with_options(file, options)?;
55
56        let schema = reader.metadata().file_metadata().schema_descr();
57        let column_path = cli::parse_column_path(&self.column).map_err(ParquetError::General)?;
58        let column_idx = schema
59            .columns()
60            .iter()
61            .position(|x| x.path().parts() == column_path.as_slice())
62            .ok_or_else(|| {
63                ParquetError::General(format!("Failed to find column {}", self.column))
64            })?;
65
66        let page_index = reader
67            .metadata()
68            .page_index()
69            .ok_or_else(|| ParquetError::General("Page index not found".to_string()))?;
70
71        // Iterate through each row group
72        for row_group_idx in 0..reader.metadata().num_row_groups() {
73            let row_group = reader.metadata().row_group(row_group_idx);
74            println!("Row Group: {row_group_idx}");
75            let offset_index = page_index
76                .offset_index(row_group_idx, column_idx)
77                .ok_or_else(|| {
78                    ParquetError::General(format!(
79                        "No offset index for row group {row_group_idx} column chunk {column_idx}"
80                    ))
81                })?;
82
83            let row_counts =
84                compute_row_counts(offset_index.page_locations(), row_group.num_rows());
85            match page_index.column_index(row_group_idx, column_idx) {
86                None => println!("NO INDEX"),
87                Some(ColumnIndexMetaData::BOOLEAN(v)) => {
88                    print_index::<bool>(v, offset_index, &row_counts)?
89                }
90                Some(ColumnIndexMetaData::INT32(v)) => print_index(v, offset_index, &row_counts)?,
91                Some(ColumnIndexMetaData::INT64(v)) => print_index(v, offset_index, &row_counts)?,
92                Some(ColumnIndexMetaData::INT96(v)) => print_index(v, offset_index, &row_counts)?,
93                Some(ColumnIndexMetaData::FLOAT(v)) => print_index(v, offset_index, &row_counts)?,
94                Some(ColumnIndexMetaData::DOUBLE(v)) => print_index(v, offset_index, &row_counts)?,
95                Some(ColumnIndexMetaData::BYTE_ARRAY(v)) => {
96                    print_bytes_index(v, offset_index, &row_counts)?
97                }
98                Some(ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(v)) => {
99                    print_bytes_index(v, offset_index, &row_counts)?
100                }
101            }
102        }
103        Ok(())
104    }
105}
106
107/// Computes the number of rows in each page within a column chunk
108fn compute_row_counts(offset_index: &[PageLocation], rows: i64) -> Vec<i64> {
109    if offset_index.is_empty() {
110        return vec![];
111    }
112
113    let mut last = offset_index[0].first_row_index;
114    let mut out = Vec::with_capacity(offset_index.len());
115    for o in offset_index.iter().skip(1) {
116        out.push(o.first_row_index - last);
117        last = o.first_row_index;
118    }
119    out.push(rows - last);
120    out
121}
122
123/// Prints index information for a single column chunk
124fn print_index<T: std::fmt::Display>(
125    column_index: &PrimitiveColumnIndex<T>,
126    offset_index: &OffsetIndexMetaData,
127    row_counts: &[i64],
128) -> Result<()> {
129    if column_index.num_pages() as usize != offset_index.page_locations().len() {
130        return Err(ParquetError::General(format!(
131            "Index length mismatch, got {} and {}",
132            column_index.num_pages(),
133            offset_index.page_locations().len()
134        )));
135    }
136
137    for (idx, (((min, max), o), row_count)) in column_index
138        .min_values_iter()
139        .zip(column_index.max_values_iter())
140        .zip(offset_index.page_locations())
141        .zip(row_counts)
142        .enumerate()
143    {
144        print!(
145            "Page {:>5} at offset {:>10} with length {:>10} and row count {:>10}",
146            idx, o.offset, o.compressed_page_size, row_count
147        );
148        match min {
149            Some(m) => print!(", min {m:>10}"),
150            None => print!(", min {:>10}", "NONE"),
151        }
152
153        match max {
154            Some(m) => print!(", max {m:>10}"),
155            None => print!(", max {:>10}", "NONE"),
156        }
157        println!()
158    }
159
160    Ok(())
161}
162
163fn print_bytes_index(
164    column_index: &ByteArrayColumnIndex,
165    offset_index: &OffsetIndexMetaData,
166    row_counts: &[i64],
167) -> Result<()> {
168    if column_index.num_pages() as usize != offset_index.page_locations().len() {
169        return Err(ParquetError::General(format!(
170            "Index length mismatch, got {} and {}",
171            column_index.num_pages(),
172            offset_index.page_locations().len()
173        )));
174    }
175
176    for (idx, (((min, max), o), row_count)) in column_index
177        .min_values_iter()
178        .zip(column_index.max_values_iter())
179        .zip(offset_index.page_locations())
180        .zip(row_counts)
181        .enumerate()
182    {
183        print!(
184            "Page {:>5} at offset {:>10} with length {:>10} and row count {:>10}",
185            idx, o.offset, o.compressed_page_size, row_count
186        );
187        match min {
188            Some(m) => match String::from_utf8(m.to_vec()) {
189                Ok(s) => print!(", min {s:>10}"),
190                Err(_) => print!(", min {:>10}", ByteArray::from(m)),
191            },
192            None => print!(", min {:>10}", "NONE"),
193        }
194
195        match max {
196            Some(m) => match String::from_utf8(m.to_vec()) {
197                Ok(s) => print!(", max {s:>10}"),
198                Err(_) => print!(", min {:>10}", ByteArray::from(m)),
199            },
200            None => print!(", max {:>10}", "NONE"),
201        }
202        println!()
203    }
204
205    Ok(())
206}
207
208fn main() -> Result<()> {
209    Args::parse().run()
210}