1mod cli;
26
27use clap::Parser;
28use parquet::data_type::ByteArray;
29use parquet::errors::{ParquetError, Result};
30use parquet::file::page_index::column_index::{
31 ByteArrayColumnIndex, ColumnIndexMetaData, PrimitiveColumnIndex,
32};
33use parquet::file::page_index::offset_index::{OffsetIndexMetaData, PageLocation};
34use parquet::file::reader::{FileReader, SerializedFileReader};
35use parquet::file::serialized_reader::ReadOptionsBuilder;
36use std::fs::File;
37
38#[derive(Debug, Parser)]
39#[clap(author, version, about("Prints the page index of a parquet file"), long_about = None)]
40struct Args {
41 #[clap(help("Path to a parquet file"))]
42 file: String,
43
44 #[clap(help(
45 "Dot-separated column path to get index for. Literal dots can be escaped as `\\.`"
46 ))]
47 column: String,
48}
49
50impl Args {
51 fn run(&self) -> Result<()> {
52 let file = File::open(&self.file)?;
53 let options = ReadOptionsBuilder::new().with_page_index().build();
54 let reader = SerializedFileReader::new_with_options(file, options)?;
55
56 let schema = reader.metadata().file_metadata().schema_descr();
57 let column_path = cli::parse_column_path(&self.column).map_err(ParquetError::General)?;
58 let column_idx = schema
59 .columns()
60 .iter()
61 .position(|x| x.path().parts() == column_path.as_slice())
62 .ok_or_else(|| {
63 ParquetError::General(format!("Failed to find column {}", self.column))
64 })?;
65
66 let page_index = reader
67 .metadata()
68 .page_index()
69 .ok_or_else(|| ParquetError::General("Page index not found".to_string()))?;
70
71 for row_group_idx in 0..reader.metadata().num_row_groups() {
73 let row_group = reader.metadata().row_group(row_group_idx);
74 println!("Row Group: {row_group_idx}");
75 let offset_index = page_index
76 .offset_index(row_group_idx, column_idx)
77 .ok_or_else(|| {
78 ParquetError::General(format!(
79 "No offset index for row group {row_group_idx} column chunk {column_idx}"
80 ))
81 })?;
82
83 let row_counts =
84 compute_row_counts(offset_index.page_locations(), row_group.num_rows());
85 match page_index.column_index(row_group_idx, column_idx) {
86 None => println!("NO INDEX"),
87 Some(ColumnIndexMetaData::BOOLEAN(v)) => {
88 print_index::<bool>(v, offset_index, &row_counts)?
89 }
90 Some(ColumnIndexMetaData::INT32(v)) => print_index(v, offset_index, &row_counts)?,
91 Some(ColumnIndexMetaData::INT64(v)) => print_index(v, offset_index, &row_counts)?,
92 Some(ColumnIndexMetaData::INT96(v)) => print_index(v, offset_index, &row_counts)?,
93 Some(ColumnIndexMetaData::FLOAT(v)) => print_index(v, offset_index, &row_counts)?,
94 Some(ColumnIndexMetaData::DOUBLE(v)) => print_index(v, offset_index, &row_counts)?,
95 Some(ColumnIndexMetaData::BYTE_ARRAY(v)) => {
96 print_bytes_index(v, offset_index, &row_counts)?
97 }
98 Some(ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(v)) => {
99 print_bytes_index(v, offset_index, &row_counts)?
100 }
101 }
102 }
103 Ok(())
104 }
105}
106
107fn compute_row_counts(offset_index: &[PageLocation], rows: i64) -> Vec<i64> {
109 if offset_index.is_empty() {
110 return vec![];
111 }
112
113 let mut last = offset_index[0].first_row_index;
114 let mut out = Vec::with_capacity(offset_index.len());
115 for o in offset_index.iter().skip(1) {
116 out.push(o.first_row_index - last);
117 last = o.first_row_index;
118 }
119 out.push(rows - last);
120 out
121}
122
123fn print_index<T: std::fmt::Display>(
125 column_index: &PrimitiveColumnIndex<T>,
126 offset_index: &OffsetIndexMetaData,
127 row_counts: &[i64],
128) -> Result<()> {
129 if column_index.num_pages() as usize != offset_index.page_locations().len() {
130 return Err(ParquetError::General(format!(
131 "Index length mismatch, got {} and {}",
132 column_index.num_pages(),
133 offset_index.page_locations().len()
134 )));
135 }
136
137 for (idx, (((min, max), o), row_count)) in column_index
138 .min_values_iter()
139 .zip(column_index.max_values_iter())
140 .zip(offset_index.page_locations())
141 .zip(row_counts)
142 .enumerate()
143 {
144 print!(
145 "Page {:>5} at offset {:>10} with length {:>10} and row count {:>10}",
146 idx, o.offset, o.compressed_page_size, row_count
147 );
148 match min {
149 Some(m) => print!(", min {m:>10}"),
150 None => print!(", min {:>10}", "NONE"),
151 }
152
153 match max {
154 Some(m) => print!(", max {m:>10}"),
155 None => print!(", max {:>10}", "NONE"),
156 }
157 println!()
158 }
159
160 Ok(())
161}
162
163fn print_bytes_index(
164 column_index: &ByteArrayColumnIndex,
165 offset_index: &OffsetIndexMetaData,
166 row_counts: &[i64],
167) -> Result<()> {
168 if column_index.num_pages() as usize != offset_index.page_locations().len() {
169 return Err(ParquetError::General(format!(
170 "Index length mismatch, got {} and {}",
171 column_index.num_pages(),
172 offset_index.page_locations().len()
173 )));
174 }
175
176 for (idx, (((min, max), o), row_count)) in column_index
177 .min_values_iter()
178 .zip(column_index.max_values_iter())
179 .zip(offset_index.page_locations())
180 .zip(row_counts)
181 .enumerate()
182 {
183 print!(
184 "Page {:>5} at offset {:>10} with length {:>10} and row count {:>10}",
185 idx, o.offset, o.compressed_page_size, row_count
186 );
187 match min {
188 Some(m) => match String::from_utf8(m.to_vec()) {
189 Ok(s) => print!(", min {s:>10}"),
190 Err(_) => print!(", min {:>10}", ByteArray::from(m)),
191 },
192 None => print!(", min {:>10}", "NONE"),
193 }
194
195 match max {
196 Some(m) => match String::from_utf8(m.to_vec()) {
197 Ok(s) => print!(", max {s:>10}"),
198 Err(_) => print!(", min {:>10}", ByteArray::from(m)),
199 },
200 None => print!(", max {:>10}", "NONE"),
201 }
202 println!()
203 }
204
205 Ok(())
206}
207
208fn main() -> Result<()> {
209 Args::parse().run()
210}