parquet_show_bloom_filter/
parquet-show-bloom-filter.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Binary file to read bloom filter data from a Parquet file.
19//!
20//! # Install
21//!
22//! `parquet-show-bloom-filter` can be installed using `cargo`:
23//! ```
24//! cargo install parquet --features=cli
25//! ```
26//! After this `parquet-show-bloom-filter` should be available:
27//! ```
28//! parquet-show-bloom-filter XYZ.parquet id a
29//! ```
30//!
31//! The binary can also be built from the source code and run as follows:
32//! ```
33//! cargo run --features=cli --bin parquet-show-bloom-filter -- --file-name XYZ.parquet --column id --values a
34//! ```
35
36mod cli;
37
38use clap::Parser;
39use parquet::basic::Type;
40use parquet::bloom_filter::Sbbf;
41use parquet::file::metadata::ColumnChunkMetaData;
42use parquet::file::{
43    properties::ReaderProperties,
44    reader::{FileReader, SerializedFileReader},
45    serialized_reader::ReadOptionsBuilder,
46};
47use std::{fs::File, path::Path};
48
49#[derive(Debug, Parser)]
50#[clap(author, version, about("Binary file to read bloom filter data from a Parquet file"), long_about = None)]
51struct Args {
52    #[clap(help("Path to the parquet file"))]
53    file_name: String,
54    #[clap(help(concat!(
55        "Dot-separated column path to check bloom filter for. Literal dots can be escaped as `\\.`. ",
56        "Only string typed columns or columns with an Int32 or Int64 physical type are supported."
57    )
58    ))]
59    column: String,
60    #[clap(
61        help(
62            "Check if the given values match bloom filter, the values will be parsed to the physical type of the column"
63        ),
64        required = true
65    )]
66    values: Vec<String>,
67}
68
69fn main() {
70    let args = Args::parse();
71    let file_name = args.file_name;
72    let path = Path::new(&file_name);
73    let file = File::open(path).expect("Unable to open file");
74
75    let file_reader = SerializedFileReader::new_with_options(
76        file,
77        ReadOptionsBuilder::new()
78            .with_reader_properties(
79                ReaderProperties::builder()
80                    .set_read_bloom_filter(true)
81                    .build(),
82            )
83            .build(),
84    )
85    .expect("Unable to open file as Parquet");
86    let metadata = file_reader.metadata();
87    let column_path = match cli::parse_column_path(&args.column) {
88        Ok(path) => path,
89        Err(err) => {
90            eprintln!("{}", err);
91            std::process::exit(1);
92        }
93    };
94
95    for (ri, row_group) in metadata.row_groups().iter().enumerate() {
96        println!("Row group #{ri}");
97        println!("{}", "=".repeat(80));
98        if let Some((column_index, column)) = row_group
99            .columns()
100            .iter()
101            .enumerate()
102            .find(|(_, column)| column.column_path().parts() == column_path.as_slice())
103        {
104            let row_group_reader = file_reader
105                .get_row_group(ri)
106                .expect("Unable to read row group");
107            if let Some(sbbf) = row_group_reader.get_column_bloom_filter(column_index) {
108                args.values.iter().for_each(|value| {
109                    match check_filter(sbbf, value, column) {
110                        Ok(present) => {
111                            println!(
112                                "Value {} is {} in bloom filter",
113                                value,
114                                if present { "present" } else { "absent" }
115                            )
116                        }
117                        Err(err) => {
118                            println!("{err}");
119                        }
120                    };
121                });
122            } else {
123                println!("No bloom filter found for column {}", args.column);
124            }
125        } else {
126            println!(
127                "No column named {} found, candidate columns are: {}",
128                args.column,
129                row_group
130                    .columns()
131                    .iter()
132                    .map(|c| c.column_path().string())
133                    .collect::<Vec<_>>()
134                    .join(", ")
135            );
136        }
137    }
138}
139
140fn check_filter(sbbf: &Sbbf, value: &String, column: &ColumnChunkMetaData) -> Result<bool, String> {
141    match column.column_type() {
142        Type::INT32 => {
143            let value: i32 = value
144                .parse()
145                .map_err(|e| format!("Unable to parse value '{value}' to i32: {e}"))?;
146            Ok(sbbf.check(&value))
147        }
148        Type::INT64 => {
149            let value: i64 = value
150                .parse()
151                .map_err(|e| format!("Unable to parse value '{value}' to i64: {e}"))?;
152            Ok(sbbf.check(&value))
153        }
154        Type::BYTE_ARRAY => Ok(sbbf.check(&value.as_str())),
155        _ => Err(format!(
156            "Unsupported column type for checking bloom filter: {}",
157            column.column_type()
158        )),
159    }
160}
parquet_show_bloom_filter/parquet-show-bloom-filter.rs

parquet_show_bloom_filter/
parquet-show-bloom-filter.rs
