Skip to main content

parquet_show_bloom_filter/
parquet-show-bloom-filter.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Binary file to read bloom filter data from a Parquet file.
6//!
7//! # Install
8//!
9//! `parquet-show-bloom-filter` can be installed using `cargo`:
10//! ```
11//! cargo install parquet --features=cli
12//! ```
13//! After this `parquet-show-bloom-filter` should be available:
14//! ```
15//! parquet-show-bloom-filter XYZ.parquet id a
16//! ```
17//!
18//! The binary can also be built from the source code and run as follows:
19//! ```
20//! cargo run --features=cli --bin parquet-show-bloom-filter -- --file-name XYZ.parquet --column id --values a
21//! ```
22
23mod cli;
24
25use clap::Parser;
26use parquet::basic::Type;
27use parquet::bloom_filter::Sbbf;
28use parquet::file::metadata::ColumnChunkMetaData;
29use parquet::file::{
30    properties::ReaderProperties,
31    reader::{FileReader, SerializedFileReader},
32    serialized_reader::ReadOptionsBuilder,
33};
34use std::{fs::File, path::Path};
35
36#[derive(Debug, Parser)]
37#[clap(author, version, about("Binary file to read bloom filter data from a Parquet file"), long_about = None)]
38struct Args {
39    #[clap(help("Path to the parquet file"))]
40    file_name: String,
41    #[clap(help(concat!(
42        "Dot-separated column path to check bloom filter for. Literal dots can be escaped as `\\.`. ",
43        "Only string typed columns or columns with an Int32 or Int64 physical type are supported."
44    )
45    ))]
46    column: String,
47    #[clap(
48        help(
49            "Check if the given values match bloom filter, the values will be parsed to the physical type of the column"
50        ),
51        required = true
52    )]
53    values: Vec<String>,
54}
55
56fn main() {
57    let args = Args::parse();
58    let file_name = args.file_name;
59    let path = Path::new(&file_name);
60    let file = File::open(path).expect("Unable to open file");
61
62    let file_reader = SerializedFileReader::new_with_options(
63        file,
64        ReadOptionsBuilder::new()
65            .with_reader_properties(
66                ReaderProperties::builder()
67                    .set_read_bloom_filter(true)
68                    .build(),
69            )
70            .build(),
71    )
72    .expect("Unable to open file as Parquet");
73    let metadata = file_reader.metadata();
74    let column_path = match cli::parse_column_path(&args.column) {
75        Ok(path) => path,
76        Err(err) => {
77            eprintln!("{err}");
78            std::process::exit(1);
79        }
80    };
81
82    for (ri, row_group) in metadata.row_groups().iter().enumerate() {
83        println!("Row group #{ri}");
84        println!("{}", "=".repeat(80));
85        if let Some((column_index, column)) = row_group
86            .columns()
87            .iter()
88            .enumerate()
89            .find(|(_, column)| column.column_path().parts() == column_path.as_slice())
90        {
91            let row_group_reader = file_reader
92                .get_row_group(ri)
93                .expect("Unable to read row group");
94            if let Some(sbbf) = row_group_reader.get_column_bloom_filter(column_index) {
95                args.values
96                    .iter()
97                    .for_each(|value| match check_filter(sbbf, value, column) {
98                        Ok(present) => {
99                            println!(
100                                "Value {} is {} in bloom filter",
101                                value,
102                                if present { "present" } else { "absent" }
103                            )
104                        }
105                        Err(err) => {
106                            println!("{err}");
107                        }
108                    });
109            } else {
110                println!("No bloom filter found for column {}", args.column);
111            }
112        } else {
113            println!(
114                "No column named {} found, candidate columns are: {}",
115                args.column,
116                row_group
117                    .columns()
118                    .iter()
119                    .map(|c| c.column_path().string())
120                    .collect::<Vec<_>>()
121                    .join(", ")
122            );
123        }
124    }
125}
126
127fn check_filter(sbbf: &Sbbf, value: &String, column: &ColumnChunkMetaData) -> Result<bool, String> {
128    match column.column_type() {
129        Type::INT32 => {
130            let value: i32 = value
131                .parse()
132                .map_err(|e| format!("Unable to parse value '{value}' to i32: {e}"))?;
133            Ok(sbbf.check(&value))
134        }
135        Type::INT64 => {
136            let value: i64 = value
137                .parse()
138                .map_err(|e| format!("Unable to parse value '{value}' to i64: {e}"))?;
139            Ok(sbbf.check(&value))
140        }
141        Type::BYTE_ARRAY => Ok(sbbf.check(&value.as_str())),
142        _ => Err(format!(
143            "Unsupported column type for checking bloom filter: {}",
144            column.column_type()
145        )),
146    }
147}