parquet_show_bloom_filter/
parquet-show-bloom-filter.rs1mod cli;
24
25use clap::Parser;
26use parquet::basic::Type;
27use parquet::bloom_filter::Sbbf;
28use parquet::file::metadata::ColumnChunkMetaData;
29use parquet::file::{
30 properties::ReaderProperties,
31 reader::{FileReader, SerializedFileReader},
32 serialized_reader::ReadOptionsBuilder,
33};
34use std::{fs::File, path::Path};
35
36#[derive(Debug, Parser)]
37#[clap(author, version, about("Binary file to read bloom filter data from a Parquet file"), long_about = None)]
38struct Args {
39 #[clap(help("Path to the parquet file"))]
40 file_name: String,
41 #[clap(help(concat!(
42 "Dot-separated column path to check bloom filter for. Literal dots can be escaped as `\\.`. ",
43 "Only string typed columns or columns with an Int32 or Int64 physical type are supported."
44 )
45 ))]
46 column: String,
47 #[clap(
48 help(
49 "Check if the given values match bloom filter, the values will be parsed to the physical type of the column"
50 ),
51 required = true
52 )]
53 values: Vec<String>,
54}
55
56fn main() {
57 let args = Args::parse();
58 let file_name = args.file_name;
59 let path = Path::new(&file_name);
60 let file = File::open(path).expect("Unable to open file");
61
62 let file_reader = SerializedFileReader::new_with_options(
63 file,
64 ReadOptionsBuilder::new()
65 .with_reader_properties(
66 ReaderProperties::builder()
67 .set_read_bloom_filter(true)
68 .build(),
69 )
70 .build(),
71 )
72 .expect("Unable to open file as Parquet");
73 let metadata = file_reader.metadata();
74 let column_path = match cli::parse_column_path(&args.column) {
75 Ok(path) => path,
76 Err(err) => {
77 eprintln!("{err}");
78 std::process::exit(1);
79 }
80 };
81
82 for (ri, row_group) in metadata.row_groups().iter().enumerate() {
83 println!("Row group #{ri}");
84 println!("{}", "=".repeat(80));
85 if let Some((column_index, column)) = row_group
86 .columns()
87 .iter()
88 .enumerate()
89 .find(|(_, column)| column.column_path().parts() == column_path.as_slice())
90 {
91 let row_group_reader = file_reader
92 .get_row_group(ri)
93 .expect("Unable to read row group");
94 if let Some(sbbf) = row_group_reader.get_column_bloom_filter(column_index) {
95 args.values
96 .iter()
97 .for_each(|value| match check_filter(sbbf, value, column) {
98 Ok(present) => {
99 println!(
100 "Value {} is {} in bloom filter",
101 value,
102 if present { "present" } else { "absent" }
103 )
104 }
105 Err(err) => {
106 println!("{err}");
107 }
108 });
109 } else {
110 println!("No bloom filter found for column {}", args.column);
111 }
112 } else {
113 println!(
114 "No column named {} found, candidate columns are: {}",
115 args.column,
116 row_group
117 .columns()
118 .iter()
119 .map(|c| c.column_path().string())
120 .collect::<Vec<_>>()
121 .join(", ")
122 );
123 }
124 }
125}
126
127fn check_filter(sbbf: &Sbbf, value: &String, column: &ColumnChunkMetaData) -> Result<bool, String> {
128 match column.column_type() {
129 Type::INT32 => {
130 let value: i32 = value
131 .parse()
132 .map_err(|e| format!("Unable to parse value '{value}' to i32: {e}"))?;
133 Ok(sbbf.check(&value))
134 }
135 Type::INT64 => {
136 let value: i64 = value
137 .parse()
138 .map_err(|e| format!("Unable to parse value '{value}' to i64: {e}"))?;
139 Ok(sbbf.check(&value))
140 }
141 Type::BYTE_ARRAY => Ok(sbbf.check(&value.as_str())),
142 _ => Err(format!(
143 "Unsupported column type for checking bloom filter: {}",
144 column.column_type()
145 )),
146 }
147}