parquet_concat/
parquet-concat.rs1use clap::Parser;
27use parquet::bloom_filter::Sbbf;
28use parquet::column::writer::ColumnCloseResult;
29use parquet::errors::{ParquetError, Result};
30use parquet::file::metadata::{ColumnChunkMetaData, PageIndexPolicy, ParquetMetaDataReader};
31use parquet::file::properties::WriterProperties;
32use parquet::file::reader::ChunkReader;
33use parquet::file::writer::SerializedFileWriter;
34use std::fs::File;
35use std::sync::Arc;
36
37#[derive(Debug, Parser)]
38#[clap(author, version)]
39struct Args {
41 output: String,
43
44 input: Vec<String>,
46}
47
48fn read_bloom_filter<R: ChunkReader>(column: &ColumnChunkMetaData, input: &R) -> Option<Sbbf> {
49 Sbbf::read_from_column_chunk(column, input).ok().flatten()
50}
51
52impl Args {
53 fn run(&self) -> Result<()> {
54 if self.input.is_empty() {
55 return Err(ParquetError::General(
56 "Must provide at least one input file".into(),
57 ));
58 }
59
60 let output = File::create(&self.output)?;
61
62 let inputs = self
63 .input
64 .iter()
65 .map(|x| {
66 let reader = File::open(x)?;
67 let metadata = ParquetMetaDataReader::new()
69 .with_page_index_policy(PageIndexPolicy::Optional)
70 .parse_and_finish(&reader)?;
71 Ok((reader, metadata))
72 })
73 .collect::<Result<Vec<_>>>()?;
74
75 let expected = inputs[0].1.file_metadata().schema();
76 for (_, metadata) in inputs.iter().skip(1) {
77 let actual = metadata.file_metadata().schema();
78 if expected != actual {
79 return Err(ParquetError::General(format!(
80 "inputs must have the same schema, {expected:#?} vs {actual:#?}"
81 )));
82 }
83 }
84
85 let props = Arc::new(WriterProperties::builder().build());
86 let schema = inputs[0].1.file_metadata().schema_descr().root_schema_ptr();
87 let mut writer = SerializedFileWriter::new(output, schema, props)?;
88
89 for (input, metadata) in inputs {
90 let page_index = metadata.page_index();
91
92 for (rg_idx, rg) in metadata.row_groups().iter().enumerate() {
93 let mut rg_out = writer.next_row_group()?;
94 for (col_idx, column) in rg.columns().iter().enumerate() {
95 let bloom_filter = read_bloom_filter(column, &input);
96 let column_index = page_index
97 .and_then(|pi| pi.column_index(rg_idx, col_idx))
98 .cloned();
99 let offset_index = page_index
100 .and_then(|pi| pi.offset_index(rg_idx, col_idx))
101 .cloned();
102
103 let result = ColumnCloseResult {
104 bytes_written: column.compressed_size() as _,
105 rows_written: rg.num_rows() as _,
106 metadata: column.clone(),
107 bloom_filter,
108 column_index,
109 offset_index,
110 };
111 rg_out.append_column(&input, result)?;
112 }
113 rg_out.close()?;
114 }
115 }
116
117 writer.close()?;
118
119 Ok(())
120 }
121}
122
123fn main() -> Result<()> {
124 Args::parse().run()
125}