Skip to main content

parquet_concat/
parquet-concat.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Binary that concatenates the column data of one or more parquet files
6//!
7//! # Install
8//!
9//! `parquet-concat` can be installed using `cargo`:
10//! ```
11//! cargo install parquet --features=cli
12//! ```
13//! After this `parquet-concat` should be available:
14//! ```
15//! parquet-concat out.parquet a.parquet b.parquet
16//! ```
17//!
18//! The binary can also be built from the source code and run as follows:
19//! ```
20//! cargo run --features=cli --bin parquet-concat out.parquet a.parquet b.parquet
21//! ```
22//!
23//! Note: this does not currently support preserving the page index or bloom filters
24//!
25
26use clap::Parser;
27use parquet::bloom_filter::Sbbf;
28use parquet::column::writer::ColumnCloseResult;
29use parquet::errors::{ParquetError, Result};
30use parquet::file::metadata::{ColumnChunkMetaData, PageIndexPolicy, ParquetMetaDataReader};
31use parquet::file::properties::WriterProperties;
32use parquet::file::reader::ChunkReader;
33use parquet::file::writer::SerializedFileWriter;
34use std::fs::File;
35use std::sync::Arc;
36
37#[derive(Debug, Parser)]
38#[clap(author, version)]
39/// Concatenates one or more parquet files
40struct Args {
41    /// Path to output
42    output: String,
43
44    /// Path to input files
45    input: Vec<String>,
46}
47
48fn read_bloom_filter<R: ChunkReader>(column: &ColumnChunkMetaData, input: &R) -> Option<Sbbf> {
49    Sbbf::read_from_column_chunk(column, input).ok().flatten()
50}
51
52impl Args {
53    fn run(&self) -> Result<()> {
54        if self.input.is_empty() {
55            return Err(ParquetError::General(
56                "Must provide at least one input file".into(),
57            ));
58        }
59
60        let output = File::create(&self.output)?;
61
62        let inputs = self
63            .input
64            .iter()
65            .map(|x| {
66                let reader = File::open(x)?;
67                // Enable reading page indexes if present
68                let metadata = ParquetMetaDataReader::new()
69                    .with_page_index_policy(PageIndexPolicy::Optional)
70                    .parse_and_finish(&reader)?;
71                Ok((reader, metadata))
72            })
73            .collect::<Result<Vec<_>>>()?;
74
75        let expected = inputs[0].1.file_metadata().schema();
76        for (_, metadata) in inputs.iter().skip(1) {
77            let actual = metadata.file_metadata().schema();
78            if expected != actual {
79                return Err(ParquetError::General(format!(
80                    "inputs must have the same schema, {expected:#?} vs {actual:#?}"
81                )));
82            }
83        }
84
85        let props = Arc::new(WriterProperties::builder().build());
86        let schema = inputs[0].1.file_metadata().schema_descr().root_schema_ptr();
87        let mut writer = SerializedFileWriter::new(output, schema, props)?;
88
89        for (input, metadata) in inputs {
90            let page_index = metadata.page_index();
91
92            for (rg_idx, rg) in metadata.row_groups().iter().enumerate() {
93                let mut rg_out = writer.next_row_group()?;
94                for (col_idx, column) in rg.columns().iter().enumerate() {
95                    let bloom_filter = read_bloom_filter(column, &input);
96                    let column_index = page_index
97                        .and_then(|pi| pi.column_index(rg_idx, col_idx))
98                        .cloned();
99                    let offset_index = page_index
100                        .and_then(|pi| pi.offset_index(rg_idx, col_idx))
101                        .cloned();
102
103                    let result = ColumnCloseResult {
104                        bytes_written: column.compressed_size() as _,
105                        rows_written: rg.num_rows() as _,
106                        metadata: column.clone(),
107                        bloom_filter,
108                        column_index,
109                        offset_index,
110                    };
111                    rg_out.append_column(&input, result)?;
112                }
113                rg_out.close()?;
114            }
115        }
116
117        writer.close()?;
118
119        Ok(())
120    }
121}
122
123fn main() -> Result<()> {
124    Args::parse().run()
125}