Skip to main content

parquet_rewrite/
parquet-rewrite.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Binary file to rewrite parquet files.
6//!
7//! # Install
8//!
9//! `parquet-rewrite` can be installed using `cargo`:
10//! ```
11//! cargo install parquet --features=cli
12//! ```
13//! After this `parquet-rewrite` should be available:
14//! ```
15//! parquet-rewrite -i XYZ.parquet -o XYZ2.parquet
16//! ```
17//!
18//! The binary can also be built from the source code and run as follows:
19//! ```
20//! cargo run --features=cli --bin parquet-rewrite -- -i XYZ.parquet -o XYZ2.parquet
21//! ```
22
23use std::fs::File;
24
25use arrow_array::RecordBatchReader;
26use clap::{Parser, ValueEnum, builder::PossibleValue};
27use parquet::{
28    arrow::{ArrowWriter, arrow_reader::ParquetRecordBatchReaderBuilder},
29    basic::{BrotliLevel, Compression, Encoding, GzipLevel, ZstdLevel},
30    file::{
31        properties::{BloomFilterPosition, EnabledStatistics, WriterProperties, WriterVersion},
32        reader::FileReader,
33        serialized_reader::SerializedFileReader,
34    },
35};
36
37#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum, Debug)]
38enum CompressionArgs {
39    /// No compression.
40    None,
41
42    /// Snappy
43    Snappy,
44
45    /// GZip
46    Gzip,
47
48    /// LZO
49    Lzo,
50
51    /// Brotli
52    Brotli,
53
54    /// LZ4
55    Lz4,
56
57    /// Zstd
58    Zstd,
59
60    /// LZ4 Raw
61    Lz4Raw,
62}
63
64fn compression_from_args(codec: CompressionArgs, level: Option<u32>) -> Compression {
65    match codec {
66        CompressionArgs::None => Compression::UNCOMPRESSED,
67        CompressionArgs::Snappy => Compression::SNAPPY,
68        CompressionArgs::Gzip => match level {
69            Some(lvl) => {
70                Compression::GZIP(GzipLevel::try_new(lvl).expect("invalid gzip compression level"))
71            }
72            None => Compression::GZIP(Default::default()),
73        },
74        CompressionArgs::Lzo => Compression::LZO,
75        CompressionArgs::Brotli => match level {
76            Some(lvl) => Compression::BROTLI(
77                BrotliLevel::try_new(lvl).expect("invalid brotli compression level"),
78            ),
79            None => Compression::BROTLI(Default::default()),
80        },
81        CompressionArgs::Lz4 => Compression::LZ4,
82        CompressionArgs::Zstd => match level {
83            Some(lvl) => Compression::ZSTD(
84                ZstdLevel::try_new(lvl as i32).expect("invalid zstd compression level"),
85            ),
86            None => Compression::ZSTD(Default::default()),
87        },
88        CompressionArgs::Lz4Raw => Compression::LZ4_RAW,
89    }
90}
91
92#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum, Debug)]
93enum EncodingArgs {
94    /// Default byte encoding.
95    Plain,
96
97    /// **Deprecated** dictionary encoding.
98    PlainDictionary,
99
100    /// Group packed run length encoding.
101    Rle,
102
103    /// **Deprecated** Bit-packed encoding.
104    BitPacked,
105
106    /// Delta encoding for integers, either INT32 or INT64.
107    DeltaBinaryPacked,
108
109    /// Encoding for byte arrays to separate the length values and the data.
110    DeltaLengthByteArray,
111
112    /// Incremental encoding for byte arrays.
113    DeltaByteArray,
114
115    /// Dictionary encoding.
116    RleDictionary,
117
118    /// Encoding for fixed-width data.
119    ByteStreamSplit,
120
121    /// Adaptive Lossless floating-Point encoding for FLOAT and DOUBLE.
122    Alp,
123}
124
125#[expect(deprecated)]
126impl From<EncodingArgs> for Encoding {
127    fn from(value: EncodingArgs) -> Self {
128        match value {
129            EncodingArgs::Plain => Self::PLAIN,
130            EncodingArgs::PlainDictionary => Self::PLAIN_DICTIONARY,
131            EncodingArgs::Rle => Self::RLE,
132            EncodingArgs::BitPacked => Self::BIT_PACKED,
133            EncodingArgs::DeltaBinaryPacked => Self::DELTA_BINARY_PACKED,
134            EncodingArgs::DeltaLengthByteArray => Self::DELTA_LENGTH_BYTE_ARRAY,
135            EncodingArgs::DeltaByteArray => Self::DELTA_BYTE_ARRAY,
136            EncodingArgs::RleDictionary => Self::RLE_DICTIONARY,
137            EncodingArgs::ByteStreamSplit => Self::BYTE_STREAM_SPLIT,
138            EncodingArgs::Alp => Self::ALP,
139        }
140    }
141}
142
143#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum, Debug)]
144enum EnabledStatisticsArgs {
145    /// Compute no statistics
146    None,
147
148    /// Compute chunk-level statistics but not page-level
149    Chunk,
150
151    /// Compute page-level and chunk-level statistics
152    Page,
153}
154
155impl From<EnabledStatisticsArgs> for EnabledStatistics {
156    fn from(value: EnabledStatisticsArgs) -> Self {
157        match value {
158            EnabledStatisticsArgs::None => Self::None,
159            EnabledStatisticsArgs::Chunk => Self::Chunk,
160            EnabledStatisticsArgs::Page => Self::Page,
161        }
162    }
163}
164
165#[derive(Clone, Copy, Debug)]
166enum WriterVersionArgs {
167    Parquet1_0,
168    Parquet2_0,
169}
170
171impl ValueEnum for WriterVersionArgs {
172    fn value_variants<'a>() -> &'a [Self] {
173        &[Self::Parquet1_0, Self::Parquet2_0]
174    }
175
176    fn to_possible_value(&self) -> Option<PossibleValue> {
177        match self {
178            WriterVersionArgs::Parquet1_0 => Some(PossibleValue::new("1.0")),
179            WriterVersionArgs::Parquet2_0 => Some(PossibleValue::new("2.0")),
180        }
181    }
182}
183
184impl From<WriterVersionArgs> for WriterVersion {
185    fn from(value: WriterVersionArgs) -> Self {
186        match value {
187            WriterVersionArgs::Parquet1_0 => Self::PARQUET_1_0,
188            WriterVersionArgs::Parquet2_0 => Self::PARQUET_2_0,
189        }
190    }
191}
192
193#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum, Debug)]
194enum BloomFilterPositionArgs {
195    /// Write Bloom Filters of each row group right after the row group
196    AfterRowGroup,
197
198    /// Write Bloom Filters at the end of the file
199    End,
200}
201
202impl From<BloomFilterPositionArgs> for BloomFilterPosition {
203    fn from(value: BloomFilterPositionArgs) -> Self {
204        match value {
205            BloomFilterPositionArgs::AfterRowGroup => Self::AfterRowGroup,
206            BloomFilterPositionArgs::End => Self::End,
207        }
208    }
209}
210
211#[derive(Debug, Parser)]
212#[clap(author, version, about("Read and write parquet file with potentially different settings"), long_about = None)]
213struct Args {
214    /// Path to input parquet file.
215    #[clap(short, long)]
216    input: String,
217
218    /// Path to output parquet file.
219    #[clap(short, long)]
220    output: String,
221
222    /// Compression used for all columns.
223    #[clap(long, value_enum)]
224    compression: Option<CompressionArgs>,
225
226    /// Compression level for gzip/brotli/zstd.
227    #[clap(long)]
228    compression_level: Option<u32>,
229
230    /// Encoding used for all columns, if dictionary is not enabled.
231    #[clap(long, value_enum)]
232    encoding: Option<EncodingArgs>,
233
234    /// Sets flag to enable/disable dictionary encoding for all columns.
235    #[clap(long)]
236    dictionary_enabled: Option<bool>,
237
238    /// Sets best effort maximum dictionary page size, in bytes.
239    #[clap(long)]
240    dictionary_page_size_limit: Option<usize>,
241
242    /// Sets maximum number of rows in a row group.
243    #[clap(long)]
244    max_row_group_size: Option<usize>,
245
246    /// Sets best effort maximum number of rows in a data page.
247    #[clap(long)]
248    data_page_row_count_limit: Option<usize>,
249
250    /// Sets best effort maximum size of a data page in bytes.
251    #[clap(long)]
252    data_page_size_limit: Option<usize>,
253
254    /// Sets the max length of min/max statistics in row group and data page
255    /// header statistics for all columns.
256    ///
257    /// Applicable only if statistics are enabled.
258    #[clap(long)]
259    statistics_truncate_length: Option<usize>,
260
261    /// Sets the max length of min/max statistics in the column index.
262    ///
263    /// Applicable only if statistics are enabled.
264    #[clap(long)]
265    column_index_truncate_length: Option<usize>,
266
267    /// Write statistics to the data page headers?
268    ///
269    /// Setting this true will also enable page level statistics.
270    #[clap(long)]
271    write_page_header_statistics: Option<bool>,
272
273    /// Write path_in_schema to the column metadata.
274    #[clap(long)]
275    write_path_in_schema: Option<bool>,
276
277    /// Sets whether bloom filter is enabled for all columns.
278    #[clap(long)]
279    bloom_filter_enabled: Option<bool>,
280
281    /// Sets bloom filter false positive probability (fpp) for all columns.
282    #[clap(long)]
283    bloom_filter_fpp: Option<f64>,
284
285    /// Sets number of distinct values (ndv) for bloom filter for all columns.
286    #[clap(long)]
287    bloom_filter_ndv: Option<u64>,
288
289    /// Sets the position of bloom filter
290    #[clap(long)]
291    bloom_filter_position: Option<BloomFilterPositionArgs>,
292
293    /// Sets flag to enable/disable statistics for all columns.
294    #[clap(long)]
295    statistics_enabled: Option<EnabledStatisticsArgs>,
296
297    /// Sets writer version.
298    #[clap(long)]
299    writer_version: Option<WriterVersionArgs>,
300
301    /// Sets write batch size.
302    #[clap(long)]
303    write_batch_size: Option<usize>,
304
305    /// Sets whether to coerce Arrow types to match Parquet specification
306    #[clap(long)]
307    coerce_types: Option<bool>,
308}
309
310fn main() {
311    let args = Args::parse();
312
313    // read key-value metadata
314    let parquet_reader =
315        SerializedFileReader::new(File::open(&args.input).expect("Unable to open input file"))
316            .expect("Failed to create reader");
317    let kv_md = parquet_reader
318        .metadata()
319        .file_metadata()
320        .key_value_metadata()
321        .cloned();
322
323    // create actual parquet reader
324    let parquet_reader = ParquetRecordBatchReaderBuilder::try_new(
325        File::open(args.input).expect("Unable to open input file"),
326    )
327    .expect("parquet open")
328    .build()
329    .expect("parquet open");
330
331    let mut writer_properties_builder = WriterProperties::builder().set_key_value_metadata(kv_md);
332
333    if let Some(value) = args.compression {
334        let compression = compression_from_args(value, args.compression_level);
335        writer_properties_builder = writer_properties_builder.set_compression(compression);
336    }
337
338    // setup encoding
339    if let Some(value) = args.encoding {
340        writer_properties_builder = writer_properties_builder.set_encoding(value.into());
341    }
342    if let Some(value) = args.dictionary_enabled {
343        writer_properties_builder = writer_properties_builder.set_dictionary_enabled(value);
344    }
345    if let Some(value) = args.dictionary_page_size_limit {
346        writer_properties_builder = writer_properties_builder.set_dictionary_page_size_limit(value);
347    }
348
349    if let Some(value) = args.max_row_group_size {
350        writer_properties_builder =
351            writer_properties_builder.set_max_row_group_row_count(Some(value));
352    }
353    if let Some(value) = args.data_page_row_count_limit {
354        writer_properties_builder = writer_properties_builder.set_data_page_row_count_limit(value);
355    }
356    if let Some(value) = args.data_page_size_limit {
357        writer_properties_builder = writer_properties_builder.set_data_page_size_limit(value);
358    }
359    if let Some(value) = args.dictionary_page_size_limit {
360        writer_properties_builder = writer_properties_builder.set_dictionary_page_size_limit(value);
361    }
362    if let Some(value) = args.statistics_truncate_length {
363        writer_properties_builder =
364            writer_properties_builder.set_statistics_truncate_length(Some(value));
365    }
366    if let Some(value) = args.column_index_truncate_length {
367        writer_properties_builder =
368            writer_properties_builder.set_column_index_truncate_length(Some(value));
369    }
370    if let Some(value) = args.bloom_filter_enabled {
371        writer_properties_builder = writer_properties_builder.set_bloom_filter_enabled(value);
372
373        if value {
374            if let Some(value) = args.bloom_filter_fpp {
375                writer_properties_builder = writer_properties_builder.set_bloom_filter_fpp(value);
376            }
377            if let Some(value) = args.bloom_filter_ndv {
378                writer_properties_builder =
379                    writer_properties_builder.set_bloom_filter_max_ndv(value);
380            }
381            if let Some(value) = args.bloom_filter_position {
382                writer_properties_builder =
383                    writer_properties_builder.set_bloom_filter_position(value.into());
384            }
385        }
386    }
387    if let Some(value) = args.statistics_enabled {
388        writer_properties_builder = writer_properties_builder.set_statistics_enabled(value.into());
389    }
390    // set this after statistics_enabled
391    if let Some(value) = args.write_page_header_statistics {
392        writer_properties_builder =
393            writer_properties_builder.set_write_page_header_statistics(value);
394        if value {
395            writer_properties_builder =
396                writer_properties_builder.set_statistics_enabled(EnabledStatistics::Page);
397        }
398    }
399    if let Some(value) = args.writer_version {
400        writer_properties_builder = writer_properties_builder.set_writer_version(value.into());
401    }
402    if let Some(value) = args.coerce_types {
403        writer_properties_builder = writer_properties_builder.set_coerce_types(value);
404    }
405    if let Some(value) = args.write_path_in_schema {
406        writer_properties_builder = writer_properties_builder.set_write_path_in_schema(value);
407    }
408    if let Some(value) = args.write_batch_size {
409        writer_properties_builder = writer_properties_builder.set_write_batch_size(value);
410    }
411    let writer_properties = writer_properties_builder.build();
412    let mut parquet_writer = ArrowWriter::try_new(
413        File::create(&args.output).expect("Unable to open output file"),
414        parquet_reader.schema(),
415        Some(writer_properties),
416    )
417    .expect("create arrow writer");
418
419    for maybe_batch in parquet_reader {
420        let batch = maybe_batch.expect("reading batch");
421        parquet_writer.write(&batch).expect("writing data");
422    }
423
424    parquet_writer.close().expect("finalizing file");
425}