parquet/file/
properties.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Configuration via [`WriterProperties`] and [`ReaderProperties`]
19use crate::basic::{Compression, Encoding};
20use crate::compression::{CodecOptions, CodecOptionsBuilder};
21#[cfg(feature = "encryption")]
22use crate::encryption::encrypt::FileEncryptionProperties;
23use crate::file::metadata::KeyValue;
24use crate::format::SortingColumn;
25use crate::schema::types::ColumnPath;
26use std::str::FromStr;
27use std::{collections::HashMap, sync::Arc};
28
29/// Default value for [`WriterProperties::data_page_size_limit`]
30pub const DEFAULT_PAGE_SIZE: usize = 1024 * 1024;
31/// Default value for [`WriterProperties::write_batch_size`]
32pub const DEFAULT_WRITE_BATCH_SIZE: usize = 1024;
33/// Default value for [`WriterProperties::writer_version`]
34pub const DEFAULT_WRITER_VERSION: WriterVersion = WriterVersion::PARQUET_1_0;
35/// Default value for [`WriterProperties::compression`]
36pub const DEFAULT_COMPRESSION: Compression = Compression::UNCOMPRESSED;
37/// Default value for [`WriterProperties::dictionary_enabled`]
38pub const DEFAULT_DICTIONARY_ENABLED: bool = true;
39/// Default value for [`WriterProperties::dictionary_page_size_limit`]
40pub const DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT: usize = DEFAULT_PAGE_SIZE;
41/// Default value for [`WriterProperties::data_page_row_count_limit`]
42pub const DEFAULT_DATA_PAGE_ROW_COUNT_LIMIT: usize = 20_000;
43/// Default value for [`WriterProperties::statistics_enabled`]
44pub const DEFAULT_STATISTICS_ENABLED: EnabledStatistics = EnabledStatistics::Page;
45/// Default value for [`WriterProperties::write_page_header_statistics`]
46pub const DEFAULT_WRITE_PAGE_HEADER_STATISTICS: bool = false;
47/// Default value for [`WriterProperties::max_row_group_size`]
48pub const DEFAULT_MAX_ROW_GROUP_SIZE: usize = 1024 * 1024;
49/// Default value for [`WriterProperties::bloom_filter_position`]
50pub const DEFAULT_BLOOM_FILTER_POSITION: BloomFilterPosition = BloomFilterPosition::AfterRowGroup;
51/// Default value for [`WriterProperties::created_by`]
52pub const DEFAULT_CREATED_BY: &str = concat!("parquet-rs version ", env!("CARGO_PKG_VERSION"));
53/// Default value for [`WriterProperties::column_index_truncate_length`]
54pub const DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH: Option<usize> = Some(64);
55/// Default value for [`BloomFilterProperties::fpp`]
56pub const DEFAULT_BLOOM_FILTER_FPP: f64 = 0.05;
57/// Default value for [`BloomFilterProperties::ndv`]
58pub const DEFAULT_BLOOM_FILTER_NDV: u64 = 1_000_000_u64;
59/// Default values for [`WriterProperties::statistics_truncate_length`]
60pub const DEFAULT_STATISTICS_TRUNCATE_LENGTH: Option<usize> = Some(64);
61/// Default value for [`WriterProperties::offset_index_disabled`]
62pub const DEFAULT_OFFSET_INDEX_DISABLED: bool = false;
63/// Default values for [`WriterProperties::coerce_types`]
64pub const DEFAULT_COERCE_TYPES: bool = false;
65
66/// Parquet writer version.
67///
68/// Basic constant, which is not part of the Thrift definition.
69#[derive(Debug, Clone, Copy, PartialEq, Eq)]
70#[allow(non_camel_case_types)]
71pub enum WriterVersion {
72    /// Parquet format version 1.0
73    PARQUET_1_0,
74    /// Parquet format version 2.0
75    PARQUET_2_0,
76}
77
78impl WriterVersion {
79    /// Returns writer version as `i32`.
80    pub fn as_num(&self) -> i32 {
81        match self {
82            WriterVersion::PARQUET_1_0 => 1,
83            WriterVersion::PARQUET_2_0 => 2,
84        }
85    }
86}
87
88impl FromStr for WriterVersion {
89    type Err = String;
90
91    fn from_str(s: &str) -> Result<Self, Self::Err> {
92        match s {
93            "PARQUET_1_0" | "parquet_1_0" => Ok(WriterVersion::PARQUET_1_0),
94            "PARQUET_2_0" | "parquet_2_0" => Ok(WriterVersion::PARQUET_2_0),
95            _ => Err(format!("Invalid writer version: {s}")),
96        }
97    }
98}
99
100/// Where in the file [`ArrowWriter`](crate::arrow::arrow_writer::ArrowWriter) should
101/// write Bloom filters
102///
103/// Basic constant, which is not part of the Thrift definition.
104#[derive(Debug, Clone, Copy, PartialEq, Eq)]
105pub enum BloomFilterPosition {
106    /// Write Bloom Filters of each row group right after the row group
107    ///
108    /// This saves memory by writing it as soon as it is computed, at the cost
109    /// of data locality for readers
110    AfterRowGroup,
111    /// Write Bloom Filters at the end of the file
112    ///
113    /// This allows better data locality for readers, at the cost of memory usage
114    /// for writers.
115    End,
116}
117
118/// Reference counted writer properties.
119pub type WriterPropertiesPtr = Arc<WriterProperties>;
120
121/// Configuration settings for writing parquet files.
122///
123/// Use [`Self::builder`] to create a [`WriterPropertiesBuilder`] to change settings.
124///
125/// # Example
126///
127/// ```rust
128/// # use parquet::{
129/// #    basic::{Compression, Encoding},
130/// #    file::properties::*,
131/// #    schema::types::ColumnPath,
132/// # };
133/// #
134/// // Create properties with default configuration.
135/// let props = WriterProperties::default();
136///
137/// // Use properties builder to set certain options and assemble the configuration.
138/// let props = WriterProperties::builder()
139///     .set_writer_version(WriterVersion::PARQUET_1_0)
140///     .set_encoding(Encoding::PLAIN)
141///     .set_column_encoding(ColumnPath::from("col1"), Encoding::DELTA_BINARY_PACKED)
142///     .set_compression(Compression::SNAPPY)
143///     .build();
144///
145/// assert_eq!(props.writer_version(), WriterVersion::PARQUET_1_0);
146/// assert_eq!(
147///     props.encoding(&ColumnPath::from("col1")),
148///     Some(Encoding::DELTA_BINARY_PACKED)
149/// );
150/// assert_eq!(
151///     props.encoding(&ColumnPath::from("col2")),
152///     Some(Encoding::PLAIN)
153/// );
154/// ```
155#[derive(Debug, Clone)]
156pub struct WriterProperties {
157    data_page_size_limit: usize,
158    data_page_row_count_limit: usize,
159    write_batch_size: usize,
160    max_row_group_size: usize,
161    bloom_filter_position: BloomFilterPosition,
162    writer_version: WriterVersion,
163    created_by: String,
164    offset_index_disabled: bool,
165    pub(crate) key_value_metadata: Option<Vec<KeyValue>>,
166    default_column_properties: ColumnProperties,
167    column_properties: HashMap<ColumnPath, ColumnProperties>,
168    sorting_columns: Option<Vec<SortingColumn>>,
169    column_index_truncate_length: Option<usize>,
170    statistics_truncate_length: Option<usize>,
171    coerce_types: bool,
172    #[cfg(feature = "encryption")]
173    pub(crate) file_encryption_properties: Option<FileEncryptionProperties>,
174}
175
176impl Default for WriterProperties {
177    fn default() -> Self {
178        Self::builder().build()
179    }
180}
181
182impl WriterProperties {
183    /// Create a new [`WriterProperties`] with the default settings
184    ///
185    /// See [`WriterProperties::builder`] for customising settings
186    pub fn new() -> Self {
187        Self::default()
188    }
189
190    /// Returns a new default [`WriterPropertiesBuilder`] for creating writer
191    /// properties.
192    pub fn builder() -> WriterPropertiesBuilder {
193        WriterPropertiesBuilder::with_defaults()
194    }
195
196    /// Returns data page size limit.
197    ///
198    /// Note: this is a best effort limit based on the write batch size
199    ///
200    /// For more details see [`WriterPropertiesBuilder::set_data_page_size_limit`]
201    pub fn data_page_size_limit(&self) -> usize {
202        self.data_page_size_limit
203    }
204
205    /// Returns dictionary page size limit.
206    ///
207    /// Note: this is a best effort limit based on the write batch size
208    ///
209    /// For more details see [`WriterPropertiesBuilder::set_dictionary_page_size_limit`]
210    pub fn dictionary_page_size_limit(&self) -> usize {
211        self.default_column_properties
212            .dictionary_page_size_limit()
213            .unwrap_or(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT)
214    }
215
216    /// Returns dictionary page size limit for a specific column.
217    pub fn column_dictionary_page_size_limit(&self, col: &ColumnPath) -> usize {
218        self.column_properties
219            .get(col)
220            .and_then(|c| c.dictionary_page_size_limit())
221            .or_else(|| self.default_column_properties.dictionary_page_size_limit())
222            .unwrap_or(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT)
223    }
224
225    /// Returns the maximum page row count
226    ///
227    /// Note: this is a best effort limit based on the write batch size
228    ///
229    /// For more details see [`WriterPropertiesBuilder::set_data_page_row_count_limit`]
230    pub fn data_page_row_count_limit(&self) -> usize {
231        self.data_page_row_count_limit
232    }
233
234    /// Returns configured batch size for writes.
235    ///
236    /// When writing a batch of data, this setting allows to split it internally into
237    /// smaller batches so we can better estimate the size of a page currently being
238    /// written.
239    ///
240    /// For more details see [`WriterPropertiesBuilder::set_write_batch_size`]
241    pub fn write_batch_size(&self) -> usize {
242        self.write_batch_size
243    }
244
245    /// Returns maximum number of rows in a row group.
246    ///
247    /// For more details see [`WriterPropertiesBuilder::set_max_row_group_size`]
248    pub fn max_row_group_size(&self) -> usize {
249        self.max_row_group_size
250    }
251
252    /// Returns bloom filter position.
253    ///
254    /// For more details see [`WriterPropertiesBuilder::set_bloom_filter_position`]
255    pub fn bloom_filter_position(&self) -> BloomFilterPosition {
256        self.bloom_filter_position
257    }
258
259    /// Returns configured writer version.
260    ///
261    /// For more details see [`WriterPropertiesBuilder::set_writer_version`]
262    pub fn writer_version(&self) -> WriterVersion {
263        self.writer_version
264    }
265
266    /// Returns `created_by` string.
267    ///
268    /// For more details see [`WriterPropertiesBuilder::set_created_by`]
269    pub fn created_by(&self) -> &str {
270        &self.created_by
271    }
272
273    /// Returns `true` if offset index writing is disabled.
274    ///
275    /// For more details see [`WriterPropertiesBuilder::set_offset_index_disabled`]
276    pub fn offset_index_disabled(&self) -> bool {
277        // If page statistics are to be collected, then do not disable the offset indexes.
278        let default_page_stats_enabled =
279            self.default_column_properties.statistics_enabled() == Some(EnabledStatistics::Page);
280        let column_page_stats_enabled = self
281            .column_properties
282            .iter()
283            .any(|path_props| path_props.1.statistics_enabled() == Some(EnabledStatistics::Page));
284        if default_page_stats_enabled || column_page_stats_enabled {
285            return false;
286        }
287
288        self.offset_index_disabled
289    }
290
291    /// Returns `key_value_metadata` KeyValue pairs.
292    ///
293    /// For more details see [`WriterPropertiesBuilder::set_key_value_metadata`]
294    pub fn key_value_metadata(&self) -> Option<&Vec<KeyValue>> {
295        self.key_value_metadata.as_ref()
296    }
297
298    /// Returns sorting columns.
299    ///
300    /// For more details see [`WriterPropertiesBuilder::set_sorting_columns`]
301    pub fn sorting_columns(&self) -> Option<&Vec<SortingColumn>> {
302        self.sorting_columns.as_ref()
303    }
304
305    /// Returns the maximum length of truncated min/max values in the column index.
306    ///
307    /// `None` if truncation is disabled, must be greater than 0 otherwise.
308    ///
309    /// For more details see [`WriterPropertiesBuilder::set_column_index_truncate_length`]
310    pub fn column_index_truncate_length(&self) -> Option<usize> {
311        self.column_index_truncate_length
312    }
313
314    /// Returns the maximum length of truncated min/max values in [`Statistics`].
315    ///
316    /// `None` if truncation is disabled, must be greater than 0 otherwise.
317    ///
318    /// For more details see [`WriterPropertiesBuilder::set_statistics_truncate_length`]
319    ///
320    /// [`Statistics`]: crate::file::statistics::Statistics
321    pub fn statistics_truncate_length(&self) -> Option<usize> {
322        self.statistics_truncate_length
323    }
324
325    /// Returns `true` if type coercion is enabled.
326    ///
327    /// For more details see [`WriterPropertiesBuilder::set_coerce_types`]
328    pub fn coerce_types(&self) -> bool {
329        self.coerce_types
330    }
331
332    /// Returns encoding for a data page, when dictionary encoding is enabled.
333    ///
334    /// This is not configurable.
335    #[inline]
336    pub fn dictionary_data_page_encoding(&self) -> Encoding {
337        // PLAIN_DICTIONARY encoding is deprecated in writer version 1.
338        // Dictionary values are encoded using RLE_DICTIONARY encoding.
339        Encoding::RLE_DICTIONARY
340    }
341
342    /// Returns encoding for dictionary page, when dictionary encoding is enabled.
343    ///
344    /// This is not configurable.
345    #[inline]
346    pub fn dictionary_page_encoding(&self) -> Encoding {
347        // PLAIN_DICTIONARY is deprecated in writer version 1.
348        // Dictionary is encoded using plain encoding.
349        Encoding::PLAIN
350    }
351
352    /// Returns encoding for a column, if set.
353    ///
354    /// In case when dictionary is enabled, returns fallback encoding.
355    ///
356    /// If encoding is not set, then column writer will choose the best encoding
357    /// based on the column type.
358    pub fn encoding(&self, col: &ColumnPath) -> Option<Encoding> {
359        self.column_properties
360            .get(col)
361            .and_then(|c| c.encoding())
362            .or_else(|| self.default_column_properties.encoding())
363    }
364
365    /// Returns compression codec for a column.
366    ///
367    /// For more details see [`WriterPropertiesBuilder::set_column_compression`]
368    pub fn compression(&self, col: &ColumnPath) -> Compression {
369        self.column_properties
370            .get(col)
371            .and_then(|c| c.compression())
372            .or_else(|| self.default_column_properties.compression())
373            .unwrap_or(DEFAULT_COMPRESSION)
374    }
375
376    /// Returns `true` if dictionary encoding is enabled for a column.
377    ///
378    /// For more details see [`WriterPropertiesBuilder::set_dictionary_enabled`]
379    pub fn dictionary_enabled(&self, col: &ColumnPath) -> bool {
380        self.column_properties
381            .get(col)
382            .and_then(|c| c.dictionary_enabled())
383            .or_else(|| self.default_column_properties.dictionary_enabled())
384            .unwrap_or(DEFAULT_DICTIONARY_ENABLED)
385    }
386
387    /// Returns which statistics are written for a column.
388    ///
389    /// For more details see [`WriterPropertiesBuilder::set_statistics_enabled`]
390    pub fn statistics_enabled(&self, col: &ColumnPath) -> EnabledStatistics {
391        self.column_properties
392            .get(col)
393            .and_then(|c| c.statistics_enabled())
394            .or_else(|| self.default_column_properties.statistics_enabled())
395            .unwrap_or(DEFAULT_STATISTICS_ENABLED)
396    }
397
398    /// Returns `true` if [`Statistics`] are to be written to the page header for a column.
399    ///
400    /// For more details see [`WriterPropertiesBuilder::set_write_page_header_statistics`]
401    ///
402    /// [`Statistics`]: crate::file::statistics::Statistics
403    pub fn write_page_header_statistics(&self, col: &ColumnPath) -> bool {
404        self.column_properties
405            .get(col)
406            .and_then(|c| c.write_page_header_statistics())
407            .or_else(|| {
408                self.default_column_properties
409                    .write_page_header_statistics()
410            })
411            .unwrap_or(DEFAULT_WRITE_PAGE_HEADER_STATISTICS)
412    }
413
414    /// Returns the [`BloomFilterProperties`] for the given column
415    ///
416    /// Returns `None` if bloom filter is disabled
417    ///
418    /// For more details see [`WriterPropertiesBuilder::set_column_bloom_filter_enabled`]
419    pub fn bloom_filter_properties(&self, col: &ColumnPath) -> Option<&BloomFilterProperties> {
420        self.column_properties
421            .get(col)
422            .and_then(|c| c.bloom_filter_properties())
423            .or_else(|| self.default_column_properties.bloom_filter_properties())
424    }
425
426    /// Return file encryption properties
427    ///
428    /// For more details see [`WriterPropertiesBuilder::with_file_encryption_properties`]
429    #[cfg(feature = "encryption")]
430    pub fn file_encryption_properties(&self) -> Option<&FileEncryptionProperties> {
431        self.file_encryption_properties.as_ref()
432    }
433}
434
435/// Builder for  [`WriterProperties`] Parquet writer configuration.
436///
437/// See example on [`WriterProperties`]
438pub struct WriterPropertiesBuilder {
439    data_page_size_limit: usize,
440    data_page_row_count_limit: usize,
441    write_batch_size: usize,
442    max_row_group_size: usize,
443    bloom_filter_position: BloomFilterPosition,
444    writer_version: WriterVersion,
445    created_by: String,
446    offset_index_disabled: bool,
447    key_value_metadata: Option<Vec<KeyValue>>,
448    default_column_properties: ColumnProperties,
449    column_properties: HashMap<ColumnPath, ColumnProperties>,
450    sorting_columns: Option<Vec<SortingColumn>>,
451    column_index_truncate_length: Option<usize>,
452    statistics_truncate_length: Option<usize>,
453    coerce_types: bool,
454    #[cfg(feature = "encryption")]
455    file_encryption_properties: Option<FileEncryptionProperties>,
456}
457
458impl WriterPropertiesBuilder {
459    /// Returns default state of the builder.
460    fn with_defaults() -> Self {
461        Self {
462            data_page_size_limit: DEFAULT_PAGE_SIZE,
463            data_page_row_count_limit: DEFAULT_DATA_PAGE_ROW_COUNT_LIMIT,
464            write_batch_size: DEFAULT_WRITE_BATCH_SIZE,
465            max_row_group_size: DEFAULT_MAX_ROW_GROUP_SIZE,
466            bloom_filter_position: DEFAULT_BLOOM_FILTER_POSITION,
467            writer_version: DEFAULT_WRITER_VERSION,
468            created_by: DEFAULT_CREATED_BY.to_string(),
469            offset_index_disabled: DEFAULT_OFFSET_INDEX_DISABLED,
470            key_value_metadata: None,
471            default_column_properties: Default::default(),
472            column_properties: HashMap::new(),
473            sorting_columns: None,
474            column_index_truncate_length: DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH,
475            statistics_truncate_length: DEFAULT_STATISTICS_TRUNCATE_LENGTH,
476            coerce_types: DEFAULT_COERCE_TYPES,
477            #[cfg(feature = "encryption")]
478            file_encryption_properties: None,
479        }
480    }
481
482    /// Finalizes the configuration and returns immutable writer properties struct.
483    pub fn build(self) -> WriterProperties {
484        WriterProperties {
485            data_page_size_limit: self.data_page_size_limit,
486            data_page_row_count_limit: self.data_page_row_count_limit,
487            write_batch_size: self.write_batch_size,
488            max_row_group_size: self.max_row_group_size,
489            bloom_filter_position: self.bloom_filter_position,
490            writer_version: self.writer_version,
491            created_by: self.created_by,
492            offset_index_disabled: self.offset_index_disabled,
493            key_value_metadata: self.key_value_metadata,
494            default_column_properties: self.default_column_properties,
495            column_properties: self.column_properties,
496            sorting_columns: self.sorting_columns,
497            column_index_truncate_length: self.column_index_truncate_length,
498            statistics_truncate_length: self.statistics_truncate_length,
499            coerce_types: self.coerce_types,
500            #[cfg(feature = "encryption")]
501            file_encryption_properties: self.file_encryption_properties,
502        }
503    }
504
505    // ----------------------------------------------------------------------
506    // Writer properties related to a file
507
508    /// Sets the `WriterVersion` written into the parquet metadata (defaults to [`PARQUET_1_0`]
509    /// via [`DEFAULT_WRITER_VERSION`])
510    ///
511    /// This value can determine what features some readers will support.
512    ///
513    /// [`PARQUET_1_0`]: [WriterVersion::PARQUET_1_0]
514    pub fn set_writer_version(mut self, value: WriterVersion) -> Self {
515        self.writer_version = value;
516        self
517    }
518
519    /// Sets best effort maximum size of a data page in bytes (defaults to `1024 * 1024`
520    /// via [`DEFAULT_PAGE_SIZE`]).
521    ///
522    /// The parquet writer will attempt to limit the sizes of each
523    /// `DataPage` to this many bytes. Reducing this value will result
524    /// in larger parquet files, but may improve the effectiveness of
525    /// page index based predicate pushdown during reading.
526    ///
527    /// Note: this is a best effort limit based on value of
528    /// [`set_write_batch_size`](Self::set_write_batch_size).
529    pub fn set_data_page_size_limit(mut self, value: usize) -> Self {
530        self.data_page_size_limit = value;
531        self
532    }
533
534    /// Sets best effort maximum number of rows in a data page (defaults to `20_000`
535    /// via [`DEFAULT_DATA_PAGE_ROW_COUNT_LIMIT`]).
536    ///
537    /// The parquet writer will attempt to limit the number of rows in
538    /// each `DataPage` to this value. Reducing this value will result
539    /// in larger parquet files, but may improve the effectiveness of
540    /// page index based predicate pushdown during reading.
541    ///
542    /// Note: this is a best effort limit based on value of
543    /// [`set_write_batch_size`](Self::set_write_batch_size).
544    pub fn set_data_page_row_count_limit(mut self, value: usize) -> Self {
545        self.data_page_row_count_limit = value;
546        self
547    }
548
549    /// Sets write batch size (defaults to 1024 via [`DEFAULT_WRITE_BATCH_SIZE`]).
550    ///
551    /// For performance reasons, data for each column is written in
552    /// batches of this size.
553    ///
554    /// Additional limits such as such as
555    /// [`set_data_page_row_count_limit`](Self::set_data_page_row_count_limit)
556    /// are checked between batches, and thus the write batch size value acts as an
557    /// upper-bound on the enforcement granularity of other limits.
558    pub fn set_write_batch_size(mut self, value: usize) -> Self {
559        self.write_batch_size = value;
560        self
561    }
562
563    /// Sets maximum number of rows in a row group (defaults to `1024 * 1024`
564    /// via [`DEFAULT_MAX_ROW_GROUP_SIZE`]).
565    ///
566    /// # Panics
567    /// If the value is set to 0.
568    pub fn set_max_row_group_size(mut self, value: usize) -> Self {
569        assert!(value > 0, "Cannot have a 0 max row group size");
570        self.max_row_group_size = value;
571        self
572    }
573
574    /// Sets where in the final file Bloom Filters are written (defaults to  [`AfterRowGroup`]
575    /// via [`DEFAULT_BLOOM_FILTER_POSITION`])
576    ///
577    /// [`AfterRowGroup`]: BloomFilterPosition::AfterRowGroup
578    pub fn set_bloom_filter_position(mut self, value: BloomFilterPosition) -> Self {
579        self.bloom_filter_position = value;
580        self
581    }
582
583    /// Sets "created by" property (defaults to `parquet-rs version <VERSION>` via
584    /// [`DEFAULT_CREATED_BY`]).
585    ///
586    /// This is a string that will be written into the file metadata
587    pub fn set_created_by(mut self, value: String) -> Self {
588        self.created_by = value;
589        self
590    }
591
592    /// Sets whether the writing of offset indexes is disabled (defaults to `false` via
593    /// [`DEFAULT_OFFSET_INDEX_DISABLED`]).
594    ///
595    /// If statistics level is set to [`Page`] this setting will be overridden with `false`.
596    ///
597    /// Note: As the offset indexes are useful for accessing data by row number,
598    /// they are always written by default, regardless of whether other statistics
599    /// are enabled. Disabling this metadata may result in a degradation in read
600    /// performance, so use this option with care.
601    ///
602    /// [`Page`]: EnabledStatistics::Page
603    pub fn set_offset_index_disabled(mut self, value: bool) -> Self {
604        self.offset_index_disabled = value;
605        self
606    }
607
608    /// Sets "key_value_metadata" property (defaults to `None`).
609    pub fn set_key_value_metadata(mut self, value: Option<Vec<KeyValue>>) -> Self {
610        self.key_value_metadata = value;
611        self
612    }
613
614    /// Sets sorting order of rows in the row group if any (defaults to `None`).
615    pub fn set_sorting_columns(mut self, value: Option<Vec<SortingColumn>>) -> Self {
616        self.sorting_columns = value;
617        self
618    }
619
620    /// Sets the max length of min/max value fields when writing the column
621    /// [`Index`] (defaults to `Some(64)` via [`DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH`]).
622    ///
623    /// This can be used to prevent columns with very long values (hundreds of
624    /// bytes long) from causing the parquet metadata to become huge.
625    ///
626    /// # Notes
627    ///
628    /// The column [`Index`] is written when [`Self::set_statistics_enabled`] is
629    /// set to [`EnabledStatistics::Page`].
630    ///
631    /// * If `Some`, must be greater than 0, otherwise will panic
632    /// * If `None`, there's no effective limit.
633    ///
634    /// [`Index`]: crate::file::page_index::index::Index
635    pub fn set_column_index_truncate_length(mut self, max_length: Option<usize>) -> Self {
636        if let Some(value) = max_length {
637            assert!(value > 0, "Cannot have a 0 column index truncate length. If you wish to disable min/max value truncation, set it to `None`.");
638        }
639
640        self.column_index_truncate_length = max_length;
641        self
642    }
643
644    /// Sets the max length of min/max value fields in row group and data page header
645    /// [`Statistics`] (defaults to `Some(64)` via [`DEFAULT_STATISTICS_TRUNCATE_LENGTH`]).
646    ///
647    /// # Notes
648    /// Row group [`Statistics`] are written when [`Self::set_statistics_enabled`] is
649    /// set to [`EnabledStatistics::Chunk`] or [`EnabledStatistics::Page`]. Data page header
650    /// [`Statistics`] are written when [`Self::set_statistics_enabled`] is set to
651    /// [`EnabledStatistics::Page`].
652    ///
653    /// * If `Some`, must be greater than 0, otherwise will panic
654    /// * If `None`, there's no effective limit.
655    ///
656    /// # See also
657    /// Truncation of Page Index statistics is controlled separately via
658    /// [`WriterPropertiesBuilder::set_column_index_truncate_length`]
659    ///
660    /// [`Statistics`]: crate::file::statistics::Statistics
661    pub fn set_statistics_truncate_length(mut self, max_length: Option<usize>) -> Self {
662        if let Some(value) = max_length {
663            assert!(value > 0, "Cannot have a 0 statistics truncate length. If you wish to disable min/max value truncation, set it to `None`.");
664        }
665
666        self.statistics_truncate_length = max_length;
667        self
668    }
669
670    /// Should the writer coerce types to parquet native types (defaults to `false` via
671    /// [`DEFAULT_COERCE_TYPES`]).
672    ///
673    /// Leaving this option the default `false` will ensure the exact same data
674    /// written to parquet using this library will be read.
675    ///
676    /// Setting this option to `true` will result in parquet files that can be
677    /// read by more readers, but potentially lose information in the process.
678    ///
679    /// * Types such as [`DataType::Date64`], which have no direct corresponding
680    ///   Parquet type, may be stored with lower precision.
681    ///
682    /// * The internal field names of `List` and `Map` types will be renamed if
683    ///   necessary to match what is required by the newest Parquet specification.
684    ///
685    /// See [`ArrowToParquetSchemaConverter::with_coerce_types`] for more details
686    ///
687    /// [`DataType::Date64`]: arrow_schema::DataType::Date64
688    /// [`ArrowToParquetSchemaConverter::with_coerce_types`]: crate::arrow::ArrowSchemaConverter::with_coerce_types
689    pub fn set_coerce_types(mut self, coerce_types: bool) -> Self {
690        self.coerce_types = coerce_types;
691        self
692    }
693
694    /// Sets FileEncryptionProperties (defaults to `None`)
695    #[cfg(feature = "encryption")]
696    pub fn with_file_encryption_properties(
697        mut self,
698        file_encryption_properties: FileEncryptionProperties,
699    ) -> Self {
700        self.file_encryption_properties = Some(file_encryption_properties);
701        self
702    }
703
704    // ----------------------------------------------------------------------
705    // Setters for any column (global)
706
707    /// Sets default encoding for all columns.
708    ///
709    /// If dictionary is not enabled, this is treated as a primary encoding for all
710    /// columns. In case when dictionary is enabled for any column, this value is
711    /// considered to be a fallback encoding for that column.
712    ///
713    /// # Panics
714    ///
715    /// if dictionary encoding is specified, regardless of dictionary
716    /// encoding flag being set.
717    pub fn set_encoding(mut self, value: Encoding) -> Self {
718        self.default_column_properties.set_encoding(value);
719        self
720    }
721
722    /// Sets default compression codec for all columns (default to [`UNCOMPRESSED`] via
723    /// [`DEFAULT_COMPRESSION`]).
724    ///
725    /// [`UNCOMPRESSED`]: Compression::UNCOMPRESSED
726    pub fn set_compression(mut self, value: Compression) -> Self {
727        self.default_column_properties.set_compression(value);
728        self
729    }
730
731    /// Sets default flag to enable/disable dictionary encoding for all columns (defaults to `true`
732    /// via [`DEFAULT_DICTIONARY_ENABLED`]).
733    ///
734    /// Use this method to set dictionary encoding, instead of explicitly specifying
735    /// encoding in `set_encoding` method.
736    pub fn set_dictionary_enabled(mut self, value: bool) -> Self {
737        self.default_column_properties.set_dictionary_enabled(value);
738        self
739    }
740
741    /// Sets best effort maximum dictionary page size, in bytes (defaults to `1024 * 1024`
742    /// via [`DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT`]).
743    ///
744    /// The parquet writer will attempt to limit the size of each
745    /// `DataPage` used to store dictionaries to this many
746    /// bytes. Reducing this value will result in larger parquet
747    /// files, but may improve the effectiveness of page index based
748    /// predicate pushdown during reading.
749    ///
750    /// Note: this is a best effort limit based on value of
751    /// [`set_write_batch_size`](Self::set_write_batch_size).
752    pub fn set_dictionary_page_size_limit(mut self, value: usize) -> Self {
753        self.default_column_properties
754            .set_dictionary_page_size_limit(value);
755        self
756    }
757
758    /// Sets default [`EnabledStatistics`] level for all columns (defaults to [`Page`] via
759    /// [`DEFAULT_STATISTICS_ENABLED`]).
760    ///
761    /// [`Page`]: EnabledStatistics::Page
762    pub fn set_statistics_enabled(mut self, value: EnabledStatistics) -> Self {
763        self.default_column_properties.set_statistics_enabled(value);
764        self
765    }
766
767    /// enable/disable writing [`Statistics`] in the page header
768    /// (defaults to `false` via [`DEFAULT_WRITE_PAGE_HEADER_STATISTICS`]).
769    ///
770    /// Only applicable if [`Page`] level statistics are gathered.
771    ///
772    /// Setting this value to `true` can greatly increase the size of the resulting Parquet
773    /// file while yielding very little added benefit. Most modern Parquet implementations
774    /// will use the min/max values stored in the [`ParquetColumnIndex`] rather than
775    /// those in the page header.
776    ///
777    /// # Note
778    ///
779    /// Prior to version 56.0.0, the `parquet` crate always wrote these
780    /// statistics (the equivalent of setting this option to `true`). This was
781    /// changed in 56.0.0 to follow the recommendation in the Parquet
782    /// specification. See [issue #7580] for more details.
783    ///
784    /// [`Statistics`]: crate::file::statistics::Statistics
785    /// [`ParquetColumnIndex`]: crate::file::metadata::ParquetColumnIndex
786    /// [`Page`]: EnabledStatistics::Page
787    /// [issue #7580]: https://github.com/apache/arrow-rs/issues/7580
788    pub fn set_write_page_header_statistics(mut self, value: bool) -> Self {
789        self.default_column_properties
790            .set_write_page_header_statistics(value);
791        self
792    }
793
794    /// Sets if bloom filter should be written for all columns (defaults to `false`).
795    ///
796    /// # Notes
797    ///
798    /// * If the bloom filter is enabled previously then it is a no-op.
799    ///
800    /// * If the bloom filter is not enabled, default values for ndv and fpp
801    ///   value are used used. See [`set_bloom_filter_ndv`] and
802    ///   [`set_bloom_filter_fpp`] to further adjust the ndv and fpp.
803    ///
804    /// [`set_bloom_filter_ndv`]: Self::set_bloom_filter_ndv
805    /// [`set_bloom_filter_fpp`]: Self::set_bloom_filter_fpp
806    pub fn set_bloom_filter_enabled(mut self, value: bool) -> Self {
807        self.default_column_properties
808            .set_bloom_filter_enabled(value);
809        self
810    }
811
812    /// Sets the default target bloom filter false positive probability (fpp)
813    /// for all columns (defaults to `0.05` via [`DEFAULT_BLOOM_FILTER_FPP`]).
814    ///
815    /// Implicitly enables bloom writing, as if [`set_bloom_filter_enabled`] had
816    /// been called.
817    ///
818    /// [`set_bloom_filter_enabled`]: Self::set_bloom_filter_enabled
819    pub fn set_bloom_filter_fpp(mut self, value: f64) -> Self {
820        self.default_column_properties.set_bloom_filter_fpp(value);
821        self
822    }
823
824    /// Sets default number of distinct values (ndv) for bloom filter for all
825    /// columns (defaults to `1_000_000` via [`DEFAULT_BLOOM_FILTER_NDV`]).
826    ///
827    /// Implicitly enables bloom writing, as if [`set_bloom_filter_enabled`] had
828    /// been called.
829    ///
830    /// [`set_bloom_filter_enabled`]: Self::set_bloom_filter_enabled
831    pub fn set_bloom_filter_ndv(mut self, value: u64) -> Self {
832        self.default_column_properties.set_bloom_filter_ndv(value);
833        self
834    }
835
836    // ----------------------------------------------------------------------
837    // Setters for a specific column
838
839    /// Helper method to get existing or new mutable reference of column properties.
840    #[inline]
841    fn get_mut_props(&mut self, col: ColumnPath) -> &mut ColumnProperties {
842        self.column_properties.entry(col).or_default()
843    }
844
845    /// Sets encoding for a specific column.
846    ///
847    /// Takes precedence over [`Self::set_encoding`].
848    ///
849    /// If dictionary is not enabled, this is treated as a primary encoding for this
850    /// column. In case when dictionary is enabled for this column, either through
851    /// global defaults or explicitly, this value is considered to be a fallback
852    /// encoding for this column.
853    ///
854    /// # Panics
855    /// If user tries to set dictionary encoding here, regardless of dictionary
856    /// encoding flag being set.
857    pub fn set_column_encoding(mut self, col: ColumnPath, value: Encoding) -> Self {
858        self.get_mut_props(col).set_encoding(value);
859        self
860    }
861
862    /// Sets compression codec for a specific column.
863    ///
864    /// Takes precedence over [`Self::set_compression`].
865    pub fn set_column_compression(mut self, col: ColumnPath, value: Compression) -> Self {
866        self.get_mut_props(col).set_compression(value);
867        self
868    }
869
870    /// Sets flag to enable/disable dictionary encoding for a specific column.
871    ///
872    /// Takes precedence over [`Self::set_dictionary_enabled`].
873    pub fn set_column_dictionary_enabled(mut self, col: ColumnPath, value: bool) -> Self {
874        self.get_mut_props(col).set_dictionary_enabled(value);
875        self
876    }
877
878    /// Sets dictionary page size limit for a specific column.
879    ///
880    /// Takes precedence over [`Self::set_dictionary_page_size_limit`].
881    pub fn set_column_dictionary_page_size_limit(mut self, col: ColumnPath, value: usize) -> Self {
882        self.get_mut_props(col)
883            .set_dictionary_page_size_limit(value);
884        self
885    }
886
887    /// Sets [`EnabledStatistics`] level for a specific column.
888    ///
889    /// Takes precedence over [`Self::set_statistics_enabled`].
890    pub fn set_column_statistics_enabled(
891        mut self,
892        col: ColumnPath,
893        value: EnabledStatistics,
894    ) -> Self {
895        self.get_mut_props(col).set_statistics_enabled(value);
896        self
897    }
898
899    /// Sets whether to write [`Statistics`] in the page header for a specific column.
900    ///
901    /// Takes precedence over [`Self::set_write_page_header_statistics`].
902    ///
903    /// [`Statistics`]: crate::file::statistics::Statistics
904    pub fn set_column_write_page_header_statistics(mut self, col: ColumnPath, value: bool) -> Self {
905        self.get_mut_props(col)
906            .set_write_page_header_statistics(value);
907        self
908    }
909
910    /// Sets whether a bloom filter should be written for a specific column.
911    ///
912    /// Takes precedence over [`Self::set_bloom_filter_enabled`].
913    pub fn set_column_bloom_filter_enabled(mut self, col: ColumnPath, value: bool) -> Self {
914        self.get_mut_props(col).set_bloom_filter_enabled(value);
915        self
916    }
917
918    /// Sets the false positive probability for bloom filter for a specific column.
919    ///
920    /// Takes precedence over [`Self::set_bloom_filter_fpp`].
921    pub fn set_column_bloom_filter_fpp(mut self, col: ColumnPath, value: f64) -> Self {
922        self.get_mut_props(col).set_bloom_filter_fpp(value);
923        self
924    }
925
926    /// Sets the number of distinct values for bloom filter for a specific column.
927    ///
928    /// Takes precedence over [`Self::set_bloom_filter_ndv`].
929    pub fn set_column_bloom_filter_ndv(mut self, col: ColumnPath, value: u64) -> Self {
930        self.get_mut_props(col).set_bloom_filter_ndv(value);
931        self
932    }
933}
934
935/// Controls the level of statistics to be computed by the writer and stored in
936/// the parquet file.
937///
938/// Enabling statistics makes the resulting Parquet file larger and requires
939/// more time to read the parquet footer.
940///
941/// Statistics can be used to improve query performance by pruning row groups
942/// and pages during query execution if the query engine supports evaluating the
943/// predicate using the statistics.
944#[derive(Debug, Clone, Copy, Eq, PartialEq)]
945pub enum EnabledStatistics {
946    /// Compute no statistics.
947    None,
948    /// Compute column chunk-level statistics but not page-level.
949    ///
950    /// Setting this option will store one set of statistics for each relevant
951    /// column for each row group. The more row groups written, the more
952    /// statistics will be stored.
953    Chunk,
954    /// Compute page-level and column chunk-level statistics.
955    ///
956    /// Setting this option will store one set of statistics for each relevant
957    /// column for each row group. In addition, this will enable the writing
958    /// of the column index (the offset index is always written regardless of
959    /// this setting). See [`ParquetColumnIndex`] for
960    /// more information.
961    ///
962    /// [`ParquetColumnIndex`]: crate::file::metadata::ParquetColumnIndex
963    Page,
964}
965
966impl FromStr for EnabledStatistics {
967    type Err = String;
968
969    fn from_str(s: &str) -> Result<Self, Self::Err> {
970        match s {
971            "NONE" | "none" => Ok(EnabledStatistics::None),
972            "CHUNK" | "chunk" => Ok(EnabledStatistics::Chunk),
973            "PAGE" | "page" => Ok(EnabledStatistics::Page),
974            _ => Err(format!("Invalid statistics arg: {s}")),
975        }
976    }
977}
978
979impl Default for EnabledStatistics {
980    fn default() -> Self {
981        DEFAULT_STATISTICS_ENABLED
982    }
983}
984
985/// Controls the bloom filter to be computed by the writer.
986#[derive(Debug, Clone, PartialEq)]
987pub struct BloomFilterProperties {
988    /// False positive probability. This should be always between 0 and 1 exclusive. Defaults to [`DEFAULT_BLOOM_FILTER_FPP`].
989    ///
990    /// You should set this value by calling [`WriterPropertiesBuilder::set_bloom_filter_fpp`].
991    ///
992    /// The bloom filter data structure is a trade of between disk and memory space versus fpp, the
993    /// smaller the fpp, the more memory and disk space is required, thus setting it to a reasonable value
994    /// e.g. 0.1, 0.05, or 0.001 is recommended.
995    ///
996    /// Setting to a very small number diminishes the value of the filter itself, as the bitset size is
997    /// even larger than just storing the whole value. You are also expected to set `ndv` if it can
998    /// be known in advance to greatly reduce space usage.
999    pub fpp: f64,
1000    /// Number of distinct values, should be non-negative to be meaningful. Defaults to [`DEFAULT_BLOOM_FILTER_NDV`].
1001    ///
1002    /// You should set this value by calling [`WriterPropertiesBuilder::set_bloom_filter_ndv`].
1003    ///
1004    /// Usage of bloom filter is most beneficial for columns with large cardinality, so a good heuristic
1005    /// is to set ndv to the number of rows. However, it can reduce disk size if you know in advance a smaller
1006    /// number of distinct values. For very small ndv value it is probably not worth it to use bloom filter
1007    /// anyway.
1008    ///
1009    /// Increasing this value (without increasing fpp) will result in an increase in disk or memory size.
1010    pub ndv: u64,
1011}
1012
1013impl Default for BloomFilterProperties {
1014    fn default() -> Self {
1015        BloomFilterProperties {
1016            fpp: DEFAULT_BLOOM_FILTER_FPP,
1017            ndv: DEFAULT_BLOOM_FILTER_NDV,
1018        }
1019    }
1020}
1021
1022/// Container for column properties that can be changed as part of writer.
1023///
1024/// If a field is `None`, it means that no specific value has been set for this column,
1025/// so some subsequent or default value must be used.
1026#[derive(Debug, Clone, Default, PartialEq)]
1027struct ColumnProperties {
1028    encoding: Option<Encoding>,
1029    codec: Option<Compression>,
1030    dictionary_page_size_limit: Option<usize>,
1031    dictionary_enabled: Option<bool>,
1032    statistics_enabled: Option<EnabledStatistics>,
1033    write_page_header_statistics: Option<bool>,
1034    /// bloom filter related properties
1035    bloom_filter_properties: Option<BloomFilterProperties>,
1036}
1037
1038impl ColumnProperties {
1039    /// Sets encoding for this column.
1040    ///
1041    /// If dictionary is not enabled, this is treated as a primary encoding for a column.
1042    /// In case when dictionary is enabled for a column, this value is considered to
1043    /// be a fallback encoding.
1044    ///
1045    /// Panics if user tries to set dictionary encoding here, regardless of dictionary
1046    /// encoding flag being set. Use `set_dictionary_enabled` method to enable dictionary
1047    /// for a column.
1048    fn set_encoding(&mut self, value: Encoding) {
1049        if value == Encoding::PLAIN_DICTIONARY || value == Encoding::RLE_DICTIONARY {
1050            panic!("Dictionary encoding can not be used as fallback encoding");
1051        }
1052        self.encoding = Some(value);
1053    }
1054
1055    /// Sets compression codec for this column.
1056    fn set_compression(&mut self, value: Compression) {
1057        self.codec = Some(value);
1058    }
1059
1060    /// Sets whether dictionary encoding is enabled for this column.
1061    fn set_dictionary_enabled(&mut self, enabled: bool) {
1062        self.dictionary_enabled = Some(enabled);
1063    }
1064
1065    /// Sets dictionary page size limit for this column.
1066    fn set_dictionary_page_size_limit(&mut self, value: usize) {
1067        self.dictionary_page_size_limit = Some(value);
1068    }
1069
1070    /// Sets the statistics level for this column.
1071    fn set_statistics_enabled(&mut self, enabled: EnabledStatistics) {
1072        self.statistics_enabled = Some(enabled);
1073    }
1074
1075    /// Sets whether to write statistics in the page header for this column.
1076    fn set_write_page_header_statistics(&mut self, enabled: bool) {
1077        self.write_page_header_statistics = Some(enabled);
1078    }
1079
1080    /// If `value` is `true`, sets bloom filter properties to default values if not previously set,
1081    /// otherwise it is a no-op.
1082    /// If `value` is `false`, resets bloom filter properties to `None`.
1083    fn set_bloom_filter_enabled(&mut self, value: bool) {
1084        if value && self.bloom_filter_properties.is_none() {
1085            self.bloom_filter_properties = Some(Default::default())
1086        } else if !value {
1087            self.bloom_filter_properties = None
1088        }
1089    }
1090
1091    /// Sets the false positive probability for bloom filter for this column, and implicitly enables
1092    /// bloom filter if not previously enabled.
1093    ///
1094    /// # Panics
1095    ///
1096    /// Panics if the `value` is not between 0 and 1 exclusive
1097    fn set_bloom_filter_fpp(&mut self, value: f64) {
1098        assert!(
1099            value > 0. && value < 1.0,
1100            "fpp must be between 0 and 1 exclusive, got {value}"
1101        );
1102
1103        self.bloom_filter_properties
1104            .get_or_insert_with(Default::default)
1105            .fpp = value;
1106    }
1107
1108    /// Sets the number of distinct (unique) values for bloom filter for this column, and implicitly
1109    /// enables bloom filter if not previously enabled.
1110    fn set_bloom_filter_ndv(&mut self, value: u64) {
1111        self.bloom_filter_properties
1112            .get_or_insert_with(Default::default)
1113            .ndv = value;
1114    }
1115
1116    /// Returns optional encoding for this column.
1117    fn encoding(&self) -> Option<Encoding> {
1118        self.encoding
1119    }
1120
1121    /// Returns optional compression codec for this column.
1122    fn compression(&self) -> Option<Compression> {
1123        self.codec
1124    }
1125
1126    /// Returns `Some(true)` if dictionary encoding is enabled for this column, if
1127    /// disabled then returns `Some(false)`. If result is `None`, then no setting has
1128    /// been provided.
1129    fn dictionary_enabled(&self) -> Option<bool> {
1130        self.dictionary_enabled
1131    }
1132
1133    /// Returns optional dictionary page size limit for this column.
1134    fn dictionary_page_size_limit(&self) -> Option<usize> {
1135        self.dictionary_page_size_limit
1136    }
1137
1138    /// Returns optional statistics level requested for this column. If result is `None`,
1139    /// then no setting has been provided.
1140    fn statistics_enabled(&self) -> Option<EnabledStatistics> {
1141        self.statistics_enabled
1142    }
1143
1144    /// Returns `Some(true)` if [`Statistics`] are to be written to the page header for this
1145    /// column.
1146    ///
1147    /// [`Statistics`]: crate::file::statistics::Statistics
1148    fn write_page_header_statistics(&self) -> Option<bool> {
1149        self.write_page_header_statistics
1150    }
1151
1152    /// Returns the bloom filter properties, or `None` if not enabled
1153    fn bloom_filter_properties(&self) -> Option<&BloomFilterProperties> {
1154        self.bloom_filter_properties.as_ref()
1155    }
1156}
1157
1158/// Reference counted reader properties.
1159pub type ReaderPropertiesPtr = Arc<ReaderProperties>;
1160
1161const DEFAULT_READ_BLOOM_FILTER: bool = false;
1162
1163/// Configuration settings for reading parquet files.
1164///
1165/// All properties are immutable and `Send` + `Sync`.
1166/// Use [`ReaderPropertiesBuilder`] to assemble these properties.
1167///
1168/// # Example
1169///
1170/// ```rust
1171/// use parquet::file::properties::ReaderProperties;
1172///
1173/// // Create properties with default configuration.
1174/// let props = ReaderProperties::builder().build();
1175///
1176/// // Use properties builder to set certain options and assemble the configuration.
1177/// let props = ReaderProperties::builder()
1178///     .set_backward_compatible_lz4(false)
1179///     .build();
1180/// ```
1181pub struct ReaderProperties {
1182    codec_options: CodecOptions,
1183    read_bloom_filter: bool,
1184}
1185
1186impl ReaderProperties {
1187    /// Returns builder for reader properties with default values.
1188    pub fn builder() -> ReaderPropertiesBuilder {
1189        ReaderPropertiesBuilder::with_defaults()
1190    }
1191
1192    /// Returns codec options.
1193    pub(crate) fn codec_options(&self) -> &CodecOptions {
1194        &self.codec_options
1195    }
1196
1197    /// Returns whether to read bloom filter
1198    pub(crate) fn read_bloom_filter(&self) -> bool {
1199        self.read_bloom_filter
1200    }
1201}
1202
1203/// Builder for parquet file reader configuration. See example on
1204/// [`ReaderProperties`]
1205pub struct ReaderPropertiesBuilder {
1206    codec_options_builder: CodecOptionsBuilder,
1207    read_bloom_filter: Option<bool>,
1208}
1209
1210/// Reader properties builder.
1211impl ReaderPropertiesBuilder {
1212    /// Returns default state of the builder.
1213    fn with_defaults() -> Self {
1214        Self {
1215            codec_options_builder: CodecOptionsBuilder::default(),
1216            read_bloom_filter: None,
1217        }
1218    }
1219
1220    /// Finalizes the configuration and returns immutable reader properties struct.
1221    pub fn build(self) -> ReaderProperties {
1222        ReaderProperties {
1223            codec_options: self.codec_options_builder.build(),
1224            read_bloom_filter: self.read_bloom_filter.unwrap_or(DEFAULT_READ_BLOOM_FILTER),
1225        }
1226    }
1227
1228    /// Enable/disable backward compatible LZ4.
1229    ///
1230    /// If backward compatible LZ4 is enable, on LZ4_HADOOP error it will fallback
1231    /// to the older versions LZ4 algorithms. That is LZ4_FRAME, for backward compatibility
1232    /// with files generated by older versions of this library, and LZ4_RAW, for backward
1233    /// compatibility with files generated by older versions of parquet-cpp.
1234    ///
1235    /// If backward compatible LZ4 is disabled, on LZ4_HADOOP error it will return the error.
1236    pub fn set_backward_compatible_lz4(mut self, value: bool) -> Self {
1237        self.codec_options_builder = self
1238            .codec_options_builder
1239            .set_backward_compatible_lz4(value);
1240        self
1241    }
1242
1243    /// Enable/disable reading bloom filter
1244    ///
1245    /// If reading bloom filter is enabled, bloom filter will be read from the file.
1246    /// If reading bloom filter is disabled, bloom filter will not be read from the file.
1247    ///
1248    /// By default bloom filter is set to be read.
1249    pub fn set_read_bloom_filter(mut self, value: bool) -> Self {
1250        self.read_bloom_filter = Some(value);
1251        self
1252    }
1253}
1254
1255#[cfg(test)]
1256mod tests {
1257    use super::*;
1258
1259    #[test]
1260    fn test_writer_version() {
1261        assert_eq!(WriterVersion::PARQUET_1_0.as_num(), 1);
1262        assert_eq!(WriterVersion::PARQUET_2_0.as_num(), 2);
1263    }
1264
1265    #[test]
1266    fn test_writer_properties_default_settings() {
1267        let props = WriterProperties::default();
1268        assert_eq!(props.data_page_size_limit(), DEFAULT_PAGE_SIZE);
1269        assert_eq!(
1270            props.dictionary_page_size_limit(),
1271            DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT
1272        );
1273        assert_eq!(props.write_batch_size(), DEFAULT_WRITE_BATCH_SIZE);
1274        assert_eq!(props.max_row_group_size(), DEFAULT_MAX_ROW_GROUP_SIZE);
1275        assert_eq!(props.bloom_filter_position(), DEFAULT_BLOOM_FILTER_POSITION);
1276        assert_eq!(props.writer_version(), DEFAULT_WRITER_VERSION);
1277        assert_eq!(props.created_by(), DEFAULT_CREATED_BY);
1278        assert_eq!(props.key_value_metadata(), None);
1279        assert_eq!(props.encoding(&ColumnPath::from("col")), None);
1280        assert_eq!(
1281            props.compression(&ColumnPath::from("col")),
1282            DEFAULT_COMPRESSION
1283        );
1284        assert_eq!(
1285            props.dictionary_enabled(&ColumnPath::from("col")),
1286            DEFAULT_DICTIONARY_ENABLED
1287        );
1288        assert_eq!(
1289            props.statistics_enabled(&ColumnPath::from("col")),
1290            DEFAULT_STATISTICS_ENABLED
1291        );
1292        assert!(props
1293            .bloom_filter_properties(&ColumnPath::from("col"))
1294            .is_none());
1295    }
1296
1297    #[test]
1298    fn test_writer_properties_dictionary_encoding() {
1299        // dictionary encoding is not configurable, and it should be the same for both
1300        // writer version 1 and 2.
1301        for version in &[WriterVersion::PARQUET_1_0, WriterVersion::PARQUET_2_0] {
1302            let props = WriterProperties::builder()
1303                .set_writer_version(*version)
1304                .build();
1305            assert_eq!(props.dictionary_page_encoding(), Encoding::PLAIN);
1306            assert_eq!(
1307                props.dictionary_data_page_encoding(),
1308                Encoding::RLE_DICTIONARY
1309            );
1310        }
1311    }
1312
1313    #[test]
1314    #[should_panic(expected = "Dictionary encoding can not be used as fallback encoding")]
1315    fn test_writer_properties_panic_when_plain_dictionary_is_fallback() {
1316        // Should panic when user specifies dictionary encoding as fallback encoding.
1317        WriterProperties::builder()
1318            .set_encoding(Encoding::PLAIN_DICTIONARY)
1319            .build();
1320    }
1321
1322    #[test]
1323    #[should_panic(expected = "Dictionary encoding can not be used as fallback encoding")]
1324    fn test_writer_properties_panic_when_rle_dictionary_is_fallback() {
1325        // Should panic when user specifies dictionary encoding as fallback encoding.
1326        WriterProperties::builder()
1327            .set_encoding(Encoding::RLE_DICTIONARY)
1328            .build();
1329    }
1330
1331    #[test]
1332    #[should_panic(expected = "Dictionary encoding can not be used as fallback encoding")]
1333    fn test_writer_properties_panic_when_dictionary_is_enabled() {
1334        WriterProperties::builder()
1335            .set_dictionary_enabled(true)
1336            .set_column_encoding(ColumnPath::from("col"), Encoding::RLE_DICTIONARY)
1337            .build();
1338    }
1339
1340    #[test]
1341    #[should_panic(expected = "Dictionary encoding can not be used as fallback encoding")]
1342    fn test_writer_properties_panic_when_dictionary_is_disabled() {
1343        WriterProperties::builder()
1344            .set_dictionary_enabled(false)
1345            .set_column_encoding(ColumnPath::from("col"), Encoding::RLE_DICTIONARY)
1346            .build();
1347    }
1348
1349    #[test]
1350    fn test_writer_properties_builder() {
1351        let props = WriterProperties::builder()
1352            // file settings
1353            .set_writer_version(WriterVersion::PARQUET_2_0)
1354            .set_data_page_size_limit(10)
1355            .set_dictionary_page_size_limit(20)
1356            .set_write_batch_size(30)
1357            .set_max_row_group_size(40)
1358            .set_created_by("default".to_owned())
1359            .set_key_value_metadata(Some(vec![KeyValue::new(
1360                "key".to_string(),
1361                "value".to_string(),
1362            )]))
1363            // global column settings
1364            .set_encoding(Encoding::DELTA_BINARY_PACKED)
1365            .set_compression(Compression::GZIP(Default::default()))
1366            .set_dictionary_enabled(false)
1367            .set_statistics_enabled(EnabledStatistics::None)
1368            // specific column settings
1369            .set_column_encoding(ColumnPath::from("col"), Encoding::RLE)
1370            .set_column_compression(ColumnPath::from("col"), Compression::SNAPPY)
1371            .set_column_dictionary_enabled(ColumnPath::from("col"), true)
1372            .set_column_statistics_enabled(ColumnPath::from("col"), EnabledStatistics::Chunk)
1373            .set_column_bloom_filter_enabled(ColumnPath::from("col"), true)
1374            .set_column_bloom_filter_ndv(ColumnPath::from("col"), 100_u64)
1375            .set_column_bloom_filter_fpp(ColumnPath::from("col"), 0.1)
1376            .build();
1377
1378        assert_eq!(props.writer_version(), WriterVersion::PARQUET_2_0);
1379        assert_eq!(props.data_page_size_limit(), 10);
1380        assert_eq!(props.dictionary_page_size_limit(), 20);
1381        assert_eq!(props.write_batch_size(), 30);
1382        assert_eq!(props.max_row_group_size(), 40);
1383        assert_eq!(props.created_by(), "default");
1384        assert_eq!(
1385            props.key_value_metadata(),
1386            Some(&vec![
1387                KeyValue::new("key".to_string(), "value".to_string(),)
1388            ])
1389        );
1390
1391        assert_eq!(
1392            props.encoding(&ColumnPath::from("a")),
1393            Some(Encoding::DELTA_BINARY_PACKED)
1394        );
1395        assert_eq!(
1396            props.compression(&ColumnPath::from("a")),
1397            Compression::GZIP(Default::default())
1398        );
1399        assert!(!props.dictionary_enabled(&ColumnPath::from("a")));
1400        assert_eq!(
1401            props.statistics_enabled(&ColumnPath::from("a")),
1402            EnabledStatistics::None
1403        );
1404
1405        assert_eq!(
1406            props.encoding(&ColumnPath::from("col")),
1407            Some(Encoding::RLE)
1408        );
1409        assert_eq!(
1410            props.compression(&ColumnPath::from("col")),
1411            Compression::SNAPPY
1412        );
1413        assert!(props.dictionary_enabled(&ColumnPath::from("col")));
1414        assert_eq!(
1415            props.statistics_enabled(&ColumnPath::from("col")),
1416            EnabledStatistics::Chunk
1417        );
1418        assert_eq!(
1419            props.bloom_filter_properties(&ColumnPath::from("col")),
1420            Some(&BloomFilterProperties { fpp: 0.1, ndv: 100 })
1421        );
1422    }
1423
1424    #[test]
1425    fn test_writer_properties_builder_partial_defaults() {
1426        let props = WriterProperties::builder()
1427            .set_encoding(Encoding::DELTA_BINARY_PACKED)
1428            .set_compression(Compression::GZIP(Default::default()))
1429            .set_bloom_filter_enabled(true)
1430            .set_column_encoding(ColumnPath::from("col"), Encoding::RLE)
1431            .build();
1432
1433        assert_eq!(
1434            props.encoding(&ColumnPath::from("col")),
1435            Some(Encoding::RLE)
1436        );
1437        assert_eq!(
1438            props.compression(&ColumnPath::from("col")),
1439            Compression::GZIP(Default::default())
1440        );
1441        assert_eq!(
1442            props.dictionary_enabled(&ColumnPath::from("col")),
1443            DEFAULT_DICTIONARY_ENABLED
1444        );
1445        assert_eq!(
1446            props.bloom_filter_properties(&ColumnPath::from("col")),
1447            Some(&BloomFilterProperties {
1448                fpp: 0.05,
1449                ndv: 1_000_000_u64
1450            })
1451        );
1452    }
1453
1454    #[test]
1455    fn test_writer_properties_bloom_filter_ndv_fpp_set() {
1456        assert_eq!(
1457            WriterProperties::builder()
1458                .build()
1459                .bloom_filter_properties(&ColumnPath::from("col")),
1460            None
1461        );
1462        assert_eq!(
1463            WriterProperties::builder()
1464                .set_bloom_filter_ndv(100)
1465                .build()
1466                .bloom_filter_properties(&ColumnPath::from("col")),
1467            Some(&BloomFilterProperties {
1468                fpp: 0.05,
1469                ndv: 100
1470            })
1471        );
1472        assert_eq!(
1473            WriterProperties::builder()
1474                .set_bloom_filter_fpp(0.1)
1475                .build()
1476                .bloom_filter_properties(&ColumnPath::from("col")),
1477            Some(&BloomFilterProperties {
1478                fpp: 0.1,
1479                ndv: 1_000_000_u64
1480            })
1481        );
1482    }
1483
1484    #[test]
1485    fn test_writer_properties_column_dictionary_page_size_limit() {
1486        let props = WriterProperties::builder()
1487            .set_dictionary_page_size_limit(100)
1488            .set_column_dictionary_page_size_limit(ColumnPath::from("col"), 10)
1489            .build();
1490
1491        assert_eq!(props.dictionary_page_size_limit(), 100);
1492        assert_eq!(
1493            props.column_dictionary_page_size_limit(&ColumnPath::from("col")),
1494            10
1495        );
1496        assert_eq!(
1497            props.column_dictionary_page_size_limit(&ColumnPath::from("other")),
1498            100
1499        );
1500    }
1501
1502    #[test]
1503    fn test_reader_properties_default_settings() {
1504        let props = ReaderProperties::builder().build();
1505
1506        let codec_options = CodecOptionsBuilder::default()
1507            .set_backward_compatible_lz4(true)
1508            .build();
1509
1510        assert_eq!(props.codec_options(), &codec_options);
1511        assert!(!props.read_bloom_filter());
1512    }
1513
1514    #[test]
1515    fn test_reader_properties_builder() {
1516        let props = ReaderProperties::builder()
1517            .set_backward_compatible_lz4(false)
1518            .build();
1519
1520        let codec_options = CodecOptionsBuilder::default()
1521            .set_backward_compatible_lz4(false)
1522            .build();
1523
1524        assert_eq!(props.codec_options(), &codec_options);
1525    }
1526
1527    #[test]
1528    fn test_parse_writerversion() {
1529        let mut writer_version = "PARQUET_1_0".parse::<WriterVersion>().unwrap();
1530        assert_eq!(writer_version, WriterVersion::PARQUET_1_0);
1531        writer_version = "PARQUET_2_0".parse::<WriterVersion>().unwrap();
1532        assert_eq!(writer_version, WriterVersion::PARQUET_2_0);
1533
1534        // test lowercase
1535        writer_version = "parquet_1_0".parse::<WriterVersion>().unwrap();
1536        assert_eq!(writer_version, WriterVersion::PARQUET_1_0);
1537
1538        // test invalid version
1539        match "PARQUET_-1_0".parse::<WriterVersion>() {
1540            Ok(_) => panic!("Should not be able to parse PARQUET_-1_0"),
1541            Err(e) => {
1542                assert_eq!(e, "Invalid writer version: PARQUET_-1_0");
1543            }
1544        }
1545    }
1546
1547    #[test]
1548    fn test_parse_enabledstatistics() {
1549        let mut enabled_statistics = "NONE".parse::<EnabledStatistics>().unwrap();
1550        assert_eq!(enabled_statistics, EnabledStatistics::None);
1551        enabled_statistics = "CHUNK".parse::<EnabledStatistics>().unwrap();
1552        assert_eq!(enabled_statistics, EnabledStatistics::Chunk);
1553        enabled_statistics = "PAGE".parse::<EnabledStatistics>().unwrap();
1554        assert_eq!(enabled_statistics, EnabledStatistics::Page);
1555
1556        // test lowercase
1557        enabled_statistics = "none".parse::<EnabledStatistics>().unwrap();
1558        assert_eq!(enabled_statistics, EnabledStatistics::None);
1559
1560        //test invalid statistics
1561        match "ChunkAndPage".parse::<EnabledStatistics>() {
1562            Ok(_) => panic!("Should not be able to parse ChunkAndPage"),
1563            Err(e) => {
1564                assert_eq!(e, "Invalid statistics arg: ChunkAndPage");
1565            }
1566        }
1567    }
1568}