parquet/file/properties.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Configuration via [`WriterProperties`] and [`ReaderProperties`]
6use crate::basic::{Compression, Encoding};
7use crate::compression::{CodecOptions, CodecOptionsBuilder};
8pub use crate::encodings::encoding::DeltaBinaryPackedEncoderOptions;
9#[cfg(feature = "encryption")]
10use crate::encryption::encrypt::FileEncryptionProperties;
11use crate::errors::{ParquetError, Result};
12use crate::file::metadata::{KeyValue, SortingColumn};
13use crate::schema::types::ColumnPath;
14use std::str::FromStr;
15use std::{collections::HashMap, sync::Arc};
16
17/// Default value for [`WriterProperties::data_page_size_limit`]
18pub const DEFAULT_PAGE_SIZE: usize = 1024 * 1024;
19/// Default value for [`WriterProperties::write_batch_size`]
20pub const DEFAULT_WRITE_BATCH_SIZE: usize = 1024;
21/// Default value for [`WriterProperties::writer_version`]
22pub const DEFAULT_WRITER_VERSION: WriterVersion = WriterVersion::PARQUET_1_0;
23/// Default value for [`WriterProperties::compression`]
24pub const DEFAULT_COMPRESSION: Compression = Compression::UNCOMPRESSED;
25/// Default value for [`WriterProperties::dictionary_enabled`]
26pub const DEFAULT_DICTIONARY_ENABLED: bool = true;
27/// Default value for [`WriterProperties::dictionary_page_size_limit`]
28pub const DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT: usize = DEFAULT_PAGE_SIZE;
29/// Default value for [`WriterProperties::data_page_row_count_limit`]
30pub const DEFAULT_DATA_PAGE_ROW_COUNT_LIMIT: usize = 20_000;
31/// Default value for [`WriterProperties::statistics_enabled`]
32pub const DEFAULT_STATISTICS_ENABLED: EnabledStatistics = EnabledStatistics::Page;
33/// Default value for [`WriterProperties::write_page_header_statistics`]
34pub const DEFAULT_WRITE_PAGE_HEADER_STATISTICS: bool = false;
35/// Default value for [`WriterProperties::max_row_group_row_count`]
36pub const DEFAULT_MAX_ROW_GROUP_ROW_COUNT: usize = 1024 * 1024;
37/// Default value for [`WriterProperties::bloom_filter_for_dictionary_encoded_chunks`]
38pub const DEFAULT_BLOOM_FILTER_FOR_DICTIONARY_ENCODED_CHUNKS: bool = true;
39/// Default value for [`WriterProperties::bloom_filter_position`]
40pub const DEFAULT_BLOOM_FILTER_POSITION: BloomFilterPosition = BloomFilterPosition::AfterRowGroup;
41/// Default value for [`WriterProperties::created_by`]
42pub const DEFAULT_CREATED_BY: &str = concat!("parquet-rs version ", env!("CARGO_PKG_VERSION"));
43/// Default value for [`WriterProperties::column_index_truncate_length`]
44pub const DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH: Option<usize> = Some(64);
45/// Default value for [`BloomFilterProperties::fpp()`]
46pub const DEFAULT_BLOOM_FILTER_FPP: f64 = 0.05;
47/// Default value for [`BloomFilterProperties::ndv()`].
48///
49/// Note: this is only the fallback default used when constructing [`BloomFilterProperties`]
50/// directly. When using [`WriterPropertiesBuilder`], columns with bloom filters enabled
51/// but without an explicit NDV will have their NDV resolved at build time to
52/// [`WriterProperties::max_row_group_row_count`], which may differ from this constant
53/// if the user configured a custom row group size.
54pub const DEFAULT_BLOOM_FILTER_NDV: u64 = DEFAULT_MAX_ROW_GROUP_ROW_COUNT as u64;
55/// Default values for [`WriterProperties::statistics_truncate_length`]
56pub const DEFAULT_STATISTICS_TRUNCATE_LENGTH: Option<usize> = Some(64);
57/// Default value for [`WriterProperties::offset_index_disabled`]
58pub const DEFAULT_OFFSET_INDEX_DISABLED: bool = false;
59/// Default values for [`WriterProperties::coerce_types`]
60pub const DEFAULT_COERCE_TYPES: bool = false;
61/// Default value for [`WriterProperties::write_row_group_number_distinct_values`]
62pub const DEFAULT_WRITE_ROW_GROUP_NUMBER_DISTINCT_VALUES: bool = false;
63/// Default value for [`WriterProperties::data_page_v2_compression_ratio_threshold`]
64pub const DEFAULT_DATA_PAGE_V2_COMPRESSION_RATIO_THRESHOLD: f64 = 1.0;
65/// Default value for [`WriterProperties::write_path_in_schema`]
66pub const DEFAULT_WRITE_PATH_IN_SCHEMA: bool = true;
67/// Default minimum chunk size for content-defined chunking: 256 KiB.
68pub const DEFAULT_CDC_MIN_CHUNK_SIZE: usize = 256 * 1024;
69/// Default maximum chunk size for content-defined chunking: 1024 KiB.
70pub const DEFAULT_CDC_MAX_CHUNK_SIZE: usize = 1024 * 1024;
71/// Default normalization level for content-defined chunking.
72pub const DEFAULT_CDC_NORM_LEVEL: i32 = 0;
73
74/// EXPERIMENTAL: Options for content-defined chunking (CDC).
75///
76/// Content-defined chunking is an experimental feature that optimizes parquet
77/// files for content addressable storage (CAS) systems by writing data pages
78/// according to content-defined chunk boundaries. This allows for more
79/// efficient deduplication of data across files, hence more efficient network
80/// transfers and storage.
81///
82/// Each content-defined chunk is written as a separate parquet data page. The
83/// following options control the chunks' size and the chunking process. Note
84/// that the chunk size is calculated based on the logical value of the data,
85/// before any encoding or compression is applied.
86#[derive(Debug, Clone, Copy, PartialEq, Eq)]
87pub struct CdcOptions {
88 /// Minimum chunk size in bytes, default is 256 KiB.
89 /// The rolling hash will not be updated until this size is reached for each chunk.
90 /// Note that all data sent through the hash function is counted towards the chunk
91 /// size, including definition and repetition levels if present.
92 pub min_chunk_size: usize,
93 /// Maximum chunk size in bytes, default is 1024 KiB.
94 /// The chunker will create a new chunk whenever the chunk size exceeds this value.
95 /// Note that the parquet writer has a related [`data_page_size_limit`] property that
96 /// controls the maximum size of a parquet data page after encoding. While setting
97 /// `data_page_size_limit` to a smaller value than `max_chunk_size` doesn't affect
98 /// the chunking effectiveness, it results in more small parquet data pages.
99 ///
100 /// [`data_page_size_limit`]: WriterPropertiesBuilder::set_data_page_size_limit
101 pub max_chunk_size: usize,
102 /// Number of bit adjustment to the gearhash mask in order to center the chunk size
103 /// around the average size more aggressively, default is 0.
104 /// Increasing the normalization level increases the probability of finding a chunk,
105 /// improving the deduplication ratio, but also increasing the number of small chunks
106 /// resulting in many small parquet data pages. The default value provides a good
107 /// balance between deduplication ratio and fragmentation.
108 /// Use norm_level=1 or norm_level=2 to reach a higher deduplication ratio at the
109 /// expense of fragmentation. Negative values can also be used to reduce the
110 /// probability of finding a chunk, resulting in larger chunks and fewer data pages.
111 /// Note that values outside [-3, 3] are not recommended, prefer using the default
112 /// value of 0 for most use cases.
113 pub norm_level: i32,
114}
115
116impl Default for CdcOptions {
117 fn default() -> Self {
118 Self {
119 min_chunk_size: DEFAULT_CDC_MIN_CHUNK_SIZE,
120 max_chunk_size: DEFAULT_CDC_MAX_CHUNK_SIZE,
121 norm_level: DEFAULT_CDC_NORM_LEVEL,
122 }
123 }
124}
125
126/// Parquet writer version.
127///
128/// Basic constant, which is not part of the Thrift definition.
129#[derive(Debug, Clone, Copy, PartialEq, Eq)]
130#[expect(non_camel_case_types)]
131pub enum WriterVersion {
132 /// Parquet format version 1.0
133 PARQUET_1_0,
134 /// Parquet format version 2.0
135 PARQUET_2_0,
136}
137
138impl WriterVersion {
139 /// Returns writer version as `i32`.
140 pub fn as_num(&self) -> i32 {
141 match self {
142 WriterVersion::PARQUET_1_0 => 1,
143 WriterVersion::PARQUET_2_0 => 2,
144 }
145 }
146}
147
148impl FromStr for WriterVersion {
149 type Err = String;
150
151 fn from_str(s: &str) -> Result<Self, Self::Err> {
152 match s {
153 "PARQUET_1_0" | "parquet_1_0" => Ok(WriterVersion::PARQUET_1_0),
154 "PARQUET_2_0" | "parquet_2_0" => Ok(WriterVersion::PARQUET_2_0),
155 _ => Err(format!("Invalid writer version: {s}")),
156 }
157 }
158}
159
160/// Where in the file [`ArrowWriter`](crate::arrow::arrow_writer::ArrowWriter) should
161/// write Bloom filters
162///
163/// Basic constant, which is not part of the Thrift definition.
164#[derive(Debug, Clone, Copy, PartialEq, Eq)]
165pub enum BloomFilterPosition {
166 /// Write Bloom Filters of each row group right after the row group
167 ///
168 /// This saves memory by writing it as soon as it is computed, at the cost
169 /// of data locality for readers
170 AfterRowGroup,
171 /// Write Bloom Filters at the end of the file
172 ///
173 /// This allows better data locality for readers, at the cost of memory usage
174 /// for writers.
175 End,
176}
177
178/// Reference counted writer properties.
179pub type WriterPropertiesPtr = Arc<WriterProperties>;
180
181/// Resolved state of [`WriterPropertiesBuilder::set_offset_index_disabled`].
182///
183/// When a user disables offset indexes but page-level statistics are enabled,
184/// the setting is overridden (offset indexes remain enabled). This enum
185/// preserves the user's original intent so that a round-trip through
186/// `WriterPropertiesBuilder` does not lose it.
187#[derive(Debug, Clone, Copy, PartialEq, Eq)]
188enum OffsetIndexSetting {
189 /// Offset indexes are enabled (the default).
190 Enabled,
191 /// User disabled offset indexes and no page-level statistics override it.
192 Disabled,
193 /// User disabled offset indexes, but page-level statistics require them,
194 /// so they remain enabled.
195 DisabledOverridden,
196}
197
198/// Configuration settings for writing parquet files.
199///
200/// Use [`Self::builder`] to create a [`WriterPropertiesBuilder`] to change settings.
201///
202/// # Example
203///
204/// ```rust
205/// # use parquet::{
206/// # basic::{Compression, Encoding},
207/// # file::properties::*,
208/// # schema::types::ColumnPath,
209/// # };
210/// #
211/// // Create properties with default configuration.
212/// let props = WriterProperties::default();
213///
214/// // Use properties builder to set certain options and assemble the configuration.
215/// let props = WriterProperties::builder()
216/// .set_writer_version(WriterVersion::PARQUET_1_0)
217/// .set_encoding(Encoding::PLAIN)
218/// .set_column_encoding(ColumnPath::from("col1"), Encoding::DELTA_BINARY_PACKED)
219/// .set_compression(Compression::SNAPPY)
220/// .build();
221///
222/// assert_eq!(props.writer_version(), WriterVersion::PARQUET_1_0);
223/// assert_eq!(
224/// props.encoding(&ColumnPath::from("col1")),
225/// Some(Encoding::DELTA_BINARY_PACKED)
226/// );
227/// assert_eq!(
228/// props.encoding(&ColumnPath::from("col2")),
229/// Some(Encoding::PLAIN)
230/// );
231/// ```
232#[derive(Debug, Clone)]
233pub struct WriterProperties {
234 write_batch_size: usize,
235 max_row_group_row_count: Option<usize>,
236 max_row_group_bytes: Option<usize>,
237 bloom_filter_position: BloomFilterPosition,
238 bloom_filter_for_dictionary_encoded_chunks: bool,
239 writer_version: WriterVersion,
240 created_by: String,
241 offset_index_setting: OffsetIndexSetting,
242 pub(crate) key_value_metadata: Option<Vec<KeyValue>>,
243 default_column_properties: ColumnProperties,
244 column_properties: HashMap<ColumnPath, ColumnProperties>,
245 sorting_columns: Option<Vec<SortingColumn>>,
246 column_index_truncate_length: Option<usize>,
247 statistics_truncate_length: Option<usize>,
248 coerce_types: bool,
249 write_row_group_number_distinct_values: bool,
250 content_defined_chunking: Option<CdcOptions>,
251 write_path_in_schema: bool,
252 #[cfg(feature = "encryption")]
253 pub(crate) file_encryption_properties: Option<Arc<FileEncryptionProperties>>,
254}
255
256impl Default for WriterProperties {
257 fn default() -> Self {
258 Self::builder().build()
259 }
260}
261
262impl WriterProperties {
263 /// Create a new [`WriterProperties`] with the default settings
264 ///
265 /// See [`WriterProperties::builder`] for customising settings
266 pub fn new() -> Self {
267 Self::default()
268 }
269
270 /// Returns a new default [`WriterPropertiesBuilder`] for creating writer
271 /// properties.
272 pub fn builder() -> WriterPropertiesBuilder {
273 WriterPropertiesBuilder::default()
274 }
275
276 /// Converts this [`WriterProperties`] into a [`WriterPropertiesBuilder`]
277 /// Used for mutating existing property settings
278 pub fn into_builder(self) -> WriterPropertiesBuilder {
279 self.into()
280 }
281
282 /// Returns data page size limit.
283 ///
284 /// Note: this is a best effort limit based on the write batch size
285 ///
286 /// For more details see [`WriterPropertiesBuilder::set_data_page_size_limit`]
287 pub fn data_page_size_limit(&self) -> usize {
288 self.default_column_properties
289 .data_page_size_limit()
290 .unwrap_or(DEFAULT_PAGE_SIZE)
291 }
292
293 /// Returns data page size limit for a specific column.
294 ///
295 /// Takes precedence over [`Self::data_page_size_limit`].
296 ///
297 /// Note: this is a best effort limit based on the write batch size.
298 pub fn column_data_page_size_limit(&self, col: &ColumnPath) -> usize {
299 resolve_data_page_size_limit(self.column_override(col), &self.default_column_properties)
300 }
301
302 /// Returns dictionary page size limit.
303 ///
304 /// Note: this is a best effort limit based on the write batch size
305 ///
306 /// For more details see [`WriterPropertiesBuilder::set_dictionary_page_size_limit`]
307 pub fn dictionary_page_size_limit(&self) -> usize {
308 self.default_column_properties
309 .dictionary_page_size_limit()
310 .unwrap_or(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT)
311 }
312
313 /// Returns dictionary page size limit for a specific column.
314 pub fn column_dictionary_page_size_limit(&self, col: &ColumnPath) -> usize {
315 resolve_dictionary_page_size_limit(
316 self.column_override(col),
317 &self.default_column_properties,
318 )
319 }
320
321 /// Returns the maximum page row count
322 ///
323 /// Note: this is a best effort limit based on the write batch size
324 ///
325 /// For more details see [`WriterPropertiesBuilder::set_data_page_row_count_limit`]
326 pub fn data_page_row_count_limit(&self) -> usize {
327 self.default_column_properties
328 .data_page_row_count_limit()
329 .unwrap_or(DEFAULT_DATA_PAGE_ROW_COUNT_LIMIT)
330 }
331
332 /// Returns data page row count limit for a specific column.
333 ///
334 /// Takes precedence over [`Self::data_page_row_count_limit`].
335 ///
336 /// Note: this is a best effort limit based on the write batch size.
337 pub fn column_data_page_row_count_limit(&self, col: &ColumnPath) -> usize {
338 resolve_data_page_row_count_limit(
339 self.column_override(col),
340 &self.default_column_properties,
341 )
342 }
343
344 /// Returns configured batch size for writes.
345 ///
346 /// When writing a batch of data, this setting allows to split it internally into
347 /// smaller batches so we can better estimate the size of a page currently being
348 /// written.
349 ///
350 /// For more details see [`WriterPropertiesBuilder::set_write_batch_size`]
351 pub fn write_batch_size(&self) -> usize {
352 self.write_batch_size
353 }
354
355 /// Returns maximum number of rows in a row group, or `None` if unlimited.
356 ///
357 /// For more details see [`WriterPropertiesBuilder::set_max_row_group_row_count`]
358 pub fn max_row_group_row_count(&self) -> Option<usize> {
359 self.max_row_group_row_count
360 }
361
362 /// Returns maximum size of a row group in bytes, or `None` if unlimited.
363 ///
364 /// For more details see [`WriterPropertiesBuilder::set_max_row_group_bytes`]
365 pub fn max_row_group_bytes(&self) -> Option<usize> {
366 self.max_row_group_bytes
367 }
368
369 /// Returns bloom filter position.
370 ///
371 /// For more details see [`WriterPropertiesBuilder::set_bloom_filter_position`]
372 pub fn bloom_filter_position(&self) -> BloomFilterPosition {
373 self.bloom_filter_position
374 }
375
376 /// Returns whether a column chunk whose data pages are all dictionary encoded gets a
377 /// bloom filter.
378 ///
379 /// For more details see [`WriterPropertiesBuilder::set_bloom_filter_for_dictionary_encoded_chunks`]
380 pub fn bloom_filter_for_dictionary_encoded_chunks(&self) -> bool {
381 self.bloom_filter_for_dictionary_encoded_chunks
382 }
383
384 /// Returns configured writer version.
385 ///
386 /// For more details see [`WriterPropertiesBuilder::set_writer_version`]
387 pub fn writer_version(&self) -> WriterVersion {
388 self.writer_version
389 }
390
391 /// Returns `created_by` string.
392 ///
393 /// For more details see [`WriterPropertiesBuilder::set_created_by`]
394 pub fn created_by(&self) -> &str {
395 &self.created_by
396 }
397
398 /// Returns `true` if offset index writing is disabled.
399 ///
400 /// For more details see [`WriterPropertiesBuilder::set_offset_index_disabled`]
401 pub fn offset_index_disabled(&self) -> bool {
402 matches!(self.offset_index_setting, OffsetIndexSetting::Disabled)
403 }
404
405 /// Returns `key_value_metadata` KeyValue pairs.
406 ///
407 /// For more details see [`WriterPropertiesBuilder::set_key_value_metadata`]
408 pub fn key_value_metadata(&self) -> Option<&Vec<KeyValue>> {
409 self.key_value_metadata.as_ref()
410 }
411
412 /// Returns sorting columns.
413 ///
414 /// For more details see [`WriterPropertiesBuilder::set_sorting_columns`]
415 pub fn sorting_columns(&self) -> Option<&Vec<SortingColumn>> {
416 self.sorting_columns.as_ref()
417 }
418
419 /// Returns the maximum length of truncated min/max values in the column index.
420 ///
421 /// `None` if truncation is disabled, must be greater than 0 otherwise.
422 ///
423 /// For more details see [`WriterPropertiesBuilder::set_column_index_truncate_length`]
424 pub fn column_index_truncate_length(&self) -> Option<usize> {
425 self.column_index_truncate_length
426 }
427
428 /// Returns the maximum length of truncated min/max values in [`Statistics`].
429 ///
430 /// `None` if truncation is disabled, must be greater than 0 otherwise.
431 ///
432 /// For more details see [`WriterPropertiesBuilder::set_statistics_truncate_length`]
433 ///
434 /// [`Statistics`]: crate::file::statistics::Statistics
435 pub fn statistics_truncate_length(&self) -> Option<usize> {
436 self.statistics_truncate_length
437 }
438
439 /// Returns `true` if type coercion is enabled.
440 ///
441 /// For more details see [`WriterPropertiesBuilder::set_coerce_types`]
442 pub fn coerce_types(&self) -> bool {
443 self.coerce_types
444 }
445
446 /// Returns `true` if the writer should compute and store the distinct count
447 /// (`num_distinct_values`) in row group column chunk statistics.
448 ///
449 /// For more details see [`WriterPropertiesBuilder::set_write_row_group_number_distinct_values`]
450 pub fn write_row_group_number_distinct_values(&self) -> bool {
451 self.write_row_group_number_distinct_values
452 }
453
454 /// Returns `true` if the `path_in_schema` field of the `ColumnMetaData` Thrift struct
455 /// should be written.
456 ///
457 /// For more details see [`WriterPropertiesBuilder::set_write_path_in_schema`]
458 pub fn write_path_in_schema(&self) -> bool {
459 self.write_path_in_schema
460 }
461
462 /// EXPERIMENTAL: Returns content-defined chunking options, or `None` if CDC is disabled.
463 ///
464 /// For more details see [`WriterPropertiesBuilder::set_content_defined_chunking`]
465 pub fn content_defined_chunking(&self) -> Option<&CdcOptions> {
466 self.content_defined_chunking.as_ref()
467 }
468
469 /// Returns the compression ratio threshold at or above which a Data Page v2's
470 /// compressed values are discarded in favor of writing the values uncompressed.
471 ///
472 /// For more details see [`WriterPropertiesBuilder::set_data_page_v2_compression_ratio_threshold`]
473 pub fn data_page_v2_compression_ratio_threshold(&self) -> f64 {
474 self.default_column_properties
475 .data_page_v2_compression_ratio_threshold()
476 .unwrap_or(DEFAULT_DATA_PAGE_V2_COMPRESSION_RATIO_THRESHOLD)
477 }
478
479 /// Returns the Data Page v2 compression ratio threshold for a specific column.
480 ///
481 /// Takes precedence over [`Self::data_page_v2_compression_ratio_threshold`].
482 pub fn column_data_page_v2_compression_ratio_threshold(&self, col: &ColumnPath) -> f64 {
483 resolve_data_page_v2_compression_ratio_threshold(
484 self.column_override(col),
485 &self.default_column_properties,
486 )
487 }
488
489 /// Returns custom delta binary packed encoder options for a specific column.
490 ///
491 /// See [`DeltaBinaryPackedEncoderOptions`] for layout trade-offs. These options also apply to
492 /// the integer sub-encoders used by `DELTA_LENGTH_BYTE_ARRAY` and `DELTA_BYTE_ARRAY`.
493 pub fn delta_binary_packed_encoder_options(
494 &self,
495 col: &ColumnPath,
496 ) -> Option<DeltaBinaryPackedEncoderOptions> {
497 resolve_delta_binary_packed_encoder_options(
498 self.column_override(col),
499 &self.default_column_properties,
500 )
501 }
502
503 /// Returns encoding for a data page, when dictionary encoding is enabled.
504 ///
505 /// This is not configurable.
506 #[inline]
507 pub fn dictionary_data_page_encoding(&self) -> Encoding {
508 // PLAIN_DICTIONARY encoding is deprecated in writer version 1.
509 // Dictionary values are encoded using RLE_DICTIONARY encoding.
510 Encoding::RLE_DICTIONARY
511 }
512
513 /// Returns encoding for dictionary page, when dictionary encoding is enabled.
514 ///
515 /// This is not configurable.
516 #[inline]
517 pub fn dictionary_page_encoding(&self) -> Encoding {
518 // PLAIN_DICTIONARY is deprecated in writer version 1.
519 // Dictionary is encoded using plain encoding.
520 Encoding::PLAIN
521 }
522
523 /// Returns encoding for a column, if set.
524 ///
525 /// In case when dictionary is enabled, returns fallback encoding.
526 ///
527 /// If encoding is not set, then column writer will choose the best encoding
528 /// based on the column type.
529 pub fn encoding(&self, col: &ColumnPath) -> Option<Encoding> {
530 resolve_encoding(self.column_override(col), &self.default_column_properties)
531 }
532
533 /// Returns compression codec for a column.
534 ///
535 /// For more details see [`WriterPropertiesBuilder::set_column_compression`]
536 pub fn compression(&self, col: &ColumnPath) -> Compression {
537 resolve_compression(self.column_override(col), &self.default_column_properties)
538 }
539
540 /// Returns `true` if dictionary encoding is enabled for a column.
541 ///
542 /// For more details see [`WriterPropertiesBuilder::set_dictionary_enabled`]
543 pub fn dictionary_enabled(&self, col: &ColumnPath) -> bool {
544 resolve_dictionary_enabled(self.column_override(col), &self.default_column_properties)
545 }
546
547 /// Returns which statistics are written for a column.
548 ///
549 /// For more details see [`WriterPropertiesBuilder::set_statistics_enabled`]
550 pub fn statistics_enabled(&self, col: &ColumnPath) -> EnabledStatistics {
551 resolve_statistics_enabled(self.column_override(col), &self.default_column_properties)
552 }
553
554 /// Returns `true` if [`Statistics`] are to be written to the page header for a column.
555 ///
556 /// For more details see [`WriterPropertiesBuilder::set_write_page_header_statistics`]
557 ///
558 /// [`Statistics`]: crate::file::statistics::Statistics
559 pub fn write_page_header_statistics(&self, col: &ColumnPath) -> bool {
560 resolve_write_page_header_statistics(
561 self.column_override(col),
562 &self.default_column_properties,
563 )
564 }
565
566 /// Returns the [`BloomFilterProperties`] for the given column
567 ///
568 /// Returns `None` if bloom filter is disabled
569 ///
570 /// For more details see [`WriterPropertiesBuilder::set_column_bloom_filter_enabled`]
571 pub fn bloom_filter_properties(&self, col: &ColumnPath) -> Option<&BloomFilterProperties> {
572 resolve_bloom_filter_properties(self.column_override(col), &self.default_column_properties)
573 }
574
575 /// Returns the per-column override entry for `col`, if any.
576 ///
577 /// This is the only place the per-column map is searched. Searching it hashes
578 /// `col`, which is a `Vec<String>`, so callers that need more than one setting
579 /// should go through [`Self::resolve_column_properties`] rather than call
580 /// several single-setting accessors.
581 #[inline]
582 fn column_override(&self, col: &ColumnPath) -> Option<&ColumnProperties> {
583 self.column_properties.get(col)
584 }
585
586 /// Resolves every per-column writer setting for `col` with a single search of
587 /// the per-column override map.
588 ///
589 /// A column writer needs most of these settings, and needs some of them again
590 /// on every batch and every page, so it resolves them once when it is created
591 /// and reads the result from then on.
592 pub(crate) fn resolve_column_properties(&self, col: &ColumnPath) -> ResolvedColumnProperties {
593 let column = self.column_override(col);
594 let default = &self.default_column_properties;
595 ResolvedColumnProperties {
596 encoding: resolve_encoding(column, default),
597 compression: resolve_compression(column, default),
598 dictionary_enabled: resolve_dictionary_enabled(column, default),
599 statistics_enabled: resolve_statistics_enabled(column, default),
600 write_page_header_statistics: resolve_write_page_header_statistics(column, default),
601 data_page_size_limit: resolve_data_page_size_limit(column, default),
602 data_page_row_count_limit: resolve_data_page_row_count_limit(column, default),
603 dictionary_page_size_limit: resolve_dictionary_page_size_limit(column, default),
604 data_page_v2_compression_ratio_threshold:
605 resolve_data_page_v2_compression_ratio_threshold(column, default),
606 bloom_filter_properties: resolve_bloom_filter_properties(column, default).cloned(),
607 delta_binary_packed_encoder_options: resolve_delta_binary_packed_encoder_options(
608 column, default,
609 ),
610 }
611 }
612
613 /// Return file encryption properties
614 ///
615 /// For more details see [`WriterPropertiesBuilder::with_file_encryption_properties`]
616 #[cfg(feature = "encryption")]
617 pub fn file_encryption_properties(&self) -> Option<&Arc<FileEncryptionProperties>> {
618 self.file_encryption_properties.as_ref()
619 }
620}
621
622/// Builder for [`WriterProperties`] Parquet writer configuration.
623///
624/// See example on [`WriterProperties`]
625#[derive(Debug, Clone)]
626pub struct WriterPropertiesBuilder {
627 write_batch_size: usize,
628 max_row_group_row_count: Option<usize>,
629 max_row_group_bytes: Option<usize>,
630 bloom_filter_position: BloomFilterPosition,
631 bloom_filter_for_dictionary_encoded_chunks: bool,
632 writer_version: WriterVersion,
633 created_by: String,
634 offset_index_disabled: bool,
635 key_value_metadata: Option<Vec<KeyValue>>,
636 default_column_properties: ColumnProperties,
637 column_properties: HashMap<ColumnPath, ColumnProperties>,
638 sorting_columns: Option<Vec<SortingColumn>>,
639 column_index_truncate_length: Option<usize>,
640 statistics_truncate_length: Option<usize>,
641 coerce_types: bool,
642 write_row_group_number_distinct_values: bool,
643 content_defined_chunking: Option<CdcOptions>,
644 write_path_in_schema: bool,
645 #[cfg(feature = "encryption")]
646 file_encryption_properties: Option<Arc<FileEncryptionProperties>>,
647}
648
649impl Default for WriterPropertiesBuilder {
650 /// Returns default state of the builder.
651 fn default() -> Self {
652 Self {
653 write_batch_size: DEFAULT_WRITE_BATCH_SIZE,
654 max_row_group_row_count: Some(DEFAULT_MAX_ROW_GROUP_ROW_COUNT),
655 max_row_group_bytes: None,
656 bloom_filter_position: DEFAULT_BLOOM_FILTER_POSITION,
657 bloom_filter_for_dictionary_encoded_chunks:
658 DEFAULT_BLOOM_FILTER_FOR_DICTIONARY_ENCODED_CHUNKS,
659 writer_version: DEFAULT_WRITER_VERSION,
660 created_by: DEFAULT_CREATED_BY.to_string(),
661 offset_index_disabled: DEFAULT_OFFSET_INDEX_DISABLED,
662 key_value_metadata: None,
663 default_column_properties: Default::default(),
664 column_properties: HashMap::new(),
665 sorting_columns: None,
666 column_index_truncate_length: DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH,
667 statistics_truncate_length: DEFAULT_STATISTICS_TRUNCATE_LENGTH,
668 coerce_types: DEFAULT_COERCE_TYPES,
669 write_row_group_number_distinct_values: DEFAULT_WRITE_ROW_GROUP_NUMBER_DISTINCT_VALUES,
670 content_defined_chunking: None,
671 write_path_in_schema: DEFAULT_WRITE_PATH_IN_SCHEMA,
672 #[cfg(feature = "encryption")]
673 file_encryption_properties: None,
674 }
675 }
676}
677
678impl WriterPropertiesBuilder {
679 /// Finalizes the configuration and returns immutable writer properties struct.
680 pub fn build(self) -> WriterProperties {
681 // Pre-compute offset_index_setting
682 let offset_index_setting = if self.offset_index_disabled {
683 let default_page_stats_enabled = self.default_column_properties.statistics_enabled()
684 == Some(EnabledStatistics::Page);
685 let column_page_stats_enabled = self.column_properties.iter().any(|path_props| {
686 path_props.1.statistics_enabled() == Some(EnabledStatistics::Page)
687 });
688 if default_page_stats_enabled || column_page_stats_enabled {
689 OffsetIndexSetting::DisabledOverridden
690 } else {
691 OffsetIndexSetting::Disabled
692 }
693 } else {
694 OffsetIndexSetting::Enabled
695 };
696
697 // Resolve bloom filter NDV for columns where it wasn't explicitly set:
698 // default to max_row_group_row_count so the filter is never undersized.
699 let default_ndv = self
700 .max_row_group_row_count
701 .unwrap_or(DEFAULT_MAX_ROW_GROUP_ROW_COUNT) as u64;
702 let mut default_column_properties = self.default_column_properties;
703 default_column_properties.resolve_bloom_filter_ndv(default_ndv);
704 let mut column_properties = self.column_properties;
705 for props in column_properties.values_mut() {
706 props.resolve_bloom_filter_ndv(default_ndv);
707 }
708
709 WriterProperties {
710 write_batch_size: self.write_batch_size,
711 max_row_group_row_count: self.max_row_group_row_count,
712 max_row_group_bytes: self.max_row_group_bytes,
713 bloom_filter_position: self.bloom_filter_position,
714 bloom_filter_for_dictionary_encoded_chunks: self
715 .bloom_filter_for_dictionary_encoded_chunks,
716 writer_version: self.writer_version,
717 created_by: self.created_by,
718 offset_index_setting,
719 key_value_metadata: self.key_value_metadata,
720 default_column_properties,
721 column_properties,
722 sorting_columns: self.sorting_columns,
723 column_index_truncate_length: self.column_index_truncate_length,
724 statistics_truncate_length: self.statistics_truncate_length,
725 coerce_types: self.coerce_types,
726 write_row_group_number_distinct_values: self.write_row_group_number_distinct_values,
727 content_defined_chunking: self.content_defined_chunking,
728 write_path_in_schema: self.write_path_in_schema,
729 #[cfg(feature = "encryption")]
730 file_encryption_properties: self.file_encryption_properties,
731 }
732 }
733
734 // ----------------------------------------------------------------------
735 // Writer properties related to a file
736
737 /// Sets the `WriterVersion` written into the parquet metadata (defaults to [`PARQUET_1_0`]
738 /// via [`DEFAULT_WRITER_VERSION`])
739 ///
740 /// This value can determine what features some readers will support.
741 ///
742 /// [`PARQUET_1_0`]: [WriterVersion::PARQUET_1_0]
743 pub fn set_writer_version(mut self, value: WriterVersion) -> Self {
744 self.writer_version = value;
745 self
746 }
747
748 /// Sets best effort maximum number of rows in a data page (defaults to `20_000`
749 /// via [`DEFAULT_DATA_PAGE_ROW_COUNT_LIMIT`]).
750 ///
751 /// The parquet writer will attempt to limit the number of rows in
752 /// each `DataPage` to this value. Reducing this value will result
753 /// in larger parquet files, but may improve the effectiveness of
754 /// page index based predicate pushdown during reading.
755 ///
756 /// Note: this is a best effort limit based on value of
757 /// [`set_write_batch_size`](Self::set_write_batch_size).
758 ///
759 /// # Panics
760 /// If the value is `0`.
761 pub fn set_data_page_row_count_limit(mut self, value: usize) -> Self {
762 assert_ne!(value, 0, "Cannot have a 0 data page row count limit");
763 self.default_column_properties
764 .set_data_page_row_count_limit(value);
765 self
766 }
767
768 /// Sets write batch size (defaults to 1024 via [`DEFAULT_WRITE_BATCH_SIZE`]).
769 ///
770 /// For performance reasons, data for each column is written in
771 /// batches of this size.
772 ///
773 /// Additional limits such as such as
774 /// [`set_data_page_row_count_limit`](Self::set_data_page_row_count_limit)
775 /// are checked between batches, and thus the write batch size value acts as an
776 /// upper-bound on the enforcement granularity of other limits.
777 ///
778 /// # Panics
779 /// If the value is `0`.
780 pub fn set_write_batch_size(mut self, value: usize) -> Self {
781 assert_ne!(value, 0, "Cannot have a 0 write batch size");
782 self.write_batch_size = value;
783 self
784 }
785
786 /// Sets maximum number of rows in a row group, or `None` for unlimited.
787 ///
788 /// If both `max_row_group_row_count` and `max_row_group_bytes` are set,
789 /// the row group with the smaller limit will be produced.
790 ///
791 /// # Panics
792 /// If the value is `Some(0)`.
793 pub fn set_max_row_group_row_count(mut self, value: Option<usize>) -> Self {
794 assert_ne!(value, Some(0), "Cannot have a 0 max row group row count");
795 self.max_row_group_row_count = value;
796 self
797 }
798
799 /// Sets maximum size of a row group in bytes, or `None` for unlimited.
800 ///
801 /// Row groups are flushed when their estimated encoded size exceeds this threshold.
802 /// This is similar to the official Java implementation for `parquet.block.size`'s behavior.
803 ///
804 /// If both `max_row_group_row_count` and `max_row_group_bytes` are set,
805 /// the row group with the smaller limit will be produced.
806 ///
807 /// # Panics
808 /// If the value is `Some(0)`.
809 pub fn set_max_row_group_bytes(mut self, value: Option<usize>) -> Self {
810 assert_ne!(value, Some(0), "Cannot have a 0 max row group bytes");
811 self.max_row_group_bytes = value;
812 self
813 }
814
815 /// Sets where in the final file Bloom Filters are written (defaults to [`AfterRowGroup`]
816 /// via [`DEFAULT_BLOOM_FILTER_POSITION`])
817 ///
818 /// [`AfterRowGroup`]: BloomFilterPosition::AfterRowGroup
819 pub fn set_bloom_filter_position(mut self, value: BloomFilterPosition) -> Self {
820 self.bloom_filter_position = value;
821 self
822 }
823
824 /// Sets whether a column chunk whose data pages are all dictionary encoded gets a bloom
825 /// filter (defaults to `true` via [`DEFAULT_BLOOM_FILTER_FOR_DICTIONARY_ENCODED_CHUNKS`]).
826 ///
827 /// The dictionary page of such a chunk already holds every distinct value, so its bloom
828 /// filter is redundant; set this to `false` to skip writing it and save the space.
829 pub fn set_bloom_filter_for_dictionary_encoded_chunks(mut self, value: bool) -> Self {
830 self.bloom_filter_for_dictionary_encoded_chunks = value;
831 self
832 }
833
834 /// Sets "created by" property (defaults to `parquet-rs version <VERSION>` via
835 /// [`DEFAULT_CREATED_BY`]).
836 ///
837 /// This is a string that will be written into the file metadata
838 pub fn set_created_by(mut self, value: String) -> Self {
839 self.created_by = value;
840 self
841 }
842
843 /// Sets whether the writing of offset indexes is disabled (defaults to `false` via
844 /// [`DEFAULT_OFFSET_INDEX_DISABLED`]).
845 ///
846 /// If statistics level is set to [`Page`] this setting will be overridden with `false`.
847 ///
848 /// Note: As the offset indexes are useful for accessing data by row number,
849 /// they are always written by default, regardless of whether other statistics
850 /// are enabled. Disabling this metadata may result in a degradation in read
851 /// performance, so use this option with care.
852 ///
853 /// [`Page`]: EnabledStatistics::Page
854 pub fn set_offset_index_disabled(mut self, value: bool) -> Self {
855 self.offset_index_disabled = value;
856 self
857 }
858
859 /// Sets "key_value_metadata" property (defaults to `None`).
860 pub fn set_key_value_metadata(mut self, value: Option<Vec<KeyValue>>) -> Self {
861 self.key_value_metadata = value;
862 self
863 }
864
865 /// Sets sorting order of rows in the row group if any (defaults to `None`).
866 pub fn set_sorting_columns(mut self, value: Option<Vec<SortingColumn>>) -> Self {
867 self.sorting_columns = value;
868 self
869 }
870
871 /// Sets the max length of min/max value fields when writing the column
872 /// [`Index`] (defaults to `Some(64)` via [`DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH`]).
873 ///
874 /// This can be used to prevent columns with very long values (hundreds of
875 /// bytes long) from causing the parquet metadata to become huge.
876 ///
877 /// # Notes
878 ///
879 /// The column [`Index`] is written when [`Self::set_statistics_enabled`] is
880 /// set to [`EnabledStatistics::Page`].
881 ///
882 /// * If `Some`, must be greater than 0, otherwise will panic
883 /// * If `None`, there's no effective limit.
884 ///
885 /// [`Index`]: crate::file::page_index::column_index::ColumnIndexMetaData
886 ///
887 /// # Panics
888 ///
889 /// Panics if `max_length` is `Some(0)`
890 pub fn set_column_index_truncate_length(mut self, max_length: Option<usize>) -> Self {
891 if let Some(value) = max_length {
892 assert!(
893 value > 0,
894 "Cannot have a 0 column index truncate length. If you wish to disable min/max value truncation, set it to `None`."
895 );
896 }
897
898 self.column_index_truncate_length = max_length;
899 self
900 }
901
902 /// Sets the max length of min/max value fields in row group and data page header
903 /// [`Statistics`] (defaults to `Some(64)` via [`DEFAULT_STATISTICS_TRUNCATE_LENGTH`]).
904 ///
905 /// # Notes
906 /// Row group [`Statistics`] are written when [`Self::set_statistics_enabled`] is
907 /// set to [`EnabledStatistics::Chunk`] or [`EnabledStatistics::Page`]. Data page header
908 /// [`Statistics`] are written when [`Self::set_statistics_enabled`] is set to
909 /// [`EnabledStatistics::Page`].
910 ///
911 /// * If `Some`, must be greater than 0, otherwise will panic
912 /// * If `None`, there's no effective limit.
913 ///
914 /// # See also
915 /// Truncation of Page Index statistics is controlled separately via
916 /// [`WriterPropertiesBuilder::set_column_index_truncate_length`]
917 ///
918 /// [`Statistics`]: crate::file::statistics::Statistics
919 ///
920 /// # Panics
921 ///
922 /// Panics if `max_length` is `Some(0)`
923 pub fn set_statistics_truncate_length(mut self, max_length: Option<usize>) -> Self {
924 if let Some(value) = max_length {
925 assert!(
926 value > 0,
927 "Cannot have a 0 statistics truncate length. If you wish to disable min/max value truncation, set it to `None`."
928 );
929 }
930
931 self.statistics_truncate_length = max_length;
932 self
933 }
934
935 /// Should the writer coerce types to parquet native types (defaults to `false` via
936 /// [`DEFAULT_COERCE_TYPES`]).
937 ///
938 /// Leaving this option the default `false` will ensure the exact same data
939 /// written to parquet using this library will be read.
940 ///
941 /// Setting this option to `true` will result in parquet files that can be
942 /// read by more readers, but potentially lose information in the process.
943 ///
944 /// * Types such as [`DataType::Date64`], which have no direct corresponding
945 /// Parquet type, may be stored with lower precision.
946 ///
947 /// * The internal field names of `List` and `Map` types will be renamed if
948 /// necessary to match what is required by the newest Parquet specification.
949 ///
950 /// See [`ArrowToParquetSchemaConverter::with_coerce_types`] for more details
951 ///
952 /// [`DataType::Date64`]: arrow_schema::DataType::Date64
953 /// [`ArrowToParquetSchemaConverter::with_coerce_types`]: crate::arrow::ArrowSchemaConverter::with_coerce_types
954 pub fn set_coerce_types(mut self, coerce_types: bool) -> Self {
955 self.coerce_types = coerce_types;
956 self
957 }
958
959 /// Enable or disable writing the distinct value count (`num_distinct_values`) into
960 /// row group column chunk statistics (defaults to `false` via
961 /// [`DEFAULT_WRITE_ROW_GROUP_NUMBER_DISTINCT_VALUES`]).
962 ///
963 /// When enabled, the [`ArrowWriter`] scans each column's values before encoding
964 /// and stores the number of distinct non-null values in the row group statistics
965 /// footer.
966 ///
967 /// # Compatibility
968 ///
969 /// This setting only takes effect when using [`ArrowWriter`]. The row-based
970 /// [`SerializedFileWriter`] / [`SerializedRowGroupWriter`] APIs do not populate
971 /// `num_distinct_values` and will ignore this flag.
972 ///
973 /// # Performance
974 ///
975 /// Computing the distinct count requires hashing every non-null value in the column.
976 /// For large row groups or columns with many values this adds significant overhead.
977 /// Benchmark your workload before enabling this globally.
978 ///
979 /// [`ArrowWriter`]: crate::arrow::ArrowWriter
980 /// [`SerializedFileWriter`]: crate::file::writer::SerializedFileWriter
981 /// [`SerializedRowGroupWriter`]: crate::file::writer::SerializedRowGroupWriter
982 pub fn set_write_row_group_number_distinct_values(mut self, value: bool) -> Self {
983 self.write_row_group_number_distinct_values = value;
984 self
985 }
986
987 /// EXPERIMENTAL: Should the writer emit the `path_in_schema` element of the
988 /// `ColumnMetaData` Thrift struct. Defaults to `true` via [`DEFAULT_WRITE_PATH_IN_SCHEMA`].
989 ///
990 /// Because `path_in_schema` is a field on the `ColumnMetaData`, it is repeated
991 /// `num_columns * num_rowgroups` times. Compounding this is any level of nesting or
992 /// repetition in the schema. For instance, a top-level list column named `foo` will have
993 /// a `path_in_schema` of `["foo", "list", "element"]`. A list-of-struct is even worse,
994 /// because the necessary list wrapping is repeated for each element of the struct. A
995 /// file with a deeply nested schema and many row groups can have a large percentage of the
996 /// footer taken up by this field. For example, a file of 38 row groups with a schema containing
997 /// several lists of structs containing lists had 36% of the footer taken up by `path_in_schema`.
998 /// Removing this redundant information can greatly speed up footer parsing, which is particularly
999 /// important in scenarios where one does not wish to read the entire file (e.g. point
1000 /// lookups).
1001 ///
1002 /// <div class="warning">
1003 ///
1004 /// **WARNING:**
1005 /// Setting this to `false` will break compatibility with Parquet readers that
1006 /// still expect this field to be present. Virtually all Parquet readers (parquet-java,
1007 /// Spark, arrow-cpp, pyarrow, pandas to name a few), with the exception
1008 /// of the one in this crate, expect this field to be present, and will terminate execution
1009 /// if it is not. This will continue to be the case unless/until the Parquet format
1010 /// specification is explicitly changed to allow this field to be missing. As a consequence,
1011 /// users should only set this to `false` if they have verified that any reader(s) they plan
1012 /// to use can tolerate the absence of this field.
1013 ///
1014 /// For more context, see [GH-563].
1015 ///
1016 /// </div>
1017 ///
1018 /// [GH-563]: https://github.com/apache/parquet-format/issues/563
1019 pub fn set_write_path_in_schema(mut self, write_path_in_schema: bool) -> Self {
1020 self.write_path_in_schema = write_path_in_schema;
1021 self
1022 }
1023
1024 /// EXPERIMENTAL: Sets content-defined chunking options, or disables CDC with `None`.
1025 ///
1026 /// When enabled, data page boundaries are determined by a rolling hash of the
1027 /// column values, so unchanged data produces identical byte sequences across
1028 /// file versions. This enables efficient deduplication on content-addressable
1029 /// storage systems.
1030 ///
1031 /// Only supported through the Arrow writer interface ([`ArrowWriter`]).
1032 ///
1033 /// # Panics
1034 ///
1035 /// Panics if `min_chunk_size == 0` or `max_chunk_size <= min_chunk_size`.
1036 ///
1037 /// [`ArrowWriter`]: crate::arrow::arrow_writer::ArrowWriter
1038 pub fn set_content_defined_chunking(mut self, options: Option<CdcOptions>) -> Self {
1039 if let Some(ref options) = options {
1040 assert!(
1041 options.min_chunk_size > 0,
1042 "min_chunk_size must be positive"
1043 );
1044 assert!(
1045 options.max_chunk_size > options.min_chunk_size,
1046 "max_chunk_size ({}) must be greater than min_chunk_size ({})",
1047 options.max_chunk_size,
1048 options.min_chunk_size
1049 );
1050 }
1051 self.content_defined_chunking = options;
1052 self
1053 }
1054
1055 /// Sets the default compression ratio threshold at or above which a Data Page
1056 /// v2's compressed values are discarded in favor of writing the values
1057 /// uncompressed, for all columns (defaults to `1.0` via
1058 /// [`DEFAULT_DATA_PAGE_V2_COMPRESSION_RATIO_THRESHOLD`]).
1059 ///
1060 /// When writing a Data Page v2 with a configured compression codec, the writer
1061 /// first compresses the values and then compares the compressed size to the
1062 /// uncompressed size. If `compressed_size >= uncompressed_size * threshold`, the
1063 /// compressed buffer is discarded and the values are written uncompressed for
1064 /// that page (the page's `is_compressed` flag is set to `false`).
1065 ///
1066 /// The default of `1.0` preserves the historical behavior of only keeping
1067 /// compression when it strictly reduces the size. Setting a value below `1.0`
1068 /// requires a minimum amount of size reduction to keep the compressed page —
1069 /// for example `0.9` requires at least a 10% reduction. Setting a value above
1070 /// `1.0` keeps the compressed buffer even if it's somewhat larger than the
1071 /// uncompressed values.
1072 ///
1073 /// This setting only affects Data Page v2; Data Page v1 always stores the
1074 /// compressor's output regardless of the resulting size.
1075 ///
1076 /// # Panics
1077 /// If `value` is not finite or is not strictly positive.
1078 pub fn set_data_page_v2_compression_ratio_threshold(mut self, value: f64) -> Self {
1079 self.default_column_properties
1080 .set_data_page_v2_compression_ratio_threshold(value);
1081 self
1082 }
1083
1084 /// Sets the default delta binary packed encoder block layout for all columns.
1085 ///
1086 /// See [`DeltaBinaryPackedEncoderOptions`] for layout trade-offs. These options also apply to
1087 /// the integer sub-encoders used by `DELTA_LENGTH_BYTE_ARRAY` and `DELTA_BYTE_ARRAY`.
1088 pub fn set_delta_binary_packed_encoder_options(
1089 mut self,
1090 value: DeltaBinaryPackedEncoderOptions,
1091 ) -> Self {
1092 self.default_column_properties
1093 .set_delta_binary_packed_encoder_options(value);
1094 self
1095 }
1096
1097 /// Sets FileEncryptionProperties (defaults to `None`)
1098 ///
1099 /// # Notes
1100 ///
1101 /// Writing encrypted bloom filters is not yet supported. See
1102 /// <https://github.com/apache/arrow-rs/issues/12166>
1103 #[cfg(feature = "encryption")]
1104 pub fn with_file_encryption_properties(
1105 mut self,
1106 file_encryption_properties: Arc<FileEncryptionProperties>,
1107 ) -> Self {
1108 self.file_encryption_properties = Some(file_encryption_properties);
1109 self
1110 }
1111
1112 // ----------------------------------------------------------------------
1113 // Setters for any column (global)
1114
1115 /// Sets default encoding for all columns.
1116 ///
1117 /// If dictionary is not enabled, this is treated as a primary encoding for all
1118 /// columns. In case when dictionary is enabled for any column, this value is
1119 /// considered to be a fallback encoding for that column.
1120 ///
1121 /// # Panics
1122 ///
1123 /// if dictionary encoding is specified, regardless of dictionary
1124 /// encoding flag being set.
1125 pub fn set_encoding(mut self, value: Encoding) -> Self {
1126 self.default_column_properties.set_encoding(value);
1127 self
1128 }
1129
1130 /// Sets default compression codec for all columns (default to [`UNCOMPRESSED`] via
1131 /// [`DEFAULT_COMPRESSION`]).
1132 ///
1133 /// [`UNCOMPRESSED`]: Compression::UNCOMPRESSED
1134 pub fn set_compression(mut self, value: Compression) -> Self {
1135 self.default_column_properties.set_compression(value);
1136 self
1137 }
1138
1139 /// Sets default flag to enable/disable dictionary encoding for all columns (defaults to `true`
1140 /// via [`DEFAULT_DICTIONARY_ENABLED`]).
1141 ///
1142 /// Use this method to set dictionary encoding, instead of explicitly specifying
1143 /// encoding in `set_encoding` method.
1144 pub fn set_dictionary_enabled(mut self, value: bool) -> Self {
1145 self.default_column_properties.set_dictionary_enabled(value);
1146 self
1147 }
1148
1149 /// Sets best effort maximum dictionary page size, in bytes (defaults to `1024 * 1024`
1150 /// via [`DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT`]).
1151 ///
1152 /// The parquet writer will attempt to limit the size of each
1153 /// `DataPage` used to store dictionaries to this many
1154 /// bytes. Reducing this value will result in larger parquet
1155 /// files, but may improve the effectiveness of page index based
1156 /// predicate pushdown during reading.
1157 ///
1158 /// Note: this is a best effort limit based on value of
1159 /// [`set_write_batch_size`](Self::set_write_batch_size).
1160 pub fn set_dictionary_page_size_limit(mut self, value: usize) -> Self {
1161 self.default_column_properties
1162 .set_dictionary_page_size_limit(value);
1163 self
1164 }
1165
1166 /// Sets best effort maximum size of a data page in bytes (defaults to `1024 * 1024`
1167 /// via [`DEFAULT_PAGE_SIZE`]).
1168 ///
1169 /// The parquet writer will attempt to limit the sizes of each
1170 /// `DataPage` to this many bytes. Reducing this value will result
1171 /// in larger parquet files, but may improve the effectiveness of
1172 /// page index based predicate pushdown during reading.
1173 ///
1174 /// Note: this is a best effort limit based on value of
1175 /// [`set_write_batch_size`](Self::set_write_batch_size).
1176 pub fn set_data_page_size_limit(mut self, value: usize) -> Self {
1177 self.default_column_properties
1178 .set_data_page_size_limit(value);
1179 self
1180 }
1181
1182 /// Sets default [`EnabledStatistics`] level for all columns (defaults to [`Page`] via
1183 /// [`DEFAULT_STATISTICS_ENABLED`]).
1184 ///
1185 /// [`Page`]: EnabledStatistics::Page
1186 pub fn set_statistics_enabled(mut self, value: EnabledStatistics) -> Self {
1187 self.default_column_properties.set_statistics_enabled(value);
1188 self
1189 }
1190
1191 /// enable/disable writing [`Statistics`] in the page header
1192 /// (defaults to `false` via [`DEFAULT_WRITE_PAGE_HEADER_STATISTICS`]).
1193 ///
1194 /// Only applicable if [`Page`] level statistics are gathered.
1195 ///
1196 /// Setting this value to `true` can greatly increase the size of the resulting Parquet
1197 /// file while yielding very little added benefit. Most modern Parquet implementations
1198 /// will use the min/max values stored in the [`PageIndex`] rather than
1199 /// those in the page header.
1200 ///
1201 /// # Note
1202 ///
1203 /// Prior to version 56.0.0, the `parquet` crate always wrote these
1204 /// statistics (the equivalent of setting this option to `true`). This was
1205 /// changed in 56.0.0 to follow the recommendation in the Parquet
1206 /// specification. See [issue #7580] for more details.
1207 ///
1208 /// [`Statistics`]: crate::file::statistics::Statistics
1209 /// [`PageIndex`]: crate::file::metadata::page_index::PageIndex
1210 /// [`Page`]: EnabledStatistics::Page
1211 /// [issue #7580]: https://github.com/apache/arrow-rs/issues/7580
1212 pub fn set_write_page_header_statistics(mut self, value: bool) -> Self {
1213 self.default_column_properties
1214 .set_write_page_header_statistics(value);
1215 self
1216 }
1217
1218 /// Sets if bloom filter should be written for all columns (defaults to `false`).
1219 ///
1220 /// # Notes
1221 ///
1222 /// * If the bloom filter is enabled previously then it is a no-op.
1223 ///
1224 /// * If the bloom filter is not enabled, default values for ndv and fpp
1225 /// value are used. See [`set_bloom_filter_max_ndv`] and
1226 /// [`set_bloom_filter_fpp`] to further adjust the ndv and fpp.
1227 ///
1228 /// [`set_bloom_filter_max_ndv`]: Self::set_bloom_filter_max_ndv
1229 /// [`set_bloom_filter_fpp`]: Self::set_bloom_filter_fpp
1230 pub fn set_bloom_filter_enabled(mut self, value: bool) -> Self {
1231 self.default_column_properties
1232 .set_bloom_filter_enabled(value);
1233 self
1234 }
1235
1236 /// Sets the default target bloom filter false positive probability (fpp)
1237 /// for all columns (defaults to `0.05` via [`DEFAULT_BLOOM_FILTER_FPP`]).
1238 ///
1239 /// Implicitly enables bloom writing, as if [`set_bloom_filter_enabled`] had
1240 /// been called.
1241 ///
1242 /// [`set_bloom_filter_enabled`]: Self::set_bloom_filter_enabled
1243 pub fn set_bloom_filter_fpp(mut self, value: f64) -> Self {
1244 self.default_column_properties.set_bloom_filter_fpp(value);
1245 self
1246 }
1247
1248 /// Sets default maximum expected number of distinct values (ndv) for bloom filter
1249 /// for all columns (defaults to [`DEFAULT_BLOOM_FILTER_NDV`]).
1250 ///
1251 /// The bloom filter is initially sized for this many distinct values at the
1252 /// configured FPP, then folded down after all values are inserted to achieve
1253 /// optimal size. A good heuristic is to set this to the expected number of rows
1254 /// in the row group.
1255 ///
1256 /// Implicitly enables bloom writing, as if [`set_bloom_filter_enabled`] had
1257 /// been called.
1258 ///
1259 /// [`set_bloom_filter_enabled`]: Self::set_bloom_filter_enabled
1260 pub fn set_bloom_filter_max_ndv(mut self, value: u64) -> Self {
1261 self.default_column_properties.set_bloom_filter_ndv(value);
1262 self
1263 }
1264
1265 /// Deprecated alias for [`Self::set_bloom_filter_max_ndv`].
1266 #[deprecated(since = "59.0.0", note = "Use `set_bloom_filter_max_ndv` instead")]
1267 pub fn set_bloom_filter_ndv(self, value: u64) -> Self {
1268 self.set_bloom_filter_max_ndv(value)
1269 }
1270
1271 // ----------------------------------------------------------------------
1272 // Setters for a specific column
1273
1274 /// Helper method to get existing or new mutable reference of column properties.
1275 #[inline]
1276 fn get_mut_props(&mut self, col: ColumnPath) -> &mut ColumnProperties {
1277 self.column_properties.entry(col).or_default()
1278 }
1279
1280 /// Sets encoding for a specific column.
1281 ///
1282 /// Takes precedence over [`Self::set_encoding`].
1283 ///
1284 /// If dictionary is not enabled, this is treated as a primary encoding for this
1285 /// column. In case when dictionary is enabled for this column, either through
1286 /// global defaults or explicitly, this value is considered to be a fallback
1287 /// encoding for this column.
1288 ///
1289 /// # Panics
1290 /// If user tries to set dictionary encoding here, regardless of dictionary
1291 /// encoding flag being set.
1292 pub fn set_column_encoding(mut self, col: ColumnPath, value: Encoding) -> Self {
1293 self.get_mut_props(col).set_encoding(value);
1294 self
1295 }
1296
1297 /// Sets compression codec for a specific column.
1298 ///
1299 /// Takes precedence over [`Self::set_compression`].
1300 pub fn set_column_compression(mut self, col: ColumnPath, value: Compression) -> Self {
1301 self.get_mut_props(col).set_compression(value);
1302 self
1303 }
1304
1305 /// Sets flag to enable/disable dictionary encoding for a specific column.
1306 ///
1307 /// Takes precedence over [`Self::set_dictionary_enabled`].
1308 pub fn set_column_dictionary_enabled(mut self, col: ColumnPath, value: bool) -> Self {
1309 self.get_mut_props(col).set_dictionary_enabled(value);
1310 self
1311 }
1312
1313 /// Sets dictionary page size limit for a specific column.
1314 ///
1315 /// Takes precedence over [`Self::set_dictionary_page_size_limit`].
1316 pub fn set_column_dictionary_page_size_limit(mut self, col: ColumnPath, value: usize) -> Self {
1317 self.get_mut_props(col)
1318 .set_dictionary_page_size_limit(value);
1319 self
1320 }
1321
1322 /// Sets data page size limit for a specific column.
1323 ///
1324 /// Takes precedence over [`Self::set_data_page_size_limit`].
1325 pub fn set_column_data_page_size_limit(mut self, col: ColumnPath, value: usize) -> Self {
1326 self.get_mut_props(col).set_data_page_size_limit(value);
1327 self
1328 }
1329
1330 /// Sets data page row count limit for a specific column.
1331 ///
1332 /// Takes precedence over [`Self::set_data_page_row_count_limit`].
1333 ///
1334 /// # Panics
1335 /// If the value is `0`.
1336 pub fn set_column_data_page_row_count_limit(mut self, col: ColumnPath, value: usize) -> Self {
1337 assert_ne!(value, 0, "Cannot have a 0 data page row count limit");
1338 self.get_mut_props(col).set_data_page_row_count_limit(value);
1339 self
1340 }
1341
1342 /// Sets [`EnabledStatistics`] level for a specific column.
1343 ///
1344 /// Takes precedence over [`Self::set_statistics_enabled`].
1345 pub fn set_column_statistics_enabled(
1346 mut self,
1347 col: ColumnPath,
1348 value: EnabledStatistics,
1349 ) -> Self {
1350 self.get_mut_props(col).set_statistics_enabled(value);
1351 self
1352 }
1353
1354 /// Sets whether to write [`Statistics`] in the page header for a specific column.
1355 ///
1356 /// Takes precedence over [`Self::set_write_page_header_statistics`].
1357 ///
1358 /// [`Statistics`]: crate::file::statistics::Statistics
1359 pub fn set_column_write_page_header_statistics(mut self, col: ColumnPath, value: bool) -> Self {
1360 self.get_mut_props(col)
1361 .set_write_page_header_statistics(value);
1362 self
1363 }
1364
1365 /// Sets whether a bloom filter should be written for a specific column.
1366 ///
1367 /// Takes precedence over [`Self::set_bloom_filter_enabled`].
1368 pub fn set_column_bloom_filter_enabled(mut self, col: ColumnPath, value: bool) -> Self {
1369 self.get_mut_props(col).set_bloom_filter_enabled(value);
1370 self
1371 }
1372
1373 /// Sets the false positive probability for bloom filter for a specific column.
1374 ///
1375 /// Takes precedence over [`Self::set_bloom_filter_fpp`].
1376 pub fn set_column_bloom_filter_fpp(mut self, col: ColumnPath, value: f64) -> Self {
1377 self.get_mut_props(col).set_bloom_filter_fpp(value);
1378 self
1379 }
1380
1381 /// Sets the maximum expected number of distinct values for bloom filter for
1382 /// a specific column.
1383 ///
1384 /// Takes precedence over [`Self::set_bloom_filter_max_ndv`].
1385 pub fn set_column_bloom_filter_max_ndv(mut self, col: ColumnPath, value: u64) -> Self {
1386 self.get_mut_props(col).set_bloom_filter_ndv(value);
1387 self
1388 }
1389
1390 /// Sets the Data Page v2 compression ratio threshold for a specific column.
1391 ///
1392 /// Takes precedence over [`Self::set_data_page_v2_compression_ratio_threshold`].
1393 ///
1394 /// # Panics
1395 /// If `value` is not finite or is not strictly positive.
1396 pub fn set_column_data_page_v2_compression_ratio_threshold(
1397 mut self,
1398 col: ColumnPath,
1399 value: f64,
1400 ) -> Self {
1401 self.get_mut_props(col)
1402 .set_data_page_v2_compression_ratio_threshold(value);
1403 self
1404 }
1405
1406 /// Sets the delta binary packed encoder block layout for a specific column.
1407 ///
1408 /// Takes precedence over [`Self::set_delta_binary_packed_encoder_options`].
1409 /// See [`DeltaBinaryPackedEncoderOptions`] for layout trade-offs. These options also apply to
1410 /// the integer sub-encoders used by `DELTA_LENGTH_BYTE_ARRAY` and `DELTA_BYTE_ARRAY`.
1411 pub fn set_column_delta_binary_packed_encoder_options(
1412 mut self,
1413 col: ColumnPath,
1414 value: DeltaBinaryPackedEncoderOptions,
1415 ) -> Self {
1416 self.get_mut_props(col)
1417 .set_delta_binary_packed_encoder_options(value);
1418 self
1419 }
1420
1421 /// Deprecated alias for [`Self::set_column_bloom_filter_max_ndv`].
1422 #[deprecated(
1423 since = "59.0.0",
1424 note = "Use `set_column_bloom_filter_max_ndv` instead"
1425 )]
1426 pub fn set_column_bloom_filter_ndv(self, col: ColumnPath, value: u64) -> Self {
1427 self.set_column_bloom_filter_max_ndv(col, value)
1428 }
1429
1430 /// Sets the [`BloomFilterProperties`] for all columns, implicitly enabling
1431 /// the bloom filter.
1432 ///
1433 /// Both `fpp` and `ndv` from `value` are treated as explicit and will not
1434 /// be overridden by the build-time row-group-size NDV fallback. For
1435 /// dynamic NDV sizing (resolved to `max_row_group_row_count` at build
1436 /// time), use [`Self::set_bloom_filter_enabled`] or
1437 /// [`Self::set_bloom_filter_fpp`] instead.
1438 pub fn set_bloom_filter_properties(mut self, value: BloomFilterProperties) -> Self {
1439 self.default_column_properties
1440 .set_bloom_filter_properties(value);
1441 self
1442 }
1443
1444 /// Sets the [`BloomFilterProperties`] for a specific column.
1445 ///
1446 /// Takes precedence over [`Self::set_bloom_filter_properties`].
1447 pub fn set_column_bloom_filter_properties(
1448 mut self,
1449 col: ColumnPath,
1450 value: BloomFilterProperties,
1451 ) -> Self {
1452 self.get_mut_props(col).set_bloom_filter_properties(value);
1453 self
1454 }
1455}
1456
1457impl From<WriterProperties> for WriterPropertiesBuilder {
1458 fn from(props: WriterProperties) -> Self {
1459 WriterPropertiesBuilder {
1460 write_batch_size: props.write_batch_size,
1461 max_row_group_row_count: props.max_row_group_row_count,
1462 max_row_group_bytes: props.max_row_group_bytes,
1463 bloom_filter_position: props.bloom_filter_position,
1464 bloom_filter_for_dictionary_encoded_chunks: props
1465 .bloom_filter_for_dictionary_encoded_chunks,
1466 writer_version: props.writer_version,
1467 created_by: props.created_by,
1468 offset_index_disabled: !matches!(
1469 props.offset_index_setting,
1470 OffsetIndexSetting::Enabled
1471 ),
1472 key_value_metadata: props.key_value_metadata,
1473 default_column_properties: props.default_column_properties,
1474 column_properties: props.column_properties,
1475 sorting_columns: props.sorting_columns,
1476 column_index_truncate_length: props.column_index_truncate_length,
1477 statistics_truncate_length: props.statistics_truncate_length,
1478 coerce_types: props.coerce_types,
1479 write_row_group_number_distinct_values: props.write_row_group_number_distinct_values,
1480 content_defined_chunking: props.content_defined_chunking,
1481 write_path_in_schema: props.write_path_in_schema,
1482 #[cfg(feature = "encryption")]
1483 file_encryption_properties: props.file_encryption_properties,
1484 }
1485 }
1486}
1487
1488/// Controls the level of statistics to be computed by the writer and stored in
1489/// the parquet file.
1490///
1491/// Enabling statistics makes the resulting Parquet file larger and requires
1492/// more time to read the parquet footer.
1493///
1494/// Statistics can be used to improve query performance by pruning row groups
1495/// and pages during query execution if the query engine supports evaluating the
1496/// predicate using the statistics.
1497#[derive(Debug, Clone, Copy, Eq, PartialEq)]
1498pub enum EnabledStatistics {
1499 /// Compute no statistics.
1500 None,
1501 /// Compute column chunk-level statistics but not page-level.
1502 ///
1503 /// Setting this option will store one set of statistics for each relevant
1504 /// column for each row group. The more row groups written, the more
1505 /// statistics will be stored.
1506 Chunk,
1507 /// Compute page-level and column chunk-level statistics.
1508 ///
1509 /// Setting this option will store one set of statistics for each relevant
1510 /// column for each row group. In addition, this will enable the writing
1511 /// of the column index (the offset index is always written regardless of
1512 /// this setting). See [`PageIndex`] for
1513 /// more information.
1514 ///
1515 /// [`PageIndex`]: crate::file::metadata::page_index::PageIndex
1516 Page,
1517}
1518
1519impl FromStr for EnabledStatistics {
1520 type Err = String;
1521
1522 fn from_str(s: &str) -> Result<Self, Self::Err> {
1523 match s {
1524 "NONE" | "none" => Ok(EnabledStatistics::None),
1525 "CHUNK" | "chunk" => Ok(EnabledStatistics::Chunk),
1526 "PAGE" | "page" => Ok(EnabledStatistics::Page),
1527 _ => Err(format!("Invalid statistics arg: {s}")),
1528 }
1529 }
1530}
1531
1532impl Default for EnabledStatistics {
1533 fn default() -> Self {
1534 DEFAULT_STATISTICS_ENABLED
1535 }
1536}
1537
1538/// Controls the bloom filter to be computed by the writer.
1539///
1540/// The bloom filter is initially sized for `ndv` distinct values at the given `fpp`, then
1541/// automatically folded down after all values are inserted to achieve optimal size while
1542/// maintaining the target `fpp`. See [`Sbbf::fold_to_target_fpp`] for details on the
1543/// folding algorithm.
1544///
1545/// # Example
1546///
1547/// ```rust
1548/// # use parquet::{
1549/// # file::properties::{BloomFilterProperties, WriterProperties},
1550/// # schema::types::ColumnPath,
1551/// # };
1552/// // Build a BloomFilterProperties via the builder, then apply it to one column.
1553/// let bf = BloomFilterProperties::builder()
1554/// .with_fpp(0.01)
1555/// .with_max_ndv(10_000)
1556/// .build();
1557///
1558/// let props = WriterProperties::builder()
1559/// .set_column_bloom_filter_properties(ColumnPath::from("user_id"), bf.clone())
1560/// .build();
1561///
1562/// assert_eq!(
1563/// props.bloom_filter_properties(&ColumnPath::from("user_id")),
1564/// Some(&bf)
1565/// );
1566/// ```
1567///
1568/// [`Sbbf::fold_to_target_fpp`]: crate::bloom_filter::Sbbf::fold_to_target_fpp
1569#[derive(Debug, Clone, PartialEq)]
1570pub struct BloomFilterProperties {
1571 fpp: f64,
1572 ndv: u64,
1573}
1574
1575impl Default for BloomFilterProperties {
1576 fn default() -> Self {
1577 BloomFilterProperties {
1578 fpp: DEFAULT_BLOOM_FILTER_FPP,
1579 ndv: DEFAULT_BLOOM_FILTER_NDV,
1580 }
1581 }
1582}
1583
1584impl BloomFilterProperties {
1585 /// Returns a new [`BloomFilterPropertiesBuilder`] for constructing
1586 /// [`BloomFilterProperties`] with custom values.
1587 pub fn builder() -> BloomFilterPropertiesBuilder {
1588 BloomFilterPropertiesBuilder::new()
1589 }
1590
1591 /// False positive probability. This should be always between 0 and 1 exclusive. Defaults to [`DEFAULT_BLOOM_FILTER_FPP`].
1592 ///
1593 /// You should set this value by calling [`WriterPropertiesBuilder::set_bloom_filter_fpp`].
1594 ///
1595 /// The bloom filter data structure is a trade of between disk and memory space versus fpp, the
1596 /// smaller the fpp, the more memory and disk space is required, thus setting it to a reasonable value
1597 /// e.g. 0.1, 0.05, or 0.001 is recommended.
1598 ///
1599 /// This value also serves as the target FPP for bloom filter folding: after all values
1600 /// are inserted, the filter is folded down to the smallest size that still meets this FPP.
1601 pub fn fpp(&self) -> f64 {
1602 self.fpp
1603 }
1604
1605 /// Maximum expected number of distinct values. Defaults to [`DEFAULT_BLOOM_FILTER_NDV`].
1606 ///
1607 /// You should set this value by calling [`WriterPropertiesBuilder::set_bloom_filter_max_ndv`].
1608 ///
1609 /// When not explicitly set via the builder, this defaults to
1610 /// [`max_row_group_row_count`](WriterProperties::max_row_group_row_count) (resolved at
1611 /// build time). The bloom filter is initially sized for this many distinct values at the
1612 /// given `fpp`, then folded down after insertion to achieve optimal size. A good heuristic
1613 /// is to set this to the expected number of rows in the row group. If fewer distinct values
1614 /// are actually written, the filter will be automatically compacted via folding.
1615 ///
1616 /// Thus the only negative side of overestimating this value is that the bloom filter
1617 /// will use more memory during writing than necessary, but it will not affect the final
1618 /// bloom filter size on disk.
1619 ///
1620 /// If you wish to reduce memory usage during writing and are able to make a reasonable estimate
1621 /// of the number of distinct values in a row group, it is recommended to set this value explicitly
1622 /// rather than relying on the default dynamic sizing based on `max_row_group_row_count`.
1623 /// If you do set this value explicitly it is probably best to set it for each column
1624 /// individually via [`WriterPropertiesBuilder::set_column_bloom_filter_max_ndv`] rather than globally,
1625 /// since different columns may have different numbers of distinct values.
1626 pub fn ndv(&self) -> u64 {
1627 self.ndv
1628 }
1629}
1630
1631/// Builder for [`BloomFilterProperties`].
1632///
1633/// Use [`BloomFilterProperties::builder`] or [`BloomFilterPropertiesBuilder::new`]
1634/// as the entry point.
1635#[derive(Debug, Clone, Default)]
1636pub struct BloomFilterPropertiesBuilder {
1637 fpp: Option<f64>,
1638 ndv: Option<u64>,
1639}
1640
1641impl BloomFilterPropertiesBuilder {
1642 /// Returns a new builder with no fields set.
1643 ///
1644 /// Equivalent to [`BloomFilterProperties::builder`].
1645 pub fn new() -> Self {
1646 Self::default()
1647 }
1648
1649 /// Sets the target false positive probability.
1650 ///
1651 /// The value must be in `(0.0, 1.0)` exclusively; this is validated at
1652 /// build time by [`Self::build`] / [`Self::try_build`]. When unset, the
1653 /// default is `0.05` (5%, see [`DEFAULT_BLOOM_FILTER_FPP`]).
1654 pub fn with_fpp(mut self, fpp: f64) -> Self {
1655 self.fpp = Some(fpp);
1656 self
1657 }
1658
1659 /// Sets the maximum expected number of distinct values used to size the
1660 /// bloom filter before folding.
1661 ///
1662 /// When unset, the default is `1_048_576` (see [`DEFAULT_BLOOM_FILTER_NDV`]),
1663 /// which at the default fpp of 5% reserves roughly 1 MiB per column for the
1664 /// filter bitset, derived as follows:
1665 ///
1666 /// ```text
1667 /// ndv = 1,048,576, fpp = 0.05
1668 /// 0.05^(1/8) ≈ 0.6877
1669 /// 1 - 0.6877 ≈ 0.3123
1670 /// ln(0.3123) ≈ -1.164
1671 /// num_bits = -8 * 1,048,576 / -1.164 ≈ 7,206,000 bits
1672 /// ≈ 900,750 bytes (~900 KB)
1673 /// next_power_of_two(900 KB) = 1 MiB (= 1,048,576 bytes)
1674 /// ```
1675 pub fn with_max_ndv(mut self, ndv: u64) -> Self {
1676 self.ndv = Some(ndv);
1677 self
1678 }
1679
1680 /// Builds [`BloomFilterProperties`].
1681 ///
1682 ///
1683 /// # Panics
1684 ///
1685 /// Panics if the configured `fpp` is not in `(0.0, 1.0)` exclusive.
1686 /// Use [`Self::try_build`] for a non-panicking alternative.
1687 pub fn build(self) -> BloomFilterProperties {
1688 self.try_build().unwrap_or_else(|e| panic!("{e}"))
1689 }
1690
1691 /// Builds [`BloomFilterProperties`], returning an error instead of
1692 /// panicking when the configured `fpp` is not in `(0.0, 1.0)` exclusive.
1693 pub fn try_build(self) -> Result<BloomFilterProperties> {
1694 let fpp = self.fpp.unwrap_or(DEFAULT_BLOOM_FILTER_FPP);
1695 validate_bloom_filter_fpp(fpp).map_err(ParquetError::General)?;
1696 let ndv = self.ndv.unwrap_or(DEFAULT_BLOOM_FILTER_NDV);
1697 Ok(BloomFilterProperties { fpp, ndv })
1698 }
1699}
1700
1701/// Single source of truth for the bloom filter fpp range check, shared by
1702/// [`ColumnProperties::set_bloom_filter_fpp`] (panic path) and
1703/// [`BloomFilterPropertiesBuilder::try_build`] (Result path).
1704fn validate_bloom_filter_fpp(fpp: f64) -> std::result::Result<(), String> {
1705 if !(fpp > 0.0 && fpp < 1.0) {
1706 return Err(format!(
1707 "fpp must be between 0.0 and 1.0 exclusive, got {fpp}"
1708 ));
1709 }
1710 Ok(())
1711}
1712
1713/// Container for column properties that can be changed as part of writer.
1714///
1715/// If a field is `None`, it means that no specific value has been set for this column,
1716/// so some subsequent or default value must be used.
1717#[derive(Debug, Clone, Default, PartialEq)]
1718struct ColumnProperties {
1719 encoding: Option<Encoding>,
1720 codec: Option<Compression>,
1721 data_page_size_limit: Option<usize>,
1722 data_page_row_count_limit: Option<usize>,
1723 dictionary_page_size_limit: Option<usize>,
1724 dictionary_enabled: Option<bool>,
1725 statistics_enabled: Option<EnabledStatistics>,
1726 write_page_header_statistics: Option<bool>,
1727 /// bloom filter related properties
1728 bloom_filter_properties: Option<BloomFilterProperties>,
1729 /// Whether the bloom filter NDV was explicitly set by the user
1730 bloom_filter_ndv_is_set: bool,
1731 data_page_v2_compression_ratio_threshold: Option<f64>,
1732 delta_binary_packed_encoder_options: Option<DeltaBinaryPackedEncoderOptions>,
1733}
1734
1735impl ColumnProperties {
1736 /// Sets encoding for this column.
1737 ///
1738 /// If dictionary is not enabled, this is treated as a primary encoding for a column.
1739 /// In case when dictionary is enabled for a column, this value is considered to
1740 /// be a fallback encoding.
1741 ///
1742 /// Panics if user tries to set dictionary encoding here, regardless of dictionary
1743 /// encoding flag being set. Use `set_dictionary_enabled` method to enable dictionary
1744 /// for a column.
1745 fn set_encoding(&mut self, value: Encoding) {
1746 if value == Encoding::PLAIN_DICTIONARY || value == Encoding::RLE_DICTIONARY {
1747 panic!("Dictionary encoding can not be used as fallback encoding");
1748 }
1749 self.encoding = Some(value);
1750 }
1751
1752 /// Sets compression codec for this column.
1753 fn set_compression(&mut self, value: Compression) {
1754 self.codec = Some(value);
1755 }
1756
1757 /// Sets data page size limit for this column.
1758 fn set_data_page_size_limit(&mut self, value: usize) {
1759 self.data_page_size_limit = Some(value);
1760 }
1761
1762 /// Sets data page row count limit for this column.
1763 fn set_data_page_row_count_limit(&mut self, value: usize) {
1764 assert_ne!(value, 0, "Cannot have a 0 data page row count limit");
1765 self.data_page_row_count_limit = Some(value);
1766 }
1767
1768 /// Sets whether dictionary encoding is enabled for this column.
1769 fn set_dictionary_enabled(&mut self, enabled: bool) {
1770 self.dictionary_enabled = Some(enabled);
1771 }
1772
1773 /// Sets dictionary page size limit for this column.
1774 fn set_dictionary_page_size_limit(&mut self, value: usize) {
1775 self.dictionary_page_size_limit = Some(value);
1776 }
1777
1778 /// Sets the statistics level for this column.
1779 fn set_statistics_enabled(&mut self, enabled: EnabledStatistics) {
1780 self.statistics_enabled = Some(enabled);
1781 }
1782
1783 /// Sets whether to write statistics in the page header for this column.
1784 fn set_write_page_header_statistics(&mut self, enabled: bool) {
1785 self.write_page_header_statistics = Some(enabled);
1786 }
1787
1788 /// If `value` is `true`, sets bloom filter properties to default values if not previously set,
1789 /// otherwise it is a no-op.
1790 /// If `value` is `false`, resets bloom filter properties to `None`.
1791 fn set_bloom_filter_enabled(&mut self, value: bool) {
1792 if value && self.bloom_filter_properties.is_none() {
1793 self.bloom_filter_properties = Some(Default::default())
1794 } else if !value {
1795 self.bloom_filter_properties = None
1796 }
1797 }
1798
1799 /// Sets the false positive probability for bloom filter for this column, and implicitly enables
1800 /// bloom filter if not previously enabled.
1801 ///
1802 /// # Panics
1803 ///
1804 /// Panics if the `value` is not between 0 and 1 exclusive
1805 fn set_bloom_filter_fpp(&mut self, value: f64) {
1806 if let Err(msg) = validate_bloom_filter_fpp(value) {
1807 panic!("{msg}");
1808 }
1809 self.bloom_filter_properties
1810 .get_or_insert_with(Default::default)
1811 .fpp = value;
1812 }
1813
1814 /// Sets the maximum expected number of distinct (unique) values for bloom filter for this
1815 /// column, and implicitly enables bloom filter if not previously enabled.
1816 fn set_bloom_filter_ndv(&mut self, value: u64) {
1817 self.bloom_filter_properties
1818 .get_or_insert_with(Default::default)
1819 .ndv = value;
1820 self.bloom_filter_ndv_is_set = true;
1821 }
1822
1823 /// Sets the bloom filter properties for this column from a fully-built
1824 /// [`BloomFilterProperties`], implicitly enabling the bloom filter.
1825 ///
1826 /// Both `fpp` and `ndv` from `value` are treated as explicit, so the
1827 /// build-time row-group-size NDV fallback in
1828 /// [`WriterPropertiesBuilder::build`] will not override them.
1829 fn set_bloom_filter_properties(&mut self, value: BloomFilterProperties) {
1830 self.bloom_filter_properties = Some(value);
1831 self.bloom_filter_ndv_is_set = true;
1832 }
1833
1834 /// Sets the Data Page v2 compression ratio threshold for this column.
1835 ///
1836 /// # Panics
1837 /// If `value` is not finite or is not strictly positive.
1838 fn set_data_page_v2_compression_ratio_threshold(&mut self, value: f64) {
1839 assert!(
1840 value.is_finite() && value > 0.0,
1841 "data_page_v2_compression_ratio_threshold must be a positive finite number, got {value}"
1842 );
1843 self.data_page_v2_compression_ratio_threshold = Some(value);
1844 }
1845
1846 fn set_delta_binary_packed_encoder_options(&mut self, value: DeltaBinaryPackedEncoderOptions) {
1847 self.delta_binary_packed_encoder_options = Some(value);
1848 }
1849
1850 /// Returns optional encoding for this column.
1851 fn encoding(&self) -> Option<Encoding> {
1852 self.encoding
1853 }
1854
1855 /// Returns optional compression codec for this column.
1856 fn compression(&self) -> Option<Compression> {
1857 self.codec
1858 }
1859
1860 /// Returns `Some(true)` if dictionary encoding is enabled for this column, if
1861 /// disabled then returns `Some(false)`. If result is `None`, then no setting has
1862 /// been provided.
1863 fn dictionary_enabled(&self) -> Option<bool> {
1864 self.dictionary_enabled
1865 }
1866
1867 /// Returns optional dictionary page size limit for this column.
1868 fn dictionary_page_size_limit(&self) -> Option<usize> {
1869 self.dictionary_page_size_limit
1870 }
1871
1872 /// Returns optional data page size limit for this column.
1873 fn data_page_size_limit(&self) -> Option<usize> {
1874 self.data_page_size_limit
1875 }
1876
1877 /// Returns optional data page row count limit for this column.
1878 fn data_page_row_count_limit(&self) -> Option<usize> {
1879 self.data_page_row_count_limit
1880 }
1881
1882 /// Returns optional statistics level requested for this column. If result is `None`,
1883 /// then no setting has been provided.
1884 fn statistics_enabled(&self) -> Option<EnabledStatistics> {
1885 self.statistics_enabled
1886 }
1887
1888 /// Returns `Some(true)` if [`Statistics`] are to be written to the page header for this
1889 /// column.
1890 ///
1891 /// [`Statistics`]: crate::file::statistics::Statistics
1892 fn write_page_header_statistics(&self) -> Option<bool> {
1893 self.write_page_header_statistics
1894 }
1895
1896 /// Returns the bloom filter properties, or `None` if not enabled
1897 fn bloom_filter_properties(&self) -> Option<&BloomFilterProperties> {
1898 self.bloom_filter_properties.as_ref()
1899 }
1900
1901 /// Returns optional Data Page v2 compression ratio threshold for this column.
1902 fn data_page_v2_compression_ratio_threshold(&self) -> Option<f64> {
1903 self.data_page_v2_compression_ratio_threshold
1904 }
1905
1906 fn delta_binary_packed_encoder_options(&self) -> Option<DeltaBinaryPackedEncoderOptions> {
1907 self.delta_binary_packed_encoder_options
1908 }
1909
1910 /// If bloom filter is enabled and NDV was not explicitly set, resolve it to the
1911 /// given `default_ndv` (typically derived from `max_row_group_row_count`).
1912 fn resolve_bloom_filter_ndv(&mut self, default_ndv: u64) {
1913 if !self.bloom_filter_ndv_is_set
1914 && let Some(ref mut bf) = self.bloom_filter_properties
1915 {
1916 bf.ndv = default_ndv;
1917 }
1918 }
1919}
1920
1921/// Every per-column writer setting for one leaf column, resolved against the
1922/// per-column overrides and the file-wide defaults.
1923///
1924/// Built by [`WriterProperties::resolve_column_properties`].
1925#[derive(Debug, Clone, PartialEq)]
1926pub(crate) struct ResolvedColumnProperties {
1927 /// See [`WriterProperties::encoding`].
1928 pub(crate) encoding: Option<Encoding>,
1929 /// See [`WriterProperties::compression`].
1930 pub(crate) compression: Compression,
1931 /// See [`WriterProperties::dictionary_enabled`].
1932 pub(crate) dictionary_enabled: bool,
1933 /// See [`WriterProperties::statistics_enabled`].
1934 pub(crate) statistics_enabled: EnabledStatistics,
1935 /// See [`WriterProperties::write_page_header_statistics`].
1936 pub(crate) write_page_header_statistics: bool,
1937 /// See [`WriterProperties::column_data_page_size_limit`].
1938 pub(crate) data_page_size_limit: usize,
1939 /// See [`WriterProperties::column_data_page_row_count_limit`].
1940 pub(crate) data_page_row_count_limit: usize,
1941 /// See [`WriterProperties::column_dictionary_page_size_limit`].
1942 pub(crate) dictionary_page_size_limit: usize,
1943 /// See [`WriterProperties::column_data_page_v2_compression_ratio_threshold`].
1944 pub(crate) data_page_v2_compression_ratio_threshold: f64,
1945 /// See [`WriterProperties::bloom_filter_properties`].
1946 pub(crate) bloom_filter_properties: Option<BloomFilterProperties>,
1947 /// See [`WriterProperties::delta_binary_packed_encoder_options`].
1948 pub(crate) delta_binary_packed_encoder_options: Option<DeltaBinaryPackedEncoderOptions>,
1949}
1950
1951/// Returns the setting read by `get` for `column` if it sets one, otherwise the
1952/// setting on `default`.
1953///
1954/// `column` is the per-column override entry, if the column has one.
1955#[inline]
1956fn column_or_default<T>(
1957 column: Option<&ColumnProperties>,
1958 default: &ColumnProperties,
1959 get: impl Fn(&ColumnProperties) -> Option<T>,
1960) -> Option<T> {
1961 column.and_then(&get).or_else(|| get(default))
1962}
1963
1964fn resolve_encoding(
1965 column: Option<&ColumnProperties>,
1966 default: &ColumnProperties,
1967) -> Option<Encoding> {
1968 column_or_default(column, default, ColumnProperties::encoding)
1969}
1970
1971fn resolve_compression(
1972 column: Option<&ColumnProperties>,
1973 default: &ColumnProperties,
1974) -> Compression {
1975 column_or_default(column, default, ColumnProperties::compression).unwrap_or(DEFAULT_COMPRESSION)
1976}
1977
1978fn resolve_dictionary_enabled(
1979 column: Option<&ColumnProperties>,
1980 default: &ColumnProperties,
1981) -> bool {
1982 column_or_default(column, default, ColumnProperties::dictionary_enabled)
1983 .unwrap_or(DEFAULT_DICTIONARY_ENABLED)
1984}
1985
1986fn resolve_statistics_enabled(
1987 column: Option<&ColumnProperties>,
1988 default: &ColumnProperties,
1989) -> EnabledStatistics {
1990 column_or_default(column, default, ColumnProperties::statistics_enabled)
1991 .unwrap_or(DEFAULT_STATISTICS_ENABLED)
1992}
1993
1994fn resolve_write_page_header_statistics(
1995 column: Option<&ColumnProperties>,
1996 default: &ColumnProperties,
1997) -> bool {
1998 column_or_default(
1999 column,
2000 default,
2001 ColumnProperties::write_page_header_statistics,
2002 )
2003 .unwrap_or(DEFAULT_WRITE_PAGE_HEADER_STATISTICS)
2004}
2005
2006fn resolve_data_page_size_limit(
2007 column: Option<&ColumnProperties>,
2008 default: &ColumnProperties,
2009) -> usize {
2010 column_or_default(column, default, ColumnProperties::data_page_size_limit)
2011 .unwrap_or(DEFAULT_PAGE_SIZE)
2012}
2013
2014fn resolve_data_page_row_count_limit(
2015 column: Option<&ColumnProperties>,
2016 default: &ColumnProperties,
2017) -> usize {
2018 column_or_default(column, default, ColumnProperties::data_page_row_count_limit)
2019 .unwrap_or(DEFAULT_DATA_PAGE_ROW_COUNT_LIMIT)
2020}
2021
2022fn resolve_dictionary_page_size_limit(
2023 column: Option<&ColumnProperties>,
2024 default: &ColumnProperties,
2025) -> usize {
2026 column_or_default(
2027 column,
2028 default,
2029 ColumnProperties::dictionary_page_size_limit,
2030 )
2031 .unwrap_or(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT)
2032}
2033
2034fn resolve_data_page_v2_compression_ratio_threshold(
2035 column: Option<&ColumnProperties>,
2036 default: &ColumnProperties,
2037) -> f64 {
2038 column_or_default(
2039 column,
2040 default,
2041 ColumnProperties::data_page_v2_compression_ratio_threshold,
2042 )
2043 .unwrap_or(DEFAULT_DATA_PAGE_V2_COMPRESSION_RATIO_THRESHOLD)
2044}
2045
2046fn resolve_bloom_filter_properties<'a>(
2047 column: Option<&'a ColumnProperties>,
2048 default: &'a ColumnProperties,
2049) -> Option<&'a BloomFilterProperties> {
2050 column
2051 .and_then(ColumnProperties::bloom_filter_properties)
2052 .or_else(|| default.bloom_filter_properties())
2053}
2054
2055fn resolve_delta_binary_packed_encoder_options(
2056 column: Option<&ColumnProperties>,
2057 default: &ColumnProperties,
2058) -> Option<DeltaBinaryPackedEncoderOptions> {
2059 column_or_default(
2060 column,
2061 default,
2062 ColumnProperties::delta_binary_packed_encoder_options,
2063 )
2064}
2065
2066/// Reference counted reader properties.
2067pub type ReaderPropertiesPtr = Arc<ReaderProperties>;
2068
2069const DEFAULT_READ_BLOOM_FILTER: bool = false;
2070const DEFAULT_READ_PAGE_STATS: bool = false;
2071
2072/// Configuration settings for reading parquet files.
2073///
2074/// All properties are immutable and `Send` + `Sync`.
2075/// Use [`ReaderPropertiesBuilder`] to assemble these properties.
2076///
2077/// # Example
2078///
2079/// ```rust
2080/// use parquet::file::properties::ReaderProperties;
2081///
2082/// // Create properties with default configuration.
2083/// let props = ReaderProperties::builder().build();
2084///
2085/// // Use properties builder to set certain options and assemble the configuration.
2086/// let props = ReaderProperties::builder()
2087/// .set_backward_compatible_lz4(false)
2088/// .build();
2089/// ```
2090pub struct ReaderProperties {
2091 codec_options: CodecOptions,
2092 read_bloom_filter: bool,
2093 read_page_stats: bool,
2094}
2095
2096impl ReaderProperties {
2097 /// Returns builder for reader properties with default values.
2098 pub fn builder() -> ReaderPropertiesBuilder {
2099 ReaderPropertiesBuilder::with_defaults()
2100 }
2101
2102 /// Returns codec options.
2103 pub(crate) fn codec_options(&self) -> &CodecOptions {
2104 &self.codec_options
2105 }
2106
2107 /// Returns whether to read bloom filter
2108 pub(crate) fn read_bloom_filter(&self) -> bool {
2109 self.read_bloom_filter
2110 }
2111
2112 /// Returns whether to read page level statistics
2113 pub(crate) fn read_page_stats(&self) -> bool {
2114 self.read_page_stats
2115 }
2116}
2117
2118/// Builder for parquet file reader configuration. See example on
2119/// [`ReaderProperties`]
2120pub struct ReaderPropertiesBuilder {
2121 codec_options_builder: CodecOptionsBuilder,
2122 read_bloom_filter: Option<bool>,
2123 read_page_stats: Option<bool>,
2124}
2125
2126/// Reader properties builder.
2127impl ReaderPropertiesBuilder {
2128 /// Returns default state of the builder.
2129 fn with_defaults() -> Self {
2130 Self {
2131 codec_options_builder: CodecOptionsBuilder::default(),
2132 read_bloom_filter: None,
2133 read_page_stats: None,
2134 }
2135 }
2136
2137 /// Finalizes the configuration and returns immutable reader properties struct.
2138 pub fn build(self) -> ReaderProperties {
2139 ReaderProperties {
2140 codec_options: self.codec_options_builder.build(),
2141 read_bloom_filter: self.read_bloom_filter.unwrap_or(DEFAULT_READ_BLOOM_FILTER),
2142 read_page_stats: self.read_page_stats.unwrap_or(DEFAULT_READ_PAGE_STATS),
2143 }
2144 }
2145
2146 /// Enable/disable backward compatible LZ4.
2147 ///
2148 /// If backward compatible LZ4 is enable, on LZ4_HADOOP error it will fallback
2149 /// to the older versions LZ4 algorithms. That is LZ4_FRAME, for backward compatibility
2150 /// with files generated by older versions of this library, and LZ4_RAW, for backward
2151 /// compatibility with files generated by older versions of parquet-cpp.
2152 ///
2153 /// If backward compatible LZ4 is disabled, on LZ4_HADOOP error it will return the error.
2154 pub fn set_backward_compatible_lz4(mut self, value: bool) -> Self {
2155 self.codec_options_builder = self
2156 .codec_options_builder
2157 .set_backward_compatible_lz4(value);
2158 self
2159 }
2160
2161 /// Enable/disable reading bloom filter
2162 ///
2163 /// If reading bloom filter is enabled, bloom filter will be read from the file.
2164 /// If reading bloom filter is disabled, bloom filter will not be read from the file.
2165 ///
2166 /// By default bloom filter is set to be read.
2167 pub fn set_read_bloom_filter(mut self, value: bool) -> Self {
2168 self.read_bloom_filter = Some(value);
2169 self
2170 }
2171
2172 /// Enable/disable reading page-level statistics
2173 ///
2174 /// If set to `true`, then the reader will decode and populate the [`Statistics`] for
2175 /// each page, if present.
2176 /// If set to `false`, then the reader will skip decoding the statistics.
2177 ///
2178 /// By default statistics will not be decoded.
2179 ///
2180 /// [`Statistics`]: crate::file::statistics::Statistics
2181 pub fn set_read_page_statistics(mut self, value: bool) -> Self {
2182 self.read_page_stats = Some(value);
2183 self
2184 }
2185}
2186
2187#[cfg(test)]
2188mod tests {
2189 use super::*;
2190
2191 #[test]
2192 fn test_writer_version() {
2193 assert_eq!(WriterVersion::PARQUET_1_0.as_num(), 1);
2194 assert_eq!(WriterVersion::PARQUET_2_0.as_num(), 2);
2195 }
2196
2197 /// Every setting resolved in one pass must equal what the individual
2198 /// per-column accessors return, for a column that overrides settings, a
2199 /// column that inherits them, and settings left at their defaults.
2200 #[test]
2201 fn test_resolve_column_properties_matches_individual_accessors() {
2202 let overridden = ColumnPath::from("overridden");
2203 let inherited = ColumnPath::from("inherited");
2204
2205 let props = WriterProperties::builder()
2206 .set_encoding(Encoding::DELTA_BINARY_PACKED)
2207 .set_compression(Compression::SNAPPY)
2208 .set_dictionary_enabled(false)
2209 .set_statistics_enabled(EnabledStatistics::Chunk)
2210 .set_write_page_header_statistics(false)
2211 .set_data_page_size_limit(1111)
2212 .set_dictionary_page_size_limit(2222)
2213 .set_data_page_v2_compression_ratio_threshold(0.25)
2214 .set_bloom_filter_enabled(true)
2215 .set_column_encoding(overridden.clone(), Encoding::PLAIN)
2216 .set_column_compression(overridden.clone(), Compression::UNCOMPRESSED)
2217 .set_column_dictionary_enabled(overridden.clone(), true)
2218 .set_column_statistics_enabled(overridden.clone(), EnabledStatistics::Page)
2219 .set_column_write_page_header_statistics(overridden.clone(), true)
2220 .set_column_data_page_size_limit(overridden.clone(), 3333)
2221 .set_column_dictionary_page_size_limit(overridden.clone(), 4444)
2222 .set_column_data_page_v2_compression_ratio_threshold(overridden.clone(), 0.75)
2223 .set_column_bloom_filter_fpp(overridden.clone(), 0.5)
2224 .build();
2225
2226 // A column with no overrides at all, on properties that are themselves
2227 // entirely default.
2228 let bare = WriterProperties::builder().build();
2229
2230 for (props, col) in [
2231 (&props, &overridden),
2232 (&props, &inherited),
2233 (&bare, &inherited),
2234 ] {
2235 let resolved = props.resolve_column_properties(col);
2236 assert_eq!(resolved.encoding, props.encoding(col), "{col:?}");
2237 assert_eq!(resolved.compression, props.compression(col), "{col:?}");
2238 assert_eq!(
2239 resolved.dictionary_enabled,
2240 props.dictionary_enabled(col),
2241 "{col:?}"
2242 );
2243 assert_eq!(
2244 resolved.statistics_enabled,
2245 props.statistics_enabled(col),
2246 "{col:?}"
2247 );
2248 assert_eq!(
2249 resolved.write_page_header_statistics,
2250 props.write_page_header_statistics(col),
2251 "{col:?}"
2252 );
2253 assert_eq!(
2254 resolved.data_page_size_limit,
2255 props.column_data_page_size_limit(col),
2256 "{col:?}"
2257 );
2258 assert_eq!(
2259 resolved.dictionary_page_size_limit,
2260 props.column_dictionary_page_size_limit(col),
2261 "{col:?}"
2262 );
2263 assert_eq!(
2264 resolved.data_page_v2_compression_ratio_threshold,
2265 props.column_data_page_v2_compression_ratio_threshold(col),
2266 "{col:?}"
2267 );
2268 assert_eq!(
2269 resolved.bloom_filter_properties.as_ref(),
2270 props.bloom_filter_properties(col),
2271 "{col:?}"
2272 );
2273 }
2274 }
2275
2276 #[test]
2277 fn test_writer_properties_default_settings() {
2278 let props = WriterProperties::default();
2279 assert_eq!(props.data_page_size_limit(), DEFAULT_PAGE_SIZE);
2280 assert_eq!(
2281 props.dictionary_page_size_limit(),
2282 DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT
2283 );
2284 assert_eq!(props.write_batch_size(), DEFAULT_WRITE_BATCH_SIZE);
2285 assert_eq!(
2286 props.max_row_group_row_count(),
2287 Some(DEFAULT_MAX_ROW_GROUP_ROW_COUNT)
2288 );
2289 assert_eq!(props.max_row_group_bytes(), None);
2290 assert_eq!(props.bloom_filter_position(), DEFAULT_BLOOM_FILTER_POSITION);
2291 assert_eq!(props.writer_version(), DEFAULT_WRITER_VERSION);
2292 assert_eq!(props.created_by(), DEFAULT_CREATED_BY);
2293 assert_eq!(props.key_value_metadata(), None);
2294 assert_eq!(props.encoding(&ColumnPath::from("col")), None);
2295 assert_eq!(
2296 props.compression(&ColumnPath::from("col")),
2297 DEFAULT_COMPRESSION
2298 );
2299 assert_eq!(
2300 props.dictionary_enabled(&ColumnPath::from("col")),
2301 DEFAULT_DICTIONARY_ENABLED
2302 );
2303 assert_eq!(
2304 props.statistics_enabled(&ColumnPath::from("col")),
2305 DEFAULT_STATISTICS_ENABLED
2306 );
2307 assert!(
2308 props
2309 .bloom_filter_properties(&ColumnPath::from("col"))
2310 .is_none()
2311 );
2312 assert!(
2313 props
2314 .delta_binary_packed_encoder_options(&ColumnPath::from("col"))
2315 .is_none()
2316 );
2317 }
2318
2319 #[test]
2320 fn test_writer_properties_delta_binary_packed_encoder_options() {
2321 let default = DeltaBinaryPackedEncoderOptions::try_new(256, 4).unwrap();
2322 let overridden = DeltaBinaryPackedEncoderOptions::try_new(128, 4).unwrap();
2323 let column = ColumnPath::from("column");
2324 let props = WriterProperties::builder()
2325 .set_delta_binary_packed_encoder_options(default)
2326 .set_column_delta_binary_packed_encoder_options(column.clone(), overridden)
2327 .build();
2328
2329 assert_eq!(
2330 props.delta_binary_packed_encoder_options(&column),
2331 Some(overridden)
2332 );
2333 assert_eq!(
2334 props.delta_binary_packed_encoder_options(&ColumnPath::from("other")),
2335 Some(default)
2336 );
2337 assert_eq!(
2338 props
2339 .resolve_column_properties(&column)
2340 .delta_binary_packed_encoder_options,
2341 Some(overridden)
2342 );
2343
2344 let rebuilt = props.into_builder().build();
2345 assert_eq!(
2346 rebuilt.delta_binary_packed_encoder_options(&column),
2347 Some(overridden)
2348 );
2349 }
2350
2351 #[test]
2352 fn test_writer_properties_dictionary_encoding() {
2353 // dictionary encoding is not configurable, and it should be the same for both
2354 // writer version 1 and 2.
2355 for version in &[WriterVersion::PARQUET_1_0, WriterVersion::PARQUET_2_0] {
2356 let props = WriterProperties::builder()
2357 .set_writer_version(*version)
2358 .build();
2359 assert_eq!(props.dictionary_page_encoding(), Encoding::PLAIN);
2360 assert_eq!(
2361 props.dictionary_data_page_encoding(),
2362 Encoding::RLE_DICTIONARY
2363 );
2364 }
2365 }
2366
2367 #[test]
2368 #[should_panic(expected = "Dictionary encoding can not be used as fallback encoding")]
2369 fn test_writer_properties_panic_when_plain_dictionary_is_fallback() {
2370 // Should panic when user specifies dictionary encoding as fallback encoding.
2371 WriterProperties::builder()
2372 .set_encoding(Encoding::PLAIN_DICTIONARY)
2373 .build();
2374 }
2375
2376 #[test]
2377 #[should_panic(expected = "Dictionary encoding can not be used as fallback encoding")]
2378 fn test_writer_properties_panic_when_rle_dictionary_is_fallback() {
2379 // Should panic when user specifies dictionary encoding as fallback encoding.
2380 WriterProperties::builder()
2381 .set_encoding(Encoding::RLE_DICTIONARY)
2382 .build();
2383 }
2384
2385 #[test]
2386 #[should_panic(expected = "Dictionary encoding can not be used as fallback encoding")]
2387 fn test_writer_properties_panic_when_dictionary_is_enabled() {
2388 WriterProperties::builder()
2389 .set_dictionary_enabled(true)
2390 .set_column_encoding(ColumnPath::from("col"), Encoding::RLE_DICTIONARY)
2391 .build();
2392 }
2393
2394 #[test]
2395 #[should_panic(expected = "Dictionary encoding can not be used as fallback encoding")]
2396 fn test_writer_properties_panic_when_dictionary_is_disabled() {
2397 WriterProperties::builder()
2398 .set_dictionary_enabled(false)
2399 .set_column_encoding(ColumnPath::from("col"), Encoding::RLE_DICTIONARY)
2400 .build();
2401 }
2402
2403 #[test]
2404 fn test_writer_properties_builder() {
2405 let props = WriterProperties::builder()
2406 // file settings
2407 .set_writer_version(WriterVersion::PARQUET_2_0)
2408 .set_data_page_size_limit(10)
2409 .set_dictionary_page_size_limit(20)
2410 .set_write_batch_size(30)
2411 .set_max_row_group_row_count(Some(40))
2412 .set_created_by("default".to_owned())
2413 .set_key_value_metadata(Some(vec![KeyValue::new(
2414 "key".to_string(),
2415 "value".to_string(),
2416 )]))
2417 // global column settings
2418 .set_encoding(Encoding::DELTA_BINARY_PACKED)
2419 .set_compression(Compression::GZIP(Default::default()))
2420 .set_dictionary_enabled(false)
2421 .set_statistics_enabled(EnabledStatistics::None)
2422 // specific column settings
2423 .set_column_encoding(ColumnPath::from("col"), Encoding::RLE)
2424 .set_column_compression(ColumnPath::from("col"), Compression::SNAPPY)
2425 .set_column_dictionary_enabled(ColumnPath::from("col"), true)
2426 .set_column_statistics_enabled(ColumnPath::from("col"), EnabledStatistics::Chunk)
2427 .set_column_bloom_filter_enabled(ColumnPath::from("col"), true)
2428 .set_column_bloom_filter_max_ndv(ColumnPath::from("col"), 100_u64)
2429 .set_column_bloom_filter_fpp(ColumnPath::from("col"), 0.1)
2430 .build();
2431
2432 fn test_props(props: &WriterProperties) {
2433 assert_eq!(props.writer_version(), WriterVersion::PARQUET_2_0);
2434 assert_eq!(props.data_page_size_limit(), 10);
2435 assert_eq!(props.dictionary_page_size_limit(), 20);
2436 assert_eq!(props.write_batch_size(), 30);
2437 assert_eq!(props.max_row_group_row_count(), Some(40));
2438 assert_eq!(props.created_by(), "default");
2439 assert_eq!(
2440 props.key_value_metadata(),
2441 Some(&vec![
2442 KeyValue::new("key".to_string(), "value".to_string(),)
2443 ])
2444 );
2445
2446 assert_eq!(
2447 props.encoding(&ColumnPath::from("a")),
2448 Some(Encoding::DELTA_BINARY_PACKED)
2449 );
2450 assert_eq!(
2451 props.compression(&ColumnPath::from("a")),
2452 Compression::GZIP(Default::default())
2453 );
2454 assert!(!props.dictionary_enabled(&ColumnPath::from("a")));
2455 assert_eq!(
2456 props.statistics_enabled(&ColumnPath::from("a")),
2457 EnabledStatistics::None
2458 );
2459
2460 assert_eq!(
2461 props.encoding(&ColumnPath::from("col")),
2462 Some(Encoding::RLE)
2463 );
2464 assert_eq!(
2465 props.compression(&ColumnPath::from("col")),
2466 Compression::SNAPPY
2467 );
2468 assert!(props.dictionary_enabled(&ColumnPath::from("col")));
2469 assert_eq!(
2470 props.statistics_enabled(&ColumnPath::from("col")),
2471 EnabledStatistics::Chunk
2472 );
2473 assert_eq!(
2474 props.bloom_filter_properties(&ColumnPath::from("col")),
2475 Some(&BloomFilterProperties { fpp: 0.1, ndv: 100 })
2476 );
2477 }
2478
2479 // Test direct build of properties
2480 test_props(&props);
2481
2482 // Test that into_builder() gives the same result
2483 let props_into_builder_and_back = props.into_builder().build();
2484 test_props(&props_into_builder_and_back);
2485 }
2486
2487 #[test]
2488 fn test_writer_properties_builder_partial_defaults() {
2489 let props = WriterProperties::builder()
2490 .set_encoding(Encoding::DELTA_BINARY_PACKED)
2491 .set_compression(Compression::GZIP(Default::default()))
2492 .set_bloom_filter_enabled(true)
2493 .set_column_encoding(ColumnPath::from("col"), Encoding::RLE)
2494 .build();
2495
2496 assert_eq!(
2497 props.encoding(&ColumnPath::from("col")),
2498 Some(Encoding::RLE)
2499 );
2500 assert_eq!(
2501 props.compression(&ColumnPath::from("col")),
2502 Compression::GZIP(Default::default())
2503 );
2504 assert_eq!(
2505 props.dictionary_enabled(&ColumnPath::from("col")),
2506 DEFAULT_DICTIONARY_ENABLED
2507 );
2508 assert_eq!(
2509 props.bloom_filter_properties(&ColumnPath::from("col")),
2510 Some(&BloomFilterProperties {
2511 fpp: DEFAULT_BLOOM_FILTER_FPP,
2512 ndv: DEFAULT_BLOOM_FILTER_NDV,
2513 })
2514 );
2515 }
2516
2517 #[test]
2518 #[should_panic(expected = "Cannot have a 0 max row group row count")]
2519 fn test_writer_properties_panic_on_zero_row_group_row_count() {
2520 let _ = WriterProperties::builder().set_max_row_group_row_count(Some(0));
2521 }
2522
2523 #[test]
2524 #[should_panic(expected = "Cannot have a 0 max row group bytes")]
2525 fn test_writer_properties_panic_on_zero_row_group_bytes() {
2526 let _ = WriterProperties::builder().set_max_row_group_bytes(Some(0));
2527 }
2528
2529 #[test]
2530 #[should_panic(expected = "Cannot have a 0 write batch size")]
2531 fn test_writer_properties_panic_on_zero_write_batch_size() {
2532 let _ = WriterProperties::builder().set_write_batch_size(0);
2533 }
2534
2535 #[test]
2536 #[should_panic(expected = "Cannot have a 0 data page row count limit")]
2537 fn test_writer_properties_panic_on_zero_data_page_row_count_limit() {
2538 let _ = WriterProperties::builder().set_data_page_row_count_limit(0);
2539 }
2540
2541 #[test]
2542 fn test_writer_properties_bloom_filter_ndv_fpp_set() {
2543 assert_eq!(
2544 WriterProperties::builder()
2545 .build()
2546 .bloom_filter_properties(&ColumnPath::from("col")),
2547 None
2548 );
2549 assert_eq!(
2550 WriterProperties::builder()
2551 .set_bloom_filter_max_ndv(100)
2552 .build()
2553 .bloom_filter_properties(&ColumnPath::from("col")),
2554 Some(&BloomFilterProperties {
2555 fpp: DEFAULT_BLOOM_FILTER_FPP,
2556 ndv: 100,
2557 })
2558 );
2559 assert_eq!(
2560 WriterProperties::builder()
2561 .set_bloom_filter_fpp(0.1)
2562 .build()
2563 .bloom_filter_properties(&ColumnPath::from("col")),
2564 Some(&BloomFilterProperties {
2565 fpp: 0.1,
2566 ndv: DEFAULT_BLOOM_FILTER_NDV,
2567 })
2568 );
2569 }
2570
2571 #[test]
2572 fn test_writer_properties_column_data_page_v2_compression_ratio_threshold() {
2573 let props = WriterProperties::builder()
2574 .set_data_page_v2_compression_ratio_threshold(0.5)
2575 .set_column_data_page_v2_compression_ratio_threshold(ColumnPath::from("col"), 0.1)
2576 .build();
2577
2578 assert_eq!(props.data_page_v2_compression_ratio_threshold(), 0.5);
2579 assert_eq!(
2580 props.column_data_page_v2_compression_ratio_threshold(&ColumnPath::from("col")),
2581 0.1
2582 );
2583 assert_eq!(
2584 props.column_data_page_v2_compression_ratio_threshold(&ColumnPath::from("other")),
2585 0.5
2586 );
2587 }
2588
2589 #[test]
2590 #[should_panic(
2591 expected = "data_page_v2_compression_ratio_threshold must be a positive finite number"
2592 )]
2593 fn test_writer_properties_panic_on_invalid_data_page_v2_compression_ratio_threshold() {
2594 WriterProperties::builder()
2595 .set_data_page_v2_compression_ratio_threshold(0.0)
2596 .build();
2597 }
2598
2599 #[test]
2600 #[expect(deprecated)]
2601 fn test_writer_properties_deprecated_bloom_filter_ndv_setters_still_work() {
2602 let col = ColumnPath::from("col");
2603 let props = WriterProperties::builder()
2604 .set_bloom_filter_ndv(100)
2605 .set_column_bloom_filter_ndv(col.clone(), 200)
2606 .build();
2607 assert_eq!(
2608 props.bloom_filter_properties(&ColumnPath::from("other")),
2609 Some(&BloomFilterProperties {
2610 fpp: DEFAULT_BLOOM_FILTER_FPP,
2611 ndv: 100,
2612 })
2613 );
2614 assert_eq!(
2615 props.bloom_filter_properties(&col),
2616 Some(&BloomFilterProperties {
2617 fpp: DEFAULT_BLOOM_FILTER_FPP,
2618 ndv: 200,
2619 })
2620 );
2621 }
2622
2623 #[test]
2624 fn test_writer_properties_column_dictionary_page_size_limit() {
2625 let props = WriterProperties::builder()
2626 .set_dictionary_page_size_limit(100)
2627 .set_column_dictionary_page_size_limit(ColumnPath::from("col"), 10)
2628 .build();
2629
2630 assert_eq!(props.dictionary_page_size_limit(), 100);
2631 assert_eq!(
2632 props.column_dictionary_page_size_limit(&ColumnPath::from("col")),
2633 10
2634 );
2635 assert_eq!(
2636 props.column_dictionary_page_size_limit(&ColumnPath::from("other")),
2637 100
2638 );
2639 }
2640
2641 #[test]
2642 fn test_writer_properties_column_data_page_size_limit() {
2643 let props = WriterProperties::builder()
2644 .set_data_page_size_limit(100)
2645 .set_column_data_page_size_limit(ColumnPath::from("col"), 10)
2646 .build();
2647
2648 assert_eq!(props.data_page_size_limit(), 100);
2649 assert_eq!(
2650 props.column_data_page_size_limit(&ColumnPath::from("col")),
2651 10
2652 );
2653 assert_eq!(
2654 props.column_data_page_size_limit(&ColumnPath::from("other")),
2655 100
2656 );
2657 }
2658
2659 #[test]
2660 fn test_writer_properties_column_data_page_row_count_limit() {
2661 let props = WriterProperties::builder()
2662 .set_data_page_row_count_limit(100)
2663 .set_column_data_page_row_count_limit(ColumnPath::from("col"), 10)
2664 .build();
2665
2666 assert_eq!(props.data_page_row_count_limit(), 100);
2667 assert_eq!(
2668 props.column_data_page_row_count_limit(&ColumnPath::from("col")),
2669 10
2670 );
2671 assert_eq!(
2672 props.column_data_page_row_count_limit(&ColumnPath::from("other")),
2673 100
2674 );
2675 }
2676
2677 #[test]
2678 #[should_panic(expected = "Cannot have a 0 data page row count limit")]
2679 fn test_writer_properties_panic_on_zero_column_data_page_row_count_limit() {
2680 let _ = WriterProperties::builder()
2681 .set_column_data_page_row_count_limit(ColumnPath::from("col"), 0);
2682 }
2683
2684 #[test]
2685 fn test_reader_properties_default_settings() {
2686 let props = ReaderProperties::builder().build();
2687
2688 let codec_options = CodecOptionsBuilder::default()
2689 .set_backward_compatible_lz4(true)
2690 .build();
2691
2692 assert_eq!(props.codec_options(), &codec_options);
2693 assert!(!props.read_bloom_filter());
2694 }
2695
2696 #[test]
2697 fn test_reader_properties_builder() {
2698 let props = ReaderProperties::builder()
2699 .set_backward_compatible_lz4(false)
2700 .build();
2701
2702 let codec_options = CodecOptionsBuilder::default()
2703 .set_backward_compatible_lz4(false)
2704 .build();
2705
2706 assert_eq!(props.codec_options(), &codec_options);
2707 }
2708
2709 #[test]
2710 fn test_parse_writerversion() {
2711 let mut writer_version = "PARQUET_1_0".parse::<WriterVersion>().unwrap();
2712 assert_eq!(writer_version, WriterVersion::PARQUET_1_0);
2713 writer_version = "PARQUET_2_0".parse::<WriterVersion>().unwrap();
2714 assert_eq!(writer_version, WriterVersion::PARQUET_2_0);
2715
2716 // test lowercase
2717 writer_version = "parquet_1_0".parse::<WriterVersion>().unwrap();
2718 assert_eq!(writer_version, WriterVersion::PARQUET_1_0);
2719
2720 // test invalid version
2721 match "PARQUET_-1_0".parse::<WriterVersion>() {
2722 Ok(_) => panic!("Should not be able to parse PARQUET_-1_0"),
2723 Err(e) => {
2724 assert_eq!(e, "Invalid writer version: PARQUET_-1_0");
2725 }
2726 }
2727 }
2728
2729 #[test]
2730 fn test_parse_enabledstatistics() {
2731 let mut enabled_statistics = "NONE".parse::<EnabledStatistics>().unwrap();
2732 assert_eq!(enabled_statistics, EnabledStatistics::None);
2733 enabled_statistics = "CHUNK".parse::<EnabledStatistics>().unwrap();
2734 assert_eq!(enabled_statistics, EnabledStatistics::Chunk);
2735 enabled_statistics = "PAGE".parse::<EnabledStatistics>().unwrap();
2736 assert_eq!(enabled_statistics, EnabledStatistics::Page);
2737
2738 // test lowercase
2739 enabled_statistics = "none".parse::<EnabledStatistics>().unwrap();
2740 assert_eq!(enabled_statistics, EnabledStatistics::None);
2741
2742 //test invalid statistics
2743 match "ChunkAndPage".parse::<EnabledStatistics>() {
2744 Ok(_) => panic!("Should not be able to parse ChunkAndPage"),
2745 Err(e) => {
2746 assert_eq!(e, "Invalid statistics arg: ChunkAndPage");
2747 }
2748 }
2749 }
2750
2751 #[test]
2752 fn test_cdc_options_equality() {
2753 let opts = CdcOptions::default();
2754 assert_eq!(opts, CdcOptions::default());
2755
2756 let custom = CdcOptions {
2757 min_chunk_size: 1024,
2758 max_chunk_size: 8192,
2759 norm_level: 1,
2760 };
2761 assert_eq!(custom, custom);
2762 assert_ne!(opts, custom);
2763 }
2764
2765 #[test]
2766 #[expect(clippy::float_cmp_const)]
2767 fn test_bloom_filter_builder_default() {
2768 let props = BloomFilterProperties::builder().build();
2769 assert_eq!(props.fpp, DEFAULT_BLOOM_FILTER_FPP);
2770 assert_eq!(props.ndv, DEFAULT_BLOOM_FILTER_NDV);
2771 assert_eq!(props, BloomFilterProperties::default());
2772 assert_eq!(
2773 BloomFilterPropertiesBuilder::new().build(),
2774 BloomFilterProperties::default()
2775 );
2776 }
2777
2778 #[test]
2779 fn test_bloom_filter_builder_explicit_fpp() {
2780 let props = BloomFilterProperties::builder().with_fpp(0.01).build();
2781 assert_eq!(props.fpp, 0.01);
2782 assert_eq!(props.ndv, DEFAULT_BLOOM_FILTER_NDV);
2783 }
2784
2785 #[test]
2786 #[expect(clippy::float_cmp_const)]
2787 fn test_bloom_filter_builder_explicit_ndv() {
2788 let props = BloomFilterProperties::builder().with_max_ndv(1000).build();
2789 assert_eq!(props.fpp, DEFAULT_BLOOM_FILTER_FPP);
2790 assert_eq!(props.ndv, 1000);
2791 }
2792
2793 #[test]
2794 fn test_bloom_filter_builder_validates_fpp() {
2795 for wrong_val in [0.0_f64, 1.0, -0.5, 2.0] {
2796 let result = std::panic::catch_unwind(|| {
2797 BloomFilterProperties::builder().with_fpp(wrong_val).build()
2798 });
2799 assert!(
2800 result.is_err(),
2801 "with_fpp({wrong_val}).build() should reject value outside (0, 1)"
2802 );
2803 }
2804 }
2805
2806 #[test]
2807 fn test_bloom_filter_builder_try_build_validates_fpp() {
2808 for wrong_val in [0.0_f64, 1.0, -0.5, 2.0] {
2809 let result = BloomFilterProperties::builder()
2810 .with_fpp(wrong_val)
2811 .try_build();
2812 assert!(
2813 result.is_err(),
2814 "try_build() should return Err for fpp outside (0, 1)"
2815 );
2816 }
2817
2818 let ok = BloomFilterProperties::builder()
2819 .with_fpp(0.01)
2820 .with_max_ndv(1000)
2821 .try_build()
2822 .expect("valid fpp should yield Ok");
2823 assert_eq!(ok.fpp, 0.01);
2824 assert_eq!(ok.ndv, 1000);
2825 }
2826
2827 #[test]
2828 #[expect(clippy::float_cmp_const)]
2829 fn test_column_specific_implicit_ndv_uses_row_group_size() {
2830 let custom_row_group_size: usize = 7777;
2831 let col = ColumnPath::from("col");
2832 let props = WriterProperties::builder()
2833 .set_max_row_group_row_count(Some(custom_row_group_size))
2834 .set_column_bloom_filter_enabled(col.clone(), true)
2835 .build();
2836 let bf = props
2837 .bloom_filter_properties(&col)
2838 .expect("bloom filter should be enabled for col");
2839
2840 assert_eq!(bf.ndv, custom_row_group_size as u64);
2841 assert_eq!(bf.fpp, DEFAULT_BLOOM_FILTER_FPP);
2842 }
2843
2844 #[test]
2845 fn test_set_bloom_filter_properties_applied_globally() {
2846 let bf = BloomFilterProperties::builder()
2847 .with_fpp(0.01)
2848 .with_max_ndv(500)
2849 .build();
2850 let props = WriterProperties::builder()
2851 .set_bloom_filter_properties(bf.clone())
2852 .build();
2853
2854 assert_eq!(
2855 props.bloom_filter_properties(&ColumnPath::from("a")),
2856 Some(&bf),
2857 );
2858 assert_eq!(
2859 props.bloom_filter_properties(&ColumnPath::from("b")),
2860 Some(&bf),
2861 );
2862 }
2863
2864 #[test]
2865 fn test_set_column_bloom_filter_properties_overrides_global() {
2866 let global = BloomFilterProperties::builder()
2867 .with_fpp(0.01)
2868 .with_max_ndv(500)
2869 .build();
2870 let tailored = BloomFilterProperties::builder()
2871 .with_fpp(0.02)
2872 .with_max_ndv(1000)
2873 .build();
2874
2875 let col = ColumnPath::from("col");
2876 let props = WriterProperties::builder()
2877 .set_bloom_filter_properties(global.clone())
2878 .set_column_bloom_filter_properties(col.clone(), tailored.clone())
2879 .build();
2880
2881 assert_eq!(props.bloom_filter_properties(&col), Some(&tailored));
2882 assert_eq!(
2883 props.bloom_filter_properties(&ColumnPath::from("other")),
2884 Some(&global)
2885 );
2886 }
2887
2888 #[test]
2889 fn test_set_bloom_filter_properties_preserve_explicit_ndv() {
2890 let bf = BloomFilterProperties::builder().with_max_ndv(42).build();
2891 let props = WriterProperties::builder()
2892 .set_max_row_group_row_count(Some(99_999))
2893 .set_bloom_filter_properties(bf)
2894 .build();
2895 let result = props
2896 .bloom_filter_properties(&ColumnPath::from("col"))
2897 .expect("bloom filter should be enabled");
2898
2899 assert_eq!(
2900 result.ndv, 42,
2901 "explicit ndv must not be overridden by row-group-size fallback"
2902 );
2903 }
2904}