Struct WriterProperties 

pub struct WriterProperties {

    data_page_row_count_limit: usize,
    write_batch_size: usize,
    max_row_group_row_count: Option<usize>,
    max_row_group_bytes: Option<usize>,
    bloom_filter_position: BloomFilterPosition,
    bloom_filter_for_dictionary_encoded_chunks: bool,
    writer_version: WriterVersion,
    created_by: String,
    offset_index_setting: OffsetIndexSetting,
    pub(crate) key_value_metadata: Option<Vec<KeyValue>>,
    default_column_properties: ColumnProperties,
    column_properties: HashMap<ColumnPath, ColumnProperties>,
    sorting_columns: Option<Vec<SortingColumn>>,
    column_index_truncate_length: Option<usize>,
    statistics_truncate_length: Option<usize>,
    coerce_types: bool,
    write_row_group_number_distinct_values: bool,
    content_defined_chunking: Option<CdcOptions>,
    write_path_in_schema: bool,
    pub(crate) file_encryption_properties: Option<Arc<FileEncryptionProperties>>,
}

Configuration settings for writing parquet files.

Use Self::builder to create a WriterPropertiesBuilder to change settings.

Example

// Create properties with default configuration.
let props = WriterProperties::default();

// Use properties builder to set certain options and assemble the configuration.
let props = WriterProperties::builder()
    .set_writer_version(WriterVersion::PARQUET_1_0)
    .set_encoding(Encoding::PLAIN)
    .set_column_encoding(ColumnPath::from("col1"), Encoding::DELTA_BINARY_PACKED)
    .set_compression(Compression::SNAPPY)
    .build();

assert_eq!(props.writer_version(), WriterVersion::PARQUET_1_0);
assert_eq!(
    props.encoding(&ColumnPath::from("col1")),
    Some(Encoding::DELTA_BINARY_PACKED)
);
assert_eq!(
    props.encoding(&ColumnPath::from("col2")),
    Some(Encoding::PLAIN)
);

Fields

data_page_row_count_limit: usize

write_batch_size: usize

max_row_group_row_count: Option<usize>

max_row_group_bytes: Option<usize>

bloom_filter_position: BloomFilterPosition

bloom_filter_for_dictionary_encoded_chunks: bool

writer_version: WriterVersion

created_by: String

offset_index_setting: OffsetIndexSetting

key_value_metadata: Option<Vec<KeyValue>>

default_column_properties: ColumnProperties

column_properties: HashMap<ColumnPath, ColumnProperties>

sorting_columns: Option<Vec<SortingColumn>>

column_index_truncate_length: Option<usize>

statistics_truncate_length: Option<usize>

coerce_types: bool

write_row_group_number_distinct_values: bool

content_defined_chunking: Option<CdcOptions>

write_path_in_schema: bool

file_encryption_properties: Option<Arc<FileEncryptionProperties>>

Implementations

impl WriterProperties

pub fn new() -> Self

Create a new WriterProperties with the default settings

See WriterProperties::builder for customising settings

pub fn builder() -> WriterPropertiesBuilder

Returns a new default WriterPropertiesBuilder for creating writer properties.

pub fn into_builder(self) -> WriterPropertiesBuilder

Converts this WriterProperties into a WriterPropertiesBuilder Used for mutating existing property settings

pub fn data_page_size_limit(&self) -> usize

Returns data page size limit.

Note: this is a best effort limit based on the write batch size

For more details see WriterPropertiesBuilder::set_data_page_size_limit

pub fn column_data_page_size_limit(&self, col: &ColumnPath) -> usize

Returns data page size limit for a specific column.

Takes precedence over Self::data_page_size_limit.

Note: this is a best effort limit based on the write batch size.

pub fn dictionary_page_size_limit(&self) -> usize

Returns dictionary page size limit.

Note: this is a best effort limit based on the write batch size

For more details see WriterPropertiesBuilder::set_dictionary_page_size_limit

pub fn column_dictionary_page_size_limit(&self, col: &ColumnPath) -> usize

Returns dictionary page size limit for a specific column.

pub fn data_page_row_count_limit(&self) -> usize

Returns the maximum page row count

Note: this is a best effort limit based on the write batch size

For more details see WriterPropertiesBuilder::set_data_page_row_count_limit

pub fn write_batch_size(&self) -> usize

Returns configured batch size for writes.

When writing a batch of data, this setting allows to split it internally into smaller batches so we can better estimate the size of a page currently being written.

For more details see WriterPropertiesBuilder::set_write_batch_size

pub fn max_row_group_row_count(&self) -> Option<usize>

Returns maximum number of rows in a row group, or None if unlimited.

For more details see WriterPropertiesBuilder::set_max_row_group_row_count

pub fn max_row_group_bytes(&self) -> Option<usize>

Returns maximum size of a row group in bytes, or None if unlimited.

For more details see WriterPropertiesBuilder::set_max_row_group_bytes

pub fn bloom_filter_position(&self) -> BloomFilterPosition

Returns bloom filter position.

For more details see WriterPropertiesBuilder::set_bloom_filter_position

pub fn bloom_filter_for_dictionary_encoded_chunks(&self) -> bool

Returns whether a column chunk whose data pages are all dictionary encoded gets a bloom filter.

For more details see WriterPropertiesBuilder::set_bloom_filter_for_dictionary_encoded_chunks

pub fn writer_version(&self) -> WriterVersion

Returns configured writer version.

For more details see WriterPropertiesBuilder::set_writer_version

pub fn created_by(&self) -> &str

Returns created_by string.

For more details see WriterPropertiesBuilder::set_created_by

pub fn offset_index_disabled(&self) -> bool

Returns true if offset index writing is disabled.

For more details see WriterPropertiesBuilder::set_offset_index_disabled

pub fn key_value_metadata(&self) -> Option<&Vec<KeyValue>>

Returns key_value_metadata KeyValue pairs.

For more details see WriterPropertiesBuilder::set_key_value_metadata

pub fn sorting_columns(&self) -> Option<&Vec<SortingColumn>>

Returns sorting columns.

For more details see WriterPropertiesBuilder::set_sorting_columns

pub fn column_index_truncate_length(&self) -> Option<usize>

Returns the maximum length of truncated min/max values in the column index.

None if truncation is disabled, must be greater than 0 otherwise.

For more details see WriterPropertiesBuilder::set_column_index_truncate_length

pub fn statistics_truncate_length(&self) -> Option<usize>

Returns the maximum length of truncated min/max values in Statistics.

None if truncation is disabled, must be greater than 0 otherwise.

For more details see WriterPropertiesBuilder::set_statistics_truncate_length

pub fn coerce_types(&self) -> bool

Returns true if type coercion is enabled.

For more details see WriterPropertiesBuilder::set_coerce_types

pub fn write_row_group_number_distinct_values(&self) -> bool

Returns true if the writer should compute and store the distinct count (num_distinct_values) in row group column chunk statistics.

For more details see WriterPropertiesBuilder::set_write_row_group_number_distinct_values

pub fn write_path_in_schema(&self) -> bool

Returns true if the path_in_schema field of the ColumnMetaData Thrift struct should be written.

For more details see WriterPropertiesBuilder::set_write_path_in_schema

pub fn content_defined_chunking(&self) -> Option<&CdcOptions>

EXPERIMENTAL: Returns content-defined chunking options, or None if CDC is disabled.

For more details see WriterPropertiesBuilder::set_content_defined_chunking

pub fn data_page_v2_compression_ratio_threshold(&self) -> f64

Returns the compression ratio threshold at or above which a Data Page v2’s compressed values are discarded in favor of writing the values uncompressed.

For more details see WriterPropertiesBuilder::set_data_page_v2_compression_ratio_threshold

pub fn column_data_page_v2_compression_ratio_threshold( &self, col: &ColumnPath, ) -> f64

Returns the Data Page v2 compression ratio threshold for a specific column.

Takes precedence over Self::data_page_v2_compression_ratio_threshold.

pub fn dictionary_data_page_encoding(&self) -> Encoding

Returns encoding for a data page, when dictionary encoding is enabled.

This is not configurable.

pub fn dictionary_page_encoding(&self) -> Encoding

Returns encoding for dictionary page, when dictionary encoding is enabled.

This is not configurable.

pub fn encoding(&self, col: &ColumnPath) -> Option<Encoding>

Returns encoding for a column, if set.

In case when dictionary is enabled, returns fallback encoding.

If encoding is not set, then column writer will choose the best encoding based on the column type.

pub fn compression(&self, col: &ColumnPath) -> Compression

Returns compression codec for a column.

For more details see WriterPropertiesBuilder::set_column_compression

pub fn dictionary_enabled(&self, col: &ColumnPath) -> bool

Returns true if dictionary encoding is enabled for a column.

For more details see WriterPropertiesBuilder::set_dictionary_enabled

pub fn statistics_enabled(&self, col: &ColumnPath) -> EnabledStatistics

Returns which statistics are written for a column.

For more details see WriterPropertiesBuilder::set_statistics_enabled

pub fn write_page_header_statistics(&self, col: &ColumnPath) -> bool

Returns true if Statistics are to be written to the page header for a column.

For more details see WriterPropertiesBuilder::set_write_page_header_statistics

pub fn bloom_filter_properties( &self, col: &ColumnPath, ) -> Option<&BloomFilterProperties>

Returns the BloomFilterProperties for the given column

Returns None if bloom filter is disabled

For more details see WriterPropertiesBuilder::set_column_bloom_filter_enabled

fn column_override(&self, col: &ColumnPath) -> Option<&ColumnProperties>

Returns the per-column override entry for col, if any.

This is the only place the per-column map is searched. Searching it hashes col, which is a Vec<String>, so callers that need more than one setting should go through Self::resolve_column_properties rather than call several single-setting accessors.

pub(crate) fn resolve_column_properties( &self, col: &ColumnPath, ) -> ResolvedColumnProperties

Resolves every per-column writer setting for col with a single search of the per-column override map.

A column writer needs most of these settings, and needs some of them again on every batch and every page, so it resolves them once when it is created and reads the result from then on.

pub fn file_encryption_properties( &self, ) -> Option<&Arc<FileEncryptionProperties>>

Return file encryption properties

For more details see WriterPropertiesBuilder::with_file_encryption_properties

Trait Implementations

impl Clone for WriterProperties

fn clone(&self) -> WriterProperties

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for WriterProperties

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for WriterProperties

fn default() -> Self

Returns the “default value” for a type.

impl From<WriterProperties> for WriterPropertiesBuilder

fn from(props: WriterProperties) -> Self

Converts to this type from the input type.