Struct WriterProperties

pub struct WriterProperties {

    data_page_size_limit: usize,
    data_page_row_count_limit: usize,
    write_batch_size: usize,
    max_row_group_size: usize,
    bloom_filter_position: BloomFilterPosition,
    writer_version: WriterVersion,
    created_by: String,
    offset_index_disabled: bool,
    pub(crate) key_value_metadata: Option<Vec<KeyValue>>,
    default_column_properties: ColumnProperties,
    column_properties: HashMap<ColumnPath, ColumnProperties>,
    sorting_columns: Option<Vec<SortingColumn>>,
    column_index_truncate_length: Option<usize>,
    statistics_truncate_length: Option<usize>,
    coerce_types: bool,
    pub(crate) file_encryption_properties: Option<FileEncryptionProperties>,
}

Configuration settings for writing parquet files.

Use Self::builder to create a WriterPropertiesBuilder to change settings.

Example

// Create properties with default configuration.
let props = WriterProperties::default();

// Use properties builder to set certain options and assemble the configuration.
let props = WriterProperties::builder()
    .set_writer_version(WriterVersion::PARQUET_1_0)
    .set_encoding(Encoding::PLAIN)
    .set_column_encoding(ColumnPath::from("col1"), Encoding::DELTA_BINARY_PACKED)
    .set_compression(Compression::SNAPPY)
    .build();

assert_eq!(props.writer_version(), WriterVersion::PARQUET_1_0);
assert_eq!(
    props.encoding(&ColumnPath::from("col1")),
    Some(Encoding::DELTA_BINARY_PACKED)
);
assert_eq!(
    props.encoding(&ColumnPath::from("col2")),
    Some(Encoding::PLAIN)
);

Fields

data_page_size_limit: usize

data_page_row_count_limit: usize

write_batch_size: usize

max_row_group_size: usize

bloom_filter_position: BloomFilterPosition

writer_version: WriterVersion

created_by: String

offset_index_disabled: bool

key_value_metadata: Option<Vec<KeyValue>>

default_column_properties: ColumnProperties

column_properties: HashMap<ColumnPath, ColumnProperties>

sorting_columns: Option<Vec<SortingColumn>>

column_index_truncate_length: Option<usize>

statistics_truncate_length: Option<usize>

coerce_types: bool

file_encryption_properties: Option<FileEncryptionProperties>

Implementations

impl WriterProperties

pub fn new() -> Self

Create a new WriterProperties with the default settings

See WriterProperties::builder for customising settings

pub fn builder() -> WriterPropertiesBuilder

Returns a new default WriterPropertiesBuilder for creating writer properties.

pub fn data_page_size_limit(&self) -> usize

Returns data page size limit.

Note: this is a best effort limit based on the write batch size

For more details see WriterPropertiesBuilder::set_data_page_size_limit

pub fn dictionary_page_size_limit(&self) -> usize

Returns dictionary page size limit.

Note: this is a best effort limit based on the write batch size

For more details see WriterPropertiesBuilder::set_dictionary_page_size_limit

pub fn column_dictionary_page_size_limit(&self, col: &ColumnPath) -> usize

Returns dictionary page size limit for a specific column.

pub fn data_page_row_count_limit(&self) -> usize

Returns the maximum page row count

Note: this is a best effort limit based on the write batch size

For more details see WriterPropertiesBuilder::set_data_page_row_count_limit

pub fn write_batch_size(&self) -> usize

Returns configured batch size for writes.

When writing a batch of data, this setting allows to split it internally into smaller batches so we can better estimate the size of a page currently being written.

For more details see WriterPropertiesBuilder::set_write_batch_size

pub fn max_row_group_size(&self) -> usize

Returns maximum number of rows in a row group.

For more details see WriterPropertiesBuilder::set_max_row_group_size

pub fn bloom_filter_position(&self) -> BloomFilterPosition

Returns bloom filter position.

For more details see WriterPropertiesBuilder::set_bloom_filter_position

pub fn writer_version(&self) -> WriterVersion

Returns configured writer version.

For more details see WriterPropertiesBuilder::set_writer_version

pub fn created_by(&self) -> &str

Returns created_by string.

For more details see WriterPropertiesBuilder::set_created_by

pub fn offset_index_disabled(&self) -> bool

Returns true if offset index writing is disabled.

For more details see WriterPropertiesBuilder::set_offset_index_disabled

pub fn key_value_metadata(&self) -> Option<&Vec<KeyValue>>

Returns key_value_metadata KeyValue pairs.

For more details see WriterPropertiesBuilder::set_key_value_metadata

pub fn sorting_columns(&self) -> Option<&Vec<SortingColumn>>

Returns sorting columns.

For more details see WriterPropertiesBuilder::set_sorting_columns

pub fn column_index_truncate_length(&self) -> Option<usize>

Returns the maximum length of truncated min/max values in the column index.

None if truncation is disabled, must be greater than 0 otherwise.

For more details see WriterPropertiesBuilder::set_column_index_truncate_length

pub fn statistics_truncate_length(&self) -> Option<usize>

Returns the maximum length of truncated min/max values in Statistics.

None if truncation is disabled, must be greater than 0 otherwise.

For more details see WriterPropertiesBuilder::set_statistics_truncate_length

pub fn coerce_types(&self) -> bool

Returns true if type coercion is enabled.

For more details see WriterPropertiesBuilder::set_coerce_types

pub fn dictionary_data_page_encoding(&self) -> Encoding

Returns encoding for a data page, when dictionary encoding is enabled.

This is not configurable.

pub fn dictionary_page_encoding(&self) -> Encoding

Returns encoding for dictionary page, when dictionary encoding is enabled.

This is not configurable.

pub fn encoding(&self, col: &ColumnPath) -> Option<Encoding>

Returns encoding for a column, if set.

In case when dictionary is enabled, returns fallback encoding.

If encoding is not set, then column writer will choose the best encoding based on the column type.

pub fn compression(&self, col: &ColumnPath) -> Compression

Returns compression codec for a column.

For more details see WriterPropertiesBuilder::set_column_compression

pub fn dictionary_enabled(&self, col: &ColumnPath) -> bool

Returns true if dictionary encoding is enabled for a column.

For more details see WriterPropertiesBuilder::set_dictionary_enabled

pub fn statistics_enabled(&self, col: &ColumnPath) -> EnabledStatistics

Returns which statistics are written for a column.

For more details see WriterPropertiesBuilder::set_statistics_enabled

pub fn write_page_header_statistics(&self, col: &ColumnPath) -> bool

Returns true if Statistics are to be written to the page header for a column.

For more details see WriterPropertiesBuilder::set_write_page_header_statistics

pub fn bloom_filter_properties( &self, col: &ColumnPath, ) -> Option<&BloomFilterProperties>

Returns the BloomFilterProperties for the given column

Returns None if bloom filter is disabled

For more details see WriterPropertiesBuilder::set_column_bloom_filter_enabled

pub fn file_encryption_properties(&self) -> Option<&FileEncryptionProperties>

Return file encryption properties

For more details see WriterPropertiesBuilder::with_file_encryption_properties

Trait Implementations

impl Clone for WriterProperties

fn clone(&self) -> WriterProperties

Returns a duplicate of the value.

const fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for WriterProperties

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for WriterProperties

fn default() -> Self

Returns the “default value” for a type.