Skip to main content

parquet/column/writer/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Contains column writer API.
6
7use bytes::Bytes;
8use half::f16;
9
10use crate::bloom_filter::Sbbf;
11use crate::file::page_index::column_index::ColumnIndexMetaData;
12use crate::file::page_index::offset_index::OffsetIndexMetaData;
13use std::cmp::Ordering;
14use std::collections::{BTreeSet, VecDeque};
15use std::str;
16
17use crate::basic::{
18    BoundaryOrder, Compression, ConvertedType, Encoding, EncodingMask, LogicalType, PageType,
19    SortOrder, Type,
20};
21use crate::column::page::{CompressedPage, Page, PageWriteSpec, PageWriter};
22use crate::column::writer::encoder::{ColumnValueEncoder, ColumnValueEncoderImpl, ColumnValues};
23use crate::compression::{Codec, CodecOptionsBuilder, create_codec};
24use crate::data_type::private::ParquetValueType;
25use crate::data_type::*;
26use crate::encodings::levels::LevelEncoder;
27#[cfg(feature = "encryption")]
28use crate::encryption::encrypt::get_column_crypto_metadata;
29use crate::errors::{ParquetError, Result};
30use crate::file::metadata::{
31    ColumnChunkMetaData, ColumnChunkMetaDataBuilder, ColumnIndexBuilder, LevelHistogram,
32    OffsetIndexBuilder, PageEncodingStats,
33};
34use crate::file::properties::{
35    EnabledStatistics, ResolvedColumnProperties, WriterProperties, WriterPropertiesPtr,
36    WriterVersion,
37};
38use crate::file::statistics::{Statistics, ValueStatistics};
39use crate::schema::types::{BasicTypeInfo, ColumnDescPtr, ColumnDescriptor};
40
41mod byte_budget_chunker;
42pub(crate) mod encoder;
43pub(crate) mod min_max;
44
45use byte_budget_chunker::{ByteBudgetChunker, SubBatchStrategy};
46
47macro_rules! downcast_writer {
48    ($e:expr, $i:ident, $b:expr) => {
49        match $e {
50            Self::BoolColumnWriter($i) => $b,
51            Self::Int32ColumnWriter($i) => $b,
52            Self::Int64ColumnWriter($i) => $b,
53            Self::Int96ColumnWriter($i) => $b,
54            Self::FloatColumnWriter($i) => $b,
55            Self::DoubleColumnWriter($i) => $b,
56            Self::ByteArrayColumnWriter($i) => $b,
57            Self::FixedLenByteArrayColumnWriter($i) => $b,
58        }
59    };
60}
61
62/// Column writer for a Parquet type.
63///
64/// See [`get_column_writer`] to create instances of this type
65pub enum ColumnWriter<'a> {
66    /// Column writer for boolean type
67    BoolColumnWriter(ColumnWriterImpl<'a, BoolType>),
68    /// Column writer for int32 type
69    Int32ColumnWriter(ColumnWriterImpl<'a, Int32Type>),
70    /// Column writer for int64 type
71    Int64ColumnWriter(ColumnWriterImpl<'a, Int64Type>),
72    /// Column writer for int96 (timestamp) type
73    Int96ColumnWriter(ColumnWriterImpl<'a, Int96Type>),
74    /// Column writer for float type
75    FloatColumnWriter(ColumnWriterImpl<'a, FloatType>),
76    /// Column writer for double type
77    DoubleColumnWriter(ColumnWriterImpl<'a, DoubleType>),
78    /// Column writer for byte array type
79    ByteArrayColumnWriter(ColumnWriterImpl<'a, ByteArrayType>),
80    /// Column writer for fixed length byte array type
81    FixedLenByteArrayColumnWriter(ColumnWriterImpl<'a, FixedLenByteArrayType>),
82}
83
84impl ColumnWriter<'_> {
85    /// Returns the estimated total memory usage
86    #[cfg(feature = "arrow")]
87    pub(crate) fn memory_size(&self) -> usize {
88        downcast_writer!(self, typed, typed.memory_size())
89    }
90
91    /// Returns the estimated total encoded bytes for this column writer
92    #[cfg(feature = "arrow")]
93    pub(crate) fn get_estimated_total_bytes(&self) -> u64 {
94        downcast_writer!(self, typed, typed.get_estimated_total_bytes())
95    }
96
97    /// Finalize the currently buffered values as a data page.
98    ///
99    /// This is used by content-defined chunking to force a page boundary at
100    /// content-determined positions.
101    #[cfg(feature = "arrow")]
102    pub(crate) fn add_data_page(&mut self) -> Result<()> {
103        downcast_writer!(self, typed, typed.add_data_page())
104    }
105
106    /// Sets a pre-computed distinct count on this column writer.
107    ///
108    /// See [`GenericColumnWriter::set_distinct_count_override`] for details.
109    #[cfg(feature = "arrow")]
110    pub(crate) fn set_distinct_count_override(&mut self, count: u64) {
111        downcast_writer!(self, typed, typed.set_distinct_count_override(count))
112    }
113
114    /// Close this [`ColumnWriter`], returning the metadata for the column chunk.
115    pub fn close(self) -> Result<ColumnCloseResult> {
116        downcast_writer!(self, typed, typed.close())
117    }
118}
119
120/// Create a specific column writer corresponding to column descriptor `descr`.
121pub fn get_column_writer<'a>(
122    descr: ColumnDescPtr,
123    props: WriterPropertiesPtr,
124    page_writer: Box<dyn PageWriter + 'a>,
125) -> ColumnWriter<'a> {
126    match descr.physical_type() {
127        Type::BOOLEAN => {
128            ColumnWriter::BoolColumnWriter(ColumnWriterImpl::new(descr, props, page_writer))
129        }
130        Type::INT32 => {
131            ColumnWriter::Int32ColumnWriter(ColumnWriterImpl::new(descr, props, page_writer))
132        }
133        Type::INT64 => {
134            ColumnWriter::Int64ColumnWriter(ColumnWriterImpl::new(descr, props, page_writer))
135        }
136        Type::INT96 => {
137            ColumnWriter::Int96ColumnWriter(ColumnWriterImpl::new(descr, props, page_writer))
138        }
139        Type::FLOAT => {
140            ColumnWriter::FloatColumnWriter(ColumnWriterImpl::new(descr, props, page_writer))
141        }
142        Type::DOUBLE => {
143            ColumnWriter::DoubleColumnWriter(ColumnWriterImpl::new(descr, props, page_writer))
144        }
145        Type::BYTE_ARRAY => {
146            ColumnWriter::ByteArrayColumnWriter(ColumnWriterImpl::new(descr, props, page_writer))
147        }
148        Type::FIXED_LEN_BYTE_ARRAY => ColumnWriter::FixedLenByteArrayColumnWriter(
149            ColumnWriterImpl::new(descr, props, page_writer),
150        ),
151    }
152}
153
154/// Gets a typed column writer for the specific type `T`, by "up-casting" `col_writer` of
155/// non-generic type to a generic column writer type `ColumnWriterImpl`.
156///
157/// # Panics
158///
159/// Panics if actual enum value for `col_writer` does not match the type `T`.
160pub fn get_typed_column_writer<T: DataType>(col_writer: ColumnWriter) -> ColumnWriterImpl<T> {
161    T::get_column_writer(col_writer).unwrap_or_else(|| {
162        panic!(
163            "Failed to convert column writer into a typed column writer for `{}` type",
164            T::get_physical_type()
165        )
166    })
167}
168
169/// Similar to `get_typed_column_writer` but returns a reference.
170pub fn get_typed_column_writer_ref<'a, 'b: 'a, T: DataType>(
171    col_writer: &'b ColumnWriter<'a>,
172) -> &'b ColumnWriterImpl<'a, T> {
173    T::get_column_writer_ref(col_writer).unwrap_or_else(|| {
174        panic!(
175            "Failed to convert column writer into a typed column writer for `{}` type",
176            T::get_physical_type()
177        )
178    })
179}
180
181/// Similar to `get_typed_column_writer` but returns a reference.
182pub fn get_typed_column_writer_mut<'a, 'b: 'a, T: DataType>(
183    col_writer: &'a mut ColumnWriter<'b>,
184) -> &'a mut ColumnWriterImpl<'b, T> {
185    T::get_column_writer_mut(col_writer).unwrap_or_else(|| {
186        panic!(
187            "Failed to convert column writer into a typed column writer for `{}` type",
188            T::get_physical_type()
189        )
190    })
191}
192
193/// Metadata for a column chunk of a Parquet file.
194///
195/// Note this structure is returned by [`ColumnWriter::close`].
196#[derive(Debug, Clone)]
197pub struct ColumnCloseResult {
198    /// The total number of bytes written
199    pub bytes_written: u64,
200    /// The total number of rows written
201    pub rows_written: u64,
202    /// Metadata for this column chunk
203    pub metadata: ColumnChunkMetaData,
204    /// Optional bloom filter for this column
205    pub bloom_filter: Option<Sbbf>,
206    /// Optional column index, for filtering
207    pub column_index: Option<ColumnIndexMetaData>,
208    /// Optional offset index, identifying page locations
209    pub offset_index: Option<OffsetIndexMetaData>,
210}
211
212impl ColumnCloseResult {
213    /// Rewrite the page offsets for a dictionary-first on-disk layout.
214    ///
215    /// A writer that buffers the whole column chunk and splices it later (the
216    /// Arrow path) may accept the data pages *before* the dictionary page so the
217    /// data pages can stream straight through, then emit the dictionary page
218    /// first at splice. The offsets recorded during encoding therefore assume a
219    /// data-pages-first layout; call this with the serialized length of the
220    /// dictionary page to move it to offset 0 and shift every data page after
221    /// it. A `dictionary_len` of 0 (no dictionary page) leaves the result
222    /// unchanged.
223    pub fn update_dictionary_location(mut self, dictionary_len: usize) -> Result<Self> {
224        if dictionary_len > 0 {
225            self.metadata = self
226                .metadata
227                .into_builder()
228                .set_dictionary_page_offset(Some(0))
229                .set_data_page_offset(dictionary_len as i64)
230                .build()?;
231            if let Some(offset_index) = self.offset_index.as_mut() {
232                let mut offset = dictionary_len as i64;
233                for location in &mut offset_index.page_locations {
234                    location.offset = offset;
235                    offset += location.compressed_page_size as i64;
236                }
237            }
238        }
239        Ok(self)
240    }
241}
242
243// Metrics per page
244#[derive(Default)]
245struct PageMetrics {
246    num_buffered_values: u32,
247    num_buffered_rows: u32,
248    /// Encoded bytes that the data page byte limit does not apply to,
249    /// because they belong to the page's mandatory first value and cannot be
250    /// moved elsewhere. Zero unless that value alone exceeded the limit
251    /// *and* the encoding compresses against the preceding value; see
252    /// [`ColumnValueEncoder::compresses_against_previous_value`].
253    page_size_exemption: usize,
254    num_page_nulls: u64,
255    num_page_nans: Option<u64>,
256    repetition_level_histogram: Option<LevelHistogram>,
257    definition_level_histogram: Option<LevelHistogram>,
258}
259
260impl PageMetrics {
261    fn new() -> Self {
262        Default::default()
263    }
264
265    /// Initialize the repetition level histogram
266    fn with_repetition_level_histogram(mut self, max_level: i16) -> Self {
267        self.repetition_level_histogram = LevelHistogram::try_new(max_level);
268        self
269    }
270
271    /// Initialize the definition level histogram
272    fn with_definition_level_histogram(mut self, max_level: i16) -> Self {
273        self.definition_level_histogram = LevelHistogram::try_new(max_level);
274        self
275    }
276
277    /// Resets the state of this `PageMetrics` to the initial state.
278    /// If histograms have been initialized their contents will be reset to zero.
279    fn new_page(&mut self) {
280        self.num_buffered_values = 0;
281        self.num_buffered_rows = 0;
282        self.page_size_exemption = 0;
283        self.num_page_nulls = 0;
284        self.num_page_nans = None;
285        self.repetition_level_histogram
286            .as_mut()
287            .map(LevelHistogram::reset);
288        self.definition_level_histogram
289            .as_mut()
290            .map(LevelHistogram::reset);
291    }
292}
293
294// Metrics per column writer
295#[derive(Default)]
296struct ColumnMetrics<T: Default> {
297    total_bytes_written: u64,
298    total_rows_written: u64,
299    total_uncompressed_size: u64,
300    total_compressed_size: u64,
301    total_num_values: u64,
302    dictionary_page_offset: Option<u64>,
303    data_page_offset: Option<u64>,
304    min_column_value: Option<T>,
305    max_column_value: Option<T>,
306    num_column_nulls: u64,
307    num_column_nans: Option<u64>,
308    column_distinct_count: Option<u64>,
309    variable_length_bytes: Option<i64>,
310    repetition_level_histogram: Option<LevelHistogram>,
311    definition_level_histogram: Option<LevelHistogram>,
312}
313
314impl<T: Default> ColumnMetrics<T> {
315    fn new() -> Self {
316        Default::default()
317    }
318
319    /// Initialize the repetition level histogram
320    fn with_repetition_level_histogram(mut self, max_level: i16) -> Self {
321        self.repetition_level_histogram = LevelHistogram::try_new(max_level);
322        self
323    }
324
325    /// Initialize the definition level histogram
326    fn with_definition_level_histogram(mut self, max_level: i16) -> Self {
327        self.definition_level_histogram = LevelHistogram::try_new(max_level);
328        self
329    }
330
331    /// Sum `page_histogram` into `chunk_histogram`
332    fn update_histogram(
333        chunk_histogram: &mut Option<LevelHistogram>,
334        page_histogram: Option<&LevelHistogram>,
335    ) {
336        if let (Some(page_hist), Some(chunk_hist)) = (page_histogram, chunk_histogram) {
337            chunk_hist.add(page_hist);
338        }
339    }
340
341    /// Sum the provided PageMetrics histograms into the chunk histograms. Does nothing if
342    /// page histograms are not initialized.
343    fn update_from_page_metrics(&mut self, page_metrics: &PageMetrics) {
344        ColumnMetrics::<T>::update_histogram(
345            &mut self.definition_level_histogram,
346            page_metrics.definition_level_histogram.as_ref(),
347        );
348        ColumnMetrics::<T>::update_histogram(
349            &mut self.repetition_level_histogram,
350            page_metrics.repetition_level_histogram.as_ref(),
351        );
352    }
353
354    /// Sum the provided page variable_length_bytes into the chunk variable_length_bytes
355    fn update_variable_length_bytes(&mut self, variable_length_bytes: Option<i64>) {
356        if let Some(var_bytes) = variable_length_bytes {
357            *self.variable_length_bytes.get_or_insert(0) += var_bytes;
358        }
359    }
360}
361
362/// Borrowed view of level data, analogous to `&str` for `LevelData`'s `String`.
363///
364/// `LevelDataRef` can be constructed from `LevelData` and directly from an existing
365/// `&[i16]` without allocating.
366///
367/// The variants are different physical representations of the same logical
368/// sequence of levels.
369#[derive(Debug, Clone, Copy)]
370pub(crate) enum LevelDataRef<'a> {
371    Absent,
372    Materialized(&'a [i16]),
373    Uniform { value: i16, count: usize },
374}
375
376impl<'a> From<&'a [i16]> for LevelDataRef<'a> {
377    fn from(levels: &'a [i16]) -> Self {
378        Self::Materialized(levels)
379    }
380}
381
382impl<'a> From<Option<&'a [i16]>> for LevelDataRef<'a> {
383    fn from(levels: Option<&'a [i16]>) -> Self {
384        levels.map_or(Self::Absent, Self::from)
385    }
386}
387
388impl LevelDataRef<'_> {
389    pub(crate) fn len(self) -> usize {
390        match self {
391            Self::Absent => 0,
392            Self::Materialized(values) => values.len(),
393            Self::Uniform { count, .. } => count,
394        }
395    }
396
397    pub(crate) fn first(self) -> Option<i16> {
398        match self {
399            Self::Absent => None,
400            Self::Materialized(values) => values.first().copied(),
401            Self::Uniform { value, count } => (count > 0).then_some(value),
402        }
403    }
404
405    #[cfg(feature = "arrow")]
406    pub(crate) fn value_at(self, idx: usize) -> Option<i16> {
407        match self {
408            Self::Absent => None,
409            Self::Materialized(values) => values.get(idx).copied(),
410            Self::Uniform { value, count } => (idx < count).then_some(value),
411        }
412    }
413
414    pub(crate) fn slice(self, offset: usize, len: usize) -> Self {
415        match self {
416            Self::Absent => Self::Absent,
417            Self::Materialized(values) => Self::Materialized(&values[offset..offset + len]),
418            Self::Uniform { value, .. } => Self::Uniform { value, count: len },
419        }
420    }
421
422    /// Count of positions in this slice that represent an actual value
423    /// (definition level equal to `max_def`). `Absent` means the column has
424    /// `max_def == 0` and every position is a value, so the implicit count
425    /// is the caller-supplied `total`.
426    pub(crate) fn value_count(self, total: usize, max_def: i16) -> usize {
427        match self {
428            Self::Absent => total,
429            Self::Materialized(values) => values.iter().filter(|&&d| d == max_def).count(),
430            Self::Uniform { value, count } => {
431                if value == max_def {
432                    count
433                } else {
434                    0
435                }
436            }
437        }
438    }
439}
440
441/// Typed column writer for a primitive column.
442pub type ColumnWriterImpl<'a, T> = GenericColumnWriter<'a, ColumnValueEncoderImpl<T>>;
443
444/// Generic column writer for a primitive Parquet column
445pub struct GenericColumnWriter<'a, E: ColumnValueEncoder> {
446    // Column writer properties
447    descr: ColumnDescPtr,
448    props: WriterPropertiesPtr,
449    /// Per-column settings for [`Self::descr`], resolved once here so that the
450    /// per-batch and per-page write paths never search the per-column override
451    /// map in `props` again.
452    column_props: ResolvedColumnProperties,
453
454    page_writer: Box<dyn PageWriter + 'a>,
455    codec: Compression,
456    compressor: Option<Box<dyn Codec>>,
457    encoder: E,
458
459    page_metrics: PageMetrics,
460    // Metrics per column writer
461    column_metrics: ColumnMetrics<E::T>,
462
463    /// Pre-computed distinct count to write into column chunk statistics.
464    /// When set, takes precedence over `column_metrics.column_distinct_count`.
465    distinct_count_override: Option<u64>,
466
467    /// The order of encodings within the generated metadata does not impact its meaning,
468    /// but we use a BTreeSet so that the output is deterministic
469    encodings: BTreeSet<Encoding>,
470    encoding_stats: Vec<PageEncodingStats>,
471    // Streaming level encoders for definition/repetition levels.
472    def_levels_encoder: LevelEncoder,
473    rep_levels_encoder: LevelEncoder,
474    data_pages: VecDeque<CompressedPage>,
475    // column index and offset index
476    column_index_builder: ColumnIndexBuilder,
477    offset_index_builder: Option<OffsetIndexBuilder>,
478
479    // Below fields used to incrementally check boundary order across data pages.
480    // We assume they are ascending/descending until proven wrong.
481    data_page_boundary_ascending: bool,
482    data_page_boundary_descending: bool,
483    /// (min, max)
484    last_non_null_data_page_min_max: Option<(E::T, E::T)>,
485}
486
487impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> {
488    /// Returns a new instance of [`GenericColumnWriter`].
489    pub fn new(
490        descr: ColumnDescPtr,
491        props: WriterPropertiesPtr,
492        page_writer: Box<dyn PageWriter + 'a>,
493    ) -> Self {
494        let column_props = props.resolve_column_properties(descr.path());
495        let codec = column_props.compression;
496        let codec_options = CodecOptionsBuilder::default().build();
497        let compressor = create_codec(codec, &codec_options).unwrap();
498        let encoder = E::try_new(&descr, props.as_ref(), &column_props).unwrap();
499
500        let statistics_enabled = column_props.statistics_enabled;
501
502        let encodings = BTreeSet::new();
503
504        let mut page_metrics = PageMetrics::new();
505        let mut column_metrics = ColumnMetrics::<E::T>::new();
506
507        // Initialize level histograms if collecting page or chunk statistics
508        if statistics_enabled != EnabledStatistics::None {
509            page_metrics = page_metrics
510                .with_repetition_level_histogram(descr.max_rep_level())
511                .with_definition_level_histogram(descr.max_def_level());
512            column_metrics = column_metrics
513                .with_repetition_level_histogram(descr.max_rep_level())
514                .with_definition_level_histogram(descr.max_def_level())
515        }
516
517        // Disable column_index_builder if not collecting page statistics.
518        let mut column_index_builder = ColumnIndexBuilder::new(descr.physical_type());
519        if statistics_enabled != EnabledStatistics::Page {
520            column_index_builder.to_invalid()
521        }
522
523        // Disable offset_index_builder if requested by user.
524        let offset_index_builder = match props.offset_index_disabled() {
525            false => Some(OffsetIndexBuilder::new()),
526            _ => None,
527        };
528
529        Self {
530            def_levels_encoder: Self::create_level_encoder(descr.max_def_level(), &props),
531            rep_levels_encoder: Self::create_level_encoder(descr.max_rep_level(), &props),
532            descr,
533            props,
534            column_props,
535            page_writer,
536            codec,
537            compressor,
538            encoder,
539            data_pages: VecDeque::new(),
540            page_metrics,
541            column_metrics,
542            distinct_count_override: None,
543            column_index_builder,
544            offset_index_builder,
545            encodings,
546            encoding_stats: vec![],
547            data_page_boundary_ascending: true,
548            data_page_boundary_descending: true,
549            last_non_null_data_page_min_max: None,
550        }
551    }
552
553    /// Sets a pre-computed distinct count to write into column chunk statistics.
554    ///
555    /// When set, this value is written as `distinct_count` in the row group statistics
556    /// footer. It takes precedence over any `distinct_count` passed through
557    /// [`Self::write_batch_with_statistics`].
558    #[cfg(feature = "arrow")]
559    pub(crate) fn set_distinct_count_override(&mut self, count: u64) {
560        self.distinct_count_override = Some(count);
561    }
562
563    #[expect(clippy::too_many_arguments)]
564    pub(crate) fn write_batch_internal(
565        &mut self,
566        values: &E::Values,
567        value_indices: Option<&[usize]>,
568        def_levels: LevelDataRef<'_>,
569        rep_levels: LevelDataRef<'_>,
570        min: Option<&E::T>,
571        max: Option<&E::T>,
572        distinct_count: Option<u64>,
573    ) -> Result<usize> {
574        // Check if number of definition levels is the same as number of repetition levels.
575        if def_levels.len() != 0 && rep_levels.len() != 0 && def_levels.len() != rep_levels.len() {
576            return Err(general_err!(
577                "Inconsistent length of definition and repetition levels: {} != {}",
578                def_levels.len(),
579                rep_levels.len()
580            ));
581        }
582
583        // We check for DataPage limits only after we have inserted the values. If a user
584        // writes a large number of values, the DataPage size can be well above the limit.
585        //
586        // The purpose of this chunking is to bound this. Even if a user writes large
587        // number of values, the chunking will ensure that we add data page at a
588        // reasonable pagesize limit.
589
590        // TODO: find out why we don't account for size of levels when we estimate page
591        // size.
592        let num_levels = def_levels.len().max(rep_levels.len());
593        let num_levels = if num_levels > 0 {
594            num_levels
595        } else {
596            value_indices.map_or_else(|| values.len(), |i| i.len())
597        };
598
599        if let Some(min) = min {
600            update_min(&self.descr, min, &mut self.column_metrics.min_column_value);
601        }
602        if let Some(max) = max {
603            update_max(&self.descr, max, &mut self.column_metrics.max_column_value);
604        }
605
606        // Encoder counts reset per page; row metrics retain column-wide history.
607        let has_prior_data = self.column_metrics.total_rows_written != 0
608            || self.page_metrics.num_buffered_values != 0;
609        self.column_metrics.column_distinct_count =
610            if has_prior_data { None } else { distinct_count };
611
612        let mut values_offset = 0;
613        let mut levels_offset = 0;
614        let both_levels_compact = !matches!(def_levels, LevelDataRef::Materialized(_))
615            && !matches!(rep_levels, LevelDataRef::Materialized(_));
616        let has_levels = !matches!(def_levels, LevelDataRef::Absent)
617            || !matches!(rep_levels, LevelDataRef::Absent);
618
619        // When both level vectors are compact (Uniform or Absent), there is no
620        // materialized slice to split and the per-mini-batch work is O(1), so we
621        // can safely use a much larger batch size.
622        let base_batch_size = if both_levels_compact && has_levels {
623            self.column_props.data_page_row_count_limit
624        } else {
625            self.props.write_batch_size()
626        };
627        debug_assert!(base_batch_size > 0);
628
629        let chunker = ByteBudgetChunker::new(&self.descr, &self.column_props, base_batch_size);
630        while levels_offset < num_levels {
631            let mut end_offset = num_levels.min(levels_offset + base_batch_size);
632
633            // Split at record boundary
634            if let LevelDataRef::Materialized(levels) = rep_levels {
635                while end_offset < levels.len() && levels[end_offset] != 0 {
636                    end_offset += 1;
637                }
638            }
639
640            let chunk_size = end_offset - levels_offset;
641            let chunk_def = def_levels.slice(levels_offset, chunk_size);
642            let chunk_rep = rep_levels.slice(levels_offset, chunk_size);
643
644            // Key decision point: can we write this whole chunk as one
645            // mini-batch (the common case — small or fixed-width values, no
646            // further page-size accounting needed), or must we fall back to
647            // byte-budget-aware sub-batching to keep a page from overshooting
648            // `data_page_size_limit`? `pick_sub_batch` returns `None` for
649            // the former, and otherwise how wide one mini-batch may be.
650            let sub_batch = chunker.pick_sub_batch(
651                &self.encoder,
652                values,
653                value_indices,
654                chunk_def,
655                values_offset,
656                chunk_size,
657            );
658
659            match sub_batch {
660                None => {
661                    values_offset += self.write_mini_batch(
662                        values,
663                        values_offset,
664                        value_indices,
665                        chunk_size,
666                        chunk_def,
667                        chunk_rep,
668                    )?;
669                }
670                Some(sub_batch) => {
671                    values_offset += self.write_granular_chunk(
672                        values,
673                        values_offset,
674                        value_indices,
675                        chunk_size,
676                        chunk_def,
677                        chunk_rep,
678                        sub_batch,
679                    )?;
680                }
681            }
682            levels_offset = end_offset;
683        }
684
685        // Return total number of values processed.
686        Ok(values_offset)
687    }
688
689    /// Writes batch of values, definition levels and repetition levels.
690    /// Returns number of values processed (written).
691    ///
692    /// If definition and repetition levels are provided, we write fully those levels and
693    /// select how many values to write (this number will be returned), since number of
694    /// actual written values may be smaller than provided values.
695    ///
696    /// If only values are provided, then all values are written and the length of
697    /// of the values buffer is returned.
698    ///
699    /// Definition and/or repetition levels can be omitted, if values are
700    /// non-nullable and/or non-repeated.
701    pub fn write_batch(
702        &mut self,
703        values: &E::Values,
704        def_levels: Option<&[i16]>,
705        rep_levels: Option<&[i16]>,
706    ) -> Result<usize> {
707        self.write_batch_internal(
708            values,
709            None,
710            LevelDataRef::from(def_levels),
711            LevelDataRef::from(rep_levels),
712            None,
713            None,
714            None,
715        )
716    }
717
718    /// Writer may optionally provide pre-calculated statistics for use when computing
719    /// chunk-level statistics
720    ///
721    /// NB: [`WriterProperties::statistics_enabled`] must be set to [`EnabledStatistics::Chunk`]
722    /// for these statistics to take effect. If [`EnabledStatistics::None`] they will be ignored,
723    /// and if [`EnabledStatistics::Page`] the chunk statistics will instead be computed from the
724    /// computed page statistics
725    pub fn write_batch_with_statistics(
726        &mut self,
727        values: &E::Values,
728        def_levels: Option<&[i16]>,
729        rep_levels: Option<&[i16]>,
730        min: Option<&E::T>,
731        max: Option<&E::T>,
732        distinct_count: Option<u64>,
733    ) -> Result<usize> {
734        self.write_batch_internal(
735            values,
736            None,
737            LevelDataRef::from(def_levels),
738            LevelDataRef::from(rep_levels),
739            min,
740            max,
741            distinct_count,
742        )
743    }
744
745    /// Returns the estimated total memory usage.
746    ///
747    /// Unlike [`Self::get_estimated_total_bytes`] this is an estimate
748    /// of the current memory usage and not the final anticipated encoded size.
749    #[cfg(feature = "arrow")]
750    pub(crate) fn memory_size(&self) -> usize {
751        // In-flight encoder buffers, plus any completed pages still held on the
752        // heap: the dictionary-column data pages buffered here (column-at-a-time
753        // path), plus whatever the page writer keeps resident. A page writer
754        // that spills completed pages off-heap reports far less than the bytes
755        // it was handed, so this tracks real memory rather than bytes written.
756        self.encoder.estimated_memory_size()
757            + self
758                .data_pages
759                .iter()
760                .map(|page| page.memory_usage())
761                .sum::<usize>()
762            + self.page_writer.buffered_memory_size()
763    }
764
765    /// Returns total number of bytes written by this column writer so far.
766    /// This value is also returned when column writer is closed.
767    ///
768    /// Note: this value does not include any buffered data that has not
769    /// yet been flushed to a page.
770    pub fn get_total_bytes_written(&self) -> u64 {
771        self.column_metrics.total_bytes_written
772    }
773
774    /// Returns the estimated total encoded bytes for this column writer.
775    ///
776    /// Unlike [`Self::get_total_bytes_written`] this includes an estimate
777    /// of any data that has not yet been flushed to a page, based on it's
778    /// anticipated encoded size.
779    #[cfg(feature = "arrow")]
780    pub(crate) fn get_estimated_total_bytes(&self) -> u64 {
781        self.data_pages
782            .iter()
783            .map(|page| page.data().len() as u64)
784            .sum::<u64>()
785            + self.column_metrics.total_bytes_written
786            + self.encoder.estimated_data_page_size() as u64
787            + self.encoder.estimated_dict_page_size().unwrap_or_default() as u64
788    }
789
790    /// Returns total number of rows written by this column writer so far.
791    /// This value is also returned when column writer is closed.
792    pub fn get_total_rows_written(&self) -> u64 {
793        self.column_metrics.total_rows_written
794    }
795
796    /// Returns a reference to a [`ColumnDescPtr`]
797    pub fn get_descriptor(&self) -> &ColumnDescPtr {
798        &self.descr
799    }
800
801    /// Finalizes writes and closes the column writer.
802    /// Returns total bytes written, total rows written and column chunk metadata.
803    pub fn close(mut self) -> Result<ColumnCloseResult> {
804        if self.page_metrics.num_buffered_values > 0 {
805            self.add_data_page()?;
806        }
807        if self.encoder.has_dictionary() && self.column_metrics.total_rows_written > 0 {
808            self.write_dictionary_page()?;
809        }
810        self.flush_data_pages()?;
811        let metadata = self.build_column_metadata()?;
812        self.page_writer.close()?;
813
814        let write_bloom_filter = self.props.bloom_filter_for_dictionary_encoded_chunks()
815            || self.has_non_dictionary_data_page();
816        let bloom_filter = self
817            .encoder
818            .flush_bloom_filter()
819            .filter(|_| write_bloom_filter);
820
821        let boundary_order = match (
822            self.data_page_boundary_ascending,
823            self.data_page_boundary_descending,
824        ) {
825            // If the lists are composed of equal elements then will be marked as ascending
826            // (Also the case if all pages are null pages)
827            (true, _) => BoundaryOrder::ASCENDING,
828            (false, true) => BoundaryOrder::DESCENDING,
829            (false, false) => BoundaryOrder::UNORDERED,
830        };
831        self.column_index_builder.set_boundary_order(boundary_order);
832
833        let column_index = match self.column_index_builder.valid() {
834            true => Some(self.column_index_builder.build()?),
835            false => None,
836        };
837
838        let offset_index = self.offset_index_builder.map(|b| b.build());
839
840        Ok(ColumnCloseResult {
841            bytes_written: self.column_metrics.total_bytes_written,
842            rows_written: self.column_metrics.total_rows_written,
843            bloom_filter,
844            metadata,
845            column_index,
846            offset_index,
847        })
848    }
849
850    /// Writes a chunk in sub-batches sized by `sub_batch`, checking the page
851    /// byte limit after each. This keeps the page size close to
852    /// `data_page_size_limit` instead of overshooting it by a whole chunk.
853    ///
854    /// [`SubBatchStrategy::Values`] windows are cut on value boundaries,
855    /// walking definition levels to find where the value budget is used up;
856    /// [`SubBatchStrategy::Levels`] windows are a fixed level count, capped
857    /// at a maximum number of values. See
858    /// [`SubBatchStrategy`] for which budget gets which and why.
859    ///
860    /// For repeated/nested columns sub-batches then extend to the next
861    /// `rep == 0` boundary so a record never spans data pages, matching the
862    /// parquet format rule. A record holding several over-limit values
863    /// therefore still exceeds the budget; that is inherent to the format.
864    ///
865    /// Returns the total number of values consumed across all sub-batches.
866    ///
867    /// `#[inline(never)]` keeps this slow path — only reached for
868    /// variable-width columns whose values need page splitting — out of
869    /// the hot `write_batch_internal` loop.
870    #[expect(clippy::too_many_arguments)]
871    #[inline(never)]
872    fn write_granular_chunk(
873        &mut self,
874        values: &E::Values,
875        values_offset: usize,
876        value_indices: Option<&[usize]>,
877        chunk_size: usize,
878        chunk_def: LevelDataRef<'_>,
879        chunk_rep: LevelDataRef<'_>,
880        sub_batch: SubBatchStrategy,
881    ) -> Result<usize> {
882        // The chunker always sizes a sub-batch to at least one value or one
883        // level, and a value-exact window spans at least one level, so each
884        // iteration below makes progress (`sub_end > sub_start`).
885        debug_assert!(
886            matches!(
887                sub_batch,
888                SubBatchStrategy::Values(n) | SubBatchStrategy::Levels { levels: n, max_values: _ }
889                    if n >= 1
890            ),
891            "chunker must size at least one value or level"
892        );
893        let max_def_level = self.descr.max_def_level();
894        let mut values_consumed = 0;
895        let mut sub_start = 0;
896        while sub_start < chunk_size {
897            let window_end = match sub_batch {
898                SubBatchStrategy::Values(n) => {
899                    Self::window_end_for_values(chunk_def, chunk_size, max_def_level, sub_start, n)
900                }
901                SubBatchStrategy::Levels { levels, max_values } => {
902                    // Cap the ratio-sized window at `max_values` values so
903                    // clustered values cannot pile into one window.
904                    let level_end = (sub_start + levels).min(chunk_size);
905                    Self::window_end_for_values(
906                        chunk_def,
907                        level_end,
908                        max_def_level,
909                        sub_start,
910                        max_values,
911                    )
912                }
913            };
914            let sub_end = match chunk_rep {
915                LevelDataRef::Materialized(levels) => {
916                    // Extend the window to the next record boundary
917                    // (rep == 0) so a record never spans data pages. Packing
918                    // whole records rather than stepping one record at a time
919                    // avoids calling `write_mini_batch` per record: records
920                    // average only a handful of levels, so a record-at-a-time
921                    // step would issue many more mini-batches than necessary.
922                    let mut e = window_end;
923                    while e < chunk_size && levels[e] != 0 {
924                        e += 1;
925                    }
926                    e
927                }
928                _ => window_end,
929            };
930            let sub_len = sub_end - sub_start;
931            let written = self.write_mini_batch(
932                values,
933                values_offset + values_consumed,
934                value_indices,
935                sub_len,
936                chunk_def.slice(sub_start, sub_len),
937                chunk_rep.slice(sub_start, sub_len),
938            )?;
939            values_consumed += written;
940            sub_start = sub_end;
941        }
942        Ok(values_consumed)
943    }
944
945    fn has_non_dictionary_data_page(&self) -> bool {
946        self.encoding_stats.iter().any(|stats| {
947            matches!(
948                stats.page_type,
949                PageType::DATA_PAGE | PageType::DATA_PAGE_V2
950            ) && !matches!(
951                stats.encoding,
952                Encoding::PLAIN_DICTIONARY | Encoding::RLE_DICTIONARY
953            )
954        })
955    }
956
957    /// Index one past the last level of a sub-batch window that starts at
958    /// `start` and covers at most `max_values` values, clamped to
959    /// `chunk_size`.
960    ///
961    /// Nulls trailing the last value are left to the next window, so a window
962    /// ends immediately after the value that exhausts its budget. The window
963    /// always spans at least one level, so callers make progress: `start` is
964    /// below `chunk_size` and `max_values` is at least one.
965    ///
966    /// Only nullable and nested columns pay the level walk. It runs solely in
967    /// the granular path, whose values are by definition large enough to
968    /// overflow a page budget, so touching each of the chunk's levels once is
969    /// noise next to writing those values — and that path already makes full
970    /// def-level and rep-level passes.
971    fn window_end_for_values(
972        chunk_def: LevelDataRef<'_>,
973        chunk_size: usize,
974        max_def_level: i16,
975        start: usize,
976        max_values: usize,
977    ) -> usize {
978        match chunk_def {
979            // `max_def_level == 0`: every level is a value.
980            LevelDataRef::Absent => (start + max_values).min(chunk_size),
981            LevelDataRef::Uniform { value, .. } => {
982                if value == max_def_level {
983                    (start + max_values).min(chunk_size)
984                } else {
985                    // Uniformly below max def: the chunk holds no values at
986                    // all, so no window boundary can help. (The chunker
987                    // returns `None` for such a chunk, so this is defensive.)
988                    chunk_size
989                }
990            }
991            LevelDataRef::Materialized(levels) => {
992                let mut seen = 0;
993                let mut end = start;
994                while end < chunk_size && seen < max_values {
995                    if levels[end] == max_def_level {
996                        seen += 1;
997                    }
998                    end += 1;
999                }
1000                end
1001            }
1002        }
1003    }
1004
1005    /// Creates a new streaming level encoder appropriate for the writer version.
1006    fn create_level_encoder(max_level: i16, props: &WriterProperties) -> LevelEncoder {
1007        match props.writer_version() {
1008            WriterVersion::PARQUET_1_0 => LevelEncoder::v1_streaming(max_level),
1009            WriterVersion::PARQUET_2_0 => LevelEncoder::v2_streaming(max_level),
1010        }
1011    }
1012
1013    /// Writes mini batch of values, definition and repetition levels.
1014    /// This allows fine-grained processing of values and maintaining a reasonable
1015    /// page size.
1016    fn write_mini_batch(
1017        &mut self,
1018        values: &E::Values,
1019        values_offset: usize,
1020        value_indices: Option<&[usize]>,
1021        num_levels: usize,
1022        def_levels: LevelDataRef<'_>,
1023        rep_levels: LevelDataRef<'_>,
1024    ) -> Result<usize> {
1025        // Process definition levels and determine how many values to write.
1026        let values_to_write = if self.descr.max_def_level() > 0 {
1027            let max_def = self.descr.max_def_level();
1028            match def_levels {
1029                LevelDataRef::Absent => {
1030                    return Err(general_err!(
1031                        "Definition levels are required, because max definition level = {}",
1032                        self.descr.max_def_level()
1033                    ));
1034                }
1035                LevelDataRef::Materialized(levels) => {
1036                    // General path for caller-provided or already-materialized
1037                    // level buffers.
1038                    let mut values_to_write = 0usize;
1039                    let encoder = &mut self.def_levels_encoder;
1040                    match self.page_metrics.definition_level_histogram.as_mut() {
1041                        Some(histogram) => encoder.put_with_observer(levels, |level, count| {
1042                            values_to_write += count * (level == max_def) as usize;
1043                            histogram.increment_by(level, count as i64);
1044                        }),
1045                        None => encoder.put_with_observer(levels, |level, count| {
1046                            values_to_write += count * (level == max_def) as usize;
1047                        }),
1048                    };
1049                    self.page_metrics.num_page_nulls += (levels.len() - values_to_write) as u64;
1050                    values_to_write
1051                }
1052                LevelDataRef::Uniform { value, count } => {
1053                    // Fast path for all-null, all-valid, or otherwise uniform
1054                    // definition levels without materializing a level buffer.
1055                    let encoder = &mut self.def_levels_encoder;
1056                    match self.page_metrics.definition_level_histogram.as_mut() {
1057                        Some(histogram) => {
1058                            encoder.put_n_with_observer(value, count, |level, run_len| {
1059                                histogram.increment_by(level, run_len as i64);
1060                            })
1061                        }
1062                        None => encoder.put_n_with_observer(value, count, |_, _| {}),
1063                    }
1064                    let values_to_write = count * (value == max_def) as usize;
1065                    self.page_metrics.num_page_nulls += (count - values_to_write) as u64;
1066                    values_to_write
1067                }
1068            }
1069        } else {
1070            num_levels
1071        };
1072
1073        // Process repetition levels and determine how many rows we are about to process.
1074        if self.descr.max_rep_level() > 0 {
1075            // A row could contain more than one value.
1076            let first_level = rep_levels.first().ok_or_else(|| {
1077                general_err!(
1078                    "Repetition levels are required, because max repetition level = {}",
1079                    self.descr.max_rep_level()
1080                )
1081            })?;
1082
1083            if first_level != 0 {
1084                return Err(general_err!(
1085                    "Write must start at a record boundary, got non-zero repetition level of {}",
1086                    first_level
1087                ));
1088            }
1089
1090            let mut new_rows = 0u32;
1091            match rep_levels {
1092                LevelDataRef::Absent => unreachable!(),
1093                LevelDataRef::Materialized(levels) => {
1094                    let encoder = &mut self.rep_levels_encoder;
1095                    match self.page_metrics.repetition_level_histogram.as_mut() {
1096                        Some(histogram) => encoder.put_with_observer(levels, |level, count| {
1097                            new_rows += (count as u32) * (level == 0) as u32;
1098                            histogram.increment_by(level, count as i64);
1099                        }),
1100                        None => encoder.put_with_observer(levels, |level, count| {
1101                            new_rows += (count as u32) * (level == 0) as u32;
1102                        }),
1103                    };
1104                }
1105                LevelDataRef::Uniform { value, count } => {
1106                    let encoder = &mut self.rep_levels_encoder;
1107                    match self.page_metrics.repetition_level_histogram.as_mut() {
1108                        Some(histogram) => {
1109                            encoder.put_n_with_observer(value, count, |level, run_len| {
1110                                new_rows += (run_len as u32) * (level == 0) as u32;
1111                                histogram.increment_by(level, run_len as i64);
1112                            })
1113                        }
1114                        None => encoder.put_n_with_observer(value, count, |level, run_len| {
1115                            new_rows += (run_len as u32) * (level == 0) as u32;
1116                        }),
1117                    }
1118                }
1119            }
1120            self.page_metrics.num_buffered_rows += new_rows;
1121        } else {
1122            // Each value is exactly one row.
1123            // Equals to the number of values, we count nulls as well.
1124            self.page_metrics.num_buffered_rows += num_levels as u32;
1125        }
1126
1127        match value_indices {
1128            Some(all_indices) => {
1129                let indices = &all_indices[values_offset..values_offset + values_to_write];
1130                self.encoder
1131                    .write_gather(values, indices, all_indices.len())?;
1132            }
1133            None => self.encoder.write(values, values_offset, values_to_write)?,
1134        }
1135
1136        let page_was_empty = self.page_metrics.num_buffered_values == 0;
1137        self.page_metrics.num_buffered_values += num_levels as u32;
1138
1139        if page_was_empty && values_to_write == 1 {
1140            self.set_page_size_exemption();
1141        }
1142
1143        if self.should_add_data_page() {
1144            self.add_data_page()?;
1145        }
1146
1147        if self.should_dict_fallback() {
1148            self.dict_fallback()?;
1149        }
1150
1151        Ok(values_to_write)
1152    }
1153
1154    /// Returns true if we need to fall back to non-dictionary encoding.
1155    ///
1156    /// We can only fall back if dictionary encoder is set and we have exceeded dictionary
1157    /// size.
1158    #[inline]
1159    fn should_dict_fallback(&self) -> bool {
1160        match self.encoder.estimated_dict_page_size() {
1161            Some(size) => size >= self.column_props.dictionary_page_size_limit,
1162            None => false,
1163        }
1164    }
1165
1166    /// Exempt a page's mandatory first value from the data page byte limit,
1167    /// when that value alone already exceeds it.
1168    ///
1169    /// Parquet requires every data page to hold at least one value, so such a
1170    /// value cannot be split out no matter how the limit is set. Counting it
1171    /// against the limit makes the limit unsatisfiable, and
1172    /// [`Self::should_add_data_page`] then cuts a page after every single
1173    /// value.
1174    ///
1175    /// For `DELTA_BYTE_ARRAY` that costs more than the extra pages. A value is
1176    /// stored as a suffix of the value before it, and a page boundary resets
1177    /// what "the value before it" refers to, so one value per page means every
1178    /// value is stored in full: a column of large values sharing long prefixes
1179    /// writes exactly the bytes `PLAIN` would
1180    /// ([#10489](https://github.com/apache/arrow-rs/issues/10489)).
1181    ///
1182    /// Only encodings that compress against the preceding value opt in, so
1183    /// `PLAIN` and `DELTA_LENGTH_BYTE_ARRAY` keep their tighter one-value page
1184    /// bound.
1185    ///
1186    /// The caller's trigger keys on a page-opening mini-batch holding exactly
1187    /// one value. `write_granular_chunk` cuts windows after an exact value
1188    /// count, so an over-limit value gets a single-value mini-batch whether
1189    /// or not the chunk contains nulls; see
1190    /// `test_column_writer_delta_byte_array_nullable_shared_prefix_dedup`.
1191    /// The exception is a repeated column, where a record holding several
1192    /// over-limit values cannot be split across pages at all.
1193    #[cold]
1194    fn set_page_size_exemption(&mut self) {
1195        if !self.encoder.compresses_against_previous_value() {
1196            return;
1197        }
1198        let size = self.encoder.estimated_data_page_size();
1199        if size >= self.column_props.data_page_size_limit {
1200            self.page_metrics.page_size_exemption = size;
1201        }
1202    }
1203
1204    /// Returns true if there is enough data for a data page, false otherwise.
1205    #[inline]
1206    fn should_add_data_page(&self) -> bool {
1207        // This is necessary in the event of a much larger dictionary size than page size
1208        //
1209        // In such a scenario the dictionary decoder may return an estimated encoded
1210        // size in excess of the page size limit, even when there are no buffered values
1211        if self.page_metrics.num_buffered_values == 0 {
1212            return false;
1213        }
1214
1215        self.page_metrics.num_buffered_rows as usize >= self.column_props.data_page_row_count_limit
1216            || self
1217                .encoder
1218                .estimated_data_page_size()
1219                .saturating_sub(self.page_metrics.page_size_exemption)
1220                >= self.column_props.data_page_size_limit
1221    }
1222
1223    /// Performs dictionary fallback.
1224    /// Prepares and writes dictionary and all data pages into page writer.
1225    fn dict_fallback(&mut self) -> Result<()> {
1226        // At this point we know that we need to fall back.
1227        if self.page_metrics.num_buffered_values > 0 {
1228            self.add_data_page()?;
1229        }
1230        self.write_dictionary_page()?;
1231        self.flush_data_pages()?;
1232        Ok(())
1233    }
1234
1235    // For float columns, always provide Some(n), even if n is 0
1236    // For non-float columns, always provide None
1237    fn get_nan_count<T: ParquetValueType>(&self) -> Option<i64> {
1238        let nan_count = || {
1239            let nan_count = self.page_metrics.num_page_nans.unwrap_or(0);
1240            match i64::try_from(nan_count) {
1241                Ok(count) => Some(count),
1242                _ => Some(i64::MAX),
1243            }
1244        };
1245        match T::PHYSICAL_TYPE {
1246            Type::FLOAT | Type::DOUBLE => nan_count(),
1247            Type::FIXED_LEN_BYTE_ARRAY
1248                if matches!(self.descr.logical_type_ref(), Some(LogicalType::Float16)) =>
1249            {
1250                nan_count()
1251            }
1252            _ => None,
1253        }
1254    }
1255
1256    /// Update the column index and offset index when adding the data page
1257    fn update_column_offset_index(
1258        &mut self,
1259        page_statistics: Option<&ValueStatistics<E::T>>,
1260        page_variable_length_bytes: Option<i64>,
1261    ) {
1262        // update the column index
1263        // `num_page_nulls` counts levels, not rows, so compare it with the number of levels:
1264        // with repetition a page can hold as many nulls as rows and still have values.
1265        let null_page =
1266            (self.page_metrics.num_buffered_values as u64) == self.page_metrics.num_page_nulls;
1267        // a page contains only null values,
1268        // and writers have to set the corresponding entries in min_values and max_values to byte[0]
1269        if null_page && self.column_index_builder.valid() {
1270            self.column_index_builder.append(
1271                null_page,
1272                vec![],
1273                vec![],
1274                self.page_metrics.num_page_nulls as i64,
1275                self.get_nan_count::<E::T>(),
1276            );
1277        } else if self.column_index_builder.valid() {
1278            // from page statistics
1279            // If can't get the page statistics, ignore this column/offset index for this column chunk
1280            match &page_statistics {
1281                None => {
1282                    self.column_index_builder.to_invalid();
1283                }
1284                Some(stat) => {
1285                    // Check if min/max are still ascending/descending across pages
1286                    let new_min = stat.min_opt().unwrap();
1287                    let new_max = stat.max_opt().unwrap();
1288                    if let Some((last_min, last_max)) = &self.last_non_null_data_page_min_max {
1289                        let basic_info = self.descr.get_basic_info();
1290                        if self.data_page_boundary_ascending {
1291                            // If last min/max are greater than new min/max then not ascending anymore
1292                            let not_ascending = compare_greater(basic_info, last_min, new_min)
1293                                || compare_greater(basic_info, last_max, new_max);
1294                            if not_ascending {
1295                                self.data_page_boundary_ascending = false;
1296                            }
1297                        }
1298
1299                        if self.data_page_boundary_descending {
1300                            // If new min/max are greater than last min/max then not descending anymore
1301                            let not_descending = compare_greater(basic_info, new_min, last_min)
1302                                || compare_greater(basic_info, new_max, last_max);
1303                            if not_descending {
1304                                self.data_page_boundary_descending = false;
1305                            }
1306                        }
1307                    }
1308                    self.last_non_null_data_page_min_max = Some((new_min.clone(), new_max.clone()));
1309
1310                    if self.can_truncate_value() {
1311                        self.column_index_builder.append(
1312                            null_page,
1313                            self.truncate_min_value(
1314                                self.props.column_index_truncate_length(),
1315                                stat.min_bytes_opt().unwrap(),
1316                            )
1317                            .0,
1318                            self.truncate_max_value(
1319                                self.props.column_index_truncate_length(),
1320                                stat.max_bytes_opt().unwrap(),
1321                            )
1322                            .0,
1323                            self.page_metrics.num_page_nulls as i64,
1324                            self.get_nan_count::<E::T>(),
1325                        );
1326                    } else {
1327                        self.column_index_builder.append(
1328                            null_page,
1329                            stat.min_bytes_opt().unwrap().to_vec(),
1330                            stat.max_bytes_opt().unwrap().to_vec(),
1331                            self.page_metrics.num_page_nulls as i64,
1332                            self.get_nan_count::<E::T>(),
1333                        );
1334                    }
1335                }
1336            }
1337        }
1338
1339        // Append page histograms to the `ColumnIndex` histograms
1340        self.column_index_builder.append_histograms(
1341            &self.page_metrics.repetition_level_histogram,
1342            &self.page_metrics.definition_level_histogram,
1343        );
1344
1345        // Update the offset index
1346        if let Some(builder) = self.offset_index_builder.as_mut() {
1347            builder.append_row_count(self.page_metrics.num_buffered_rows as i64);
1348            builder.append_unencoded_byte_array_data_bytes(page_variable_length_bytes);
1349        }
1350    }
1351
1352    /// Determine if we should allow truncating min/max values for this column's statistics
1353    fn can_truncate_value(&self) -> bool {
1354        match self.descr.physical_type() {
1355            // Don't truncate for Float16 and Decimal because their sort order is different
1356            // from that of FIXED_LEN_BYTE_ARRAY sort order.
1357            // So truncation of those types could lead to inaccurate min/max statistics
1358            Type::FIXED_LEN_BYTE_ARRAY
1359                if !is_decimal_descr(self.descr.get_basic_info())
1360                    && !matches!(self.descr.logical_type_ref(), Some(&LogicalType::Float16)) =>
1361            {
1362                true
1363            }
1364            // As with FIXED_LEN_BYTE_ARRAY, do not truncate Decimal values
1365            Type::BYTE_ARRAY if !is_decimal_descr(self.descr.get_basic_info()) => true,
1366            // Truncation only applies for fba/binary physical types
1367            _ => false,
1368        }
1369    }
1370
1371    /// Returns `true` if this column's logical type is a UTF-8 string.
1372    fn is_utf8(&self) -> bool {
1373        self.get_descriptor().logical_type_ref() == Some(&LogicalType::String)
1374            || self.get_descriptor().converted_type() == ConvertedType::UTF8
1375    }
1376
1377    /// Truncates a binary statistic to at most `truncation_length` bytes.
1378    ///
1379    /// If truncation is not possible, returns `data`.
1380    ///
1381    /// The `bool` in the returned tuple indicates whether truncation occurred or not.
1382    ///
1383    /// UTF-8 Note:
1384    /// If the column type indicates UTF-8, and `data` contains valid UTF-8, then the result will
1385    /// also remain valid UTF-8, but may be less than `truncation_length` bytes to avoid splitting
1386    /// on non-character boundaries.
1387    fn truncate_min_value(&self, truncation_length: Option<usize>, data: &[u8]) -> (Vec<u8>, bool) {
1388        truncation_length
1389            .filter(|l| data.len() > *l)
1390            .and_then(|l|
1391                // don't do extra work if this column isn't UTF-8
1392                if self.is_utf8() {
1393                    match str::from_utf8(data) {
1394                        Ok(str_data) => truncate_utf8(str_data, l),
1395                        Err(_) => Some(data[..l].to_vec()),
1396                    }
1397                } else {
1398                    Some(data[..l].to_vec())
1399                }
1400            )
1401            .map(|truncated| (truncated, true))
1402            .unwrap_or_else(|| (data.to_vec(), false))
1403    }
1404
1405    /// Truncates a binary statistic to at most `truncation_length` bytes, and then increment the
1406    /// final byte(s) to yield a valid upper bound. This may result in a result of less than
1407    /// `truncation_length` bytes if the last byte(s) overflows.
1408    ///
1409    /// If truncation is not possible, returns `data`.
1410    ///
1411    /// The `bool` in the returned tuple indicates whether truncation occurred or not.
1412    ///
1413    /// UTF-8 Note:
1414    /// If the column type indicates UTF-8, and `data` contains valid UTF-8, then the result will
1415    /// also remain valid UTF-8 (but again may be less than `truncation_length` bytes). If `data`
1416    /// does not contain valid UTF-8, then truncation will occur as if the column is non-string
1417    /// binary.
1418    fn truncate_max_value(&self, truncation_length: Option<usize>, data: &[u8]) -> (Vec<u8>, bool) {
1419        truncation_length
1420            .filter(|l| data.len() > *l)
1421            .and_then(|l|
1422                // don't do extra work if this column isn't UTF-8
1423                if self.is_utf8() {
1424                    match str::from_utf8(data) {
1425                        Ok(str_data) => truncate_and_increment_utf8(str_data, l),
1426                        Err(_) => increment(data[..l].to_vec()),
1427                    }
1428                } else {
1429                    increment(data[..l].to_vec())
1430                }
1431            )
1432            .map(|truncated| (truncated, true))
1433            .unwrap_or_else(|| (data.to_vec(), false))
1434    }
1435
1436    /// Truncate the min and max values that will be written to a data page
1437    /// header or column chunk Statistics.
1438    ///
1439    /// Binary statistics are always rebuilt from a copy of their bytes, even when they are not
1440    /// truncated, so the returned [`Statistics`] never alias the buffers the values were written
1441    /// from. Columns whose values must not be truncated (see [`Self::can_truncate_value`]) are
1442    /// copied without being shortened.
1443    fn truncate_statistics(&self, statistics: Statistics) -> Statistics {
1444        let truncation_length = self
1445            .can_truncate_value()
1446            .then(|| self.props.statistics_truncate_length())
1447            .flatten();
1448        match statistics {
1449            Statistics::ByteArray(stats) if stats._internal_has_min_max_set() => {
1450                Statistics::ByteArray(self.copy_and_truncate_min_max(stats, truncation_length))
1451            }
1452            Statistics::FixedLenByteArray(stats) if stats._internal_has_min_max_set() => {
1453                Statistics::FixedLenByteArray(
1454                    self.copy_and_truncate_min_max(stats, truncation_length),
1455                )
1456            }
1457            stats => stats,
1458        }
1459    }
1460
1461    /// Returns `stats` with min and max copied into their own buffers, truncated to
1462    /// `truncation_length` bytes if one is given. Every other field is kept as is.
1463    ///
1464    /// Values handed to [`Self::write_batch`] may be slices of a larger shared buffer (the Arrow
1465    /// writer slices all `FixedLenByteArray` values of a batch out of one allocation), and the
1466    /// running min/max are clones of those values. Copying here keeps the column chunk
1467    /// statistics, which outlive the write, from pinning those buffers.
1468    fn copy_and_truncate_min_max<T>(
1469        &self,
1470        stats: ValueStatistics<T>,
1471        truncation_length: Option<usize>,
1472    ) -> ValueStatistics<T>
1473    where
1474        T: ParquetValueType + From<Vec<u8>>,
1475    {
1476        let (min, did_truncate_min) =
1477            self.truncate_min_value(truncation_length, stats.min_bytes_opt().unwrap());
1478        let (max, did_truncate_max) =
1479            self.truncate_max_value(truncation_length, stats.max_bytes_opt().unwrap());
1480        let min_is_exact = stats.min_is_exact() && !did_truncate_min;
1481        let max_is_exact = stats.max_is_exact() && !did_truncate_max;
1482        stats
1483            .with_min_max(Some(min.into()), Some(max.into()))
1484            .with_min_is_exact(min_is_exact)
1485            .with_max_is_exact(max_is_exact)
1486    }
1487
1488    /// Adds data page.
1489    /// Data page is either buffered in case of dictionary encoding or written directly.
1490    ///
1491    /// A call with nothing buffered is a no-op. Skip empty pages to preserve
1492    /// strictly increasing first_row_index values in the OffsetIndex.
1493    pub(crate) fn add_data_page(&mut self) -> Result<()> {
1494        if self.page_metrics.num_buffered_values == 0 {
1495            return Ok(());
1496        }
1497
1498        // Extract encoded values
1499        let values_data = self.encoder.flush_data_page()?;
1500
1501        let max_def_level = self.descr.max_def_level();
1502        let max_rep_level = self.descr.max_rep_level();
1503
1504        self.column_metrics.num_column_nulls += self.page_metrics.num_page_nulls;
1505
1506        if let Some(nan_count) = values_data.nan_count {
1507            *self.column_metrics.num_column_nans.get_or_insert(0) += nan_count;
1508            self.page_metrics.num_page_nans = Some(nan_count);
1509        }
1510
1511        let page_statistics = match (values_data.min_value, values_data.max_value) {
1512            (Some(min), Some(max)) => {
1513                // Update chunk level statistics
1514                update_min(&self.descr, &min, &mut self.column_metrics.min_column_value);
1515                update_max(&self.descr, &max, &mut self.column_metrics.max_column_value);
1516
1517                (self.column_props.statistics_enabled == EnabledStatistics::Page).then_some(
1518                    ValueStatistics::new(
1519                        Some(min),
1520                        Some(max),
1521                        None,
1522                        Some(self.page_metrics.num_page_nulls),
1523                        false,
1524                    )
1525                    .with_nan_count(values_data.nan_count),
1526                )
1527            }
1528            _ => None,
1529        };
1530
1531        // update column and offset index
1532        self.update_column_offset_index(
1533            page_statistics.as_ref(),
1534            values_data.variable_length_bytes,
1535        );
1536
1537        // Update histograms and variable_length_bytes in column_metrics
1538        self.column_metrics
1539            .update_from_page_metrics(&self.page_metrics);
1540        self.column_metrics
1541            .update_variable_length_bytes(values_data.variable_length_bytes);
1542
1543        // From here on, we only need page statistics if they will be written to the page header.
1544        let page_statistics = page_statistics
1545            .filter(|_| self.column_props.write_page_header_statistics)
1546            .map(|stats| self.truncate_statistics(Statistics::from(stats)));
1547
1548        let compressed_page = match self.props.writer_version() {
1549            WriterVersion::PARQUET_1_0 => {
1550                let mut buffer = vec![];
1551
1552                if max_rep_level > 0 {
1553                    self.rep_levels_encoder
1554                        .flush_to(|data| buffer.extend_from_slice(data));
1555                }
1556
1557                if max_def_level > 0 {
1558                    self.def_levels_encoder
1559                        .flush_to(|data| buffer.extend_from_slice(data));
1560                }
1561
1562                buffer.extend_from_slice(&values_data.buf);
1563                let uncompressed_size = buffer.len();
1564
1565                if let Some(ref mut cmpr) = self.compressor {
1566                    let mut compressed_buf = Vec::with_capacity(uncompressed_size);
1567                    cmpr.compress(&buffer[..], &mut compressed_buf)?;
1568                    compressed_buf.shrink_to_fit();
1569                    buffer = compressed_buf;
1570                }
1571
1572                let data_page = Page::DataPage {
1573                    buf: buffer.into(),
1574                    num_values: self.page_metrics.num_buffered_values,
1575                    encoding: values_data.encoding,
1576                    def_level_encoding: Encoding::RLE,
1577                    rep_level_encoding: Encoding::RLE,
1578                    statistics: page_statistics,
1579                };
1580
1581                CompressedPage::new(data_page, uncompressed_size)
1582            }
1583            WriterVersion::PARQUET_2_0 => {
1584                let mut rep_levels_byte_len = 0;
1585                let mut def_levels_byte_len = 0;
1586                let mut buffer = vec![];
1587
1588                if max_rep_level > 0 {
1589                    self.rep_levels_encoder
1590                        .flush_to(|data| buffer.extend_from_slice(data));
1591                    rep_levels_byte_len = buffer.len();
1592                }
1593
1594                if max_def_level > 0 {
1595                    self.def_levels_encoder
1596                        .flush_to(|data| buffer.extend_from_slice(data));
1597                    def_levels_byte_len = buffer.len() - rep_levels_byte_len;
1598                }
1599
1600                let uncompressed_size =
1601                    rep_levels_byte_len + def_levels_byte_len + values_data.buf.len();
1602
1603                // Data Page v2 compresses values only.
1604                let is_compressed = match self.compressor {
1605                    Some(ref mut cmpr) => {
1606                        let buffer_len = buffer.len();
1607                        cmpr.compress(&values_data.buf, &mut buffer)?;
1608                        let compressed_values_size = buffer.len() - buffer_len;
1609                        let threshold = self.column_props.data_page_v2_compression_ratio_threshold;
1610                        if (compressed_values_size as f64) >= (uncompressed_size as f64) * threshold
1611                        {
1612                            buffer.truncate(buffer_len);
1613                            buffer.extend_from_slice(&values_data.buf);
1614                            false
1615                        } else {
1616                            true
1617                        }
1618                    }
1619                    None => {
1620                        buffer.extend_from_slice(&values_data.buf);
1621                        false
1622                    }
1623                };
1624
1625                let data_page = Page::DataPageV2 {
1626                    buf: buffer.into(),
1627                    num_values: self.page_metrics.num_buffered_values,
1628                    encoding: values_data.encoding,
1629                    num_nulls: self.page_metrics.num_page_nulls as u32,
1630                    num_rows: self.page_metrics.num_buffered_rows,
1631                    def_levels_byte_len: def_levels_byte_len as u32,
1632                    rep_levels_byte_len: rep_levels_byte_len as u32,
1633                    is_compressed,
1634                    statistics: page_statistics,
1635                };
1636
1637                CompressedPage::new(data_page, uncompressed_size)
1638            }
1639        };
1640
1641        // Check if we need to buffer data page or flush it to the sink directly.
1642        //
1643        // For dictionary-encoded columns the dictionary page must be written
1644        // first, but it is not final until all values are seen, so completed
1645        // data pages are normally buffered here until `close`. A page writer
1646        // that defers final layout (the Arrow path) instead orders pages itself
1647        // at flush, so we stream the data pages straight through and never let
1648        // them accumulate in memory.
1649        if self.encoder.has_dictionary() && !self.page_writer.defers_dictionary_ordering() {
1650            self.data_pages.push_back(compressed_page);
1651        } else {
1652            self.write_data_page(compressed_page)?;
1653        }
1654
1655        // Update total number of rows.
1656        self.column_metrics.total_rows_written += self.page_metrics.num_buffered_rows as u64;
1657        self.page_metrics.new_page();
1658
1659        Ok(())
1660    }
1661
1662    /// Finalises any outstanding data pages and flushes buffered data pages from
1663    /// dictionary encoding into underlying sink.
1664    #[inline]
1665    fn flush_data_pages(&mut self) -> Result<()> {
1666        // Write all outstanding data to a new page.
1667        if self.page_metrics.num_buffered_values > 0 {
1668            self.add_data_page()?;
1669        }
1670
1671        while let Some(page) = self.data_pages.pop_front() {
1672            self.write_data_page(page)?;
1673        }
1674
1675        Ok(())
1676    }
1677
1678    /// Assembles column chunk metadata.
1679    fn build_column_metadata(&mut self) -> Result<ColumnChunkMetaData> {
1680        let total_compressed_size = self.column_metrics.total_compressed_size as i64;
1681        let total_uncompressed_size = self.column_metrics.total_uncompressed_size as i64;
1682        let num_values = self.column_metrics.total_num_values as i64;
1683        let dict_page_offset = self.column_metrics.dictionary_page_offset.map(|v| v as i64);
1684        // If data page offset is not set, then no pages have been written
1685        let data_page_offset = self.column_metrics.data_page_offset.unwrap_or(0) as i64;
1686
1687        let mut builder = ColumnChunkMetaData::builder(self.descr.clone())
1688            .set_compression(self.codec)
1689            .set_encodings_mask(EncodingMask::new_from_encodings(self.encodings.iter()))
1690            .set_page_encoding_stats(self.encoding_stats.clone())
1691            .set_total_compressed_size(total_compressed_size)
1692            .set_total_uncompressed_size(total_uncompressed_size)
1693            .set_num_values(num_values)
1694            .set_data_page_offset(data_page_offset)
1695            .set_dictionary_page_offset(dict_page_offset);
1696
1697        if self.column_props.statistics_enabled != EnabledStatistics::None {
1698            let backwards_compatible_min_max = self.descr.sort_order().is_signed();
1699
1700            let distinct_count = self
1701                .distinct_count_override
1702                .or(self.column_metrics.column_distinct_count);
1703            let statistics = ValueStatistics::<E::T>::new(
1704                self.column_metrics.min_column_value.clone(),
1705                self.column_metrics.max_column_value.clone(),
1706                distinct_count,
1707                Some(self.column_metrics.num_column_nulls),
1708                false,
1709            )
1710            .with_nan_count(self.column_metrics.num_column_nans)
1711            .with_backwards_compatible_min_max(backwards_compatible_min_max)
1712            .into();
1713
1714            let statistics = self.truncate_statistics(statistics);
1715
1716            builder = builder
1717                .set_statistics(statistics)
1718                .set_unencoded_byte_array_data_bytes(self.column_metrics.variable_length_bytes)
1719                .set_repetition_level_histogram(
1720                    self.column_metrics.repetition_level_histogram.take(),
1721                )
1722                .set_definition_level_histogram(
1723                    self.column_metrics.definition_level_histogram.take(),
1724                );
1725
1726            if let Some(geo_stats) = self.encoder.flush_geospatial_statistics() {
1727                builder = builder.set_geo_statistics(geo_stats);
1728            }
1729        }
1730
1731        builder = self.set_column_chunk_encryption_properties(builder);
1732
1733        let metadata = builder.build()?;
1734        Ok(metadata)
1735    }
1736
1737    /// Writes compressed data page into underlying sink and updates global metrics.
1738    #[inline]
1739    fn write_data_page(&mut self, page: CompressedPage) -> Result<()> {
1740        self.encodings.insert(page.encoding());
1741        if self.descr.max_def_level() > 0 || self.descr.max_rep_level() > 0 {
1742            self.encodings.insert(Encoding::RLE);
1743        }
1744        match self.encoding_stats.last_mut() {
1745            Some(encoding_stats)
1746                if encoding_stats.page_type == page.page_type()
1747                    && encoding_stats.encoding == page.encoding() =>
1748            {
1749                encoding_stats.count += 1;
1750            }
1751            _ => {
1752                // data page type does not change inside a file
1753                // encoding can currently only change from dictionary to non-dictionary once
1754                self.encoding_stats.push(PageEncodingStats {
1755                    page_type: page.page_type(),
1756                    encoding: page.encoding(),
1757                    count: 1,
1758                });
1759            }
1760        }
1761        let page_spec = self.page_writer.write_page(page)?;
1762        // update offset index
1763        // compressed_size = header_size + compressed_data_size
1764        if let Some(builder) = self.offset_index_builder.as_mut() {
1765            builder
1766                .append_offset_and_size(page_spec.offset as i64, page_spec.compressed_size as i32)
1767        }
1768        self.update_metrics_for_page(page_spec);
1769        Ok(())
1770    }
1771
1772    /// Writes dictionary page into underlying sink.
1773    #[inline]
1774    fn write_dictionary_page(&mut self) -> Result<()> {
1775        let compressed_page = {
1776            let mut page = self
1777                .encoder
1778                .flush_dict_page()?
1779                .ok_or_else(|| general_err!("Dictionary encoder is not set"))?;
1780
1781            let uncompressed_size = page.buf.len();
1782
1783            if let Some(ref mut cmpr) = self.compressor {
1784                let mut output_buf = Vec::with_capacity(uncompressed_size);
1785                cmpr.compress(&page.buf, &mut output_buf)?;
1786                page.buf = Bytes::from(output_buf);
1787            }
1788
1789            let dict_page = Page::DictionaryPage {
1790                buf: page.buf,
1791                num_values: page.num_values as u32,
1792                encoding: self.props.dictionary_page_encoding(),
1793                is_sorted: page.is_sorted,
1794            };
1795            CompressedPage::new(dict_page, uncompressed_size)
1796        };
1797
1798        self.encodings.insert(compressed_page.encoding());
1799        // The dictionary page is always laid out first in the column chunk,
1800        // even when a page writer that defers dictionary ordering receives it
1801        // after the data pages. Keep `encoding_stats` in file order, as some
1802        // readers (e.g. Arrow C++) expect the dictionary page entry first.
1803        self.encoding_stats.insert(
1804            0,
1805            PageEncodingStats {
1806                page_type: PageType::DICTIONARY_PAGE,
1807                encoding: compressed_page.encoding(),
1808                count: 1,
1809            },
1810        );
1811        let page_spec = self.page_writer.write_page(compressed_page)?;
1812        self.update_metrics_for_page(page_spec);
1813        // For the directory page, don't need to update column/offset index.
1814        Ok(())
1815    }
1816
1817    /// Updates column writer metrics with each page metadata.
1818    #[inline]
1819    fn update_metrics_for_page(&mut self, page_spec: PageWriteSpec) {
1820        self.column_metrics.total_uncompressed_size += page_spec.uncompressed_size as u64;
1821        self.column_metrics.total_compressed_size += page_spec.compressed_size as u64;
1822        self.column_metrics.total_bytes_written += page_spec.bytes_written;
1823
1824        match page_spec.page_type {
1825            PageType::DATA_PAGE | PageType::DATA_PAGE_V2 => {
1826                self.column_metrics.total_num_values += page_spec.num_values as u64;
1827                if self.column_metrics.data_page_offset.is_none() {
1828                    self.column_metrics.data_page_offset = Some(page_spec.offset);
1829                }
1830            }
1831            PageType::DICTIONARY_PAGE => {
1832                assert!(
1833                    self.column_metrics.dictionary_page_offset.is_none(),
1834                    "Dictionary offset is already set"
1835                );
1836                self.column_metrics.dictionary_page_offset = Some(page_spec.offset);
1837            }
1838            PageType::INDEX_PAGE => {}
1839        }
1840    }
1841
1842    #[inline]
1843    #[cfg(feature = "encryption")]
1844    fn set_column_chunk_encryption_properties(
1845        &self,
1846        builder: ColumnChunkMetaDataBuilder,
1847    ) -> ColumnChunkMetaDataBuilder {
1848        if let Some(encryption_properties) = self.props.file_encryption_properties.as_ref() {
1849            builder.set_column_crypto_metadata(get_column_crypto_metadata(
1850                encryption_properties,
1851                &self.descr,
1852            ))
1853        } else {
1854            builder
1855        }
1856    }
1857
1858    #[inline]
1859    #[cfg(not(feature = "encryption"))]
1860    fn set_column_chunk_encryption_properties(
1861        &self,
1862        builder: ColumnChunkMetaDataBuilder,
1863    ) -> ColumnChunkMetaDataBuilder {
1864        builder
1865    }
1866}
1867
1868fn update_min<T: ParquetValueType>(descr: &ColumnDescriptor, val: &T, min: &mut Option<T>) {
1869    match min {
1870        None => *min = Some(val.clone()),
1871        Some(min) => {
1872            let basic_type_info = descr.get_basic_info();
1873            let is_min_nan = is_nan(basic_type_info, min);
1874            let is_val_nan = is_nan(basic_type_info, val);
1875            match (is_min_nan, is_val_nan) {
1876                // current min is not NaN, but incoming is NaN: skip
1877                (false, true) => {}
1878                // current min is NaN, but incoming is not: assign val to min
1879                (true, false) => *min = val.clone(),
1880                // both NaN or non-NaN, safe to call update_stat()
1881                _ => {
1882                    update_stat::<T, _>(val, min, |cur| compare_greater(basic_type_info, cur, val))
1883                }
1884            }
1885        }
1886    }
1887}
1888
1889fn update_max<T: ParquetValueType>(descr: &ColumnDescriptor, val: &T, max: &mut Option<T>) {
1890    match max {
1891        None => *max = Some(val.clone()),
1892        Some(max) => {
1893            let basic_type_info = descr.get_basic_info();
1894            let is_max_nan = is_nan(basic_type_info, max);
1895            let is_val_nan = is_nan(basic_type_info, val);
1896            match (is_max_nan, is_val_nan) {
1897                // current max is not NaN, but incoming is NaN: skip
1898                (false, true) => {}
1899                // current max is NaN, but incoming is not: assign val to max
1900                (true, false) => *max = val.clone(),
1901                // both NaN or non-NaN, safe to call update_stat()
1902                _ => {
1903                    update_stat::<T, _>(val, max, |cur| compare_greater(basic_type_info, val, cur))
1904                }
1905            }
1906        }
1907    }
1908}
1909
1910#[inline]
1911#[expect(clippy::eq_op)]
1912fn is_nan<T: ParquetValueType>(basic_type_info: &BasicTypeInfo, val: &T) -> bool {
1913    match T::PHYSICAL_TYPE {
1914        Type::FLOAT | Type::DOUBLE => val != val,
1915        Type::FIXED_LEN_BYTE_ARRAY
1916            if matches!(basic_type_info.sort_order(), SortOrder::TOTAL_ORDER) =>
1917        {
1918            // taken from f16 impl, but skips creating f16. just compare the bits as u16.
1919            let val = val.as_bytes();
1920            // Float16 is stored little endian
1921            let uval = ((val[1] as u16) << 8) | val[0] as u16;
1922            uval & 0x7FFFu16 > 0x7C00u16
1923        }
1924        _ => false,
1925    }
1926}
1927
1928/// Perform a conditional update of `cur`
1929///
1930/// Calls `should_update` with the value of `cur`, and updates `cur` to `Some(val)` if it
1931/// returns `true`. `cur` must not be `None` or this will panic.
1932fn update_stat<T: ParquetValueType, F>(val: &T, cur: &mut T, should_update: F)
1933where
1934    F: Fn(&T) -> bool,
1935{
1936    if should_update(cur) {
1937        *cur = val.clone();
1938    }
1939}
1940
1941/// Evaluate `a > b` according to underlying logical type.
1942fn compare_greater<T: ParquetValueType>(basic_type_info: &BasicTypeInfo, a: &T, b: &T) -> bool {
1943    match T::PHYSICAL_TYPE {
1944        Type::FLOAT => {
1945            let a = f32::from_le_bytes(a.as_bytes().try_into().unwrap());
1946            let b = f32::from_le_bytes(b.as_bytes().try_into().unwrap());
1947            return a.total_cmp(&b) == Ordering::Greater;
1948        }
1949        Type::DOUBLE => {
1950            let a = f64::from_le_bytes(a.as_bytes().try_into().unwrap());
1951            let b = f64::from_le_bytes(b.as_bytes().try_into().unwrap());
1952            return a.total_cmp(&b) == Ordering::Greater;
1953        }
1954        Type::INT32 | Type::INT64
1955            if matches!(basic_type_info.sort_order(), SortOrder::UNSIGNED) =>
1956        {
1957            return compare_greater_unsigned_int(a, b);
1958        }
1959        Type::FIXED_LEN_BYTE_ARRAY
1960            if matches!(basic_type_info.sort_order(), SortOrder::TOTAL_ORDER) =>
1961        {
1962            return compare_greater_f16(a.as_bytes(), b.as_bytes());
1963        }
1964        Type::FIXED_LEN_BYTE_ARRAY | Type::BYTE_ARRAY if is_decimal_descr(basic_type_info) => {
1965            return compare_greater_byte_array_decimals(a.as_bytes(), b.as_bytes());
1966        }
1967
1968        _ => {}
1969    }
1970
1971    // compare independent of logical / converted type
1972    a > b
1973}
1974
1975// ----------------------------------------------------------------------
1976// Encoding support for column writer.
1977// This mirrors parquet-mr default encodings for writes. See:
1978// https://github.com/apache/parquet-mr/blob/master/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV1ValuesWriterFactory.java
1979// https://github.com/apache/parquet-mr/blob/master/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java
1980
1981/// Returns encoding for a column when no other encoding is provided in writer properties.
1982fn fallback_encoding(kind: Type, props: &WriterProperties) -> Encoding {
1983    match (kind, props.writer_version()) {
1984        (Type::BOOLEAN, WriterVersion::PARQUET_2_0) => Encoding::RLE,
1985        (Type::INT32, WriterVersion::PARQUET_2_0) => Encoding::DELTA_BINARY_PACKED,
1986        (Type::INT64, WriterVersion::PARQUET_2_0) => Encoding::DELTA_BINARY_PACKED,
1987        (Type::BYTE_ARRAY, WriterVersion::PARQUET_2_0) => Encoding::DELTA_BYTE_ARRAY,
1988        (Type::FIXED_LEN_BYTE_ARRAY, WriterVersion::PARQUET_2_0) => Encoding::DELTA_BYTE_ARRAY,
1989        _ => Encoding::PLAIN,
1990    }
1991}
1992
1993/// Returns true if dictionary is supported for column writer, false otherwise.
1994fn has_dictionary_support(kind: Type) -> bool {
1995    match kind {
1996        // Booleans do not support dict encoding and should use a fallback encoding.
1997        Type::BOOLEAN => false,
1998        _ => true,
1999    }
2000}
2001
2002#[inline]
2003fn compare_greater_unsigned_int<T: ParquetValueType>(a: &T, b: &T) -> bool {
2004    a.as_u64().unwrap() > b.as_u64().unwrap()
2005}
2006
2007#[inline]
2008fn compare_greater_f16(a: &[u8], b: &[u8]) -> bool {
2009    let a = f16::from_le_bytes(a.try_into().unwrap());
2010    let b = f16::from_le_bytes(b.try_into().unwrap());
2011    a.total_cmp(&b) == Ordering::Greater
2012}
2013
2014/// Returns `true` if the column described by `basic_type_info` is a Decimal column
2015#[inline]
2016pub(crate) fn is_decimal_descr(basic_type_info: &BasicTypeInfo) -> bool {
2017    basic_type_info.converted_type() == ConvertedType::DECIMAL
2018        || matches!(
2019            basic_type_info.logical_type_ref(),
2020            Some(LogicalType::Decimal(_))
2021        )
2022}
2023
2024/// Signed comparison of bytes arrays
2025pub(crate) fn compare_greater_byte_array_decimals(a: &[u8], b: &[u8]) -> bool {
2026    let a_length = a.len();
2027    let b_length = b.len();
2028
2029    if a_length == 0 || b_length == 0 {
2030        return a_length > 0;
2031    }
2032
2033    let first_a: u8 = a[0];
2034    let first_b: u8 = b[0];
2035
2036    // We can short circuit for different signed numbers or
2037    // for equal length bytes arrays that have different first bytes.
2038    // The equality requirement is necessary for sign extension cases.
2039    // 0xFF10 should be equal to 0x10 (due to big endian sign extension).
2040    if (0x80 & first_a) != (0x80 & first_b) || (a_length == b_length && first_a != first_b) {
2041        return (first_a as i8) > (first_b as i8);
2042    }
2043
2044    // When the lengths are unequal and the numbers are of the same sign,
2045    // sign-extend the shorter value: if any of the longer value's extra
2046    // leading bytes differs from the sign-extension byte it has the larger
2047    // magnitude, and otherwise those bytes are redundant and the aligned
2048    // equal-length tails decide via unsigned lexicographical comparison.
2049
2050    let extension: u8 = if (first_a as i8) < 0 { 0xFF } else { 0 };
2051
2052    if a_length != b_length {
2053        let not_equal = if a_length > b_length {
2054            let lead_length = a_length - b_length;
2055            a[0..lead_length].iter().any(|&x| x != extension)
2056        } else {
2057            let lead_length = b_length - a_length;
2058            b[0..lead_length].iter().any(|&x| x != extension)
2059        };
2060
2061        if not_equal {
2062            let negative_values: bool = (first_a as i8) < 0;
2063            let a_longer: bool = a_length > b_length;
2064            return if negative_values { !a_longer } else { a_longer };
2065        }
2066    }
2067
2068    let tail_length = a_length.min(b_length);
2069    (a[a_length - tail_length..]) > (b[b_length - tail_length..])
2070}
2071
2072/// Truncate a UTF-8 slice to the longest prefix that is still a valid UTF-8 string,
2073/// while being less than `length` bytes and non-empty. Returns `None` if truncation
2074/// is not possible within those constraints.
2075///
2076/// The caller guarantees that data.len() > length.
2077fn truncate_utf8(data: &str, length: usize) -> Option<Vec<u8>> {
2078    let split = (1..=length).rfind(|x| data.is_char_boundary(*x))?;
2079    Some(data.as_bytes()[..split].to_vec())
2080}
2081
2082/// Truncate a UTF-8 slice and increment it's final character. The returned value is the
2083/// longest such slice that is still a valid UTF-8 string while being less than `length`
2084/// bytes and non-empty. Returns `None` if no such transformation is possible.
2085///
2086/// The caller guarantees that data.len() > length.
2087fn truncate_and_increment_utf8(data: &str, length: usize) -> Option<Vec<u8>> {
2088    // UTF-8 is max 4 bytes, so start search 3 back from desired length
2089    let lower_bound = length.saturating_sub(3);
2090    let split = (lower_bound..=length).rfind(|x| data.is_char_boundary(*x))?;
2091    increment_utf8(data.get(..split)?)
2092}
2093
2094/// Increment the final character in a UTF-8 string in such a way that the returned result
2095/// is still a valid UTF-8 string. The returned string may be shorter than the input if the
2096/// last character(s) cannot be incremented (due to overflow or producing invalid code points).
2097/// Returns `None` if the string cannot be incremented.
2098///
2099/// Note that this implementation will not promote an N-byte code point to (N+1) bytes.
2100fn increment_utf8(data: &str) -> Option<Vec<u8>> {
2101    for (idx, original_char) in data.char_indices().rev() {
2102        let original_len = original_char.len_utf8();
2103        if let Some(next_char) = char::from_u32(original_char as u32 + 1) {
2104            // do not allow increasing byte width of incremented char
2105            if next_char.len_utf8() == original_len {
2106                let mut result = data.as_bytes()[..idx + original_len].to_vec();
2107                next_char.encode_utf8(&mut result[idx..]);
2108                return Some(result);
2109            }
2110        }
2111    }
2112
2113    None
2114}
2115
2116/// Try and increment the bytes from right to left.
2117///
2118/// Returns `None` if all bytes are set to `u8::MAX`.
2119fn increment(mut data: Vec<u8>) -> Option<Vec<u8>> {
2120    for byte in data.iter_mut().rev() {
2121        let (incremented, overflow) = byte.overflowing_add(1);
2122        *byte = incremented;
2123
2124        if !overflow {
2125            return Some(data);
2126        }
2127    }
2128
2129    None
2130}
2131
2132#[cfg(test)]
2133mod tests {
2134    use crate::{
2135        file::{properties::DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH, writer::SerializedFileWriter},
2136        schema::parser::parse_message_type,
2137    };
2138    use core::str;
2139    use rand::distr::uniform::SampleUniform;
2140    use std::{fs::File, sync::Arc};
2141
2142    use crate::column::{
2143        page::PageReader,
2144        reader::{ColumnReaderImpl, get_column_reader, get_typed_column_reader},
2145    };
2146    use crate::file::writer::TrackedWrite;
2147    use crate::file::{
2148        properties::ReaderProperties, reader::SerializedPageReader, writer::SerializedPageWriter,
2149    };
2150    use crate::schema::types::{ColumnPath, Type as SchemaType};
2151    use crate::util::test_common::rand_gen::random_numbers_range;
2152
2153    use super::*;
2154
2155    #[test]
2156    fn test_column_writer_inconsistent_def_rep_length() {
2157        let page_writer = get_test_page_writer();
2158        let props = Default::default();
2159        let mut writer = get_test_column_writer::<Int32Type>(page_writer, 1, 1, props);
2160        let res = writer.write_batch(&[1, 2, 3, 4], Some(&[1, 1, 1]), Some(&[0, 0]));
2161        assert!(res.is_err());
2162        if let Err(err) = res {
2163            assert_eq!(
2164                format!("{err}"),
2165                "Parquet error: Inconsistent length of definition and repetition levels: 3 != 2"
2166            );
2167        }
2168    }
2169
2170    #[test]
2171    fn test_column_writer_invalid_def_levels() {
2172        let page_writer = get_test_page_writer();
2173        let props = Default::default();
2174        let mut writer = get_test_column_writer::<Int32Type>(page_writer, 1, 0, props);
2175        let res = writer.write_batch(&[1, 2, 3, 4], None, None);
2176        assert!(res.is_err());
2177        if let Err(err) = res {
2178            assert_eq!(
2179                format!("{err}"),
2180                "Parquet error: Definition levels are required, because max definition level = 1"
2181            );
2182        }
2183    }
2184
2185    #[test]
2186    fn test_column_writer_invalid_rep_levels() {
2187        let page_writer = get_test_page_writer();
2188        let props = Default::default();
2189        let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 1, props);
2190        let res = writer.write_batch(&[1, 2, 3, 4], None, None);
2191        assert!(res.is_err());
2192        if let Err(err) = res {
2193            assert_eq!(
2194                format!("{err}"),
2195                "Parquet error: Repetition levels are required, because max repetition level = 1"
2196            );
2197        }
2198    }
2199
2200    #[test]
2201    fn test_column_writer_not_enough_values_to_write() {
2202        let page_writer = get_test_page_writer();
2203        let props = Default::default();
2204        let mut writer = get_test_column_writer::<Int32Type>(page_writer, 1, 0, props);
2205        let res = writer.write_batch(&[1, 2], Some(&[1, 1, 1, 1]), None);
2206        assert!(res.is_err());
2207        if let Err(err) = res {
2208            assert_eq!(
2209                format!("{err}"),
2210                "Parquet error: Expected to write 4 values, but have only 2"
2211            );
2212        }
2213    }
2214
2215    #[test]
2216    fn test_column_writer_write_only_one_dictionary_page() {
2217        let page_writer = get_test_page_writer();
2218        let props = Default::default();
2219        let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
2220        writer.write_batch(&[1, 2, 3, 4], None, None).unwrap();
2221        // First page should be correctly written.
2222        writer.add_data_page().unwrap();
2223        writer.write_dictionary_page().unwrap();
2224        let err = writer.write_dictionary_page().unwrap_err().to_string();
2225        assert_eq!(err, "Parquet error: Dictionary encoder is not set");
2226    }
2227
2228    #[test]
2229    fn test_column_writer_error_when_writing_disabled_dictionary() {
2230        let page_writer = get_test_page_writer();
2231        let props = Arc::new(
2232            WriterProperties::builder()
2233                .set_dictionary_enabled(false)
2234                .build(),
2235        );
2236        let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
2237        writer.write_batch(&[1, 2, 3, 4], None, None).unwrap();
2238        let err = writer.write_dictionary_page().unwrap_err().to_string();
2239        assert_eq!(err, "Parquet error: Dictionary encoder is not set");
2240    }
2241
2242    #[test]
2243    fn test_column_writer_boolean_type_does_not_support_dictionary() {
2244        let page_writer = get_test_page_writer();
2245        let props = Arc::new(
2246            WriterProperties::builder()
2247                .set_dictionary_enabled(true)
2248                .build(),
2249        );
2250        let mut writer = get_test_column_writer::<BoolType>(page_writer, 0, 0, props);
2251        writer
2252            .write_batch(&[true, false, true, false], None, None)
2253            .unwrap();
2254
2255        let r = writer.close().unwrap();
2256        // PlainEncoder uses bit writer to write boolean values, which all fit into 1
2257        // byte.
2258        assert_eq!(r.bytes_written, 1);
2259        assert_eq!(r.rows_written, 4);
2260
2261        let metadata = r.metadata;
2262        assert_eq!(
2263            metadata.encodings().collect::<Vec<_>>(),
2264            vec![Encoding::PLAIN]
2265        );
2266        assert_eq!(metadata.num_values(), 4); // just values
2267        assert_eq!(metadata.dictionary_page_offset(), None);
2268    }
2269
2270    #[test]
2271    fn test_bloom_filter_for_dictionary_encoded_chunks() {
2272        fn bloom_filter_written(dictionary_enabled: bool, for_dictionary_chunks: bool) -> bool {
2273            let props = Arc::new(
2274                WriterProperties::builder()
2275                    .set_dictionary_enabled(dictionary_enabled)
2276                    .set_bloom_filter_enabled(true)
2277                    .set_bloom_filter_for_dictionary_encoded_chunks(for_dictionary_chunks)
2278                    .build(),
2279            );
2280            let mut writer =
2281                get_test_column_writer::<Int32Type>(get_test_page_writer(), 0, 0, props);
2282            writer.write_batch(&[1, 2, 1, 2], None, None).unwrap();
2283            writer.close().unwrap().bloom_filter.is_some()
2284        }
2285
2286        assert!(bloom_filter_written(true, true));
2287        assert!(bloom_filter_written(false, true));
2288        assert!(bloom_filter_written(false, false));
2289        assert!(!bloom_filter_written(true, false));
2290    }
2291
2292    #[test]
2293    fn test_column_writer_default_encoding_support_bool() {
2294        check_encoding_write_support::<BoolType>(
2295            WriterVersion::PARQUET_1_0,
2296            true,
2297            &[true, false],
2298            None,
2299            &[Encoding::PLAIN],
2300            &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)],
2301        );
2302        check_encoding_write_support::<BoolType>(
2303            WriterVersion::PARQUET_1_0,
2304            false,
2305            &[true, false],
2306            None,
2307            &[Encoding::PLAIN],
2308            &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)],
2309        );
2310        check_encoding_write_support::<BoolType>(
2311            WriterVersion::PARQUET_2_0,
2312            true,
2313            &[true, false],
2314            None,
2315            &[Encoding::RLE],
2316            &[encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE, 1)],
2317        );
2318        check_encoding_write_support::<BoolType>(
2319            WriterVersion::PARQUET_2_0,
2320            false,
2321            &[true, false],
2322            None,
2323            &[Encoding::RLE],
2324            &[encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE, 1)],
2325        );
2326    }
2327
2328    #[test]
2329    fn test_column_writer_default_encoding_support_int32() {
2330        check_encoding_write_support::<Int32Type>(
2331            WriterVersion::PARQUET_1_0,
2332            true,
2333            &[1, 2],
2334            Some(0),
2335            &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2336            &[
2337                encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2338                encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1),
2339            ],
2340        );
2341        check_encoding_write_support::<Int32Type>(
2342            WriterVersion::PARQUET_1_0,
2343            false,
2344            &[1, 2],
2345            None,
2346            &[Encoding::PLAIN],
2347            &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)],
2348        );
2349        check_encoding_write_support::<Int32Type>(
2350            WriterVersion::PARQUET_2_0,
2351            true,
2352            &[1, 2],
2353            Some(0),
2354            &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2355            &[
2356                encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2357                encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1),
2358            ],
2359        );
2360        check_encoding_write_support::<Int32Type>(
2361            WriterVersion::PARQUET_2_0,
2362            false,
2363            &[1, 2],
2364            None,
2365            &[Encoding::DELTA_BINARY_PACKED],
2366            &[encoding_stats(
2367                PageType::DATA_PAGE_V2,
2368                Encoding::DELTA_BINARY_PACKED,
2369                1,
2370            )],
2371        );
2372    }
2373
2374    #[test]
2375    fn test_column_writer_default_encoding_support_int64() {
2376        check_encoding_write_support::<Int64Type>(
2377            WriterVersion::PARQUET_1_0,
2378            true,
2379            &[1, 2],
2380            Some(0),
2381            &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2382            &[
2383                encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2384                encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1),
2385            ],
2386        );
2387        check_encoding_write_support::<Int64Type>(
2388            WriterVersion::PARQUET_1_0,
2389            false,
2390            &[1, 2],
2391            None,
2392            &[Encoding::PLAIN],
2393            &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)],
2394        );
2395        check_encoding_write_support::<Int64Type>(
2396            WriterVersion::PARQUET_2_0,
2397            true,
2398            &[1, 2],
2399            Some(0),
2400            &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2401            &[
2402                encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2403                encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1),
2404            ],
2405        );
2406        check_encoding_write_support::<Int64Type>(
2407            WriterVersion::PARQUET_2_0,
2408            false,
2409            &[1, 2],
2410            None,
2411            &[Encoding::DELTA_BINARY_PACKED],
2412            &[encoding_stats(
2413                PageType::DATA_PAGE_V2,
2414                Encoding::DELTA_BINARY_PACKED,
2415                1,
2416            )],
2417        );
2418    }
2419
2420    #[test]
2421    fn test_column_writer_default_encoding_support_int96() {
2422        check_encoding_write_support::<Int96Type>(
2423            WriterVersion::PARQUET_1_0,
2424            true,
2425            &[Int96::from(vec![1, 2, 3])],
2426            Some(0),
2427            &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2428            &[
2429                encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2430                encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1),
2431            ],
2432        );
2433        check_encoding_write_support::<Int96Type>(
2434            WriterVersion::PARQUET_1_0,
2435            false,
2436            &[Int96::from(vec![1, 2, 3])],
2437            None,
2438            &[Encoding::PLAIN],
2439            &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)],
2440        );
2441        check_encoding_write_support::<Int96Type>(
2442            WriterVersion::PARQUET_2_0,
2443            true,
2444            &[Int96::from(vec![1, 2, 3])],
2445            Some(0),
2446            &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2447            &[
2448                encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2449                encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1),
2450            ],
2451        );
2452        check_encoding_write_support::<Int96Type>(
2453            WriterVersion::PARQUET_2_0,
2454            false,
2455            &[Int96::from(vec![1, 2, 3])],
2456            None,
2457            &[Encoding::PLAIN],
2458            &[encoding_stats(PageType::DATA_PAGE_V2, Encoding::PLAIN, 1)],
2459        );
2460    }
2461
2462    #[test]
2463    fn test_column_writer_default_encoding_support_float() {
2464        check_encoding_write_support::<FloatType>(
2465            WriterVersion::PARQUET_1_0,
2466            true,
2467            &[1.0, 2.0],
2468            Some(0),
2469            &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2470            &[
2471                encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2472                encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1),
2473            ],
2474        );
2475        check_encoding_write_support::<FloatType>(
2476            WriterVersion::PARQUET_1_0,
2477            false,
2478            &[1.0, 2.0],
2479            None,
2480            &[Encoding::PLAIN],
2481            &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)],
2482        );
2483        check_encoding_write_support::<FloatType>(
2484            WriterVersion::PARQUET_2_0,
2485            true,
2486            &[1.0, 2.0],
2487            Some(0),
2488            &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2489            &[
2490                encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2491                encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1),
2492            ],
2493        );
2494        check_encoding_write_support::<FloatType>(
2495            WriterVersion::PARQUET_2_0,
2496            false,
2497            &[1.0, 2.0],
2498            None,
2499            &[Encoding::PLAIN],
2500            &[encoding_stats(PageType::DATA_PAGE_V2, Encoding::PLAIN, 1)],
2501        );
2502    }
2503
2504    #[test]
2505    fn test_column_writer_default_encoding_support_double() {
2506        check_encoding_write_support::<DoubleType>(
2507            WriterVersion::PARQUET_1_0,
2508            true,
2509            &[1.0, 2.0],
2510            Some(0),
2511            &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2512            &[
2513                encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2514                encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1),
2515            ],
2516        );
2517        check_encoding_write_support::<DoubleType>(
2518            WriterVersion::PARQUET_1_0,
2519            false,
2520            &[1.0, 2.0],
2521            None,
2522            &[Encoding::PLAIN],
2523            &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)],
2524        );
2525        check_encoding_write_support::<DoubleType>(
2526            WriterVersion::PARQUET_2_0,
2527            true,
2528            &[1.0, 2.0],
2529            Some(0),
2530            &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2531            &[
2532                encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2533                encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1),
2534            ],
2535        );
2536        check_encoding_write_support::<DoubleType>(
2537            WriterVersion::PARQUET_2_0,
2538            false,
2539            &[1.0, 2.0],
2540            None,
2541            &[Encoding::PLAIN],
2542            &[encoding_stats(PageType::DATA_PAGE_V2, Encoding::PLAIN, 1)],
2543        );
2544    }
2545
2546    #[test]
2547    fn test_column_writer_default_encoding_support_byte_array() {
2548        check_encoding_write_support::<ByteArrayType>(
2549            WriterVersion::PARQUET_1_0,
2550            true,
2551            &[ByteArray::from(vec![1u8])],
2552            Some(0),
2553            &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2554            &[
2555                encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2556                encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1),
2557            ],
2558        );
2559        check_encoding_write_support::<ByteArrayType>(
2560            WriterVersion::PARQUET_1_0,
2561            false,
2562            &[ByteArray::from(vec![1u8])],
2563            None,
2564            &[Encoding::PLAIN],
2565            &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)],
2566        );
2567        check_encoding_write_support::<ByteArrayType>(
2568            WriterVersion::PARQUET_2_0,
2569            true,
2570            &[ByteArray::from(vec![1u8])],
2571            Some(0),
2572            &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2573            &[
2574                encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2575                encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1),
2576            ],
2577        );
2578        check_encoding_write_support::<ByteArrayType>(
2579            WriterVersion::PARQUET_2_0,
2580            false,
2581            &[ByteArray::from(vec![1u8])],
2582            None,
2583            &[Encoding::DELTA_BYTE_ARRAY],
2584            &[encoding_stats(
2585                PageType::DATA_PAGE_V2,
2586                Encoding::DELTA_BYTE_ARRAY,
2587                1,
2588            )],
2589        );
2590    }
2591
2592    #[test]
2593    fn test_column_writer_default_encoding_support_fixed_len_byte_array() {
2594        for version in [WriterVersion::PARQUET_1_0, WriterVersion::PARQUET_2_0] {
2595            let default_props = WriterProperties::builder()
2596                .set_writer_version(version)
2597                .build();
2598            let meta = column_write_and_get_metadata::<FixedLenByteArrayType>(
2599                default_props,
2600                &[ByteArray::from(vec![1u8]).into()],
2601            );
2602            assert_eq!(meta.dictionary_page_offset(), Some(0));
2603        }
2604
2605        check_encoding_write_support::<FixedLenByteArrayType>(
2606            WriterVersion::PARQUET_1_0,
2607            true,
2608            &[ByteArray::from(vec![1u8]).into()],
2609            Some(0),
2610            &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2611            &[
2612                encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2613                encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1),
2614            ],
2615        );
2616        let column_props = WriterProperties::builder()
2617            .set_writer_version(WriterVersion::PARQUET_1_0)
2618            .set_dictionary_enabled(false)
2619            .set_column_dictionary_enabled(ColumnPath::from("col"), true)
2620            .build();
2621        let meta = column_write_and_get_metadata::<FixedLenByteArrayType>(
2622            column_props,
2623            &[ByteArray::from(vec![1u8]).into()],
2624        );
2625        assert_eq!(meta.dictionary_page_offset(), Some(0));
2626        check_encoding_write_support::<FixedLenByteArrayType>(
2627            WriterVersion::PARQUET_1_0,
2628            false,
2629            &[ByteArray::from(vec![1u8]).into()],
2630            None,
2631            &[Encoding::PLAIN],
2632            &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)],
2633        );
2634        check_encoding_write_support::<FixedLenByteArrayType>(
2635            WriterVersion::PARQUET_2_0,
2636            true,
2637            &[ByteArray::from(vec![1u8]).into()],
2638            Some(0),
2639            &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2640            &[
2641                encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2642                encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1),
2643            ],
2644        );
2645        check_encoding_write_support::<FixedLenByteArrayType>(
2646            WriterVersion::PARQUET_2_0,
2647            false,
2648            &[ByteArray::from(vec![1u8]).into()],
2649            None,
2650            &[Encoding::DELTA_BYTE_ARRAY],
2651            &[encoding_stats(
2652                PageType::DATA_PAGE_V2,
2653                Encoding::DELTA_BYTE_ARRAY,
2654                1,
2655            )],
2656        );
2657    }
2658
2659    #[test]
2660    fn test_column_writer_no_dictionary_page_for_empty_required() {
2661        let props = WriterProperties::builder()
2662            .set_writer_version(WriterVersion::PARQUET_1_0)
2663            .set_dictionary_enabled(true)
2664            .build();
2665        let meta = column_write_and_get_metadata_with_def_levels::<Int64Type>(props, &[], &[], 0);
2666        assert_eq!(meta.dictionary_page_offset(), None);
2667        assert_eq!(meta.data_page_offset(), 0);
2668        assert_eq!(meta.uncompressed_size(), 0);
2669        assert_eq!(meta.compressed_size(), 0);
2670        assert_eq!(meta.encodings().collect::<Vec<_>>(), &[]);
2671        assert_eq!(meta.page_encoding_stats().unwrap(), &[]);
2672    }
2673
2674    #[test]
2675    fn test_column_writer_no_dictionary_page_for_empty_optional() {
2676        let props = WriterProperties::builder()
2677            .set_writer_version(WriterVersion::PARQUET_1_0)
2678            .set_dictionary_enabled(true)
2679            .build();
2680        let meta = column_write_and_get_metadata_with_def_levels::<Int64Type>(props, &[], &[], 1);
2681        assert_eq!(meta.dictionary_page_offset(), None);
2682        assert_eq!(meta.data_page_offset(), 0);
2683        assert_eq!(meta.uncompressed_size(), 0);
2684        assert_eq!(meta.compressed_size(), 0);
2685        assert_eq!(meta.encodings().collect::<Vec<_>>(), &[]);
2686        assert_eq!(meta.page_encoding_stats().unwrap(), &[]);
2687    }
2688
2689    #[test]
2690    fn test_column_writer_encoding_includes_rle_for_all_null_data() {
2691        let props = WriterProperties::builder()
2692            .set_writer_version(WriterVersion::PARQUET_1_0)
2693            .set_dictionary_enabled(true)
2694            .build();
2695        let meta = column_write_and_get_metadata_with_def_levels::<Int64Type>(props, &[], &[0], 1);
2696        // this writes an empty dictionary page because we write the all-null data page with RLE_DICTIONARY encoding
2697        assert_eq!(meta.dictionary_page_offset(), Some(0));
2698        assert_eq!(meta.data_page_offset(), 0);
2699        assert_eq!(meta.uncompressed_size(), 7);
2700        assert_eq!(meta.compressed_size(), 7);
2701        assert_eq!(
2702            meta.encodings().collect::<Vec<_>>(),
2703            &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY]
2704        );
2705        assert_eq!(
2706            meta.page_encoding_stats().unwrap(),
2707            &[
2708                encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2709                encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1)
2710            ]
2711        );
2712    }
2713
2714    #[test]
2715    fn test_column_writer_encoding_includes_rle_for_mixed_null_data() {
2716        let props = WriterProperties::builder()
2717            .set_writer_version(WriterVersion::PARQUET_1_0)
2718            .set_dictionary_enabled(true)
2719            .build();
2720        let meta = column_write_and_get_metadata_with_def_levels::<Int64Type>(
2721            props,
2722            &[42_i64],
2723            &[1, 0],
2724            1,
2725        );
2726        assert_eq!(meta.dictionary_page_offset(), Some(0));
2727        assert_eq!(meta.data_page_offset(), 8);
2728        assert_eq!(meta.uncompressed_size(), 16);
2729        assert_eq!(meta.compressed_size(), 16);
2730        assert_eq!(
2731            meta.encodings().collect::<Vec<_>>(),
2732            &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY]
2733        );
2734        assert_eq!(
2735            meta.page_encoding_stats().unwrap(),
2736            &[
2737                encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2738                encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1)
2739            ]
2740        );
2741    }
2742
2743    #[test]
2744    fn test_column_writer_check_metadata() {
2745        let page_writer = get_test_page_writer();
2746        let props = Default::default();
2747        let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
2748        writer.write_batch(&[1, 2, 3, 4], None, None).unwrap();
2749
2750        let r = writer.close().unwrap();
2751        assert_eq!(r.bytes_written, 20);
2752        assert_eq!(r.rows_written, 4);
2753
2754        let metadata = r.metadata;
2755        assert_eq!(
2756            metadata.encodings().collect::<Vec<_>>(),
2757            vec![Encoding::PLAIN, Encoding::RLE_DICTIONARY]
2758        );
2759        assert_eq!(metadata.num_values(), 4);
2760        assert_eq!(metadata.compressed_size(), 20);
2761        assert_eq!(metadata.uncompressed_size(), 20);
2762        assert_eq!(metadata.data_page_offset(), 16);
2763        assert_eq!(metadata.dictionary_page_offset(), Some(0));
2764        if let Some(stats) = metadata.statistics() {
2765            assert_eq!(stats.null_count_opt(), Some(0));
2766            assert_eq!(stats.distinct_count_opt(), None);
2767            if let Statistics::Int32(stats) = stats {
2768                assert_eq!(stats.min_opt().unwrap(), &1);
2769                assert_eq!(stats.max_opt().unwrap(), &4);
2770            } else {
2771                panic!("expecting Statistics::Int32");
2772            }
2773        } else {
2774            panic!("metadata missing statistics");
2775        }
2776    }
2777
2778    #[test]
2779    fn test_column_writer_check_byte_array_min_max() {
2780        let page_writer = get_test_page_writer();
2781        let props = Default::default();
2782        let mut writer = get_test_decimals_column_writer::<ByteArrayType>(page_writer, 0, 0, props);
2783        writer
2784            .write_batch(
2785                &[
2786                    ByteArray::from(vec![
2787                        255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 179u8, 172u8, 19u8,
2788                        35u8, 231u8, 90u8, 0u8, 0u8,
2789                    ]),
2790                    ByteArray::from(vec![
2791                        255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 228u8, 62u8, 146u8,
2792                        152u8, 177u8, 56u8, 0u8, 0u8,
2793                    ]),
2794                    ByteArray::from(vec![
2795                        0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
2796                        0u8,
2797                    ]),
2798                    ByteArray::from(vec![
2799                        0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 41u8, 162u8, 36u8, 26u8, 246u8,
2800                        44u8, 0u8, 0u8,
2801                    ]),
2802                ],
2803                None,
2804                None,
2805            )
2806            .unwrap();
2807        let metadata = writer.close().unwrap().metadata;
2808        if let Some(stats) = metadata.statistics() {
2809            if let Statistics::ByteArray(stats) = stats {
2810                assert_eq!(
2811                    stats.min_opt().unwrap(),
2812                    &ByteArray::from(vec![
2813                        255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 179u8, 172u8, 19u8,
2814                        35u8, 231u8, 90u8, 0u8, 0u8,
2815                    ])
2816                );
2817                assert_eq!(
2818                    stats.max_opt().unwrap(),
2819                    &ByteArray::from(vec![
2820                        0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 41u8, 162u8, 36u8, 26u8, 246u8,
2821                        44u8, 0u8, 0u8,
2822                    ])
2823                );
2824            } else {
2825                panic!("expecting Statistics::ByteArray");
2826            }
2827        } else {
2828            panic!("metadata missing statistics");
2829        }
2830    }
2831
2832    #[test]
2833    fn test_column_writer_byte_array_min_max_unequal_lengths() {
2834        // Byte-array decimal min/max with values of different encoded lengths
2835        // https://github.com/apache/arrow-rs/issues/10860
2836        let page_writer = get_test_page_writer();
2837        let props = Default::default();
2838        let mut writer = get_test_decimals_column_writer::<ByteArrayType>(page_writer, 0, 0, props);
2839        writer
2840            .write_batch(
2841                &[
2842                    ByteArray::from(vec![0u8, 255u8]),      // 255
2843                    ByteArray::from(vec![0u8, 128u8, 0u8]), // 32768
2844                    ByteArray::from(vec![255u8, 127u8]),    // -129
2845                    ByteArray::from(vec![128u8]),           // -128
2846                ],
2847                None,
2848                None,
2849            )
2850            .unwrap();
2851        let metadata = writer.close().unwrap().metadata;
2852        let stats = metadata.statistics().expect("metadata missing statistics");
2853        let Statistics::ByteArray(stats) = stats else {
2854            panic!("expecting Statistics::ByteArray");
2855        };
2856        // -129
2857        assert_eq!(
2858            stats.min_opt().unwrap(),
2859            &ByteArray::from(vec![255u8, 127u8])
2860        );
2861        // 32768
2862        assert_eq!(
2863            stats.max_opt().unwrap(),
2864            &ByteArray::from(vec![0u8, 128u8, 0u8])
2865        );
2866    }
2867
2868    #[test]
2869    fn test_column_writer_uint32_converted_type_min_max() {
2870        let page_writer = get_test_page_writer();
2871        let props = Default::default();
2872        let mut writer = get_test_unsigned_int_given_as_converted_column_writer::<Int32Type>(
2873            page_writer,
2874            0,
2875            0,
2876            props,
2877        );
2878        writer.write_batch(&[0, 1, 2, 3, 4, 5], None, None).unwrap();
2879        let metadata = writer.close().unwrap().metadata;
2880        if let Some(stats) = metadata.statistics() {
2881            if let Statistics::Int32(stats) = stats {
2882                assert_eq!(stats.min_opt().unwrap(), &0,);
2883                assert_eq!(stats.max_opt().unwrap(), &5,);
2884            } else {
2885                panic!("expecting Statistics::Int32");
2886            }
2887        } else {
2888            panic!("metadata missing statistics");
2889        }
2890    }
2891
2892    #[test]
2893    fn test_column_writer_precalculated_statistics() {
2894        let page_writer = get_test_page_writer();
2895        let props = Arc::new(
2896            WriterProperties::builder()
2897                .set_statistics_enabled(EnabledStatistics::Chunk)
2898                .build(),
2899        );
2900        let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
2901        writer
2902            .write_batch_with_statistics(
2903                &[1, 2, 3, 4],
2904                None,
2905                None,
2906                Some(&-17),
2907                Some(&9000),
2908                Some(55),
2909            )
2910            .unwrap();
2911
2912        let r = writer.close().unwrap();
2913        assert_eq!(r.bytes_written, 20);
2914        assert_eq!(r.rows_written, 4);
2915
2916        let metadata = r.metadata;
2917        assert_eq!(
2918            metadata.encodings().collect::<Vec<_>>(),
2919            vec![Encoding::PLAIN, Encoding::RLE_DICTIONARY]
2920        );
2921        assert_eq!(metadata.num_values(), 4);
2922        assert_eq!(metadata.compressed_size(), 20);
2923        assert_eq!(metadata.uncompressed_size(), 20);
2924        assert_eq!(metadata.data_page_offset(), 16);
2925        assert_eq!(metadata.dictionary_page_offset(), Some(0));
2926        if let Some(stats) = metadata.statistics() {
2927            assert_eq!(stats.null_count_opt(), Some(0));
2928            assert_eq!(stats.distinct_count_opt().unwrap_or(0), 55);
2929            if let Statistics::Int32(stats) = stats {
2930                assert_eq!(stats.min_opt().unwrap(), &-17);
2931                assert_eq!(stats.max_opt().unwrap(), &9000);
2932            } else {
2933                panic!("expecting Statistics::Int32");
2934            }
2935        } else {
2936            panic!("metadata missing statistics");
2937        }
2938    }
2939
2940    #[test]
2941    fn test_mixed_precomputed_statistics() {
2942        let mut buf = Vec::with_capacity(100);
2943        let mut write = TrackedWrite::new(&mut buf);
2944        let page_writer = Box::new(SerializedPageWriter::new(&mut write));
2945        let props = Arc::new(
2946            WriterProperties::builder()
2947                .set_write_page_header_statistics(true)
2948                .set_data_page_row_count_limit(4)
2949                .build(),
2950        );
2951        let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
2952
2953        writer.write_batch(&[1, 2, 3, 4], None, None).unwrap();
2954        writer
2955            .write_batch_with_statistics(&[5, 6, 7], None, None, Some(&5), Some(&7), Some(3))
2956            .unwrap();
2957
2958        let r = writer.close().unwrap();
2959
2960        let stats = r.metadata.statistics().unwrap();
2961        assert_eq!(stats.min_bytes_opt().unwrap(), 1_i32.to_le_bytes());
2962        assert_eq!(stats.max_bytes_opt().unwrap(), 7_i32.to_le_bytes());
2963        assert_eq!(stats.null_count_opt(), Some(0));
2964        assert!(stats.distinct_count_opt().is_none());
2965
2966        drop(write);
2967
2968        let props = ReaderProperties::builder()
2969            .set_backward_compatible_lz4(false)
2970            .set_read_page_statistics(true)
2971            .build();
2972        let reader = SerializedPageReader::new_with_properties(
2973            Arc::new(Bytes::from(buf)),
2974            &r.metadata,
2975            r.rows_written as usize,
2976            None,
2977            Arc::new(props),
2978        )
2979        .unwrap();
2980
2981        let pages = reader.collect::<Result<Vec<_>>>().unwrap();
2982        assert_eq!(pages.len(), 3);
2983
2984        assert_eq!(pages[0].page_type(), PageType::DICTIONARY_PAGE);
2985        assert_eq!(pages[1].page_type(), PageType::DATA_PAGE);
2986        assert_eq!(pages[2].page_type(), PageType::DATA_PAGE);
2987        for (page, min, max) in [(&pages[1], 1_i32, 4_i32), (&pages[2], 5_i32, 7_i32)] {
2988            let stats = page.statistics().unwrap();
2989            assert_eq!(stats.min_bytes_opt().unwrap(), min.to_le_bytes());
2990            assert_eq!(stats.max_bytes_opt().unwrap(), max.to_le_bytes());
2991            assert_eq!(stats.null_count_opt(), Some(0));
2992            assert!(stats.distinct_count_opt().is_none());
2993        }
2994    }
2995
2996    #[test]
2997    fn test_disabled_statistics() {
2998        let mut buf = Vec::with_capacity(100);
2999        let mut write = TrackedWrite::new(&mut buf);
3000        let page_writer = Box::new(SerializedPageWriter::new(&mut write));
3001        let props = WriterProperties::builder()
3002            .set_statistics_enabled(EnabledStatistics::None)
3003            .set_writer_version(WriterVersion::PARQUET_2_0)
3004            .build();
3005        let props = Arc::new(props);
3006
3007        let mut writer = get_test_column_writer::<Int32Type>(page_writer, 1, 0, props);
3008        writer
3009            .write_batch(&[1, 2, 3, 4], Some(&[1, 0, 0, 1, 1, 1]), None)
3010            .unwrap();
3011
3012        let r = writer.close().unwrap();
3013        assert!(r.metadata.statistics().is_none());
3014
3015        drop(write);
3016
3017        let props = ReaderProperties::builder()
3018            .set_backward_compatible_lz4(false)
3019            .build();
3020        let reader = SerializedPageReader::new_with_properties(
3021            Arc::new(Bytes::from(buf)),
3022            &r.metadata,
3023            r.rows_written as usize,
3024            None,
3025            Arc::new(props),
3026        )
3027        .unwrap();
3028
3029        let pages = reader.collect::<Result<Vec<_>>>().unwrap();
3030        assert_eq!(pages.len(), 2);
3031
3032        assert_eq!(pages[0].page_type(), PageType::DICTIONARY_PAGE);
3033        assert_eq!(pages[1].page_type(), PageType::DATA_PAGE_V2);
3034
3035        match &pages[1] {
3036            Page::DataPageV2 {
3037                num_values,
3038                num_nulls,
3039                num_rows,
3040                statistics,
3041                ..
3042            } => {
3043                assert_eq!(*num_values, 6);
3044                assert_eq!(*num_nulls, 2);
3045                assert_eq!(*num_rows, 6);
3046                assert!(statistics.is_none());
3047            }
3048            _ => unreachable!(),
3049        }
3050    }
3051
3052    #[test]
3053    fn test_column_writer_empty_column_roundtrip() {
3054        let props = Default::default();
3055        column_roundtrip::<Int32Type>(props, &[], None, None);
3056    }
3057
3058    #[test]
3059    fn test_column_writer_non_nullable_values_roundtrip() {
3060        let props = Default::default();
3061        column_roundtrip_random::<Int32Type>(props, 1024, i32::MIN, i32::MAX, 0, 0);
3062    }
3063
3064    #[test]
3065    fn test_column_writer_nullable_non_repeated_values_roundtrip() {
3066        let props = Default::default();
3067        column_roundtrip_random::<Int32Type>(props, 1024, i32::MIN, i32::MAX, 10, 0);
3068    }
3069
3070    #[test]
3071    fn test_column_writer_nullable_repeated_values_roundtrip() {
3072        let props = Default::default();
3073        column_roundtrip_random::<Int32Type>(props, 1024, i32::MIN, i32::MAX, 10, 10);
3074    }
3075
3076    #[test]
3077    fn test_column_writer_dictionary_fallback_small_data_page() {
3078        let props = WriterProperties::builder()
3079            .set_dictionary_page_size_limit(32)
3080            .set_data_page_size_limit(32)
3081            .build();
3082        column_roundtrip_random::<Int32Type>(props, 1024, i32::MIN, i32::MAX, 10, 10);
3083    }
3084
3085    #[test]
3086    #[cfg_attr(miri, ignore)] // Takes too long
3087    fn test_column_writer_small_write_batch_size() {
3088        for i in &[1usize, 2, 5, 10, 11, 1023] {
3089            let props = WriterProperties::builder().set_write_batch_size(*i).build();
3090
3091            column_roundtrip_random::<Int32Type>(props, 1024, i32::MIN, i32::MAX, 10, 10);
3092        }
3093    }
3094
3095    #[test]
3096    fn test_column_writer_dictionary_disabled_v1() {
3097        let props = WriterProperties::builder()
3098            .set_writer_version(WriterVersion::PARQUET_1_0)
3099            .set_dictionary_enabled(false)
3100            .build();
3101        column_roundtrip_random::<Int32Type>(props, 1024, i32::MIN, i32::MAX, 10, 10);
3102    }
3103
3104    #[test]
3105    fn test_column_writer_dictionary_disabled_v2() {
3106        let props = WriterProperties::builder()
3107            .set_writer_version(WriterVersion::PARQUET_2_0)
3108            .set_dictionary_enabled(false)
3109            .build();
3110        column_roundtrip_random::<Int32Type>(props, 1024, i32::MIN, i32::MAX, 10, 10);
3111    }
3112
3113    #[test]
3114    fn test_column_writer_compression_v1() {
3115        let props = WriterProperties::builder()
3116            .set_writer_version(WriterVersion::PARQUET_1_0)
3117            .set_compression(Compression::SNAPPY)
3118            .build();
3119        column_roundtrip_random::<Int32Type>(props, 2048, i32::MIN, i32::MAX, 10, 10);
3120    }
3121
3122    #[test]
3123    fn test_column_writer_compression_v2() {
3124        let props = WriterProperties::builder()
3125            .set_writer_version(WriterVersion::PARQUET_2_0)
3126            .set_compression(Compression::SNAPPY)
3127            .build();
3128        column_roundtrip_random::<Int32Type>(props, 2048, i32::MIN, i32::MAX, 10, 10);
3129    }
3130
3131    #[test]
3132    fn test_column_writer_v2_compression_ratio_threshold() {
3133        fn write_v2_page(threshold: f64) -> bool {
3134            let mut buf = Vec::with_capacity(4096);
3135            let mut write = TrackedWrite::new(&mut buf);
3136            let page_writer = Box::new(SerializedPageWriter::new(&mut write));
3137            let props = Arc::new(
3138                WriterProperties::builder()
3139                    .set_writer_version(WriterVersion::PARQUET_2_0)
3140                    .set_compression(Compression::SNAPPY)
3141                    .set_dictionary_enabled(false)
3142                    .set_data_page_v2_compression_ratio_threshold(threshold)
3143                    .build(),
3144            );
3145
3146            let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
3147            let values: Vec<i32> = vec![42; 4096];
3148            writer.write_batch(&values, None, None).unwrap();
3149            let r = writer.close().unwrap();
3150            drop(write);
3151
3152            let reader_props = ReaderProperties::builder()
3153                .set_backward_compatible_lz4(false)
3154                .build();
3155            let reader = SerializedPageReader::new_with_properties(
3156                Arc::new(Bytes::from(buf)),
3157                &r.metadata,
3158                r.rows_written as usize,
3159                None,
3160                Arc::new(reader_props),
3161            )
3162            .unwrap();
3163            let pages = reader.collect::<Result<Vec<_>>>().unwrap();
3164            let data_page = pages
3165                .iter()
3166                .find(|p| p.page_type() == PageType::DATA_PAGE_V2)
3167                .expect("expected a v2 data page");
3168            match data_page {
3169                Page::DataPageV2 { is_compressed, .. } => *is_compressed,
3170                _ => unreachable!(),
3171            }
3172        }
3173
3174        // Default threshold keeps the compressed buffer for constant data.
3175        assert!(write_v2_page(1.0));
3176        // A strict threshold (require >1000x reduction) discards it.
3177        assert!(!write_v2_page(0.001));
3178    }
3179
3180    #[test]
3181    fn test_column_writer_add_data_pages_with_dict() {
3182        // ARROW-5129: Test verifies that we add data page in case of dictionary encoding
3183        // and no fallback occurred so far.
3184        let mut file = tempfile::tempfile().unwrap();
3185        let mut write = TrackedWrite::new(&mut file);
3186        let page_writer = Box::new(SerializedPageWriter::new(&mut write));
3187        let props = Arc::new(
3188            WriterProperties::builder()
3189                .set_data_page_size_limit(10)
3190                .set_write_batch_size(3) // write 3 values at a time
3191                .build(),
3192        );
3193        let data = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
3194        let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
3195        writer.write_batch(data, None, None).unwrap();
3196        let r = writer.close().unwrap();
3197
3198        drop(write);
3199
3200        // Read pages and check the sequence
3201        let props = ReaderProperties::builder()
3202            .set_backward_compatible_lz4(false)
3203            .build();
3204        let mut page_reader = Box::new(
3205            SerializedPageReader::new_with_properties(
3206                Arc::new(file),
3207                &r.metadata,
3208                r.rows_written as usize,
3209                None,
3210                Arc::new(props),
3211            )
3212            .unwrap(),
3213        );
3214        let mut res = Vec::new();
3215        while let Some(page) = page_reader.get_next_page().unwrap() {
3216            res.push((page.page_type(), page.num_values(), page.buffer().len()));
3217        }
3218        assert_eq!(
3219            res,
3220            vec![
3221                (PageType::DICTIONARY_PAGE, 10, 40),
3222                (PageType::DATA_PAGE, 9, 10),
3223                (PageType::DATA_PAGE, 1, 3),
3224            ]
3225        );
3226        assert_eq!(
3227            r.metadata.page_encoding_stats(),
3228            Some(&vec![
3229                PageEncodingStats {
3230                    page_type: PageType::DICTIONARY_PAGE,
3231                    encoding: Encoding::PLAIN,
3232                    count: 1
3233                },
3234                PageEncodingStats {
3235                    page_type: PageType::DATA_PAGE,
3236                    encoding: Encoding::RLE_DICTIONARY,
3237                    count: 2,
3238                }
3239            ])
3240        );
3241    }
3242
3243    #[test]
3244    fn test_column_writer_column_data_page_size_limit() {
3245        let props = Arc::new(
3246            WriterProperties::builder()
3247                .set_writer_version(WriterVersion::PARQUET_1_0)
3248                .set_dictionary_enabled(false)
3249                .set_data_page_size_limit(1000)
3250                .set_column_data_page_size_limit(ColumnPath::from("col"), 10)
3251                .set_write_batch_size(3)
3252                .build(),
3253        );
3254        let data = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
3255
3256        let col_values =
3257            write_and_collect_page_values(ColumnPath::from("col"), Arc::clone(&props), data);
3258        let other_values = write_and_collect_page_values(ColumnPath::from("other"), props, data);
3259
3260        assert_eq!(col_values, vec![3, 3, 3, 1]);
3261        assert_eq!(other_values, vec![10]);
3262    }
3263
3264    #[test]
3265    fn test_column_writer_column_data_page_row_count_limit() {
3266        let props = Arc::new(
3267            WriterProperties::builder()
3268                .set_writer_version(WriterVersion::PARQUET_1_0)
3269                .set_dictionary_enabled(false)
3270                .set_data_page_row_count_limit(100)
3271                .set_column_data_page_row_count_limit(ColumnPath::from("col"), 3)
3272                .set_write_batch_size(1)
3273                .build(),
3274        );
3275        let data = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
3276
3277        let col_values =
3278            write_and_collect_page_values(ColumnPath::from("col"), Arc::clone(&props), data);
3279        let other_values = write_and_collect_page_values(ColumnPath::from("other"), props, data);
3280
3281        assert_eq!(col_values, vec![3, 3, 3, 1]);
3282        assert_eq!(other_values, vec![10]);
3283    }
3284
3285    #[test]
3286    fn test_column_writer_caps_page_size_for_large_byte_array_values() {
3287        // Regression: the post-write data page byte limit check only fires
3288        // at mini-batch boundaries, so a 1024-row mini-batch of multi-MiB
3289        // BYTE_ARRAY values used to buffer multiple GiB into a single page
3290        // before the limit was even consulted. With the threshold-based
3291        // granular mode this batch should split into ~one page per value.
3292        let value_size = 64 * 1024; // 64 KiB per value
3293        let page_byte_limit = 16 * 1024; // 16 KiB page limit
3294        let num_rows = 64;
3295
3296        let props = WriterProperties::builder()
3297            .set_writer_version(WriterVersion::PARQUET_1_0)
3298            .set_dictionary_enabled(false)
3299            .set_encoding(Encoding::PLAIN)
3300            .set_data_page_size_limit(page_byte_limit)
3301            // Default write_batch_size (1024) — without the fix this
3302            // buffers the entire input into a single ~4 MiB page.
3303            .build();
3304
3305        let data: Vec<_> = (0..num_rows)
3306            .map(|i| ByteArray::from(vec![i as u8; value_size]))
3307            .collect();
3308        let pages = write_and_collect_pages::<ByteArrayType>(props, 0, 0, &data, None, None);
3309
3310        // Every value must end up somewhere.
3311        let total_values: u32 = pages.data_pages.iter().map(|(_, n)| n).sum();
3312        assert_eq!(total_values as usize, num_rows);
3313        // Without the fix this assertion fired with one ~4 MiB page; the
3314        // threshold splits the input so that no page holds more than a
3315        // single oversized value's worth of bytes.
3316        assert!(
3317            pages.data_pages.len() >= num_rows / 2,
3318            "expected pages to be cut close to one per value, got {:?}",
3319            pages.data_pages,
3320        );
3321        // Each page must be bounded by roughly one value's worth of bytes;
3322        // parquet allows a single oversized value to occupy a page by
3323        // itself but never lets us pile many of them together.
3324        for (size, _) in &pages.data_pages {
3325            assert!(
3326                *size <= value_size + 64,
3327                "page size {size} exceeds one-value bound ({}B) — pages {:?}",
3328                value_size + 64,
3329                pages.data_pages,
3330            );
3331        }
3332    }
3333
3334    #[test]
3335    fn test_column_writer_delta_byte_array_dedups_large_shared_prefix_values() {
3336        // Regression for https://github.com/apache/arrow-rs/issues/10489.
3337        // 16 identical 64 KiB values against a 16 KiB page limit: every value
3338        // is over the limit on its own, and `DELTA_BYTE_ARRAY` should still
3339        // dedup them down to about one value's worth of bytes in total.
3340        let value_size = 64 * 1024; // 64 KiB per value, > the page limit
3341        let page_byte_limit = 16 * 1024;
3342        let num_rows = 16;
3343
3344        let props = WriterProperties::builder()
3345            .set_writer_version(WriterVersion::PARQUET_1_0)
3346            .set_dictionary_enabled(false)
3347            .set_encoding(Encoding::DELTA_BYTE_ARRAY)
3348            .set_data_page_size_limit(page_byte_limit)
3349            .set_statistics_enabled(EnabledStatistics::None)
3350            .build();
3351
3352        // Identical values: one full value plus `num_rows - 1` zero-length
3353        // suffixes is all this column should cost.
3354        let data: Vec<_> = (0..num_rows)
3355            .map(|_| ByteArray::from(vec![b'a'; value_size]))
3356            .collect();
3357        let pages = write_and_collect_pages::<ByteArrayType>(props, 0, 0, &data, None, None);
3358
3359        // Every value must still end up somewhere.
3360        let total_values: u32 = pages.data_pages.iter().map(|(_, n)| n).sum();
3361        assert_eq!(total_values as usize, num_rows);
3362
3363        // Before the fix this was `num_rows * value_size` — byte for byte
3364        // what PLAIN produces, i.e. the encoding doing no work at all.
3365        let total_bytes: usize = pages.data_pages.iter().map(|(size, _)| size).sum();
3366        assert!(
3367            total_bytes < 2 * value_size,
3368            "expected under 2x a single value ({}B) for {num_rows} identical \
3369             values, got {total_bytes}B across pages {:?}",
3370            2 * value_size,
3371            pages.data_pages,
3372        );
3373    }
3374
3375    #[test]
3376    fn test_column_writer_delta_byte_array_bounds_pages_without_shared_prefix() {
3377        // Companion to the test above: same shape, but the values share no
3378        // prefix, so there is nothing to dedup and pages must stay bounded
3379        // by the value size. This is why the exemption covers one value
3380        // rather than dropping the byte budget altogether.
3381        let value_size = 64 * 1024;
3382        let page_byte_limit = 16 * 1024;
3383        let num_rows = 16;
3384
3385        let props = WriterProperties::builder()
3386            .set_writer_version(WriterVersion::PARQUET_1_0)
3387            .set_dictionary_enabled(false)
3388            .set_encoding(Encoding::DELTA_BYTE_ARRAY)
3389            .set_data_page_size_limit(page_byte_limit)
3390            .set_statistics_enabled(EnabledStatistics::None)
3391            .build();
3392
3393        // No two values share a prefix: they differ at the first byte.
3394        let data: Vec<_> = (0..num_rows)
3395            .map(|i| ByteArray::from(vec![i as u8; value_size]))
3396            .collect();
3397        let pages = write_and_collect_pages::<ByteArrayType>(props, 0, 0, &data, None, None);
3398
3399        let total_values: u32 = pages.data_pages.iter().map(|(_, n)| n).sum();
3400        assert_eq!(total_values as usize, num_rows);
3401
3402        // Expect at most two values per page: the exempted first value plus
3403        // one more that trips the budget.
3404        let upper_bound = 2 * value_size + 64;
3405        for (size, n_values) in &pages.data_pages {
3406            assert!(
3407                *size <= upper_bound,
3408                "page size {size} exceeds two-value bound ({upper_bound}B); pages {:?}",
3409                pages.data_pages,
3410            );
3411            assert!(
3412                *n_values <= 2,
3413                "page holds {n_values} values, expected at most 2; pages {:?}",
3414                pages.data_pages,
3415            );
3416        }
3417    }
3418
3419    #[test]
3420    fn test_column_writer_caps_page_size_with_sparse_nulls() {
3421        // `PLAIN` keeps ratio-scaled windows, so a sparsely-null chunk puts
3422        // *two* over-limit values on a page rather than one. That is the
3423        // deliberate choice: value-exact windows would cut this to one, but
3424        // `PLAIN` stores a value identically wherever it lands, so the output
3425        // is byte for byte the same either way while the page count doubles.
3426        //
3427        // What has to hold is that the bound stays a small constant and does
3428        // not scale with `write_batch_size` — the failure #9972 fixed, where
3429        // a page took a whole mini-batch. A window spans
3430        // `ceil(values * levels / values_in_chunk)` levels, which covers at
3431        // most two values however sparse the nulls are, so two is the whole
3432        // exposure. Assert it exactly: this test fails if the encoding gate
3433        // on value-exact windows is dropped (pages would hold one value) as
3434        // well as if the bound is lost (they would hold many).
3435        let value_size = 64 * 1024;
3436        let page_byte_limit = 16 * 1024;
3437        let num_values = 16;
3438
3439        let props = WriterProperties::builder()
3440            .set_dictionary_enabled(false)
3441            .set_encoding(Encoding::PLAIN)
3442            .set_data_page_size_limit(page_byte_limit)
3443            .set_statistics_enabled(EnabledStatistics::None)
3444            .build();
3445
3446        let data: Vec<_> = (0..num_values)
3447            .map(|_| ByteArray::from(vec![b'a'; value_size]))
3448            .collect();
3449        // 17 levels: a null at index 8, values everywhere else.
3450        let def_levels: Vec<i16> = (0..num_values as i16 + 1)
3451            .map(|i| i16::from(i != 8))
3452            .collect();
3453        let pages =
3454            write_and_collect_pages::<ByteArrayType>(props, 1, 0, &data, Some(&def_levels), None);
3455
3456        // At most two values' payload on any page, and never a whole
3457        // mini-batch's worth.
3458        let upper_bound = 2 * value_size + 64;
3459        for (size, n_levels) in &pages.data_pages {
3460            assert!(
3461                *size <= upper_bound,
3462                "page size {size} exceeds two-value bound ({upper_bound}B); pages {:?}",
3463                pages.data_pages,
3464            );
3465            assert!(
3466                *n_levels <= 3,
3467                "page holds {n_levels} levels, expected at most 3 (two values + a null); \
3468                 pages {:?}",
3469                pages.data_pages,
3470            );
3471        }
3472        // Two-level windows over 17 levels: eight pages carrying two levels
3473        // and a ninth holding the remainder.
3474        let num_levels: usize = num_values + 1;
3475        assert_eq!(pages.data_pages.len(), num_levels.div_ceil(2));
3476    }
3477
3478    #[test]
3479    fn test_column_writer_delta_byte_array_nullable_shared_prefix_dedup() {
3480        // A null in the chunk must not cost dedup. The first-value exemption
3481        // fires when a page's opening mini-batch holds exactly one value, and
3482        // `write_granular_chunk` cuts windows after an exact value count, so
3483        // the over-limit value that opens the page gets a single-value
3484        // mini-batch regardless of where nulls fall.
3485        //
3486        // This pinned `[2, 2, 2, 2, 9]` before #10538: the chunker scaled the
3487        // one-value budget by the chunk's 17:16 level:value ratio and rounded
3488        // up to two-level windows, so most pages opened with two values,
3489        // missed the exemption, and stored their first value in full — ~5
3490        // full values against ~1 here (`PLAIN` stores all 16).
3491        let value_size = 64 * 1024;
3492        let page_byte_limit = 16 * 1024;
3493        let num_values = 16;
3494
3495        let props = WriterProperties::builder()
3496            .set_writer_version(WriterVersion::PARQUET_1_0)
3497            .set_dictionary_enabled(false)
3498            .set_encoding(Encoding::DELTA_BYTE_ARRAY)
3499            .set_data_page_size_limit(page_byte_limit)
3500            .set_statistics_enabled(EnabledStatistics::None)
3501            .build();
3502
3503        let data: Vec<_> = (0..num_values)
3504            .map(|_| ByteArray::from(vec![b'a'; value_size]))
3505            .collect();
3506        // 17 levels: a null at index 8, values everywhere else.
3507        let def_levels: Vec<i16> = (0..num_values as i16 + 1)
3508            .map(|i| i16::from(i != 8))
3509            .collect();
3510        let pages =
3511            write_and_collect_pages::<ByteArrayType>(props, 1, 0, &data, Some(&def_levels), None);
3512
3513        // One page holding all 17 levels: the first value is exempt and every
3514        // later value dedups down to a length pair.
3515        let per_page_values: Vec<u32> = pages.data_pages.iter().map(|(_, n)| *n).collect();
3516        assert_eq!(per_page_values, vec![num_values as u32 + 1]);
3517
3518        let total_bytes: usize = pages.data_pages.iter().map(|(size, _)| size).sum();
3519        assert!(
3520            total_bytes < 2 * value_size,
3521            "expected ~one full value's worth of bytes (full dedup), \
3522             got {total_bytes}B across pages {:?}",
3523            pages.data_pages,
3524        );
3525    }
3526
3527    #[test]
3528    fn test_column_writer_caps_page_size_for_large_values_in_list() {
3529        // Coverage for the Materialized-rep branch of
3530        // `write_granular_chunk`. The flat-column regression test
3531        // exercises the per-level step; this exercises the
3532        // record-by-record step used when rep levels are present.
3533        //
3534        // Column is `list<required binary>` (max_def = 1, max_rep = 1)
3535        // with 3 records of 3 large blobs each. The page byte limit is
3536        // smaller than a single blob, so granular mode kicks in, and the
3537        // Materialized-rep arm of `write_granular_chunk` steps from one
3538        // `rep == 0` boundary to the next so a record never spans pages.
3539        let value_size = 32 * 1024;
3540        let page_byte_limit = 16 * 1024;
3541        let values_per_record = 3;
3542        let num_records = 3;
3543        let num_values = values_per_record * num_records;
3544
3545        // rep levels: 0, 1, 1, 0, 1, 1, 0, 1, 1
3546        let mut rep_levels = Vec::with_capacity(num_values);
3547        for _ in 0..num_records {
3548            rep_levels.push(0i16);
3549            rep_levels.extend(std::iter::repeat_n(1i16, values_per_record - 1));
3550        }
3551        let def_levels = vec![1i16; num_values];
3552
3553        let props = WriterProperties::builder()
3554            .set_writer_version(WriterVersion::PARQUET_1_0)
3555            .set_dictionary_enabled(false)
3556            .set_encoding(Encoding::PLAIN)
3557            .set_data_page_size_limit(page_byte_limit)
3558            .build();
3559
3560        let data: Vec<_> = (0..num_values)
3561            .map(|i| ByteArray::from(vec![i as u8; value_size]))
3562            .collect();
3563        let pages = write_and_collect_pages::<ByteArrayType>(
3564            props,
3565            1,
3566            1,
3567            &data,
3568            Some(&def_levels),
3569            Some(&rep_levels),
3570        );
3571        let data_pages = pages.data_pages;
3572
3573        // The Materialized-rep arm groups levels by record, and each
3574        // record's bytes blow the page byte limit on its own, so we get
3575        // exactly one page per record.
3576        assert_eq!(
3577            data_pages.len(),
3578            num_records,
3579            "expected one data page per record, got {data_pages:?}"
3580        );
3581        for (bytes, n_values) in &data_pages {
3582            assert_eq!(
3583                *n_values as usize, values_per_record,
3584                "each page must hold a whole record's leaves, got {data_pages:?}"
3585            );
3586            // Each page is one full record (its leaves cannot be split),
3587            // so allow up to `values_per_record` blobs of payload plus a
3588            // small fudge for level encoding overhead.
3589            let upper_bound = values_per_record * (value_size + 16);
3590            assert!(
3591                *bytes <= upper_bound,
3592                "page size {bytes} exceeds whole-record bound ({upper_bound}); pages {data_pages:?}"
3593            );
3594        }
3595    }
3596
3597    #[test]
3598    fn test_column_writer_caps_page_size_with_nullable_large_values() {
3599        // Coverage for `LevelDataRef::value_count` on Materialized def
3600        // levels: a nullable column with mixed nulls and large values.
3601        // `value_count` must return the actual non-null count so the
3602        // byte estimate reflects bytes that will actually be written,
3603        // not the level count.
3604        let value_size = 32 * 1024;
3605        let page_byte_limit = 16 * 1024;
3606        let num_levels = 32;
3607
3608        // Alternating null / non-null: 16 nulls and 16 values.
3609        let def_levels: Vec<i16> = (0..num_levels as i16).map(|i| i % 2).collect();
3610        let num_values = def_levels.iter().filter(|&&d| d == 1).count();
3611
3612        let props = WriterProperties::builder()
3613            .set_writer_version(WriterVersion::PARQUET_1_0)
3614            .set_dictionary_enabled(false)
3615            .set_encoding(Encoding::PLAIN)
3616            .set_data_page_size_limit(page_byte_limit)
3617            .build();
3618
3619        let data: Vec<_> = (0..num_values)
3620            .map(|i| ByteArray::from(vec![i as u8; value_size]))
3621            .collect();
3622        let pages =
3623            write_and_collect_pages::<ByteArrayType>(props, 1, 0, &data, Some(&def_levels), None);
3624        let data_pages: Vec<_> = pages.data_pages.iter().map(|(size, _)| *size).collect();
3625
3626        // With 16 actual values of 32 KiB each and a 16 KiB page limit,
3627        // every non-null value should get its own page (plus possibly
3628        // adjacent nulls). At minimum, the number of pages must be
3629        // roughly the value count, not 1 (which is what `main` produced).
3630        assert!(
3631            data_pages.len() >= num_values / 2,
3632            "expected at least {} pages for {num_values} large values, got {} pages: {data_pages:?}",
3633            num_values / 2,
3634            data_pages.len(),
3635        );
3636        // No page contains more than ~one value's worth of payload bytes.
3637        for size in &data_pages {
3638            assert!(
3639                *size <= value_size + 64,
3640                "page size {size} exceeds one-value bound; pages {data_pages:?}"
3641            );
3642        }
3643    }
3644
3645    #[test]
3646    fn test_column_writer_dict_enabled_large_values_post_spill() {
3647        // While dictionary encoding is active, `has_dictionary()` short-
3648        // circuits `estimated_value_bytes` — the byte estimate is plain-
3649        // encoded size but dict-encoded pages only store small RLE
3650        // indices, so we'd otherwise shrink pages spuriously. Once the
3651        // dictionary spills (each value is large + unique), plain
3652        // encoding takes over and the byte-budget sub-batch kicks in.
3653        //
3654        // This test makes sure the writer survives that transition and
3655        // produces bounded pages thereafter.
3656        let value_size = 64 * 1024;
3657        let page_byte_limit = 16 * 1024;
3658        let num_rows = 32;
3659
3660        let props = WriterProperties::builder()
3661            .set_writer_version(WriterVersion::PARQUET_1_0)
3662            .set_dictionary_enabled(true)
3663            // Force a small dict so it spills quickly even though
3664            // each value here is unique.
3665            .set_dictionary_page_size_limit(1024)
3666            .set_data_page_size_limit(page_byte_limit)
3667            // Small mini-batches so dict fallback happens part-way
3668            // through the input, leaving subsequent mini-batches to
3669            // exercise the post-spill plain-encoding path that the
3670            // page-size fix actually targets.
3671            .set_write_batch_size(4)
3672            .build();
3673
3674        let data: Vec<_> = (0..num_rows)
3675            .map(|i| ByteArray::from(vec![i as u8; value_size]))
3676            .collect();
3677        let pages = write_and_collect_pages::<ByteArrayType>(props, 0, 0, &data, None, None);
3678        let data_pages: Vec<_> = pages.data_pages.iter().map(|(size, _)| *size).collect();
3679
3680        // After spill, plain encoding writes one ~64 KiB value per page.
3681        // Without the fix, post-spill writes still buffered all 32
3682        // values into a single ~2 MiB page.
3683        assert!(
3684            data_pages.len() >= num_rows / 2,
3685            "expected >= {} data pages after dict spill, got {} ({data_pages:?})",
3686            num_rows / 2,
3687            data_pages.len(),
3688        );
3689        for size in &data_pages {
3690            assert!(
3691                *size <= value_size + 64,
3692                "page size {size} exceeds one-value bound; pages {data_pages:?}"
3693            );
3694        }
3695    }
3696
3697    #[test]
3698    fn test_column_writer_caps_dictionary_page_size() {
3699        // A column of large *distinct* values with dictionary encoding on:
3700        // the dictionary page accumulates the values themselves, and its
3701        // spill check runs only once per mini-batch. Without bounding the
3702        // dictionary-encoding mini-batch, one `write_batch_size` mini-batch
3703        // would intern `write_batch_size * value_size` bytes into the
3704        // dictionary page before the check fires (~16 MiB here). The chunker
3705        // must sub-batch the dictionary-encoding phase too.
3706        let value_size = 8 * 1024;
3707        let dict_page_limit = 64 * 1024;
3708        let num_rows = 2048;
3709
3710        let props = WriterProperties::builder()
3711            .set_writer_version(WriterVersion::PARQUET_1_0)
3712            .set_dictionary_enabled(true)
3713            .set_dictionary_page_size_limit(dict_page_limit)
3714            .build();
3715
3716        let data: Vec<_> = (0..num_rows)
3717            .map(|i| {
3718                // each value distinct, so the dictionary cannot dedup them
3719                let mut v = vec![0u8; value_size];
3720                v[..8].copy_from_slice(&(i as u64).to_le_bytes());
3721                ByteArray::from(v)
3722            })
3723            .collect();
3724        let pages = write_and_collect_pages::<ByteArrayType>(props, 0, 0, &data, None, None);
3725        let dict_page_size = pages.dict_page_size;
3726
3727        assert!(
3728            dict_page_size > 0,
3729            "expected the column to dictionary-encode"
3730        );
3731        // Bounded near the limit (~2x from the post-mini-batch check). Before
3732        // the fix the dictionary page reached num_rows * value_size (~16 MiB,
3733        // 256x the limit).
3734        assert!(
3735            dict_page_size <= 3 * dict_page_limit,
3736            "dictionary page {dict_page_size} exceeds 3x the {dict_page_limit} limit",
3737        );
3738    }
3739
3740    #[test]
3741    fn test_column_writer_caps_dictionary_page_size_with_clustered_nulls() {
3742        // Regression test for #11994. Large distinct values followed by a run
3743        // of nulls: the chunk's average level:value ratio is ~36 levels per
3744        // value, so a ratio-sized window over the leading values covers all
3745        // of them. Each window must also be capped by value count, or every
3746        // value lands in the dictionary page before its spill check runs.
3747        let value_size = 64 * 1024;
3748        let num_values = 28;
3749        let num_nulls = 996;
3750
3751        let props = WriterProperties::builder()
3752            .set_writer_version(WriterVersion::PARQUET_1_0)
3753            .set_dictionary_enabled(true)
3754            .set_dictionary_page_size_limit(value_size)
3755            .build();
3756
3757        let data: Vec<_> = (0..num_values)
3758            .map(|i| {
3759                let mut v = vec![0u8; value_size];
3760                v[..8].copy_from_slice(&(i as u64).to_le_bytes());
3761                ByteArray::from(v)
3762            })
3763            .collect();
3764        let mut def_levels = vec![1i16; num_values];
3765        def_levels.extend(std::iter::repeat_n(0i16, num_nulls));
3766
3767        let pages =
3768            write_and_collect_pages::<ByteArrayType>(props, 1, 0, &data, Some(&def_levels), None);
3769        assert!(
3770            pages.dict_page_size > 0,
3771            "expected the column to dictionary-encode"
3772        );
3773        // At most two values per window. Before the fix the dictionary page
3774        // held all 28 values (~1.8 MiB).
3775        assert!(
3776            pages.dict_page_size <= 3 * value_size,
3777            "dictionary page {} exceeds three values ({}B)",
3778            pages.dict_page_size,
3779            3 * value_size,
3780        );
3781    }
3782
3783    #[test]
3784    fn test_column_writer_caps_page_size_for_fixed_len_byte_array() {
3785        // Coverage for `ParquetValueType::byte_size` override on
3786        // `FixedLenByteArray`. With `type_length = 1`, each plain-encoded
3787        // value is one byte, so a 4-byte page byte limit forces the
3788        // sub-batch sizer to write ~4 values per page rather than one
3789        // page for the whole batch.
3790        let page_byte_limit = 4;
3791        let num_values = 128;
3792
3793        let props = WriterProperties::builder()
3794            .set_writer_version(WriterVersion::PARQUET_1_0)
3795            .set_dictionary_enabled(false)
3796            .set_encoding(Encoding::PLAIN)
3797            .set_data_page_size_limit(page_byte_limit)
3798            .build();
3799
3800        let data: Vec<_> = (0..num_values)
3801            .map(|i| {
3802                let mut fla = FixedLenByteArray::default();
3803                fla.set_data(Bytes::from(vec![i as u8]));
3804                fla
3805            })
3806            .collect();
3807        let pages =
3808            write_and_collect_pages::<FixedLenByteArrayType>(props, 0, 0, &data, None, None);
3809        let data_pages: Vec<_> = pages.data_pages.iter().map(|(size, _)| *size).collect();
3810
3811        // Without the fix this is a single 128-byte page; with the fix
3812        // the byte budget caps each page at ~`page_byte_limit` bytes.
3813        assert!(
3814            data_pages.len() >= num_values / 8,
3815            "expected pages capped by byte budget, got {data_pages:?}"
3816        );
3817        for size in &data_pages {
3818            assert!(
3819                *size <= page_byte_limit * 4,
3820                "page size {size} larger than expected; pages {data_pages:?}"
3821            );
3822        }
3823    }
3824
3825    #[test]
3826    fn test_bool_statistics() {
3827        let stats = statistics_roundtrip::<BoolType>(&[true, false, false, true]);
3828        // Booleans have an unsigned sort order and so are not compatible
3829        // with the deprecated `min` and `max` statistics
3830        assert!(!stats.is_min_max_backwards_compatible());
3831        if let Statistics::Boolean(stats) = stats {
3832            assert_eq!(stats.min_opt().unwrap(), &false);
3833            assert_eq!(stats.max_opt().unwrap(), &true);
3834        } else {
3835            panic!("expecting Statistics::Boolean, got {stats:?}");
3836        }
3837    }
3838
3839    #[test]
3840    fn test_int32_statistics() {
3841        let stats = statistics_roundtrip::<Int32Type>(&[-1, 3, -2, 2]);
3842        assert!(stats.is_min_max_backwards_compatible());
3843        if let Statistics::Int32(stats) = stats {
3844            assert_eq!(stats.min_opt().unwrap(), &-2);
3845            assert_eq!(stats.max_opt().unwrap(), &3);
3846        } else {
3847            panic!("expecting Statistics::Int32, got {stats:?}");
3848        }
3849    }
3850
3851    #[test]
3852    fn test_int64_statistics() {
3853        let stats = statistics_roundtrip::<Int64Type>(&[-1, 3, -2, 2]);
3854        assert!(stats.is_min_max_backwards_compatible());
3855        if let Statistics::Int64(stats) = stats {
3856            assert_eq!(stats.min_opt().unwrap(), &-2);
3857            assert_eq!(stats.max_opt().unwrap(), &3);
3858        } else {
3859            panic!("expecting Statistics::Int64, got {stats:?}");
3860        }
3861    }
3862
3863    #[test]
3864    fn test_int96_statistics() {
3865        let input = vec![
3866            Int96::from(vec![1, 20, 30]),
3867            Int96::from(vec![3, 20, 10]),
3868            Int96::from(vec![0, 20, 30]),
3869            Int96::from(vec![2, 20, 30]),
3870        ]
3871        .into_iter()
3872        .collect::<Vec<Int96>>();
3873
3874        let stats = statistics_roundtrip::<Int96Type>(&input);
3875        assert!(!stats.is_min_max_backwards_compatible());
3876        if let Statistics::Int96(stats) = stats {
3877            assert_eq!(stats.min_opt().unwrap(), &Int96::from(vec![3, 20, 10]));
3878            assert_eq!(stats.max_opt().unwrap(), &Int96::from(vec![2, 20, 30]));
3879        } else {
3880            panic!("expecting Statistics::Int96, got {stats:?}");
3881        }
3882    }
3883
3884    #[test]
3885    fn test_float_statistics() {
3886        let stats = statistics_roundtrip::<FloatType>(&[-1.0, 3.0, -2.0, 2.0]);
3887        assert!(!stats.is_min_max_backwards_compatible());
3888        if let Statistics::Float(stats) = stats {
3889            assert_eq!(stats.min_opt().unwrap(), &-2.0);
3890            assert_eq!(stats.max_opt().unwrap(), &3.0);
3891        } else {
3892            panic!("expecting Statistics::Float, got {stats:?}");
3893        }
3894    }
3895
3896    #[test]
3897    fn test_double_statistics() {
3898        let stats = statistics_roundtrip::<DoubleType>(&[-1.0, 3.0, -2.0, 2.0]);
3899        assert!(!stats.is_min_max_backwards_compatible());
3900        if let Statistics::Double(stats) = stats {
3901            assert_eq!(stats.min_opt().unwrap(), &-2.0);
3902            assert_eq!(stats.max_opt().unwrap(), &3.0);
3903        } else {
3904            panic!("expecting Statistics::Double, got {stats:?}");
3905        }
3906    }
3907
3908    #[test]
3909    fn test_byte_array_statistics() {
3910        let input = ["aawaa", "zz", "aaw", "m", "qrs"]
3911            .iter()
3912            .map(|&s| s.into())
3913            .collect::<Vec<_>>();
3914
3915        let stats = statistics_roundtrip::<ByteArrayType>(&input);
3916        assert!(!stats.is_min_max_backwards_compatible());
3917        if let Statistics::ByteArray(stats) = stats {
3918            assert_eq!(stats.min_opt().unwrap(), &ByteArray::from("aaw"));
3919            assert_eq!(stats.max_opt().unwrap(), &ByteArray::from("zz"));
3920        } else {
3921            panic!("expecting Statistics::ByteArray, got {stats:?}");
3922        }
3923    }
3924
3925    #[test]
3926    fn test_fixed_len_byte_array_statistics() {
3927        let input = ["aawaa", "zz   ", "aaw  ", "m    ", "qrs  "]
3928            .iter()
3929            .map(|&s| ByteArray::from(s).into())
3930            .collect::<Vec<_>>();
3931
3932        let stats = statistics_roundtrip::<FixedLenByteArrayType>(&input);
3933        assert!(!stats.is_min_max_backwards_compatible());
3934        if let Statistics::FixedLenByteArray(stats) = stats {
3935            let expected_min: FixedLenByteArray = ByteArray::from("aaw  ").into();
3936            assert_eq!(stats.min_opt().unwrap(), &expected_min);
3937            let expected_max: FixedLenByteArray = ByteArray::from("zz   ").into();
3938            assert_eq!(stats.max_opt().unwrap(), &expected_max);
3939        } else {
3940            panic!("expecting Statistics::FixedLenByteArray, got {stats:?}");
3941        }
3942    }
3943
3944    #[test]
3945    fn test_ieee754_total_order_float() {
3946        // Test IEEE 754 total order for f32
3947        // Order should be: -NaN < -Inf < -1.0 < -0.0 < +0.0 < 1.0 < +Inf < +NaN
3948        let neg_nan = f32::from_bits(0xffc00000); // a NaN with the sign bit set
3949        let neg_inf = f32::NEG_INFINITY;
3950        let neg_one = -1.0_f32;
3951        let neg_zero = -0.0_f32;
3952        let pos_zero = 0.0_f32;
3953        let pos_one = 1.0_f32;
3954        let pos_inf = f32::INFINITY;
3955        let pos_nan = f32::from_bits(0x7fc00000); // a NaN with the sign bit unset
3956
3957        let values = vec![
3958            pos_nan, neg_zero, pos_inf, neg_one, neg_nan, pos_one, neg_inf, pos_zero,
3959        ];
3960
3961        let stats = statistics_roundtrip::<FloatType>(&values);
3962        if let Statistics::Float(stats) = stats {
3963            // With IEEE 754 total order, min should be -NaN, max should be +NaN
3964            // But since we filter out NaN values, min should be -Inf, max should be +Inf
3965            assert_eq!(stats.min_opt().unwrap(), &neg_inf);
3966            assert_eq!(stats.max_opt().unwrap(), &pos_inf);
3967            assert_eq!(stats.nan_count_opt(), Some(2)); // neg_nan and pos_nan
3968        } else {
3969            panic!("Expected float statistics");
3970        }
3971    }
3972
3973    #[test]
3974    fn test_ieee754_total_order_float_only_nan() {
3975        // Test IEEE 754 total order for various NaN representations
3976        // They should be ordered by the significand
3977        let neg_nan1 = f32::from_bits(0xffc00000); // sign bit set, significand x400000
3978        let neg_nan2 = f32::from_bits(0xffc00001); // sign bit set, significand x400001
3979        let neg_nan3 = f32::from_bits(0xffc00002); // sign bit set, significand x400002
3980        let pos_nan1 = f32::from_bits(0x7fc00000); // sign bit unset, significand x400000
3981        let pos_nan2 = f32::from_bits(0x7fc00001); // sign bit unset, significand x400001
3982        let pos_nan3 = f32::from_bits(0x7fc00002); // sign bit unset, significand x400002
3983
3984        let values = vec![neg_nan1, neg_nan2, neg_nan3, pos_nan1, pos_nan2, pos_nan3];
3985
3986        let stats = statistics_roundtrip::<FloatType>(&values);
3987        if let Statistics::Float(stats) = stats {
3988            // With IEEE 754 total order, min should be `neg_nan3`, max `pos_nan3`
3989            assert_eq!(
3990                stats.min_opt().unwrap().total_cmp(&neg_nan3),
3991                Ordering::Equal
3992            );
3993            assert_eq!(
3994                stats.max_opt().unwrap().total_cmp(&pos_nan3),
3995                Ordering::Equal
3996            );
3997            assert_eq!(stats.nan_count_opt(), Some(6));
3998        } else {
3999            panic!("Expected float statistics");
4000        }
4001    }
4002
4003    #[test]
4004    fn test_ieee754_total_order_double() {
4005        // Test IEEE 754 total order for f64
4006        let neg_nan = f64::from_bits(0xfff8000000000000);
4007        let neg_inf = f64::NEG_INFINITY;
4008        let neg_one = -1.0_f64;
4009        let neg_zero = -0.0_f64;
4010        let pos_zero = 0.0_f64;
4011        let pos_one = 1.0_f64;
4012        let pos_inf = f64::INFINITY;
4013        let pos_nan = f64::from_bits(0x7ff8000000000000);
4014
4015        let values = vec![
4016            pos_nan, neg_zero, pos_inf, neg_one, neg_nan, pos_one, neg_inf, pos_zero,
4017        ];
4018
4019        let stats = statistics_roundtrip::<DoubleType>(&values);
4020        if let Statistics::Double(stats) = stats {
4021            // With IEEE 754 total order, and NaN filtering
4022            assert_eq!(stats.min_opt().unwrap(), &neg_inf);
4023            assert_eq!(stats.max_opt().unwrap(), &pos_inf);
4024            assert_eq!(stats.nan_count_opt(), Some(2));
4025        } else {
4026            panic!("Expected double statistics");
4027        }
4028    }
4029
4030    #[test]
4031    fn test_ieee754_total_order_double_only_nan() {
4032        // Test IEEE 754 total order for various NaN representations
4033        // They should be ordered by the significand
4034        let neg_nan1 = f64::from_bits(0xfff8000000000000);
4035        let neg_nan2 = f64::from_bits(0xfff8000000000001);
4036        let neg_nan3 = f64::from_bits(0xfff8000000000002);
4037        let pos_nan1 = f64::from_bits(0x7ff8000000000000);
4038        let pos_nan2 = f64::from_bits(0x7ff8000000000001);
4039        let pos_nan3 = f64::from_bits(0x7ff8000000000002);
4040
4041        let values = vec![neg_nan1, neg_nan2, neg_nan3, pos_nan1, pos_nan2, pos_nan3];
4042
4043        let stats = statistics_roundtrip::<DoubleType>(&values);
4044        if let Statistics::Double(stats) = stats {
4045            // With IEEE 754 total order, min should be `neg_nan3`, max `pos_nan3`
4046            assert_eq!(
4047                stats.min_opt().unwrap().total_cmp(&neg_nan3),
4048                Ordering::Equal
4049            );
4050            assert_eq!(
4051                stats.max_opt().unwrap().total_cmp(&pos_nan3),
4052                Ordering::Equal
4053            );
4054            assert_eq!(stats.nan_count_opt(), Some(6));
4055        } else {
4056            panic!("Expected float statistics");
4057        }
4058    }
4059
4060    #[test]
4061    fn test_ieee754_total_order_zeros() {
4062        // Test that -0.0 and +0.0 are handled correctly
4063        let values = vec![-0.0_f32, 0.0_f32, -0.0_f32, 0.0_f32];
4064
4065        let stats = statistics_roundtrip::<FloatType>(&values);
4066        if let Statistics::Float(stats) = stats {
4067            // With IEEE 754 total order, -0.0 < +0.0
4068            assert_eq!(stats.min_opt().unwrap().to_bits(), (-0.0_f32).to_bits());
4069            assert_eq!(stats.max_opt().unwrap().to_bits(), 0.0_f32.to_bits());
4070        } else {
4071            panic!("Expected float statistics");
4072        }
4073    }
4074
4075    #[test]
4076    #[cfg_attr(miri, ignore)] // inline assembly is not supported
4077    fn test_column_writer_check_float16_min_max() {
4078        let input = [
4079            -f16::ONE,
4080            f16::from_f32(3.0),
4081            -f16::from_f32(2.0),
4082            f16::from_f32(2.0),
4083        ]
4084        .into_iter()
4085        .map(|s| ByteArray::from(s).into())
4086        .collect::<Vec<_>>();
4087
4088        let stats = float16_statistics_roundtrip(&input);
4089        assert!(!stats.is_min_max_backwards_compatible());
4090        assert_eq!(
4091            stats.min_opt().unwrap(),
4092            &ByteArray::from(-f16::from_f32(2.0))
4093        );
4094        assert_eq!(
4095            stats.max_opt().unwrap(),
4096            &ByteArray::from(f16::from_f32(3.0))
4097        );
4098    }
4099
4100    #[test]
4101    #[cfg_attr(miri, ignore)] // inline assembly is not supported
4102    fn test_column_writer_check_float16_nan_middle() {
4103        let input = [f16::ONE, f16::NAN, f16::ONE + f16::ONE]
4104            .into_iter()
4105            .map(|s| ByteArray::from(s).into())
4106            .collect::<Vec<_>>();
4107
4108        let stats = float16_statistics_roundtrip(&input);
4109        assert!(!stats.is_min_max_backwards_compatible());
4110        assert_eq!(stats.min_opt().unwrap(), &ByteArray::from(f16::ONE));
4111        assert_eq!(
4112            stats.max_opt().unwrap(),
4113            &ByteArray::from(f16::ONE + f16::ONE)
4114        );
4115        assert_eq!(stats.nan_count_opt(), Some(1));
4116    }
4117
4118    #[test]
4119    #[cfg_attr(miri, ignore)] // inline assembly is not supported
4120    fn test_float16_statistics_nan_middle() {
4121        let input = [f16::ONE, f16::NAN, f16::ONE + f16::ONE]
4122            .into_iter()
4123            .map(|s| ByteArray::from(s).into())
4124            .collect::<Vec<_>>();
4125
4126        let stats = float16_statistics_roundtrip(&input);
4127        assert!(!stats.is_min_max_backwards_compatible());
4128        assert_eq!(stats.min_opt().unwrap(), &ByteArray::from(f16::ONE));
4129        assert_eq!(
4130            stats.max_opt().unwrap(),
4131            &ByteArray::from(f16::ONE + f16::ONE)
4132        );
4133        assert_eq!(stats.nan_count_opt(), Some(1));
4134    }
4135
4136    #[test]
4137    #[cfg_attr(miri, ignore)] // inline assembly is not supported
4138    fn test_float16_statistics_nan_start() {
4139        let input = [f16::NAN, f16::ONE, f16::ONE + f16::ONE]
4140            .into_iter()
4141            .map(|s| ByteArray::from(s).into())
4142            .collect::<Vec<_>>();
4143
4144        let stats = float16_statistics_roundtrip(&input);
4145        assert!(!stats.is_min_max_backwards_compatible());
4146        assert_eq!(stats.min_opt().unwrap(), &ByteArray::from(f16::ONE));
4147        assert_eq!(
4148            stats.max_opt().unwrap(),
4149            &ByteArray::from(f16::ONE + f16::ONE)
4150        );
4151        assert_eq!(stats.nan_count_opt(), Some(1));
4152    }
4153
4154    #[test]
4155    fn test_float16_statistics_nan_only() {
4156        let input = [f16::NAN, f16::NAN]
4157            .into_iter()
4158            .map(|s| ByteArray::from(s).into())
4159            .collect::<Vec<_>>();
4160
4161        let stats = float16_statistics_roundtrip(&input);
4162        assert_eq!(
4163            stats.min_bytes_opt(),
4164            Some(ByteArray::from(f16::NAN).as_bytes())
4165        );
4166        assert_eq!(
4167            stats.max_bytes_opt(),
4168            Some(ByteArray::from(f16::NAN).as_bytes())
4169        );
4170        assert!(!stats.is_min_max_backwards_compatible());
4171        assert_eq!(stats.nan_count_opt(), Some(2));
4172    }
4173
4174    #[test]
4175    fn test_float16_statistics_zero_only() {
4176        let input = std::iter::once(f16::ZERO)
4177            .map(|s| ByteArray::from(s).into())
4178            .collect::<Vec<_>>();
4179
4180        let stats = float16_statistics_roundtrip(&input);
4181        assert!(!stats.is_min_max_backwards_compatible());
4182        assert_eq!(stats.min_opt().unwrap(), &ByteArray::from(f16::ZERO));
4183        assert_eq!(stats.max_opt().unwrap(), &ByteArray::from(f16::ZERO));
4184    }
4185
4186    #[test]
4187    fn test_float16_statistics_neg_zero_only() {
4188        let input = std::iter::once(f16::NEG_ZERO)
4189            .map(|s| ByteArray::from(s).into())
4190            .collect::<Vec<_>>();
4191
4192        let stats = float16_statistics_roundtrip(&input);
4193        assert!(!stats.is_min_max_backwards_compatible());
4194        assert_eq!(stats.min_opt().unwrap(), &ByteArray::from(f16::NEG_ZERO));
4195        assert_eq!(stats.max_opt().unwrap(), &ByteArray::from(f16::NEG_ZERO));
4196    }
4197
4198    #[test]
4199    fn test_float16_statistics_zero_min() {
4200        let input = [f16::ZERO, f16::ONE, f16::NAN, f16::PI]
4201            .into_iter()
4202            .map(|s| ByteArray::from(s).into())
4203            .collect::<Vec<_>>();
4204
4205        let stats = float16_statistics_roundtrip(&input);
4206        assert!(!stats.is_min_max_backwards_compatible());
4207        assert_eq!(stats.min_opt().unwrap(), &ByteArray::from(f16::ZERO));
4208        assert_eq!(stats.max_opt().unwrap(), &ByteArray::from(f16::PI));
4209    }
4210
4211    #[test]
4212    fn test_float16_statistics_neg_zero_max() {
4213        let input = [f16::NEG_ZERO, f16::NEG_ONE, f16::NAN, -f16::PI]
4214            .into_iter()
4215            .map(|s| ByteArray::from(s).into())
4216            .collect::<Vec<_>>();
4217
4218        let stats = float16_statistics_roundtrip(&input);
4219        assert!(!stats.is_min_max_backwards_compatible());
4220        assert_eq!(stats.min_opt().unwrap(), &ByteArray::from(-f16::PI));
4221        assert_eq!(stats.max_opt().unwrap(), &ByteArray::from(f16::NEG_ZERO));
4222    }
4223
4224    #[test]
4225    fn test_float_statistics_nan_middle() {
4226        let stats = statistics_roundtrip::<FloatType>(&[1.0, f32::NAN, 2.0]);
4227        assert!(!stats.is_min_max_backwards_compatible());
4228        if let Statistics::Float(stats) = stats {
4229            assert_eq!(stats.min_opt().unwrap(), &1.0);
4230            assert_eq!(stats.max_opt().unwrap(), &2.0);
4231            assert_eq!(stats.nan_count_opt(), Some(1))
4232        } else {
4233            panic!("expecting Statistics::Float");
4234        }
4235    }
4236
4237    #[test]
4238    fn test_float_statistics_nan_start() {
4239        let stats = statistics_roundtrip::<FloatType>(&[f32::NAN, 1.0, 2.0]);
4240        assert!(!stats.is_min_max_backwards_compatible());
4241        if let Statistics::Float(stats) = stats {
4242            assert_eq!(stats.min_opt().unwrap(), &1.0);
4243            assert_eq!(stats.max_opt().unwrap(), &2.0);
4244            assert_eq!(stats.nan_count_opt(), Some(1))
4245        } else {
4246            panic!("expecting Statistics::Float");
4247        }
4248    }
4249
4250    #[test]
4251    fn test_float_statistics_nan_only() {
4252        let stats = statistics_roundtrip::<FloatType>(&[f32::NAN, f32::NAN]);
4253        assert_eq!(stats.min_bytes_opt(), Some(f32::NAN.as_bytes()));
4254        assert_eq!(stats.max_bytes_opt(), Some(f32::NAN.as_bytes()));
4255        assert_eq!(stats.nan_count_opt(), Some(2));
4256        assert!(!stats.is_min_max_backwards_compatible());
4257        assert!(matches!(stats, Statistics::Float(_)));
4258    }
4259
4260    #[test]
4261    fn test_float_statistics_zero_only() {
4262        let stats = statistics_roundtrip::<FloatType>(&[0.0]);
4263        assert!(!stats.is_min_max_backwards_compatible());
4264        if let Statistics::Float(stats) = stats {
4265            assert_eq!(stats.min_opt().unwrap(), &0.0);
4266            assert!(stats.min_opt().unwrap().is_sign_positive());
4267            assert_eq!(stats.max_opt().unwrap(), &0.0);
4268            assert!(stats.max_opt().unwrap().is_sign_positive());
4269        } else {
4270            panic!("expecting Statistics::Float");
4271        }
4272    }
4273
4274    #[test]
4275    fn test_float_statistics_neg_zero_only() {
4276        let stats = statistics_roundtrip::<FloatType>(&[-0.0]);
4277        assert!(!stats.is_min_max_backwards_compatible());
4278        if let Statistics::Float(stats) = stats {
4279            assert_eq!(stats.min_opt().unwrap(), &-0.0);
4280            assert!(stats.min_opt().unwrap().is_sign_negative());
4281            assert_eq!(stats.max_opt().unwrap(), &-0.0);
4282            assert!(stats.max_opt().unwrap().is_sign_negative());
4283        } else {
4284            panic!("expecting Statistics::Float");
4285        }
4286    }
4287
4288    #[test]
4289    fn test_float_statistics_zero_min() {
4290        let stats = statistics_roundtrip::<FloatType>(&[0.0, 1.0, f32::NAN, 2.0]);
4291        assert!(!stats.is_min_max_backwards_compatible());
4292        if let Statistics::Float(stats) = stats {
4293            assert_eq!(stats.min_opt().unwrap(), &0.0);
4294            assert!(stats.min_opt().unwrap().is_sign_positive());
4295            assert_eq!(stats.max_opt().unwrap(), &2.0);
4296        } else {
4297            panic!("expecting Statistics::Float");
4298        }
4299    }
4300
4301    #[test]
4302    fn test_float_statistics_neg_zero_max() {
4303        let stats = statistics_roundtrip::<FloatType>(&[-0.0, -1.0, f32::NAN, -2.0]);
4304        assert!(!stats.is_min_max_backwards_compatible());
4305        if let Statistics::Float(stats) = stats {
4306            assert_eq!(stats.min_opt().unwrap(), &-2.0);
4307            assert_eq!(stats.max_opt().unwrap(), &-0.0);
4308            assert!(stats.max_opt().unwrap().is_sign_negative());
4309        } else {
4310            panic!("expecting Statistics::Float");
4311        }
4312    }
4313
4314    #[test]
4315    fn test_double_statistics_nan_middle() {
4316        let stats = statistics_roundtrip::<DoubleType>(&[1.0, f64::NAN, 2.0]);
4317        assert!(!stats.is_min_max_backwards_compatible());
4318        if let Statistics::Double(stats) = stats {
4319            assert_eq!(stats.min_opt().unwrap(), &1.0);
4320            assert_eq!(stats.max_opt().unwrap(), &2.0);
4321            assert_eq!(stats.nan_count_opt(), Some(1))
4322        } else {
4323            panic!("expecting Statistics::Double");
4324        }
4325    }
4326
4327    #[test]
4328    fn test_double_statistics_nan_start() {
4329        let stats = statistics_roundtrip::<DoubleType>(&[f64::NAN, 1.0, 2.0]);
4330        assert!(!stats.is_min_max_backwards_compatible());
4331        if let Statistics::Double(stats) = stats {
4332            assert_eq!(stats.min_opt().unwrap(), &1.0);
4333            assert_eq!(stats.max_opt().unwrap(), &2.0);
4334            assert_eq!(stats.nan_count_opt(), Some(1))
4335        } else {
4336            panic!("expecting Statistics::Double");
4337        }
4338    }
4339
4340    #[test]
4341    fn test_double_statistics_nan_only() {
4342        let stats = statistics_roundtrip::<DoubleType>(&[f64::NAN, f64::NAN]);
4343        assert_eq!(stats.min_bytes_opt(), Some(f64::NAN.as_bytes()));
4344        assert_eq!(stats.max_bytes_opt(), Some(f64::NAN.as_bytes()));
4345        assert_eq!(stats.nan_count_opt(), Some(2));
4346        assert!(matches!(stats, Statistics::Double(_)));
4347        assert!(!stats.is_min_max_backwards_compatible());
4348    }
4349
4350    #[test]
4351    fn test_double_statistics_zero_only() {
4352        let stats = statistics_roundtrip::<DoubleType>(&[0.0]);
4353        assert!(!stats.is_min_max_backwards_compatible());
4354        if let Statistics::Double(stats) = stats {
4355            assert_eq!(stats.min_opt().unwrap(), &0.0);
4356            assert!(stats.min_opt().unwrap().is_sign_positive());
4357            assert_eq!(stats.max_opt().unwrap(), &0.0);
4358            assert!(stats.max_opt().unwrap().is_sign_positive());
4359        } else {
4360            panic!("expecting Statistics::Double");
4361        }
4362    }
4363
4364    #[test]
4365    fn test_double_statistics_neg_zero_only() {
4366        let stats = statistics_roundtrip::<DoubleType>(&[-0.0]);
4367        assert!(!stats.is_min_max_backwards_compatible());
4368        if let Statistics::Double(stats) = stats {
4369            assert_eq!(stats.min_opt().unwrap(), &-0.0);
4370            assert!(stats.min_opt().unwrap().is_sign_negative());
4371            assert_eq!(stats.max_opt().unwrap(), &-0.0);
4372            assert!(stats.max_opt().unwrap().is_sign_negative());
4373        } else {
4374            panic!("expecting Statistics::Double");
4375        }
4376    }
4377
4378    #[test]
4379    fn test_double_statistics_zero_min() {
4380        let stats = statistics_roundtrip::<DoubleType>(&[0.0, 1.0, f64::NAN, 2.0]);
4381        assert!(!stats.is_min_max_backwards_compatible());
4382        if let Statistics::Double(stats) = stats {
4383            assert_eq!(stats.min_opt().unwrap(), &0.0);
4384            assert!(stats.min_opt().unwrap().is_sign_positive());
4385            assert_eq!(stats.max_opt().unwrap(), &2.0);
4386        } else {
4387            panic!("expecting Statistics::Double");
4388        }
4389    }
4390
4391    #[test]
4392    fn test_double_statistics_neg_zero_max() {
4393        let stats = statistics_roundtrip::<DoubleType>(&[-0.0, -1.0, f64::NAN, -2.0]);
4394        assert!(!stats.is_min_max_backwards_compatible());
4395        if let Statistics::Double(stats) = stats {
4396            assert_eq!(stats.min_opt().unwrap(), &-2.0);
4397            assert_eq!(stats.max_opt().unwrap(), &-0.0);
4398            assert!(stats.max_opt().unwrap().is_sign_negative());
4399        } else {
4400            panic!("expecting Statistics::Double");
4401        }
4402    }
4403
4404    #[test]
4405    fn test_compare_greater_byte_array_decimals() {
4406        assert!(!compare_greater_byte_array_decimals(&[], &[],),);
4407        assert!(compare_greater_byte_array_decimals(&[1u8,], &[],),);
4408        assert!(!compare_greater_byte_array_decimals(&[], &[1u8,],),);
4409        assert!(compare_greater_byte_array_decimals(&[1u8,], &[0u8,],),);
4410        assert!(!compare_greater_byte_array_decimals(&[1u8,], &[1u8,],),);
4411        assert!(compare_greater_byte_array_decimals(&[1u8, 0u8,], &[0u8,],),);
4412        assert!(!compare_greater_byte_array_decimals(
4413            &[0u8, 1u8,],
4414            &[1u8, 0u8,],
4415        ),);
4416        assert!(!compare_greater_byte_array_decimals(
4417            &[255u8, 35u8, 0u8, 0u8,],
4418            &[0u8,],
4419        ),);
4420        assert!(compare_greater_byte_array_decimals(
4421            &[0u8,],
4422            &[255u8, 35u8, 0u8, 0u8,],
4423        ),);
4424
4425        // Unequal lengths where the longer value's extra leading bytes are all
4426        // sign extension, so the aligned tails decide.
4427        // https://github.com/apache/arrow-rs/issues/10860
4428
4429        // 32768 > 255
4430        assert!(compare_greater_byte_array_decimals(
4431            &[0u8, 128u8, 0u8,],
4432            &[0u8, 255u8,],
4433        ),);
4434        assert!(!compare_greater_byte_array_decimals(
4435            &[0u8, 255u8,],
4436            &[0u8, 128u8, 0u8,],
4437        ),);
4438        // -128 > -129
4439        assert!(compare_greater_byte_array_decimals(
4440            &[128u8,],
4441            &[255u8, 127u8,],
4442        ),);
4443        assert!(!compare_greater_byte_array_decimals(
4444            &[255u8, 127u8,],
4445            &[128u8,],
4446        ),);
4447        // -128 > -256
4448        assert!(compare_greater_byte_array_decimals(
4449            &[128u8,],
4450            &[255u8, 0u8,],
4451        ),);
4452        // 10 (with a redundant leading zero) > 5
4453        assert!(compare_greater_byte_array_decimals(&[0u8, 10u8,], &[5u8,],),);
4454        assert!(compare_greater_byte_array_decimals(&[10u8,], &[0u8, 5u8,],),);
4455        // equal values of different lengths are not greater in either direction
4456        assert!(!compare_greater_byte_array_decimals(
4457            &[255u8, 128u8,],
4458            &[128u8,],
4459        ),);
4460        assert!(!compare_greater_byte_array_decimals(
4461            &[128u8,],
4462            &[255u8, 128u8,],
4463        ),);
4464    }
4465
4466    #[test]
4467    fn test_column_index_with_null_pages() {
4468        // write a single page of all nulls
4469        let page_writer = get_test_page_writer();
4470        let props = Default::default();
4471        let mut writer = get_test_column_writer::<Int32Type>(page_writer, 1, 0, props);
4472        writer.write_batch(&[], Some(&[0, 0, 0, 0]), None).unwrap();
4473
4474        let r = writer.close().unwrap();
4475        assert!(r.column_index.is_some());
4476        let col_idx = r.column_index.unwrap();
4477        let ColumnIndexMetaData::INT32(col_idx) = col_idx else {
4478            panic!("wrong stats type")
4479        };
4480        // null_pages should be true for page 0
4481        assert!(col_idx.is_null_page(0));
4482        // min and max should be empty byte arrays
4483        assert!(col_idx.min_value(0).is_none());
4484        assert!(col_idx.max_value(0).is_none());
4485        // null_counts should be defined and be 4 for page 0
4486        assert!(col_idx.null_count(0).is_some());
4487        assert_eq!(col_idx.null_count(0), Some(4));
4488        // there is no repetition so rep histogram should be absent
4489        assert!(col_idx.repetition_level_histogram(0).is_none());
4490        // definition_level_histogram should be present and should be 0:4, 1:0
4491        assert!(col_idx.definition_level_histogram(0).is_some());
4492        assert_eq!(col_idx.definition_level_histogram(0).unwrap(), &[4, 0]);
4493    }
4494
4495    #[test]
4496    fn test_column_index_repeated_page_with_values_not_null() {
4497        // four lists of [null, 7]: as many null levels as rows, but the page has values
4498        let page_writer = get_test_page_writer();
4499        let props = Default::default();
4500        let mut writer = get_test_column_writer::<Int32Type>(page_writer, 2, 1, props);
4501        writer
4502            .write_batch(
4503                &[7, 7, 7, 7],
4504                Some(&[1, 2, 1, 2, 1, 2, 1, 2]),
4505                Some(&[0, 1, 0, 1, 0, 1, 0, 1]),
4506            )
4507            .unwrap();
4508
4509        let r = writer.close().unwrap();
4510        let ColumnIndexMetaData::INT32(col_idx) = r.column_index.unwrap() else {
4511            panic!("wrong stats type")
4512        };
4513        assert!(!col_idx.is_null_page(0));
4514        assert_eq!(col_idx.min_value(0), Some(&7));
4515        assert_eq!(col_idx.max_value(0), Some(&7));
4516        assert_eq!(col_idx.null_count(0), Some(4));
4517    }
4518
4519    #[test]
4520    fn test_column_index_repeated_page_of_nulls() {
4521        // two lists of [null, null]: no values, more null levels than rows
4522        let page_writer = get_test_page_writer();
4523        let props = Default::default();
4524        let mut writer = get_test_column_writer::<Int32Type>(page_writer, 2, 1, props);
4525        writer
4526            .write_batch(&[], Some(&[1, 1, 1, 1]), Some(&[0, 1, 0, 1]))
4527            .unwrap();
4528
4529        let r = writer.close().unwrap();
4530        let ColumnIndexMetaData::INT32(col_idx) = r.column_index.unwrap() else {
4531            panic!("wrong stats type")
4532        };
4533        assert!(col_idx.is_null_page(0));
4534        assert!(col_idx.min_value(0).is_none());
4535        assert!(col_idx.max_value(0).is_none());
4536        assert_eq!(col_idx.null_count(0), Some(4));
4537    }
4538
4539    #[test]
4540    fn test_column_offset_index_metadata() {
4541        // write data
4542        // and check the offset index and column index
4543        let page_writer = get_test_page_writer();
4544        let props = Default::default();
4545        let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
4546        writer.write_batch(&[1, 2, 3, 4], None, None).unwrap();
4547        // first page
4548        writer.flush_data_pages().unwrap();
4549        // second page
4550        writer.write_batch(&[4, 8, 2, -5], None, None).unwrap();
4551
4552        let r = writer.close().unwrap();
4553        let column_index = r.column_index.unwrap();
4554        let offset_index = r.offset_index.unwrap();
4555
4556        assert_eq!(8, r.rows_written);
4557
4558        // column index
4559        let ColumnIndexMetaData::INT32(column_index) = column_index else {
4560            panic!("wrong stats type")
4561        };
4562        assert_eq!(2, column_index.num_pages());
4563        assert_eq!(2, offset_index.page_locations.len());
4564        assert_eq!(BoundaryOrder::UNORDERED, column_index.boundary_order);
4565        for idx in 0..2 {
4566            assert!(!column_index.is_null_page(idx));
4567            assert_eq!(0, column_index.null_count(0).unwrap());
4568        }
4569
4570        if let Some(stats) = r.metadata.statistics() {
4571            assert_eq!(stats.null_count_opt(), Some(0));
4572            assert_eq!(stats.distinct_count_opt(), None);
4573            if let Statistics::Int32(stats) = stats {
4574                // first page is [1,2,3,4]
4575                // second page is [-5,2,4,8]
4576                // note that we don't increment here, as this is a non BinaryArray type.
4577                assert_eq!(stats.min_opt(), column_index.min_value(1));
4578                assert_eq!(stats.max_opt(), column_index.max_value(1));
4579            } else {
4580                panic!("expecting Statistics::Int32");
4581            }
4582        } else {
4583            panic!("metadata missing statistics");
4584        }
4585
4586        // page location
4587        assert_eq!(0, offset_index.page_locations[0].first_row_index);
4588        assert_eq!(4, offset_index.page_locations[1].first_row_index);
4589    }
4590
4591    /// Verify min/max value truncation in the column index works as expected
4592    #[test]
4593    fn test_column_offset_index_metadata_truncating() {
4594        // write data
4595        // and check the offset index and column index
4596        let page_writer = get_test_page_writer();
4597        let props = WriterProperties::builder()
4598            .set_statistics_truncate_length(None) // disable column index truncation
4599            .build()
4600            .into();
4601        let mut writer = get_test_column_writer::<FixedLenByteArrayType>(page_writer, 0, 0, props);
4602
4603        let mut data = vec![FixedLenByteArray::default(); 3];
4604        // This is the expected min value - "aaa..."
4605        data[0].set_data(Bytes::from(vec![97_u8; 200]));
4606        // This is the expected max value - "ZZZ..."
4607        data[1].set_data(Bytes::from(vec![112_u8; 200]));
4608        data[2].set_data(Bytes::from(vec![98_u8; 200]));
4609
4610        writer.write_batch(&data, None, None).unwrap();
4611
4612        writer.flush_data_pages().unwrap();
4613
4614        let r = writer.close().unwrap();
4615        let column_index = r.column_index.unwrap();
4616        let offset_index = r.offset_index.unwrap();
4617
4618        let ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(column_index) = column_index else {
4619            panic!("wrong stats type")
4620        };
4621
4622        assert_eq!(3, r.rows_written);
4623
4624        // column index
4625        assert_eq!(1, column_index.num_pages());
4626        assert_eq!(1, offset_index.page_locations.len());
4627        assert_eq!(BoundaryOrder::ASCENDING, column_index.boundary_order);
4628        assert!(!column_index.is_null_page(0));
4629        assert_eq!(Some(0), column_index.null_count(0));
4630
4631        if let Some(stats) = r.metadata.statistics() {
4632            assert_eq!(stats.null_count_opt(), Some(0));
4633            assert_eq!(stats.distinct_count_opt(), None);
4634            if let Statistics::FixedLenByteArray(stats) = stats {
4635                let column_index_min_value = column_index.min_value(0).unwrap();
4636                let column_index_max_value = column_index.max_value(0).unwrap();
4637
4638                // Column index stats are truncated, while the column chunk's aren't.
4639                assert_ne!(stats.min_bytes_opt().unwrap(), column_index_min_value);
4640                assert_ne!(stats.max_bytes_opt().unwrap(), column_index_max_value);
4641
4642                assert_eq!(
4643                    column_index_min_value.len(),
4644                    DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH.unwrap()
4645                );
4646                assert_eq!(column_index_min_value, &[97_u8; 64]);
4647                assert_eq!(
4648                    column_index_max_value.len(),
4649                    DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH.unwrap()
4650                );
4651
4652                // We expect the last byte to be incremented
4653                assert_eq!(
4654                    *column_index_max_value.last().unwrap(),
4655                    *column_index_max_value.first().unwrap() + 1
4656                );
4657            } else {
4658                panic!("expecting Statistics::FixedLenByteArray");
4659            }
4660        } else {
4661            panic!("metadata missing statistics");
4662        }
4663    }
4664
4665    #[test]
4666    fn test_column_offset_index_truncating_spec_example() {
4667        // write data
4668        // and check the offset index and column index
4669        let page_writer = get_test_page_writer();
4670
4671        // Truncate values at 1 byte
4672        let builder = WriterProperties::builder().set_column_index_truncate_length(Some(1));
4673        let props = Arc::new(builder.build());
4674        let mut writer = get_test_column_writer::<FixedLenByteArrayType>(page_writer, 0, 0, props);
4675
4676        let mut data = vec![FixedLenByteArray::default(); 1];
4677        // This is the expected min value
4678        data[0].set_data(Bytes::from(String::from("Blart Versenwald III")));
4679
4680        writer.write_batch(&data, None, None).unwrap();
4681
4682        writer.flush_data_pages().unwrap();
4683
4684        let r = writer.close().unwrap();
4685        let column_index = r.column_index.unwrap();
4686        let offset_index = r.offset_index.unwrap();
4687
4688        let ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(column_index) = column_index else {
4689            panic!("wrong stats type")
4690        };
4691
4692        assert_eq!(1, r.rows_written);
4693
4694        // column index
4695        assert_eq!(1, column_index.num_pages());
4696        assert_eq!(1, offset_index.page_locations.len());
4697        assert_eq!(BoundaryOrder::ASCENDING, column_index.boundary_order);
4698        assert!(!column_index.is_null_page(0));
4699        assert_eq!(Some(0), column_index.null_count(0));
4700
4701        if let Some(stats) = r.metadata.statistics() {
4702            assert_eq!(stats.null_count_opt(), Some(0));
4703            assert_eq!(stats.distinct_count_opt(), None);
4704            if let Statistics::FixedLenByteArray(_stats) = stats {
4705                let column_index_min_value = column_index.min_value(0).unwrap();
4706                let column_index_max_value = column_index.max_value(0).unwrap();
4707
4708                assert_eq!(column_index_min_value.len(), 1);
4709                assert_eq!(column_index_max_value.len(), 1);
4710
4711                assert_eq!(b"B", column_index_min_value);
4712                assert_eq!(b"C", column_index_max_value);
4713
4714                assert_ne!(column_index_min_value, stats.min_bytes_opt().unwrap());
4715                assert_ne!(column_index_max_value, stats.max_bytes_opt().unwrap());
4716            } else {
4717                panic!("expecting Statistics::FixedLenByteArray");
4718            }
4719        } else {
4720            panic!("metadata missing statistics");
4721        }
4722    }
4723
4724    #[test]
4725    fn test_float16_min_max_no_truncation() {
4726        // Even if we set truncation to occur at 1 byte, we should not truncate for Float16
4727        let builder = WriterProperties::builder().set_column_index_truncate_length(Some(1));
4728        let props = Arc::new(builder.build());
4729        let page_writer = get_test_page_writer();
4730        let mut writer = get_test_float16_column_writer(page_writer, props);
4731
4732        let expected_value = f16::PI.to_le_bytes().to_vec();
4733        let data = vec![ByteArray::from(expected_value.clone()).into()];
4734        writer.write_batch(&data, None, None).unwrap();
4735        writer.flush_data_pages().unwrap();
4736
4737        let r = writer.close().unwrap();
4738
4739        // stats should still be written
4740        // ensure bytes weren't truncated for column index
4741        let column_index = r.column_index.unwrap();
4742        let ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(column_index) = column_index else {
4743            panic!("wrong stats type")
4744        };
4745        let column_index_min_bytes = column_index.min_value(0).unwrap();
4746        let column_index_max_bytes = column_index.max_value(0).unwrap();
4747        assert_eq!(expected_value, column_index_min_bytes);
4748        assert_eq!(expected_value, column_index_max_bytes);
4749
4750        // ensure bytes weren't truncated for statistics
4751        let stats = r.metadata.statistics().unwrap();
4752        if let Statistics::FixedLenByteArray(stats) = stats {
4753            let stats_min_bytes = stats.min_bytes_opt().unwrap();
4754            let stats_max_bytes = stats.max_bytes_opt().unwrap();
4755            assert_eq!(expected_value, stats_min_bytes);
4756            assert_eq!(expected_value, stats_max_bytes);
4757        } else {
4758            panic!("expecting Statistics::FixedLenByteArray");
4759        }
4760    }
4761
4762    #[test]
4763    fn test_decimal_min_max_no_truncation() {
4764        // Even if we set truncation to occur at 1 byte, we should not truncate for Decimal
4765        let builder = WriterProperties::builder().set_column_index_truncate_length(Some(1));
4766        let props = Arc::new(builder.build());
4767        let page_writer = get_test_page_writer();
4768        let mut writer =
4769            get_test_decimals_column_writer::<FixedLenByteArrayType>(page_writer, 0, 0, props);
4770
4771        let expected_value = vec![
4772            255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 179u8, 172u8, 19u8, 35u8,
4773            231u8, 90u8, 0u8, 0u8,
4774        ];
4775        let data = vec![ByteArray::from(expected_value.clone()).into()];
4776        writer.write_batch(&data, None, None).unwrap();
4777        writer.flush_data_pages().unwrap();
4778
4779        let r = writer.close().unwrap();
4780
4781        // stats should still be written
4782        // ensure bytes weren't truncated for column index
4783        let column_index = r.column_index.unwrap();
4784        let ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(column_index) = column_index else {
4785            panic!("wrong stats type")
4786        };
4787        let column_index_min_bytes = column_index.min_value(0).unwrap();
4788        let column_index_max_bytes = column_index.max_value(0).unwrap();
4789        assert_eq!(expected_value, column_index_min_bytes);
4790        assert_eq!(expected_value, column_index_max_bytes);
4791
4792        // ensure bytes weren't truncated for statistics
4793        let stats = r.metadata.statistics().unwrap();
4794        if let Statistics::FixedLenByteArray(stats) = stats {
4795            let stats_min_bytes = stats.min_bytes_opt().unwrap();
4796            let stats_max_bytes = stats.max_bytes_opt().unwrap();
4797            assert_eq!(expected_value, stats_min_bytes);
4798            assert_eq!(expected_value, stats_max_bytes);
4799        } else {
4800            panic!("expecting Statistics::FixedLenByteArray");
4801        }
4802    }
4803
4804    #[test]
4805    fn test_decimal_statistics_do_not_alias_written_values() {
4806        // See https://github.com/apache/arrow-rs/issues/12027
4807        //
4808        // The Arrow writer slices all FixedLenByteArray values of a batch out of one shared
4809        // buffer. Decimal and Float16 statistics are never truncated and used to be stored as
4810        // clones of the written values, so the column chunk statistics kept every such buffer
4811        // alive for as long as the file metadata existed.
4812        let page_writer = get_test_page_writer();
4813        let mut writer = get_test_decimals_column_writer::<FixedLenByteArrayType>(
4814            page_writer,
4815            0,
4816            0,
4817            Default::default(),
4818        );
4819
4820        // Four 16-byte big-endian non-negative values in ascending order, in one buffer
4821        let arena = Bytes::from((0..64u8).collect::<Vec<_>>());
4822        let values: Vec<FixedLenByteArray> = (0..4)
4823            .map(|i| ByteArray::from(arena.slice(i * 16..(i + 1) * 16)).into())
4824            .collect();
4825        writer.write_batch(&values, None, None).unwrap();
4826        let r = writer.close().unwrap();
4827        drop(values);
4828
4829        let statistics = r.metadata.statistics().unwrap();
4830        // Copying must not change how the statistics are written: Decimal has a signed sort
4831        // order, so both min_value/max_value and the deprecated min/max are emitted
4832        assert!(!statistics.is_min_max_deprecated());
4833        assert!(statistics.is_min_max_backwards_compatible());
4834        let Statistics::FixedLenByteArray(stats) = statistics else {
4835            panic!("expecting Statistics::FixedLenByteArray");
4836        };
4837        let min = stats.min_bytes_opt().unwrap();
4838        let max = stats.max_bytes_opt().unwrap();
4839        assert_eq!(min, &arena[..16]);
4840        assert_eq!(max, &arena[48..]);
4841
4842        // The statistics must own their bytes rather than point into the written buffer
4843        let arena_range = arena.as_ptr_range();
4844        assert!(
4845            !arena_range.contains(&min.as_ptr()),
4846            "min aliases the written buffer"
4847        );
4848        assert!(
4849            !arena_range.contains(&max.as_ptr()),
4850            "max aliases the written buffer"
4851        );
4852
4853        assert!(stats.min_is_exact());
4854        assert!(stats.max_is_exact());
4855    }
4856
4857    #[test]
4858    fn test_decimal_byte_array_min_max_no_statistics_truncation() {
4859        // See https://github.com/apache/arrow-rs/issues/11073
4860        let page_writer = get_test_page_writer();
4861
4862        // Truncate at 1 byte -- far shorter than either encoded value below.
4863        let builder = WriterProperties::builder().set_statistics_truncate_length(Some(1));
4864        let props = Arc::new(builder.build());
4865        let mut writer = get_test_decimals_column_writer::<ByteArrayType>(page_writer, 0, 0, props);
4866
4867        // Two's-complement, big-endian encodings of 200 and -200 (2 bytes each).
4868        // Naive unsigned byte-wise truncation to 1 byte followed by an unsigned
4869        // increment of the max value would corrupt these:
4870        //   max: truncate([0x00, 0xC8], 1) -> [0x00], increment -> [0x01],
4871        //        which decodes to 1 -- far less than the true max of 200.
4872        let positive_200 = vec![0x00u8, 0xC8u8];
4873        let negative_200 = vec![0xFFu8, 0x38u8];
4874
4875        let data = vec![
4876            ByteArray::from(positive_200.clone()),
4877            ByteArray::from(negative_200.clone()),
4878        ];
4879        writer.write_batch(&data, None, None).unwrap();
4880        writer.flush_data_pages().unwrap();
4881
4882        let r = writer.close().unwrap();
4883        assert_eq!(2, r.rows_written);
4884
4885        let stats = r.metadata.statistics().expect("statistics");
4886        if let Statistics::ByteArray(stats) = stats {
4887            // Decimal columns must never be truncated, regardless of
4888            // `statistics_truncate_length`.
4889            assert!(stats.min_is_exact());
4890            assert!(stats.max_is_exact());
4891
4892            let min_value = stats.min_bytes_opt().unwrap();
4893            let max_value = stats.max_bytes_opt().unwrap();
4894
4895            // Correct signed (two's-complement) ordering: -200 < 200.
4896            assert_eq!(negative_200, min_value);
4897            assert_eq!(positive_200, max_value);
4898        } else {
4899            panic!("expecting Statistics::ByteArray");
4900        }
4901    }
4902
4903    #[test]
4904    fn test_statistics_truncating_byte_array_default() {
4905        let page_writer = get_test_page_writer();
4906
4907        // The default truncate length is 64 bytes
4908        let props = WriterProperties::builder().build().into();
4909        let mut writer = get_test_column_writer::<ByteArrayType>(page_writer, 0, 0, props);
4910
4911        let mut data = vec![ByteArray::default(); 1];
4912        data[0].set_data(Bytes::from(String::from(
4913            "This string is longer than 64 bytes, so it will almost certainly be truncated.",
4914        )));
4915        writer.write_batch(&data, None, None).unwrap();
4916        writer.flush_data_pages().unwrap();
4917
4918        let r = writer.close().unwrap();
4919
4920        assert_eq!(1, r.rows_written);
4921
4922        let stats = r.metadata.statistics().expect("statistics");
4923        if let Statistics::ByteArray(stats) = stats {
4924            let min_value = stats.min_opt().unwrap();
4925            let max_value = stats.max_opt().unwrap();
4926
4927            assert!(!stats.min_is_exact());
4928            assert!(!stats.max_is_exact());
4929
4930            let expected_len = 64;
4931            assert_eq!(min_value.len(), expected_len);
4932            assert_eq!(max_value.len(), expected_len);
4933
4934            let expected_min =
4935                "This string is longer than 64 bytes, so it will almost certainly".as_bytes();
4936            assert_eq!(expected_min, min_value.as_bytes());
4937            // note the max value is different from the min value: the last byte is incremented
4938            let expected_max =
4939                "This string is longer than 64 bytes, so it will almost certainlz".as_bytes();
4940            assert_eq!(expected_max, max_value.as_bytes());
4941        } else {
4942            panic!("expecting Statistics::ByteArray");
4943        }
4944    }
4945
4946    #[test]
4947    fn test_statistics_truncating_byte_array() {
4948        let page_writer = get_test_page_writer();
4949
4950        const TEST_TRUNCATE_LENGTH: usize = 1;
4951
4952        // Truncate values at 1 byte
4953        let builder =
4954            WriterProperties::builder().set_statistics_truncate_length(Some(TEST_TRUNCATE_LENGTH));
4955        let props = Arc::new(builder.build());
4956        let mut writer = get_test_column_writer::<ByteArrayType>(page_writer, 0, 0, props);
4957
4958        let mut data = vec![ByteArray::default(); 1];
4959        // This is the expected min value
4960        data[0].set_data(Bytes::from(String::from("Blart Versenwald III")));
4961
4962        writer.write_batch(&data, None, None).unwrap();
4963
4964        writer.flush_data_pages().unwrap();
4965
4966        let r = writer.close().unwrap();
4967
4968        assert_eq!(1, r.rows_written);
4969
4970        let stats = r.metadata.statistics().expect("statistics");
4971        assert_eq!(stats.null_count_opt(), Some(0));
4972        assert_eq!(stats.distinct_count_opt(), None);
4973        if let Statistics::ByteArray(stats) = stats {
4974            let min_value = stats.min_opt().unwrap();
4975            let max_value = stats.max_opt().unwrap();
4976
4977            assert!(!stats.min_is_exact());
4978            assert!(!stats.max_is_exact());
4979
4980            assert_eq!(min_value.len(), TEST_TRUNCATE_LENGTH);
4981            assert_eq!(max_value.len(), TEST_TRUNCATE_LENGTH);
4982
4983            assert_eq!(b"B", min_value.as_bytes());
4984            assert_eq!(b"C", max_value.as_bytes());
4985        } else {
4986            panic!("expecting Statistics::ByteArray");
4987        }
4988    }
4989
4990    #[test]
4991    fn test_statistics_truncating_fixed_len_byte_array() {
4992        let page_writer = get_test_page_writer();
4993
4994        const TEST_TRUNCATE_LENGTH: usize = 1;
4995
4996        // Truncate values at 1 byte
4997        let builder =
4998            WriterProperties::builder().set_statistics_truncate_length(Some(TEST_TRUNCATE_LENGTH));
4999        let props = Arc::new(builder.build());
5000        let mut writer = get_test_column_writer::<FixedLenByteArrayType>(page_writer, 0, 0, props);
5001
5002        let mut data = vec![FixedLenByteArray::default(); 1];
5003
5004        const PSEUDO_DECIMAL_VALUE: i128 = 6541894651216648486512564456564654;
5005        const PSEUDO_DECIMAL_BYTES: [u8; 16] = PSEUDO_DECIMAL_VALUE.to_be_bytes();
5006
5007        const EXPECTED_MIN: [u8; TEST_TRUNCATE_LENGTH] = [PSEUDO_DECIMAL_BYTES[0]]; // parquet specifies big-endian order for decimals
5008        const EXPECTED_MAX: [u8; TEST_TRUNCATE_LENGTH] =
5009            [PSEUDO_DECIMAL_BYTES[0].overflowing_add(1).0];
5010
5011        // This is the expected min value
5012        data[0].set_data(Bytes::from(PSEUDO_DECIMAL_BYTES.as_slice()));
5013
5014        writer.write_batch(&data, None, None).unwrap();
5015
5016        writer.flush_data_pages().unwrap();
5017
5018        let r = writer.close().unwrap();
5019
5020        assert_eq!(1, r.rows_written);
5021
5022        let stats = r.metadata.statistics().expect("statistics");
5023        assert_eq!(stats.null_count_opt(), Some(0));
5024        assert_eq!(stats.distinct_count_opt(), None);
5025        if let Statistics::FixedLenByteArray(stats) = stats {
5026            let min_value = stats.min_opt().unwrap();
5027            let max_value = stats.max_opt().unwrap();
5028
5029            assert!(!stats.min_is_exact());
5030            assert!(!stats.max_is_exact());
5031
5032            assert_eq!(min_value.len(), TEST_TRUNCATE_LENGTH);
5033            assert_eq!(max_value.len(), TEST_TRUNCATE_LENGTH);
5034
5035            assert_eq!(EXPECTED_MIN.as_slice(), min_value.as_bytes());
5036            assert_eq!(EXPECTED_MAX.as_slice(), max_value.as_bytes());
5037
5038            let reconstructed_min = i128::from_be_bytes([
5039                min_value.as_bytes()[0],
5040                0,
5041                0,
5042                0,
5043                0,
5044                0,
5045                0,
5046                0,
5047                0,
5048                0,
5049                0,
5050                0,
5051                0,
5052                0,
5053                0,
5054                0,
5055            ]);
5056
5057            let reconstructed_max = i128::from_be_bytes([
5058                max_value.as_bytes()[0],
5059                0,
5060                0,
5061                0,
5062                0,
5063                0,
5064                0,
5065                0,
5066                0,
5067                0,
5068                0,
5069                0,
5070                0,
5071                0,
5072                0,
5073                0,
5074            ]);
5075
5076            // check that the inner value is correctly bounded by the min/max
5077            println!("min: {reconstructed_min} {PSEUDO_DECIMAL_VALUE}");
5078            assert!(reconstructed_min <= PSEUDO_DECIMAL_VALUE);
5079            println!("max {reconstructed_max} {PSEUDO_DECIMAL_VALUE}");
5080            assert!(reconstructed_max >= PSEUDO_DECIMAL_VALUE);
5081        } else {
5082            panic!("expecting Statistics::FixedLenByteArray");
5083        }
5084    }
5085
5086    #[test]
5087    fn test_send() {
5088        fn test<T: Send>() {}
5089        test::<ColumnWriterImpl<Int32Type>>();
5090    }
5091
5092    #[test]
5093    fn test_increment() {
5094        let v = increment(vec![0, 0, 0]).unwrap();
5095        assert_eq!(&v, &[0, 0, 1]);
5096
5097        // Handle overflow
5098        let v = increment(vec![0, 255, 255]).unwrap();
5099        assert_eq!(&v, &[1, 0, 0]);
5100
5101        // Return `None` if all bytes are u8::MAX
5102        let v = increment(vec![255, 255, 255]);
5103        assert!(v.is_none());
5104    }
5105
5106    #[test]
5107    fn test_increment_utf8() {
5108        let test_inc = |o: &str, expected: &str| {
5109            if let Ok(v) = String::from_utf8(increment_utf8(o).unwrap()) {
5110                // Got the expected result...
5111                assert_eq!(v, expected);
5112                // and it's greater than the original string
5113                assert!(*v > *o);
5114                // Also show that BinaryArray level comparison works here
5115                let mut greater = ByteArray::new();
5116                greater.set_data(Bytes::from(v));
5117                let mut original = ByteArray::new();
5118                original.set_data(Bytes::from(o.as_bytes().to_vec()));
5119                assert!(greater > original);
5120            } else {
5121                panic!("Expected incremented UTF8 string to also be valid.");
5122            }
5123        };
5124
5125        // Basic ASCII case
5126        test_inc("hello", "hellp");
5127
5128        // 1-byte ending in max 1-byte
5129        test_inc("a\u{7f}", "b");
5130
5131        // 1-byte max should not truncate as it would need 2-byte code points
5132        assert!(increment_utf8("\u{7f}\u{7f}").is_none());
5133
5134        // UTF8 string
5135        test_inc("❤️🧡💛💚💙💜", "❤️🧡💛💚💙💝");
5136
5137        // 2-byte without overflow
5138        test_inc("éééé", "éééê");
5139
5140        // 2-byte that overflows lowest byte
5141        test_inc("\u{ff}\u{ff}", "\u{ff}\u{100}");
5142
5143        // 2-byte ending in max 2-byte
5144        test_inc("a\u{7ff}", "b");
5145
5146        // Max 2-byte should not truncate as it would need 3-byte code points
5147        assert!(increment_utf8("\u{7ff}\u{7ff}").is_none());
5148
5149        // 3-byte without overflow [U+800, U+800] -> [U+800, U+801] (note that these
5150        // characters should render right to left).
5151        test_inc("ࠀࠀ", "ࠀࠁ");
5152
5153        // 3-byte ending in max 3-byte
5154        test_inc("a\u{ffff}", "b");
5155
5156        // Max 3-byte should not truncate as it would need 4-byte code points
5157        assert!(increment_utf8("\u{ffff}\u{ffff}").is_none());
5158
5159        // 4-byte without overflow
5160        test_inc("𐀀𐀀", "𐀀𐀁");
5161
5162        // 4-byte ending in max unicode
5163        test_inc("a\u{10ffff}", "b");
5164
5165        // Max 4-byte should not truncate
5166        assert!(increment_utf8("\u{10ffff}\u{10ffff}").is_none());
5167
5168        // Skip over surrogate pair range (0xD800..=0xDFFF)
5169        //test_inc("a\u{D7FF}", "a\u{e000}");
5170        test_inc("a\u{D7FF}", "b");
5171    }
5172
5173    #[test]
5174    fn test_truncate_utf8() {
5175        // No-op
5176        let data = "❤️🧡💛💚💙💜";
5177        let r = truncate_utf8(data, data.len()).unwrap();
5178        assert_eq!(r.len(), data.len());
5179        assert_eq!(&r, data.as_bytes());
5180
5181        // We slice it away from the UTF8 boundary
5182        let r = truncate_utf8(data, 13).unwrap();
5183        assert_eq!(r.len(), 10);
5184        assert_eq!(&r, "❤️🧡".as_bytes());
5185
5186        // One multi-byte code point, and a length shorter than it, so we can't slice it
5187        let r = truncate_utf8("\u{0836}", 1);
5188        assert!(r.is_none());
5189
5190        // Test truncate and increment for max bounds on UTF-8 statistics
5191        // 7-bit (i.e. ASCII)
5192        let r = truncate_and_increment_utf8("yyyyyyyyy", 8).unwrap();
5193        assert_eq!(&r, b"yyyyyyyz");
5194
5195        // 2-byte without overflow
5196        let r = truncate_and_increment_utf8("ééééé", 7).unwrap();
5197        assert_eq!(&r, "ééê".as_bytes());
5198
5199        // 2-byte that overflows lowest byte
5200        let r = truncate_and_increment_utf8("\u{ff}\u{ff}\u{ff}\u{ff}\u{ff}", 8).unwrap();
5201        assert_eq!(&r, "\u{ff}\u{ff}\u{ff}\u{100}".as_bytes());
5202
5203        // max 2-byte should not truncate as it would need 3-byte code points
5204        let r = truncate_and_increment_utf8("߿߿߿߿߿", 8);
5205        assert!(r.is_none());
5206
5207        // 3-byte without overflow [U+800, U+800, U+800] -> [U+800, U+801] (note that these
5208        // characters should render right to left).
5209        let r = truncate_and_increment_utf8("ࠀࠀࠀࠀ", 8).unwrap();
5210        assert_eq!(&r, "ࠀࠁ".as_bytes());
5211
5212        // max 3-byte should not truncate as it would need 4-byte code points
5213        let r = truncate_and_increment_utf8("\u{ffff}\u{ffff}\u{ffff}", 8);
5214        assert!(r.is_none());
5215
5216        // 4-byte without overflow
5217        let r = truncate_and_increment_utf8("𐀀𐀀𐀀𐀀", 9).unwrap();
5218        assert_eq!(&r, "𐀀𐀁".as_bytes());
5219
5220        // max 4-byte should not truncate
5221        let r = truncate_and_increment_utf8("\u{10ffff}\u{10ffff}", 8);
5222        assert!(r.is_none());
5223    }
5224
5225    #[test]
5226    // Check fallback truncation of statistics that should be UTF-8, but aren't
5227    // (see https://github.com/apache/arrow-rs/pull/6870).
5228    fn test_byte_array_truncate_invalid_utf8_statistics() {
5229        let message_type = "
5230            message test_schema {
5231                OPTIONAL BYTE_ARRAY a (UTF8);
5232            }
5233        ";
5234        let schema = Arc::new(parse_message_type(message_type).unwrap());
5235
5236        // Create Vec<ByteArray> containing non-UTF8 bytes
5237        let data = vec![ByteArray::from(vec![128u8; 32]); 7];
5238        let def_levels = [1, 1, 1, 1, 0, 1, 0, 1, 0, 1];
5239        let file: File = tempfile::tempfile().unwrap();
5240        let props = Arc::new(
5241            WriterProperties::builder()
5242                .set_statistics_enabled(EnabledStatistics::Chunk)
5243                .set_statistics_truncate_length(Some(8))
5244                .build(),
5245        );
5246
5247        let mut writer = SerializedFileWriter::new(&file, schema, props).unwrap();
5248        let mut row_group_writer = writer.next_row_group().unwrap();
5249
5250        let mut col_writer = row_group_writer.next_column().unwrap().unwrap();
5251        col_writer
5252            .typed::<ByteArrayType>()
5253            .write_batch(&data, Some(&def_levels), None)
5254            .unwrap();
5255        col_writer.close().unwrap();
5256        row_group_writer.close().unwrap();
5257        let file_metadata = writer.close().unwrap();
5258        let stats = file_metadata.row_group(0).column(0).statistics().unwrap();
5259        assert!(!stats.max_is_exact());
5260        // Truncation of invalid UTF-8 should fall back to binary truncation, so last byte should
5261        // be incremented by 1.
5262        assert_eq!(
5263            stats.max_bytes_opt().map(|v| v.to_vec()),
5264            Some([128, 128, 128, 128, 128, 128, 128, 129].to_vec())
5265        );
5266    }
5267
5268    #[test]
5269    fn test_increment_max_binary_chars() {
5270        let r = increment(vec![0xFF, 0xFE, 0xFD, 0xFF, 0xFF]);
5271        assert_eq!(&r.unwrap(), &[0xFF, 0xFE, 0xFE, 0x00, 0x00]);
5272
5273        let incremented = increment(vec![0xFF, 0xFF, 0xFF]);
5274        assert!(incremented.is_none())
5275    }
5276
5277    #[test]
5278    fn test_no_column_index_when_stats_disabled() {
5279        // https://github.com/apache/arrow-rs/issues/6010
5280        // Test that column index is not created/written for all-nulls column when page
5281        // statistics are disabled.
5282        let descr = Arc::new(get_test_column_descr::<Int32Type>(1, 0));
5283        let props = Arc::new(
5284            WriterProperties::builder()
5285                .set_statistics_enabled(EnabledStatistics::None)
5286                .build(),
5287        );
5288        let column_writer = get_column_writer(descr, props, get_test_page_writer());
5289        let mut writer = get_typed_column_writer::<Int32Type>(column_writer);
5290
5291        let data = Vec::new();
5292        let def_levels = vec![0; 10];
5293        writer.write_batch(&data, Some(&def_levels), None).unwrap();
5294        writer.flush_data_pages().unwrap();
5295
5296        let column_close_result = writer.close().unwrap();
5297        assert!(column_close_result.offset_index.is_some());
5298        assert!(column_close_result.column_index.is_none());
5299    }
5300
5301    #[test]
5302    fn test_no_offset_index_when_disabled() {
5303        // Test that offset indexes can be disabled
5304        let descr = Arc::new(get_test_column_descr::<Int32Type>(1, 0));
5305        let props = Arc::new(
5306            WriterProperties::builder()
5307                .set_statistics_enabled(EnabledStatistics::None)
5308                .set_offset_index_disabled(true)
5309                .build(),
5310        );
5311        let column_writer = get_column_writer(descr, props, get_test_page_writer());
5312        let mut writer = get_typed_column_writer::<Int32Type>(column_writer);
5313
5314        let data = Vec::new();
5315        let def_levels = vec![0; 10];
5316        writer.write_batch(&data, Some(&def_levels), None).unwrap();
5317        writer.flush_data_pages().unwrap();
5318
5319        let column_close_result = writer.close().unwrap();
5320        assert!(column_close_result.offset_index.is_none());
5321        assert!(column_close_result.column_index.is_none());
5322    }
5323
5324    #[test]
5325    fn test_offset_index_overridden() {
5326        // Test that offset indexes are not disabled when gathering page statistics
5327        let descr = Arc::new(get_test_column_descr::<Int32Type>(1, 0));
5328        let props = Arc::new(
5329            WriterProperties::builder()
5330                .set_statistics_enabled(EnabledStatistics::Page)
5331                .set_offset_index_disabled(true)
5332                .build(),
5333        );
5334        let column_writer = get_column_writer(descr, props, get_test_page_writer());
5335        let mut writer = get_typed_column_writer::<Int32Type>(column_writer);
5336
5337        let data = Vec::new();
5338        let def_levels = vec![0; 10];
5339        writer.write_batch(&data, Some(&def_levels), None).unwrap();
5340        writer.flush_data_pages().unwrap();
5341
5342        let column_close_result = writer.close().unwrap();
5343        assert!(column_close_result.offset_index.is_some());
5344        assert!(column_close_result.column_index.is_some());
5345    }
5346
5347    #[test]
5348    fn test_boundary_order() -> Result<()> {
5349        let descr = Arc::new(get_test_column_descr::<Int32Type>(1, 0));
5350        // min max both ascending
5351        let column_close_result = write_multiple_pages::<Int32Type>(
5352            &descr,
5353            &[
5354                &[Some(-10), Some(10)],
5355                &[Some(-5), Some(11)],
5356                &[None],
5357                &[Some(-5), Some(11)],
5358            ],
5359        )?;
5360        let boundary_order = column_close_result
5361            .column_index
5362            .unwrap()
5363            .get_boundary_order();
5364        assert_eq!(boundary_order, Some(BoundaryOrder::ASCENDING));
5365
5366        // min max both descending
5367        let column_close_result = write_multiple_pages::<Int32Type>(
5368            &descr,
5369            &[
5370                &[Some(10), Some(11)],
5371                &[Some(5), Some(11)],
5372                &[None],
5373                &[Some(-5), Some(0)],
5374            ],
5375        )?;
5376        let boundary_order = column_close_result
5377            .column_index
5378            .unwrap()
5379            .get_boundary_order();
5380        assert_eq!(boundary_order, Some(BoundaryOrder::DESCENDING));
5381
5382        // min max both equal
5383        let column_close_result = write_multiple_pages::<Int32Type>(
5384            &descr,
5385            &[&[Some(10), Some(11)], &[None], &[Some(10), Some(11)]],
5386        )?;
5387        let boundary_order = column_close_result
5388            .column_index
5389            .unwrap()
5390            .get_boundary_order();
5391        assert_eq!(boundary_order, Some(BoundaryOrder::ASCENDING));
5392
5393        // only nulls
5394        let column_close_result =
5395            write_multiple_pages::<Int32Type>(&descr, &[&[None], &[None], &[None]])?;
5396        let boundary_order = column_close_result
5397            .column_index
5398            .unwrap()
5399            .get_boundary_order();
5400        assert_eq!(boundary_order, Some(BoundaryOrder::ASCENDING));
5401
5402        // one page
5403        let column_close_result =
5404            write_multiple_pages::<Int32Type>(&descr, &[&[Some(-10), Some(10)]])?;
5405        let boundary_order = column_close_result
5406            .column_index
5407            .unwrap()
5408            .get_boundary_order();
5409        assert_eq!(boundary_order, Some(BoundaryOrder::ASCENDING));
5410
5411        // one non-null page
5412        let column_close_result =
5413            write_multiple_pages::<Int32Type>(&descr, &[&[Some(-10), Some(10)], &[None]])?;
5414        let boundary_order = column_close_result
5415            .column_index
5416            .unwrap()
5417            .get_boundary_order();
5418        assert_eq!(boundary_order, Some(BoundaryOrder::ASCENDING));
5419
5420        // min max both unordered
5421        let column_close_result = write_multiple_pages::<Int32Type>(
5422            &descr,
5423            &[
5424                &[Some(10), Some(11)],
5425                &[Some(11), Some(16)],
5426                &[None],
5427                &[Some(-5), Some(0)],
5428            ],
5429        )?;
5430        let boundary_order = column_close_result
5431            .column_index
5432            .unwrap()
5433            .get_boundary_order();
5434        assert_eq!(boundary_order, Some(BoundaryOrder::UNORDERED));
5435
5436        // min max both ordered in different orders
5437        let column_close_result = write_multiple_pages::<Int32Type>(
5438            &descr,
5439            &[
5440                &[Some(1), Some(9)],
5441                &[Some(2), Some(8)],
5442                &[None],
5443                &[Some(3), Some(7)],
5444            ],
5445        )?;
5446        let boundary_order = column_close_result
5447            .column_index
5448            .unwrap()
5449            .get_boundary_order();
5450        assert_eq!(boundary_order, Some(BoundaryOrder::UNORDERED));
5451
5452        Ok(())
5453    }
5454
5455    #[test]
5456    fn test_boundary_order_logical_type() -> Result<()> {
5457        // ensure that logical types account for different sort order than underlying
5458        // physical type representation
5459        let f16_descr = Arc::new(get_test_float16_column_descr(1, 0));
5460        let fba_descr = {
5461            let type_ = SchemaType::primitive_type_builder(
5462                "col",
5463                FixedLenByteArrayType::get_physical_type(),
5464            )
5465            .with_length(2)
5466            .build()?;
5467            Arc::new(ColumnDescriptor::new(
5468                Arc::new(type_),
5469                1,
5470                0,
5471                ColumnPath::from("col"),
5472            ))
5473        };
5474
5475        let values: &[&[Option<FixedLenByteArray>]] = &[
5476            &[Some(FixedLenByteArray::from(ByteArray::from(f16::ONE)))],
5477            &[Some(FixedLenByteArray::from(ByteArray::from(f16::ZERO)))],
5478            &[Some(FixedLenByteArray::from(ByteArray::from(
5479                f16::NEG_ZERO,
5480            )))],
5481            &[Some(FixedLenByteArray::from(ByteArray::from(f16::NEG_ONE)))],
5482        ];
5483
5484        // f16 descending
5485        let column_close_result =
5486            write_multiple_pages::<FixedLenByteArrayType>(&f16_descr, values)?;
5487        let boundary_order = column_close_result
5488            .column_index
5489            .unwrap()
5490            .get_boundary_order();
5491        assert_eq!(boundary_order, Some(BoundaryOrder::DESCENDING));
5492
5493        // same bytes, but fba unordered
5494        let column_close_result =
5495            write_multiple_pages::<FixedLenByteArrayType>(&fba_descr, values)?;
5496        let boundary_order = column_close_result
5497            .column_index
5498            .unwrap()
5499            .get_boundary_order();
5500        assert_eq!(boundary_order, Some(BoundaryOrder::UNORDERED));
5501
5502        Ok(())
5503    }
5504
5505    #[test]
5506    fn test_interval_stats_should_not_have_min_max() {
5507        let input = [
5508            vec![0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
5509            vec![0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
5510            vec![0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2],
5511        ]
5512        .into_iter()
5513        .map(|s| ByteArray::from(s).into())
5514        .collect::<Vec<_>>();
5515
5516        let page_writer = get_test_page_writer();
5517        let mut writer = get_test_interval_column_writer(page_writer);
5518        writer.write_batch(&input, None, None).unwrap();
5519
5520        let metadata = writer.close().unwrap().metadata;
5521        let stats = if let Some(Statistics::FixedLenByteArray(stats)) = metadata.statistics() {
5522            stats.clone()
5523        } else {
5524            panic!("metadata missing statistics");
5525        };
5526        assert!(stats.min_bytes_opt().is_none());
5527        assert!(stats.max_bytes_opt().is_none());
5528    }
5529
5530    #[test]
5531    #[cfg(feature = "arrow")]
5532    fn test_column_writer_get_estimated_total_bytes() {
5533        let page_writer = get_test_page_writer();
5534        let props = Default::default();
5535        let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
5536        assert_eq!(writer.get_estimated_total_bytes(), 0);
5537
5538        writer.write_batch(&[1, 2, 3, 4], None, None).unwrap();
5539        writer.add_data_page().unwrap();
5540        let size_with_one_page = writer.get_estimated_total_bytes();
5541        assert_eq!(size_with_one_page, 20);
5542
5543        writer.write_batch(&[5, 6, 7, 8], None, None).unwrap();
5544        writer.add_data_page().unwrap();
5545        let size_with_two_pages = writer.get_estimated_total_bytes();
5546        // different pages have different compressed lengths
5547        assert_eq!(size_with_two_pages, 20 + 21);
5548    }
5549
5550    fn write_multiple_pages<T: DataType>(
5551        column_descr: &Arc<ColumnDescriptor>,
5552        pages: &[&[Option<T::T>]],
5553    ) -> Result<ColumnCloseResult> {
5554        let column_writer = get_column_writer(
5555            column_descr.clone(),
5556            Default::default(),
5557            get_test_page_writer(),
5558        );
5559        let mut writer = get_typed_column_writer::<T>(column_writer);
5560
5561        for &page in pages {
5562            let values = page.iter().filter_map(Clone::clone).collect::<Vec<_>>();
5563            let def_levels = page
5564                .iter()
5565                .map(|maybe_value| i16::from(maybe_value.is_some()))
5566                .collect::<Vec<_>>();
5567            writer.write_batch(&values, Some(&def_levels), None)?;
5568            writer.flush_data_pages()?;
5569        }
5570
5571        writer.close()
5572    }
5573
5574    /// Performs write-read roundtrip with randomly generated values and levels.
5575    /// `max_size` is maximum number of values or levels (if `max_def_level` > 0) to write
5576    /// for a column.
5577    fn column_roundtrip_random<T: DataType>(
5578        props: WriterProperties,
5579        max_size: usize,
5580        min_value: T::T,
5581        max_value: T::T,
5582        max_def_level: i16,
5583        max_rep_level: i16,
5584    ) where
5585        T::T: PartialOrd + SampleUniform + Copy,
5586    {
5587        let mut num_values: usize = 0;
5588
5589        let mut buf: Vec<i16> = Vec::new();
5590        let def_levels = if max_def_level > 0 {
5591            random_numbers_range(max_size, 0, max_def_level + 1, &mut buf);
5592            for &dl in &buf[..] {
5593                if dl == max_def_level {
5594                    num_values += 1;
5595                }
5596            }
5597            Some(&buf[..])
5598        } else {
5599            num_values = max_size;
5600            None
5601        };
5602
5603        let mut buf: Vec<i16> = Vec::new();
5604        let rep_levels = if max_rep_level > 0 {
5605            random_numbers_range(max_size, 0, max_rep_level + 1, &mut buf);
5606            buf[0] = 0; // Must start on record boundary
5607            Some(&buf[..])
5608        } else {
5609            None
5610        };
5611
5612        let mut values: Vec<T::T> = Vec::new();
5613        random_numbers_range(num_values, min_value, max_value, &mut values);
5614
5615        column_roundtrip::<T>(props, &values[..], def_levels, rep_levels);
5616    }
5617
5618    /// Performs write-read roundtrip and asserts written values and levels.
5619    fn column_roundtrip<T: DataType>(
5620        props: WriterProperties,
5621        values: &[T::T],
5622        def_levels: Option<&[i16]>,
5623        rep_levels: Option<&[i16]>,
5624    ) {
5625        let mut file = tempfile::tempfile().unwrap();
5626        let mut write = TrackedWrite::new(&mut file);
5627        let page_writer = Box::new(SerializedPageWriter::new(&mut write));
5628
5629        let max_def_level = match def_levels {
5630            Some(buf) => *buf.iter().max().unwrap_or(&0i16),
5631            None => 0i16,
5632        };
5633
5634        let max_rep_level = match rep_levels {
5635            Some(buf) => *buf.iter().max().unwrap_or(&0i16),
5636            None => 0i16,
5637        };
5638
5639        let mut max_batch_size = values.len();
5640        if let Some(levels) = def_levels {
5641            max_batch_size = max_batch_size.max(levels.len());
5642        }
5643        if let Some(levels) = rep_levels {
5644            max_batch_size = max_batch_size.max(levels.len());
5645        }
5646
5647        let mut writer =
5648            get_test_column_writer::<T>(page_writer, max_def_level, max_rep_level, Arc::new(props));
5649
5650        let values_written = writer.write_batch(values, def_levels, rep_levels).unwrap();
5651        assert_eq!(values_written, values.len());
5652        let result = writer.close().unwrap();
5653
5654        drop(write);
5655
5656        let props = ReaderProperties::builder()
5657            .set_backward_compatible_lz4(false)
5658            .build();
5659        let page_reader = Box::new(
5660            SerializedPageReader::new_with_properties(
5661                Arc::new(file),
5662                &result.metadata,
5663                result.rows_written as usize,
5664                None,
5665                Arc::new(props),
5666            )
5667            .unwrap(),
5668        );
5669        let mut reader = get_test_column_reader::<T>(page_reader, max_def_level, max_rep_level);
5670
5671        let mut actual_values = Vec::with_capacity(max_batch_size);
5672        let mut actual_def_levels = def_levels.map(|_| Vec::with_capacity(max_batch_size));
5673        let mut actual_rep_levels = rep_levels.map(|_| Vec::with_capacity(max_batch_size));
5674
5675        let (_, values_read, levels_read) = reader
5676            .read_records(
5677                max_batch_size,
5678                actual_def_levels.as_mut(),
5679                actual_rep_levels.as_mut(),
5680                &mut actual_values,
5681            )
5682            .unwrap();
5683
5684        // Assert values, definition and repetition levels.
5685
5686        assert_eq!(&actual_values[..values_read], values);
5687        match actual_def_levels {
5688            Some(ref vec) => assert_eq!(Some(&vec[..levels_read]), def_levels),
5689            None => assert_eq!(None, def_levels),
5690        }
5691        match actual_rep_levels {
5692            Some(ref vec) => assert_eq!(Some(&vec[..levels_read]), rep_levels),
5693            None => assert_eq!(None, rep_levels),
5694        }
5695
5696        // Assert written rows.
5697
5698        if let Some(levels) = actual_rep_levels {
5699            let mut actual_rows_written = 0;
5700            for l in levels {
5701                if l == 0 {
5702                    actual_rows_written += 1;
5703                }
5704            }
5705            assert_eq!(actual_rows_written, result.rows_written);
5706        } else if actual_def_levels.is_some() {
5707            assert_eq!(levels_read as u64, result.rows_written);
5708        } else {
5709            assert_eq!(values_read as u64, result.rows_written);
5710        }
5711    }
5712
5713    /// Performs write of provided values and returns column metadata of those values.
5714    /// Used to test encoding support for column writer.
5715    fn column_write_and_get_metadata<T: DataType>(
5716        props: WriterProperties,
5717        values: &[T::T],
5718    ) -> ColumnChunkMetaData {
5719        let page_writer = get_test_page_writer();
5720        let props = Arc::new(props);
5721        let mut writer = get_test_column_writer::<T>(page_writer, 0, 0, props);
5722        writer.write_batch(values, None, None).unwrap();
5723        writer.close().unwrap().metadata
5724    }
5725
5726    fn column_write_and_get_metadata_with_def_levels<T: DataType>(
5727        props: WriterProperties,
5728        values: &[T::T],
5729        def_levels: &[i16],
5730        max_def_level: i16,
5731    ) -> ColumnChunkMetaData {
5732        let page_writer = get_test_page_writer();
5733        let props = Arc::new(props);
5734        let mut writer = get_test_column_writer::<T>(page_writer, max_def_level, 0, props);
5735        writer.write_batch(values, Some(def_levels), None).unwrap();
5736        writer.close().unwrap().metadata
5737    }
5738
5739    // Helper function to more compactly create a PageEncodingStats struct.
5740    fn encoding_stats(page_type: PageType, encoding: Encoding, count: i32) -> PageEncodingStats {
5741        PageEncodingStats {
5742            page_type,
5743            encoding,
5744            count,
5745        }
5746    }
5747
5748    // Function to use in tests for EncodingWriteSupport. This checks that dictionary
5749    // offset and encodings to make sure that column writer uses provided by trait
5750    // encodings.
5751    fn check_encoding_write_support<T: DataType>(
5752        version: WriterVersion,
5753        dict_enabled: bool,
5754        data: &[T::T],
5755        dictionary_page_offset: Option<i64>,
5756        encodings: &[Encoding],
5757        page_encoding_stats: &[PageEncodingStats],
5758    ) {
5759        let props = WriterProperties::builder()
5760            .set_writer_version(version)
5761            .set_dictionary_enabled(dict_enabled)
5762            .build();
5763        let meta = column_write_and_get_metadata::<T>(props, data);
5764        assert_eq!(meta.dictionary_page_offset(), dictionary_page_offset);
5765        assert_eq!(meta.encodings().collect::<Vec<_>>(), encodings);
5766        assert_eq!(meta.page_encoding_stats().unwrap(), page_encoding_stats);
5767    }
5768
5769    /// Returns column writer.
5770    fn get_test_column_writer<'a, T: DataType>(
5771        page_writer: Box<dyn PageWriter + 'a>,
5772        max_def_level: i16,
5773        max_rep_level: i16,
5774        props: WriterPropertiesPtr,
5775    ) -> ColumnWriterImpl<'a, T> {
5776        let descr = Arc::new(get_test_column_descr::<T>(max_def_level, max_rep_level));
5777        let column_writer = get_column_writer(descr, props, page_writer);
5778        get_typed_column_writer::<T>(column_writer)
5779    }
5780
5781    fn get_test_column_writer_with_path<'a, T: DataType>(
5782        page_writer: Box<dyn PageWriter + 'a>,
5783        max_def_level: i16,
5784        max_rep_level: i16,
5785        props: WriterPropertiesPtr,
5786        path: ColumnPath,
5787    ) -> ColumnWriterImpl<'a, T> {
5788        let descr = Arc::new(get_test_column_descr_with_path::<T>(
5789            max_def_level,
5790            max_rep_level,
5791            path,
5792        ));
5793        let column_writer = get_column_writer(descr, props, page_writer);
5794        get_typed_column_writer::<T>(column_writer)
5795    }
5796
5797    /// Pages collected by [`write_and_collect_pages`].
5798    struct CollectedPages {
5799        /// `(compressed byte size, value count)` for every data page, in order.
5800        data_pages: Vec<(usize, u32)>,
5801        /// Largest dictionary page seen, or 0 if the column wasn't dict-encoded.
5802        dict_page_size: usize,
5803    }
5804
5805    /// Writes `data` (with optional def/rep levels) through a raw
5806    /// `ColumnWriterImpl` configured by `props`, then re-reads the file and
5807    /// returns its page layout. Shared by the page-size regression tests so
5808    /// each only has to express its props, input, and assertions.
5809    fn write_and_collect_pages<T: DataType>(
5810        props: WriterProperties,
5811        max_def_level: i16,
5812        max_rep_level: i16,
5813        data: &[T::T],
5814        def_levels: Option<&[i16]>,
5815        rep_levels: Option<&[i16]>,
5816    ) -> CollectedPages {
5817        let mut file = tempfile::tempfile().unwrap();
5818        let mut write = TrackedWrite::new(&mut file);
5819        let page_writer = Box::new(SerializedPageWriter::new(&mut write));
5820        let mut writer =
5821            get_test_column_writer::<T>(page_writer, max_def_level, max_rep_level, Arc::new(props));
5822        writer.write_batch(data, def_levels, rep_levels).unwrap();
5823        let r = writer.close().unwrap();
5824        drop(write);
5825
5826        let read_props = ReaderProperties::builder()
5827            .set_backward_compatible_lz4(false)
5828            .build();
5829        let mut page_reader = Box::new(
5830            SerializedPageReader::new_with_properties(
5831                Arc::new(file),
5832                &r.metadata,
5833                r.rows_written as usize,
5834                None,
5835                Arc::new(read_props),
5836            )
5837            .unwrap(),
5838        );
5839
5840        let mut collected = CollectedPages {
5841            data_pages: Vec::new(),
5842            dict_page_size: 0,
5843        };
5844        while let Some(page) = page_reader.get_next_page().unwrap() {
5845            match page.page_type() {
5846                PageType::DATA_PAGE | PageType::DATA_PAGE_V2 => {
5847                    collected
5848                        .data_pages
5849                        .push((page.buffer().len(), page.num_values()));
5850                }
5851                PageType::DICTIONARY_PAGE => {
5852                    collected.dict_page_size = collected.dict_page_size.max(page.buffer().len());
5853                }
5854                PageType::INDEX_PAGE => {}
5855            }
5856        }
5857        collected
5858    }
5859
5860    /// Returns column reader.
5861    fn get_test_column_reader<T: DataType>(
5862        page_reader: Box<dyn PageReader>,
5863        max_def_level: i16,
5864        max_rep_level: i16,
5865    ) -> ColumnReaderImpl<T> {
5866        let descr = Arc::new(get_test_column_descr::<T>(max_def_level, max_rep_level));
5867        let column_reader = get_column_reader(descr, page_reader);
5868        get_typed_column_reader::<T>(column_reader)
5869    }
5870
5871    /// Returns descriptor for primitive column.
5872    fn get_test_column_descr<T: DataType>(
5873        max_def_level: i16,
5874        max_rep_level: i16,
5875    ) -> ColumnDescriptor {
5876        let path = ColumnPath::from("col");
5877        let type_ = SchemaType::primitive_type_builder("col", T::get_physical_type())
5878            // length is set for "encoding support" tests for FIXED_LEN_BYTE_ARRAY type,
5879            // it should be no-op for other types
5880            .with_length(1)
5881            .build()
5882            .unwrap();
5883        ColumnDescriptor::new(Arc::new(type_), max_def_level, max_rep_level, path)
5884    }
5885
5886    fn get_test_column_descr_with_path<T: DataType>(
5887        max_def_level: i16,
5888        max_rep_level: i16,
5889        path: ColumnPath,
5890    ) -> ColumnDescriptor {
5891        let name = path.string();
5892        let type_ = SchemaType::primitive_type_builder(&name, T::get_physical_type())
5893            // length is set for "encoding support" tests for FIXED_LEN_BYTE_ARRAY type,
5894            // it should be no-op for other types
5895            .with_length(1)
5896            .build()
5897            .unwrap();
5898        ColumnDescriptor::new(Arc::new(type_), max_def_level, max_rep_level, path)
5899    }
5900
5901    fn write_and_collect_page_values(
5902        path: ColumnPath,
5903        props: WriterPropertiesPtr,
5904        data: &[i32],
5905    ) -> Vec<u32> {
5906        let mut file = tempfile::tempfile().unwrap();
5907        let mut write = TrackedWrite::new(&mut file);
5908        let page_writer = Box::new(SerializedPageWriter::new(&mut write));
5909        let mut writer =
5910            get_test_column_writer_with_path::<Int32Type>(page_writer, 0, 0, props, path);
5911        writer.write_batch(data, None, None).unwrap();
5912        let r = writer.close().unwrap();
5913
5914        drop(write);
5915
5916        let props = ReaderProperties::builder()
5917            .set_backward_compatible_lz4(false)
5918            .build();
5919        let mut page_reader = Box::new(
5920            SerializedPageReader::new_with_properties(
5921                Arc::new(file),
5922                &r.metadata,
5923                r.rows_written as usize,
5924                None,
5925                Arc::new(props),
5926            )
5927            .unwrap(),
5928        );
5929
5930        let mut values_per_page = Vec::new();
5931        while let Some(page) = page_reader.get_next_page().unwrap() {
5932            assert_eq!(page.page_type(), PageType::DATA_PAGE);
5933            values_per_page.push(page.num_values());
5934        }
5935
5936        values_per_page
5937    }
5938
5939    /// Returns page writer that collects pages without serializing them.
5940    fn get_test_page_writer() -> Box<dyn PageWriter> {
5941        Box::new(TestPageWriter::default())
5942    }
5943
5944    #[derive(Default)]
5945    struct TestPageWriter {
5946        bytes_written: u64,
5947    }
5948
5949    impl PageWriter for TestPageWriter {
5950        fn write_page(&mut self, page: CompressedPage) -> Result<PageWriteSpec> {
5951            let mut res = PageWriteSpec::new();
5952            res.page_type = page.page_type();
5953            res.uncompressed_size = page.uncompressed_size();
5954            res.compressed_size = page.compressed_size();
5955            res.num_values = page.num_values();
5956            res.offset = self.bytes_written;
5957            res.bytes_written = page.data().len() as u64;
5958            self.bytes_written += res.bytes_written;
5959            Ok(res)
5960        }
5961
5962        fn close(&mut self) -> Result<()> {
5963            Ok(())
5964        }
5965    }
5966
5967    /// Write data into parquet using [`get_test_page_writer`] and [`get_test_column_writer`] and returns generated statistics.
5968    fn statistics_roundtrip<T: DataType>(values: &[<T as DataType>::T]) -> Statistics {
5969        let page_writer = get_test_page_writer();
5970        let props = Default::default();
5971        let mut writer = get_test_column_writer::<T>(page_writer, 0, 0, props);
5972        writer.write_batch(values, None, None).unwrap();
5973
5974        let metadata = writer.close().unwrap().metadata;
5975        if let Some(stats) = metadata.statistics() {
5976            stats.clone()
5977        } else {
5978            panic!("metadata missing statistics");
5979        }
5980    }
5981
5982    /// Returns Decimals column writer.
5983    fn get_test_decimals_column_writer<T: DataType>(
5984        page_writer: Box<dyn PageWriter>,
5985        max_def_level: i16,
5986        max_rep_level: i16,
5987        props: WriterPropertiesPtr,
5988    ) -> ColumnWriterImpl<'static, T> {
5989        let descr = Arc::new(get_test_decimals_column_descr::<T>(
5990            max_def_level,
5991            max_rep_level,
5992        ));
5993        let column_writer = get_column_writer(descr, props, page_writer);
5994        get_typed_column_writer::<T>(column_writer)
5995    }
5996
5997    /// Returns descriptor for Decimal type with primitive column.
5998    fn get_test_decimals_column_descr<T: DataType>(
5999        max_def_level: i16,
6000        max_rep_level: i16,
6001    ) -> ColumnDescriptor {
6002        let path = ColumnPath::from("col");
6003        let type_ = SchemaType::primitive_type_builder("col", T::get_physical_type())
6004            .with_length(16)
6005            .with_logical_type(Some(LogicalType::decimal(2, 3)))
6006            .with_scale(2)
6007            .with_precision(3)
6008            .build()
6009            .unwrap();
6010        ColumnDescriptor::new(Arc::new(type_), max_def_level, max_rep_level, path)
6011    }
6012
6013    fn float16_statistics_roundtrip(
6014        values: &[FixedLenByteArray],
6015    ) -> ValueStatistics<FixedLenByteArray> {
6016        let page_writer = get_test_page_writer();
6017        let mut writer = get_test_float16_column_writer(page_writer, Default::default());
6018        writer.write_batch(values, None, None).unwrap();
6019
6020        let metadata = writer.close().unwrap().metadata;
6021        if let Some(Statistics::FixedLenByteArray(stats)) = metadata.statistics() {
6022            stats.clone()
6023        } else {
6024            panic!("metadata missing statistics");
6025        }
6026    }
6027
6028    fn get_test_float16_column_writer(
6029        page_writer: Box<dyn PageWriter>,
6030        props: WriterPropertiesPtr,
6031    ) -> ColumnWriterImpl<'static, FixedLenByteArrayType> {
6032        let descr = Arc::new(get_test_float16_column_descr(0, 0));
6033        let column_writer = get_column_writer(descr, props, page_writer);
6034        get_typed_column_writer::<FixedLenByteArrayType>(column_writer)
6035    }
6036
6037    fn get_test_float16_column_descr(max_def_level: i16, max_rep_level: i16) -> ColumnDescriptor {
6038        let path = ColumnPath::from("col");
6039        let type_ =
6040            SchemaType::primitive_type_builder("col", FixedLenByteArrayType::get_physical_type())
6041                .with_length(2)
6042                .with_logical_type(Some(LogicalType::Float16))
6043                .build()
6044                .unwrap();
6045        ColumnDescriptor::new(Arc::new(type_), max_def_level, max_rep_level, path)
6046    }
6047
6048    fn get_test_interval_column_writer(
6049        page_writer: Box<dyn PageWriter>,
6050    ) -> ColumnWriterImpl<'static, FixedLenByteArrayType> {
6051        let descr = Arc::new(get_test_interval_column_descr());
6052        let column_writer = get_column_writer(descr, Default::default(), page_writer);
6053        get_typed_column_writer::<FixedLenByteArrayType>(column_writer)
6054    }
6055
6056    fn get_test_interval_column_descr() -> ColumnDescriptor {
6057        let path = ColumnPath::from("col");
6058        let type_ =
6059            SchemaType::primitive_type_builder("col", FixedLenByteArrayType::get_physical_type())
6060                .with_length(12)
6061                .with_converted_type(ConvertedType::INTERVAL)
6062                .build()
6063                .unwrap();
6064        ColumnDescriptor::new(Arc::new(type_), 0, 0, path)
6065    }
6066
6067    /// Returns column writer for UINT32 Column provided as ConvertedType only
6068    fn get_test_unsigned_int_given_as_converted_column_writer<'a, T: DataType>(
6069        page_writer: Box<dyn PageWriter + 'a>,
6070        max_def_level: i16,
6071        max_rep_level: i16,
6072        props: WriterPropertiesPtr,
6073    ) -> ColumnWriterImpl<'a, T> {
6074        let descr = Arc::new(get_test_converted_type_unsigned_integer_column_descr::<T>(
6075            max_def_level,
6076            max_rep_level,
6077        ));
6078        let column_writer = get_column_writer(descr, props, page_writer);
6079        get_typed_column_writer::<T>(column_writer)
6080    }
6081
6082    /// Returns column descriptor for UINT32 Column provided as ConvertedType only
6083    fn get_test_converted_type_unsigned_integer_column_descr<T: DataType>(
6084        max_def_level: i16,
6085        max_rep_level: i16,
6086    ) -> ColumnDescriptor {
6087        let path = ColumnPath::from("col");
6088        let type_ = SchemaType::primitive_type_builder("col", T::get_physical_type())
6089            .with_converted_type(ConvertedType::UINT_32)
6090            .build()
6091            .unwrap();
6092        ColumnDescriptor::new(Arc::new(type_), max_def_level, max_rep_level, path)
6093    }
6094
6095    #[test]
6096    fn test_page_v2_snappy_compression_fallback() {
6097        // Test that PageV2 sets is_compressed to false when Snappy compression increases data size
6098        let page_writer = TestPageWriter::default();
6099
6100        // Create WriterProperties with PageV2 and Snappy compression
6101        let props = WriterProperties::builder()
6102            .set_writer_version(WriterVersion::PARQUET_2_0)
6103            // Disable dictionary to ensure data is written directly
6104            .set_dictionary_enabled(false)
6105            .set_compression(Compression::SNAPPY)
6106            .build();
6107
6108        let mut column_writer =
6109            get_test_column_writer::<ByteArrayType>(Box::new(page_writer), 0, 0, Arc::new(props));
6110
6111        // Create small, simple data that Snappy compression will likely increase in size
6112        // due to compression overhead for very small data
6113        let values = vec![ByteArray::from("a")];
6114
6115        column_writer.write_batch(&values, None, None).unwrap();
6116
6117        let result = column_writer.close().unwrap();
6118        assert_eq!(
6119            result.metadata.uncompressed_size(),
6120            result.metadata.compressed_size()
6121        );
6122    }
6123
6124    struct ColumnRoundTripUniform<'a, T: DataType> {
6125        props: WriterProperties,
6126        values: &'a [T::T],
6127        def_levels: LevelDataRef<'a>,
6128        rep_levels: LevelDataRef<'a>,
6129        max_def_level: i16,
6130        max_rep_level: i16,
6131        expected_values: &'a [T::T],
6132        expected_def_levels: Option<&'a [i16]>,
6133        expected_rep_levels: Option<&'a [i16]>,
6134    }
6135
6136    impl<'a, T: DataType> ColumnRoundTripUniform<'a, T>
6137    where
6138        T::T: PartialEq + std::fmt::Debug,
6139    {
6140        fn new() -> Self {
6141            Self {
6142                props: Default::default(),
6143                values: &[],
6144                def_levels: LevelDataRef::Absent,
6145                rep_levels: LevelDataRef::Absent,
6146                max_def_level: 0,
6147                max_rep_level: 0,
6148                expected_values: &[],
6149                expected_def_levels: None,
6150                expected_rep_levels: None,
6151            }
6152        }
6153
6154        fn with_props(mut self, props: WriterProperties) -> Self {
6155            self.props = props;
6156            self
6157        }
6158
6159        fn with_values(mut self, values: &'a [T::T]) -> Self {
6160            self.values = values;
6161            self
6162        }
6163
6164        fn with_def_levels(mut self, def_levels: LevelDataRef<'a>) -> Self {
6165            self.def_levels = def_levels;
6166            self
6167        }
6168
6169        fn with_rep_levels(mut self, rep_levels: LevelDataRef<'a>) -> Self {
6170            self.rep_levels = rep_levels;
6171            self
6172        }
6173
6174        fn with_max_def_level(mut self, max_def_level: i16) -> Self {
6175            self.max_def_level = max_def_level;
6176            self
6177        }
6178
6179        fn with_max_rep_level(mut self, max_rep_level: i16) -> Self {
6180            self.max_rep_level = max_rep_level;
6181            self
6182        }
6183
6184        fn with_expected_values(mut self, expected_values: &'a [T::T]) -> Self {
6185            self.expected_values = expected_values;
6186            self
6187        }
6188
6189        fn with_expected_def_levels(mut self, expected_def_levels: &'a [i16]) -> Self {
6190            self.expected_def_levels = Some(expected_def_levels);
6191            self
6192        }
6193
6194        fn with_expected_rep_levels(mut self, expected_rep_levels: &'a [i16]) -> Self {
6195            self.expected_rep_levels = Some(expected_rep_levels);
6196            self
6197        }
6198
6199        /// Write-then-read roundtrip using `write_batch_internal` with the given
6200        /// [`LevelDataRef`] variants, and assert the read-back matches `expected_*`.
6201        fn run(self) {
6202            let mut file = tempfile::tempfile().unwrap();
6203            let mut write = TrackedWrite::new(&mut file);
6204            let page_writer = Box::new(SerializedPageWriter::new(&mut write));
6205            let mut writer = get_test_column_writer::<T>(
6206                page_writer,
6207                self.max_def_level,
6208                self.max_rep_level,
6209                Arc::new(self.props),
6210            );
6211
6212            writer
6213                .write_batch_internal(
6214                    self.values,
6215                    None,
6216                    self.def_levels,
6217                    self.rep_levels,
6218                    None,
6219                    None,
6220                    None,
6221                )
6222                .unwrap();
6223            let result = writer.close().unwrap();
6224            drop(write);
6225
6226            let props = ReaderProperties::builder()
6227                .set_backward_compatible_lz4(false)
6228                .build();
6229            let page_reader = Box::new(
6230                SerializedPageReader::new_with_properties(
6231                    Arc::new(file),
6232                    &result.metadata,
6233                    result.rows_written as usize,
6234                    None,
6235                    Arc::new(props),
6236                )
6237                .unwrap(),
6238            );
6239            let mut reader =
6240                get_test_column_reader::<T>(page_reader, self.max_def_level, self.max_rep_level);
6241
6242            let batch_size = self
6243                .expected_def_levels
6244                .map_or(self.expected_values.len(), |l| l.len());
6245            let mut actual_values = Vec::with_capacity(batch_size);
6246            let mut actual_def = self
6247                .expected_def_levels
6248                .map(|_| Vec::with_capacity(batch_size));
6249            let mut actual_rep = self
6250                .expected_rep_levels
6251                .map(|_| Vec::with_capacity(batch_size));
6252
6253            let (_, values_read, levels_read) = reader
6254                .read_records(
6255                    batch_size,
6256                    actual_def.as_mut(),
6257                    actual_rep.as_mut(),
6258                    &mut actual_values,
6259                )
6260                .unwrap();
6261
6262            assert_eq!(&actual_values[..values_read], self.expected_values);
6263            if let Some(ref v) = actual_def {
6264                assert_eq!(&v[..levels_read], self.expected_def_levels.unwrap());
6265            }
6266            if let Some(ref v) = actual_rep {
6267                assert_eq!(&v[..levels_read], self.expected_rep_levels.unwrap());
6268            }
6269        }
6270    }
6271
6272    #[test]
6273    fn test_level_data_ref_value_count() {
6274        // `value_count` is what the byte-budget chunker uses to convert a
6275        // chunk's level span into a leaf-value count. It must work for any
6276        // column shape — flat, nullable, or nested — because the leaf
6277        // values array is decoupled from the rep/def level stream.
6278        let max_def = 2;
6279        // Non-nullable / unrepeated: no def levels materialized — every
6280        // level is a value.
6281        assert_eq!(LevelDataRef::Absent.value_count(64, max_def), 64);
6282        // Uniform run of present values, and of nulls.
6283        assert_eq!(
6284            LevelDataRef::Uniform {
6285                value: max_def,
6286                count: 40
6287            }
6288            .value_count(40, max_def),
6289            40
6290        );
6291        assert_eq!(
6292            LevelDataRef::Uniform {
6293                value: max_def - 1,
6294                count: 40
6295            }
6296            .value_count(40, max_def),
6297            0
6298        );
6299        // Materialized def levels (nullable / nested): only levels equal to
6300        // `max_def` are values; empty-list / null levels are not.
6301        let levels = [2i16, 0, 2, 1, 2, 2, 0];
6302        assert_eq!(
6303            LevelDataRef::Materialized(&levels).value_count(levels.len(), max_def),
6304            4
6305        );
6306    }
6307
6308    #[test]
6309    fn test_uniform_def_levels_all_null() {
6310        // All-null column: def_level=0 (null) for every slot, no values written.
6311        let max_def_level = 1;
6312        let count = 100;
6313        let expected_def_levels = vec![0i16; count];
6314        ColumnRoundTripUniform::<Int32Type>::new()
6315            .with_def_levels(LevelDataRef::Uniform { value: 0, count })
6316            .with_max_def_level(max_def_level)
6317            .with_expected_def_levels(&expected_def_levels)
6318            .run();
6319    }
6320
6321    #[test]
6322    fn test_uniform_def_levels_all_valid() {
6323        // All-valid column: def_level=max for every slot, all values written.
6324        let max_def_level = 1;
6325        let values: Vec<i32> = (0..50).collect();
6326        let expected_def_levels = vec![max_def_level; values.len()];
6327        ColumnRoundTripUniform::<Int32Type>::new()
6328            .with_values(&values)
6329            .with_def_levels(LevelDataRef::Uniform {
6330                value: max_def_level,
6331                count: values.len(),
6332            })
6333            .with_max_def_level(max_def_level)
6334            .with_expected_values(&values)
6335            .with_expected_def_levels(&expected_def_levels)
6336            .run();
6337    }
6338
6339    #[test]
6340    fn test_uniform_def_and_rep_levels() {
6341        // Simulates a list column where every row is null:
6342        // def=0, rep=0 for each row (one row = one entry with no child values).
6343        let max_def_level = 2;
6344        let max_rep_level = 1;
6345        let count = 200;
6346        let expected_def_levels = vec![0i16; count];
6347        let expected_rep_levels = vec![0i16; count];
6348        ColumnRoundTripUniform::<Int32Type>::new()
6349            .with_def_levels(LevelDataRef::Uniform { value: 0, count })
6350            .with_rep_levels(LevelDataRef::Uniform { value: 0, count })
6351            .with_max_def_level(max_def_level)
6352            .with_max_rep_level(max_rep_level)
6353            .with_expected_def_levels(&expected_def_levels)
6354            .with_expected_rep_levels(&expected_rep_levels)
6355            .run();
6356    }
6357
6358    #[test]
6359    fn test_uniform_levels_v1_and_v2() {
6360        // Verify uniform levels work identically for both Parquet writer versions.
6361        for version in [WriterVersion::PARQUET_1_0, WriterVersion::PARQUET_2_0] {
6362            let props = WriterProperties::builder()
6363                .set_writer_version(version)
6364                .build();
6365            let max_def = 1;
6366            let count = 100;
6367            let expected_def_levels = vec![0i16; count];
6368            ColumnRoundTripUniform::<Int32Type>::new()
6369                .with_props(props)
6370                .with_def_levels(LevelDataRef::Uniform { value: 0, count })
6371                .with_max_def_level(max_def)
6372                .with_expected_def_levels(&expected_def_levels)
6373                .run();
6374        }
6375    }
6376}