parquet/file/
statistics.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Contains definitions for working with Parquet statistics.
6//!
7//! Though some common methods are available on enum, use pattern match to extract
8//! actual min and max values from statistics, see below:
9//!
10//! # Examples
11//! ```rust
12//! use parquet::file::statistics::Statistics;
13//!
14//! let stats = Statistics::int32(Some(1), Some(10), None, Some(3), true);
15//! assert_eq!(stats.null_count_opt(), Some(3));
16//! assert!(stats.is_min_max_deprecated());
17//! assert!(stats.min_is_exact());
18//! assert!(stats.max_is_exact());
19//!
20//! match stats {
21//!     Statistics::Int32(ref typed) => {
22//!         assert_eq!(typed.min_opt(), Some(&1));
23//!         assert_eq!(typed.max_opt(), Some(&10));
24//!     }
25//!     _ => {}
26//! }
27//! ```
28
29use std::fmt;
30
31use crate::basic::Type;
32use crate::data_type::private::ParquetValueType;
33use crate::data_type::*;
34use crate::errors::{ParquetError, Result};
35use crate::file::metadata::thrift::PageStatistics;
36use crate::util::bit_util::FromBytes;
37
38pub(crate) mod private {
39    use super::*;
40
41    pub trait MakeStatistics {
42        fn make_statistics(statistics: ValueStatistics<Self>) -> Statistics
43        where
44            Self: Sized;
45    }
46
47    macro_rules! gen_make_statistics {
48        ($value_ty:ty, $stat:ident) => {
49            impl MakeStatistics for $value_ty {
50                fn make_statistics(statistics: ValueStatistics<Self>) -> Statistics
51                where
52                    Self: Sized,
53                {
54                    Statistics::$stat(statistics)
55                }
56            }
57        };
58    }
59
60    gen_make_statistics!(bool, Boolean);
61    gen_make_statistics!(i32, Int32);
62    gen_make_statistics!(i64, Int64);
63    gen_make_statistics!(Int96, Int96);
64    gen_make_statistics!(f32, Float);
65    gen_make_statistics!(f64, Double);
66    gen_make_statistics!(ByteArray, ByteArray);
67    gen_make_statistics!(FixedLenByteArray, FixedLenByteArray);
68}
69
70/// Macro to generate methods to create Statistics.
71macro_rules! statistics_new_func {
72    ($func:ident, $vtype:ty, $stat:ident) => {
73        #[doc = concat!("Creates new statistics for `", stringify!($stat), "` column type.")]
74        pub fn $func(
75            min: $vtype,
76            max: $vtype,
77            distinct: Option<u64>,
78            nulls: Option<u64>,
79            is_deprecated: bool,
80        ) -> Self {
81            Statistics::$stat(ValueStatistics::new(
82                min,
83                max,
84                distinct,
85                nulls,
86                is_deprecated,
87            ))
88        }
89    };
90}
91
92// Macro to generate getter functions for Statistics.
93macro_rules! statistics_enum_func {
94    ($self:ident, $func:ident) => {{
95        match *$self {
96            Statistics::Boolean(ref typed) => typed.$func(),
97            Statistics::Int32(ref typed) => typed.$func(),
98            Statistics::Int64(ref typed) => typed.$func(),
99            Statistics::Int96(ref typed) => typed.$func(),
100            Statistics::Float(ref typed) => typed.$func(),
101            Statistics::Double(ref typed) => typed.$func(),
102            Statistics::ByteArray(ref typed) => typed.$func(),
103            Statistics::FixedLenByteArray(ref typed) => typed.$func(),
104        }
105    }};
106}
107
108/// Converts Thrift definition into `Statistics`.
109pub(crate) fn from_thrift_page_stats(
110    physical_type: Type,
111    thrift_stats: Option<PageStatistics>,
112) -> Result<Option<Statistics>> {
113    Ok(match thrift_stats {
114        Some(stats) => {
115            // Generic null count.
116            let null_count = stats
117                .null_count
118                .map(|null_count| {
119                    if null_count < 0 {
120                        return Err(ParquetError::General(format!(
121                            "Statistics null count is negative {null_count}",
122                        )));
123                    }
124                    Ok(null_count as u64)
125                })
126                .transpose()?;
127            // Generic distinct count (count of distinct values occurring)
128            let distinct_count = stats.distinct_count.map(|value| value as u64);
129            // Generic nan count for floating point types
130            let nan_count = stats
131                .nan_count
132                .map(|nan_count| {
133                    if nan_count < 0 {
134                        return Err(ParquetError::General(format!(
135                            "Statistics NaN count is negative {nan_count}",
136                        )));
137                    }
138                    Ok(nan_count as u64)
139                })
140                .transpose()?;
141            // Whether or not statistics use deprecated min/max fields.
142            let old_format = stats.min_value.is_none() && stats.max_value.is_none();
143            // Generic min value as bytes.
144            let min = if old_format {
145                stats.min
146            } else {
147                stats.min_value
148            };
149            // Generic max value as bytes.
150            let max = if old_format {
151                stats.max
152            } else {
153                stats.max_value
154            };
155
156            fn check_len(min: Option<&[u8]>, max: Option<&[u8]>, len: usize) -> Result<()> {
157                if let Some(min) = min
158                    && min.len() < len
159                {
160                    return Err(ParquetError::General(
161                        "Insufficient bytes to parse min statistic".to_string(),
162                    ));
163                }
164                if let Some(max) = max
165                    && max.len() < len
166                {
167                    return Err(ParquetError::General(
168                        "Insufficient bytes to parse max statistic".to_string(),
169                    ));
170                }
171                Ok(())
172            }
173
174            {
175                let (min, max) = (min.as_deref(), max.as_deref());
176                match physical_type {
177                    Type::BOOLEAN => check_len(min, max, 1),
178                    Type::INT32 | Type::FLOAT => check_len(min, max, 4),
179                    Type::INT64 | Type::DOUBLE => check_len(min, max, 8),
180                    Type::INT96 => check_len(min, max, 12),
181                    _ => Ok(()),
182                }?;
183            }
184
185            // Values are encoded using PLAIN encoding definition, except that
186            // variable-length byte arrays do not include a length prefix.
187            //
188            // Instead of using actual decoder, we manually convert values.
189            let res = match physical_type {
190                Type::BOOLEAN => Statistics::boolean(
191                    min.map(|data| data[0] != 0),
192                    max.map(|data| data[0] != 0),
193                    distinct_count,
194                    null_count,
195                    old_format,
196                ),
197                Type::INT32 => Statistics::int32(
198                    min.map(|data| i32::from_le_bytes(data[..4].try_into().unwrap())),
199                    max.map(|data| i32::from_le_bytes(data[..4].try_into().unwrap())),
200                    distinct_count,
201                    null_count,
202                    old_format,
203                ),
204                Type::INT64 => Statistics::int64(
205                    min.map(|data| i64::from_le_bytes(data[..8].try_into().unwrap())),
206                    max.map(|data| i64::from_le_bytes(data[..8].try_into().unwrap())),
207                    distinct_count,
208                    null_count,
209                    old_format,
210                ),
211                Type::INT96 => {
212                    // INT96 statistics may not be correct, because comparison is signed
213                    let min = if let Some(data) = min {
214                        if data.len() != 12 {
215                            return Err(ParquetError::General(
216                                "Incorrect Int96 min statistics".to_string(),
217                            ));
218                        }
219                        Some(Int96::try_from_le_slice(&data)?)
220                    } else {
221                        None
222                    };
223                    let max = if let Some(data) = max {
224                        if data.len() != 12 {
225                            return Err(ParquetError::General(
226                                "Incorrect Int96 max statistics".to_string(),
227                            ));
228                        }
229                        Some(Int96::try_from_le_slice(&data)?)
230                    } else {
231                        None
232                    };
233                    Statistics::int96(min, max, distinct_count, null_count, old_format)
234                }
235                Type::FLOAT => Statistics::Float(
236                    ValueStatistics::new(
237                        min.map(|data| f32::from_le_bytes(data[..4].try_into().unwrap())),
238                        max.map(|data| f32::from_le_bytes(data[..4].try_into().unwrap())),
239                        distinct_count,
240                        null_count,
241                        old_format,
242                    )
243                    .with_nan_count(nan_count)
244                    .with_max_is_exact(stats.is_max_value_exact.unwrap_or(false))
245                    .with_min_is_exact(stats.is_min_value_exact.unwrap_or(false)),
246                ),
247                Type::DOUBLE => Statistics::Double(
248                    ValueStatistics::new(
249                        min.map(|data| f64::from_le_bytes(data[..8].try_into().unwrap())),
250                        max.map(|data| f64::from_le_bytes(data[..8].try_into().unwrap())),
251                        distinct_count,
252                        null_count,
253                        old_format,
254                    )
255                    .with_nan_count(nan_count)
256                    .with_max_is_exact(stats.is_max_value_exact.unwrap_or(false))
257                    .with_min_is_exact(stats.is_min_value_exact.unwrap_or(false)),
258                ),
259                Type::BYTE_ARRAY => Statistics::ByteArray(
260                    ValueStatistics::new(
261                        min.map(ByteArray::from),
262                        max.map(ByteArray::from),
263                        distinct_count,
264                        null_count,
265                        old_format,
266                    )
267                    .with_max_is_exact(stats.is_max_value_exact.unwrap_or(false))
268                    .with_min_is_exact(stats.is_min_value_exact.unwrap_or(false)),
269                ),
270                Type::FIXED_LEN_BYTE_ARRAY => Statistics::FixedLenByteArray(
271                    ValueStatistics::new(
272                        min.map(ByteArray::from).map(FixedLenByteArray::from),
273                        max.map(ByteArray::from).map(FixedLenByteArray::from),
274                        distinct_count,
275                        null_count,
276                        old_format,
277                    )
278                    // Note: We set nan_count here even though we can't verify if this is Float16.
279                    // The spec says nan_count should only be set for Float16 logical type,
280                    // but this function doesn't have access to logical type information.
281                    // Writers should only set nan_count for Float16, and readers should
282                    // handle this gracefully.
283                    .with_nan_count(nan_count)
284                    .with_max_is_exact(stats.is_max_value_exact.unwrap_or(false))
285                    .with_min_is_exact(stats.is_min_value_exact.unwrap_or(false)),
286                ),
287            };
288
289            Some(res)
290        }
291        None => None,
292    })
293}
294
295/// Convert Statistics into Thrift definition.
296pub(crate) fn page_stats_to_thrift(stats: Option<&Statistics>) -> Option<PageStatistics> {
297    let stats = stats?;
298
299    // record null count if it can fit in i64
300    let null_count = stats
301        .null_count_opt()
302        .and_then(|value| i64::try_from(value).ok());
303
304    // record distinct count if it can fit in i64
305    let distinct_count = stats
306        .distinct_count_opt()
307        .and_then(|value| i64::try_from(value).ok());
308
309    // record nan count if it can fit in i64
310    let nan_count = stats
311        .nan_count_opt()
312        .and_then(|value| i64::try_from(value).ok());
313
314    let mut thrift_stats = PageStatistics {
315        max: None,
316        min: None,
317        null_count,
318        distinct_count,
319        max_value: None,
320        min_value: None,
321        is_max_value_exact: None,
322        is_min_value_exact: None,
323        nan_count,
324    };
325
326    // Get min/max if set.
327    let (min, max, min_exact, max_exact) = (
328        stats.min_bytes_opt().map(|x| x.to_vec()),
329        stats.max_bytes_opt().map(|x| x.to_vec()),
330        Some(stats.min_is_exact()),
331        Some(stats.max_is_exact()),
332    );
333    if stats.is_min_max_backwards_compatible() {
334        // Copy to deprecated min, max values for compatibility with older readers
335        thrift_stats.min.clone_from(&min);
336        thrift_stats.max.clone_from(&max);
337    }
338
339    if !stats.is_min_max_deprecated() {
340        thrift_stats.min_value = min;
341        thrift_stats.max_value = max;
342    }
343
344    thrift_stats.is_min_value_exact = min_exact;
345    thrift_stats.is_max_value_exact = max_exact;
346
347    Some(thrift_stats)
348}
349
350/// Strongly typed statistics for a column chunk within a row group.
351///
352/// This structure is a natively typed, in memory representation of the thrift
353/// `Statistics` structure in a Parquet file footer. The statistics stored in
354/// this structure can be used by query engines to skip decoding pages while
355/// reading parquet data.
356///
357/// Page level statistics are stored separately, in [ColumnIndexMetaData].
358///
359/// [ColumnIndexMetaData]: crate::file::page_index::column_index::ColumnIndexMetaData
360#[derive(Debug, Clone, PartialEq)]
361pub enum Statistics {
362    /// Statistics for Boolean column
363    Boolean(ValueStatistics<bool>),
364    /// Statistics for Int32 column
365    Int32(ValueStatistics<i32>),
366    /// Statistics for Int64 column
367    Int64(ValueStatistics<i64>),
368    /// Statistics for Int96 column
369    Int96(ValueStatistics<Int96>),
370    /// Statistics for Float column
371    Float(ValueStatistics<f32>),
372    /// Statistics for Double column
373    Double(ValueStatistics<f64>),
374    /// Statistics for ByteArray column
375    ByteArray(ValueStatistics<ByteArray>),
376    /// Statistics for FixedLenByteArray column
377    FixedLenByteArray(ValueStatistics<FixedLenByteArray>),
378}
379
380impl<T: ParquetValueType> From<ValueStatistics<T>> for Statistics {
381    fn from(t: ValueStatistics<T>) -> Self {
382        T::make_statistics(t)
383    }
384}
385
386impl Statistics {
387    /// Creates new statistics for a column type
388    pub fn new<T: ParquetValueType>(
389        min: Option<T>,
390        max: Option<T>,
391        distinct_count: Option<u64>,
392        null_count: Option<u64>,
393        is_deprecated: bool,
394    ) -> Self {
395        Self::from(ValueStatistics::new(
396            min,
397            max,
398            distinct_count,
399            null_count,
400            is_deprecated,
401        ))
402    }
403
404    statistics_new_func![boolean, Option<bool>, Boolean];
405
406    statistics_new_func![int32, Option<i32>, Int32];
407
408    statistics_new_func![int64, Option<i64>, Int64];
409
410    statistics_new_func![int96, Option<Int96>, Int96];
411
412    statistics_new_func![float, Option<f32>, Float];
413
414    statistics_new_func![double, Option<f64>, Double];
415
416    statistics_new_func![byte_array, Option<ByteArray>, ByteArray];
417
418    statistics_new_func![
419        fixed_len_byte_array,
420        Option<FixedLenByteArray>,
421        FixedLenByteArray
422    ];
423
424    /// Returns `true` if statistics have old `min` and `max` fields set.
425    /// This means that the column order is likely to be undefined, which, for old files
426    /// could mean a signed sort order of values.
427    ///
428    /// Refer to [`ColumnOrder`](crate::basic::ColumnOrder) and
429    /// [`SortOrder`](crate::basic::SortOrder) for more information.
430    pub fn is_min_max_deprecated(&self) -> bool {
431        statistics_enum_func![self, is_min_max_deprecated]
432    }
433
434    /// Old versions of parquet stored statistics in `min` and `max` fields, ordered
435    /// using signed comparison. This resulted in an undefined ordering for unsigned
436    /// quantities, such as booleans and unsigned integers.
437    ///
438    /// These fields were therefore deprecated in favour of `min_value` and `max_value`,
439    /// which have a type-defined sort order.
440    ///
441    /// However, not all readers have been updated. For backwards compatibility, this method
442    /// returns `true` if the statistics within this have a signed sort order, that is
443    /// compatible with being stored in the deprecated `min` and `max` fields
444    pub fn is_min_max_backwards_compatible(&self) -> bool {
445        statistics_enum_func![self, is_min_max_backwards_compatible]
446    }
447
448    /// Returns optional value of number of distinct values occurring.
449    /// When it is `None`, the value should be ignored.
450    pub fn distinct_count_opt(&self) -> Option<u64> {
451        statistics_enum_func![self, distinct_count]
452    }
453
454    /// Returns number of null values for the column, if known.
455    /// Note that this includes all nulls when column is part of the complex type.
456    ///
457    /// Note: Versions of this library prior to `58.1.0` returned `0` if the null count
458    /// was not available. This method now returns `None` in that case.
459    ///
460    /// Also, versions of this library prior to `53.1.0` did not store a null count
461    /// statistic when the null count was `0`.
462    ///
463    /// It is unsound to assume that missing nullcount stats mean the column contains no nulls,
464    /// but code that depends on the old behavior can restore it by defaulting to zero:
465    ///
466    /// ```no_run
467    /// # use parquet::file::statistics::Statistics;
468    /// # let statistics: Statistics = todo!();
469    /// let null_count = statistics.null_count_opt().unwrap_or(0);
470    /// ```
471    pub fn null_count_opt(&self) -> Option<u64> {
472        statistics_enum_func![self, null_count_opt]
473    }
474
475    /// Returns NaN count for floating point types, if known.
476    pub fn nan_count_opt(&self) -> Option<u64> {
477        statistics_enum_func![self, nan_count_opt]
478    }
479
480    /// Returns `true` if the min value is set, and is an exact min value.
481    pub fn min_is_exact(&self) -> bool {
482        statistics_enum_func![self, min_is_exact]
483    }
484
485    /// Returns `true` if the max value is set, and is an exact max value.
486    pub fn max_is_exact(&self) -> bool {
487        statistics_enum_func![self, max_is_exact]
488    }
489
490    /// Returns slice of bytes that represent min value, if min value is known.
491    pub fn min_bytes_opt(&self) -> Option<&[u8]> {
492        statistics_enum_func![self, min_bytes_opt]
493    }
494
495    /// Returns slice of bytes that represent max value, if max value is known.
496    pub fn max_bytes_opt(&self) -> Option<&[u8]> {
497        statistics_enum_func![self, max_bytes_opt]
498    }
499
500    /// Returns physical type associated with statistics.
501    pub fn physical_type(&self) -> Type {
502        match self {
503            Statistics::Boolean(_) => Type::BOOLEAN,
504            Statistics::Int32(_) => Type::INT32,
505            Statistics::Int64(_) => Type::INT64,
506            Statistics::Int96(_) => Type::INT96,
507            Statistics::Float(_) => Type::FLOAT,
508            Statistics::Double(_) => Type::DOUBLE,
509            Statistics::ByteArray(_) => Type::BYTE_ARRAY,
510            Statistics::FixedLenByteArray(_) => Type::FIXED_LEN_BYTE_ARRAY,
511        }
512    }
513}
514
515impl fmt::Display for Statistics {
516    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
517        match self {
518            Statistics::Boolean(typed) => write!(f, "{typed}"),
519            Statistics::Int32(typed) => write!(f, "{typed}"),
520            Statistics::Int64(typed) => write!(f, "{typed}"),
521            Statistics::Int96(typed) => write!(f, "{typed}"),
522            Statistics::Float(typed) => write!(f, "{typed}"),
523            Statistics::Double(typed) => write!(f, "{typed}"),
524            Statistics::ByteArray(typed) => write!(f, "{typed}"),
525            Statistics::FixedLenByteArray(typed) => write!(f, "{typed}"),
526        }
527    }
528}
529
530/// Typed implementation for [`Statistics`].
531pub type TypedStatistics<T> = ValueStatistics<<T as DataType>::T>;
532
533/// Typed statistics for one column chunk
534///
535/// See [`Statistics`] for more details
536#[derive(Clone, Eq, PartialEq)]
537pub struct ValueStatistics<T> {
538    min: Option<T>,
539    max: Option<T>,
540    // Distinct count could be omitted in some cases
541    distinct_count: Option<u64>,
542    null_count: Option<u64>,
543    // NaN count for floating point types
544    nan_count: Option<u64>,
545
546    // Whether or not the min or max values are exact, or truncated.
547    is_max_value_exact: bool,
548    is_min_value_exact: bool,
549
550    /// If `true` populate the deprecated `min` and `max` fields instead of
551    /// `min_value` and `max_value`
552    is_min_max_deprecated: bool,
553
554    /// If `true` the statistics are compatible with the deprecated `min` and
555    /// `max` fields. See [`ValueStatistics::is_min_max_backwards_compatible`]
556    is_min_max_backwards_compatible: bool,
557}
558
559impl<T> ValueStatistics<T> {
560    /// Creates new typed statistics.
561    pub fn new(
562        min: Option<T>,
563        max: Option<T>,
564        distinct_count: Option<u64>,
565        null_count: Option<u64>,
566        is_min_max_deprecated: bool,
567    ) -> Self {
568        Self {
569            is_max_value_exact: max.is_some(),
570            is_min_value_exact: min.is_some(),
571            min,
572            max,
573            distinct_count,
574            null_count,
575            nan_count: None,
576            is_min_max_deprecated,
577            is_min_max_backwards_compatible: is_min_max_deprecated,
578        }
579    }
580
581    /// Returns these statistics with `min` and `max` replaced and every other field kept.
582    pub(crate) fn with_min_max(self, min: Option<T>, max: Option<T>) -> Self {
583        Self { min, max, ..self }
584    }
585
586    /// Set whether the stored `min` field represents the exact
587    /// minimum, or just a bound on the minimum value.
588    ///
589    /// see [`Self::min_is_exact`]
590    pub fn with_min_is_exact(self, is_min_value_exact: bool) -> Self {
591        Self {
592            is_min_value_exact,
593            ..self
594        }
595    }
596
597    /// Set whether the stored `max` field represents the exact
598    /// maximum, or just a bound on the maximum value.
599    ///
600    /// see [`Self::max_is_exact`]
601    pub fn with_max_is_exact(self, is_max_value_exact: bool) -> Self {
602        Self {
603            is_max_value_exact,
604            ..self
605        }
606    }
607
608    /// Set whether to write the deprecated `min` and `max` fields
609    /// for compatibility with older parquet writers
610    ///
611    /// This should only be enabled if the field is signed,
612    /// see [`Self::is_min_max_backwards_compatible`]
613    pub fn with_backwards_compatible_min_max(self, backwards_compatible: bool) -> Self {
614        Self {
615            is_min_max_backwards_compatible: backwards_compatible,
616            ..self
617        }
618    }
619
620    /// Returns NaN count for floating point types.
621    pub fn nan_count_opt(&self) -> Option<u64> {
622        self.nan_count
623    }
624
625    /// Set the NaN count for floating point types.
626    pub fn with_nan_count(self, nan_count: Option<u64>) -> Self {
627        Self { nan_count, ..self }
628    }
629
630    /// Returns min value of the statistics, if known.
631    pub fn min_opt(&self) -> Option<&T> {
632        self.min.as_ref()
633    }
634
635    /// Returns max value of the statistics, if known.
636    pub fn max_opt(&self) -> Option<&T> {
637        self.max.as_ref()
638    }
639
640    /// Whether or not min and max values are set.
641    /// Normally both min/max values will be set to `Some(value)` or `None`.
642    pub(crate) fn _internal_has_min_max_set(&self) -> bool {
643        self.min.is_some() && self.max.is_some()
644    }
645
646    /// Whether or not max value is set, and is an exact value.
647    pub fn max_is_exact(&self) -> bool {
648        self.max.is_some() && self.is_max_value_exact
649    }
650
651    /// Whether or not min value is set, and is an exact value.
652    pub fn min_is_exact(&self) -> bool {
653        self.min.is_some() && self.is_min_value_exact
654    }
655
656    /// Returns optional value of number of distinct values occurring.
657    pub fn distinct_count(&self) -> Option<u64> {
658        self.distinct_count
659    }
660
661    /// Returns null count.
662    pub fn null_count_opt(&self) -> Option<u64> {
663        self.null_count
664    }
665
666    /// Returns `true` if statistics were created using old min/max fields.
667    fn is_min_max_deprecated(&self) -> bool {
668        self.is_min_max_deprecated
669    }
670
671    /// Old versions of parquet stored statistics in `min` and `max` fields, ordered
672    /// using signed comparison. This resulted in an undefined ordering for unsigned
673    /// quantities, such as booleans and unsigned integers.
674    ///
675    /// These fields were therefore deprecated in favour of `min_value` and `max_value`,
676    /// which have a type-defined sort order.
677    ///
678    /// However, not all readers have been updated. For backwards compatibility, this method
679    /// returns `true` if the statistics within this have a signed sort order, that is
680    /// compatible with being stored in the deprecated `min` and `max` fields
681    pub fn is_min_max_backwards_compatible(&self) -> bool {
682        self.is_min_max_backwards_compatible
683    }
684}
685
686impl<T: AsBytes> ValueStatistics<T> {
687    /// Returns min value as bytes of the statistics, if min value is known.
688    pub fn min_bytes_opt(&self) -> Option<&[u8]> {
689        self.min_opt().map(AsBytes::as_bytes)
690    }
691
692    /// Returns max value as bytes of the statistics, if max value is known.
693    pub fn max_bytes_opt(&self) -> Option<&[u8]> {
694        self.max_opt().map(AsBytes::as_bytes)
695    }
696}
697
698impl<T: ParquetValueType> fmt::Display for ValueStatistics<T> {
699    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
700        write!(f, "{{")?;
701        write!(f, "min: ")?;
702        match self.min {
703            Some(ref value) => write!(f, "{value}")?,
704            None => write!(f, "N/A")?,
705        }
706        write!(f, ", max: ")?;
707        match self.max {
708            Some(ref value) => write!(f, "{value}")?,
709            None => write!(f, "N/A")?,
710        }
711        write!(f, ", distinct_count: ")?;
712        match self.distinct_count {
713            Some(value) => write!(f, "{value}")?,
714            None => write!(f, "N/A")?,
715        }
716        write!(f, ", null_count: ")?;
717        match self.null_count {
718            Some(value) => write!(f, "{value}")?,
719            None => write!(f, "N/A")?,
720        }
721        write!(f, ", min_max_deprecated: {}", self.is_min_max_deprecated)?;
722        write!(f, ", max_value_exact: {}", self.is_max_value_exact)?;
723        write!(f, ", min_value_exact: {}", self.is_min_value_exact)?;
724        write!(f, "}}")
725    }
726}
727
728impl<T: ParquetValueType> fmt::Debug for ValueStatistics<T> {
729    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
730        write!(
731            f,
732            "{{min: {:?}, max: {:?}, distinct_count: {:?}, null_count: {:?}, \
733             min_max_deprecated: {}, min_max_backwards_compatible: {}, max_value_exact: {}, min_value_exact: {}}}",
734            self.min,
735            self.max,
736            self.distinct_count,
737            self.null_count,
738            self.is_min_max_deprecated,
739            self.is_min_max_backwards_compatible,
740            self.is_max_value_exact,
741            self.is_min_value_exact
742        )
743    }
744}
745
746#[cfg(test)]
747mod tests {
748    use super::*;
749
750    #[test]
751    fn test_statistics_min_max_bytes() {
752        let stats = Statistics::int32(Some(-123), Some(234), None, Some(1), false);
753        assert_eq!(stats.min_bytes_opt(), Some((-123).as_bytes()));
754        assert_eq!(stats.max_bytes_opt(), Some(234.as_bytes()));
755
756        let stats = Statistics::byte_array(
757            Some(ByteArray::from(vec![1, 2, 3])),
758            Some(ByteArray::from(vec![3, 4, 5])),
759            None,
760            Some(1),
761            true,
762        );
763        assert_eq!(stats.min_bytes_opt().unwrap(), &[1, 2, 3]);
764        assert_eq!(stats.max_bytes_opt().unwrap(), &[3, 4, 5]);
765    }
766
767    #[test]
768    #[should_panic(expected = "General(\"Statistics null count is negative -10\")")]
769    fn test_statistics_negative_null_count() {
770        let thrift_stats = PageStatistics {
771            max: None,
772            min: None,
773            null_count: Some(-10),
774            distinct_count: None,
775            max_value: None,
776            min_value: None,
777            is_max_value_exact: None,
778            is_min_value_exact: None,
779            nan_count: None,
780        };
781
782        from_thrift_page_stats(Type::INT32, Some(thrift_stats)).unwrap();
783    }
784
785    #[test]
786    fn test_statistics_thrift_none() {
787        assert_eq!(from_thrift_page_stats(Type::INT32, None).unwrap(), None);
788        assert_eq!(
789            from_thrift_page_stats(Type::BYTE_ARRAY, None).unwrap(),
790            None
791        );
792    }
793
794    #[test]
795    fn test_statistics_debug() {
796        let stats = Statistics::int32(Some(1), Some(12), None, Some(12), true);
797        assert_eq!(
798            format!("{stats:?}"),
799            "Int32({min: Some(1), max: Some(12), distinct_count: None, null_count: Some(12), \
800             min_max_deprecated: true, min_max_backwards_compatible: true, max_value_exact: true, min_value_exact: true})"
801        );
802
803        let stats = Statistics::int32(None, None, None, Some(7), false);
804        assert_eq!(
805            format!("{stats:?}"),
806            "Int32({min: None, max: None, distinct_count: None, null_count: Some(7), \
807             min_max_deprecated: false, min_max_backwards_compatible: false, max_value_exact: false, min_value_exact: false})"
808        )
809    }
810
811    #[test]
812    fn test_statistics_display() {
813        let stats = Statistics::int32(Some(1), Some(12), None, Some(12), true);
814        assert_eq!(
815            format!("{stats}"),
816            "{min: 1, max: 12, distinct_count: N/A, null_count: 12, min_max_deprecated: true, max_value_exact: true, min_value_exact: true}"
817        );
818
819        let stats = Statistics::int64(None, None, None, Some(7), false);
820        assert_eq!(
821            format!("{stats}"),
822            "{min: N/A, max: N/A, distinct_count: N/A, null_count: 7, min_max_deprecated: \
823             false, max_value_exact: false, min_value_exact: false}"
824        );
825
826        let stats = Statistics::int96(
827            Some(Int96::from(vec![1, 0, 0])),
828            Some(Int96::from(vec![2, 3, 4])),
829            None,
830            Some(3),
831            true,
832        );
833        assert_eq!(
834            format!("{stats}"),
835            "{min: [1, 0, 0], max: [2, 3, 4], distinct_count: N/A, null_count: 3, \
836             min_max_deprecated: true, max_value_exact: true, min_value_exact: true}"
837        );
838
839        let stats = Statistics::ByteArray(
840            ValueStatistics::new(
841                Some(ByteArray::from(vec![1u8])),
842                Some(ByteArray::from(vec![2u8])),
843                Some(5),
844                Some(7),
845                false,
846            )
847            .with_max_is_exact(false)
848            .with_min_is_exact(false),
849        );
850        assert_eq!(
851            format!("{stats}"),
852            "{min: [1], max: [2], distinct_count: 5, null_count: 7, min_max_deprecated: false, max_value_exact: false, min_value_exact: false}"
853        );
854    }
855
856    #[test]
857    fn test_statistics_partial_eq() {
858        let expected = Statistics::int32(Some(12), Some(45), None, Some(11), true);
859
860        assert_eq!(
861            Statistics::int32(Some(12), Some(45), None, Some(11), true),
862            expected
863        );
864        assert_ne!(
865            Statistics::int32(Some(11), Some(45), None, Some(11), true),
866            expected
867        );
868        assert_ne!(
869            Statistics::int32(Some(12), Some(44), None, Some(11), true),
870            expected
871        );
872        assert_ne!(
873            Statistics::int32(Some(12), Some(45), None, Some(23), true),
874            expected
875        );
876        assert_ne!(
877            Statistics::int32(Some(12), Some(45), None, Some(11), false),
878            expected
879        );
880
881        assert_ne!(
882            Statistics::int32(Some(12), Some(45), None, Some(11), false),
883            Statistics::int64(Some(12), Some(45), None, Some(11), false)
884        );
885
886        assert_ne!(
887            Statistics::boolean(Some(false), Some(true), None, None, true),
888            Statistics::double(Some(1.2), Some(4.5), None, None, true)
889        );
890
891        assert_ne!(
892            Statistics::byte_array(
893                Some(ByteArray::from(vec![1, 2, 3])),
894                Some(ByteArray::from(vec![1, 2, 3])),
895                None,
896                None,
897                true
898            ),
899            Statistics::fixed_len_byte_array(
900                Some(ByteArray::from(vec![1, 2, 3]).into()),
901                Some(ByteArray::from(vec![1, 2, 3]).into()),
902                None,
903                None,
904                true,
905            )
906        );
907
908        assert_ne!(
909            Statistics::byte_array(
910                Some(ByteArray::from(vec![1, 2, 3])),
911                Some(ByteArray::from(vec![1, 2, 3])),
912                None,
913                None,
914                true,
915            ),
916            Statistics::ByteArray(
917                ValueStatistics::new(
918                    Some(ByteArray::from(vec![1, 2, 3])),
919                    Some(ByteArray::from(vec![1, 2, 3])),
920                    None,
921                    None,
922                    true,
923                )
924                .with_max_is_exact(false)
925            )
926        );
927
928        assert_ne!(
929            Statistics::fixed_len_byte_array(
930                Some(FixedLenByteArray::from(vec![1, 2, 3])),
931                Some(FixedLenByteArray::from(vec![1, 2, 3])),
932                None,
933                None,
934                true,
935            ),
936            Statistics::FixedLenByteArray(
937                ValueStatistics::new(
938                    Some(FixedLenByteArray::from(vec![1, 2, 3])),
939                    Some(FixedLenByteArray::from(vec![1, 2, 3])),
940                    None,
941                    None,
942                    true,
943                )
944                .with_min_is_exact(false)
945            )
946        );
947    }
948
949    #[test]
950    fn test_statistics_from_thrift() {
951        // Helper method to check statistics conversion.
952        fn check_stats(stats: Statistics) {
953            let type_ = stats.physical_type();
954            let thrift_stats = page_stats_to_thrift(Some(&stats));
955            assert_eq!(
956                from_thrift_page_stats(type_, thrift_stats).unwrap(),
957                Some(stats)
958            );
959        }
960
961        check_stats(Statistics::boolean(
962            Some(false),
963            Some(true),
964            None,
965            Some(7),
966            true,
967        ));
968        check_stats(Statistics::boolean(
969            Some(false),
970            Some(true),
971            None,
972            Some(7),
973            true,
974        ));
975        check_stats(Statistics::boolean(
976            Some(false),
977            Some(true),
978            None,
979            Some(0),
980            false,
981        ));
982        check_stats(Statistics::boolean(
983            Some(true),
984            Some(true),
985            None,
986            Some(7),
987            true,
988        ));
989        check_stats(Statistics::boolean(
990            Some(false),
991            Some(false),
992            None,
993            Some(7),
994            true,
995        ));
996        check_stats(Statistics::boolean(None, None, None, Some(7), true));
997
998        check_stats(Statistics::int32(
999            Some(-100),
1000            Some(500),
1001            None,
1002            Some(7),
1003            true,
1004        ));
1005        check_stats(Statistics::int32(
1006            Some(-100),
1007            Some(500),
1008            None,
1009            Some(0),
1010            false,
1011        ));
1012        check_stats(Statistics::int32(None, None, None, Some(7), true));
1013
1014        check_stats(Statistics::int64(
1015            Some(-100),
1016            Some(200),
1017            None,
1018            Some(7),
1019            true,
1020        ));
1021        check_stats(Statistics::int64(
1022            Some(-100),
1023            Some(200),
1024            None,
1025            Some(0),
1026            false,
1027        ));
1028        check_stats(Statistics::int64(None, None, None, Some(7), true));
1029
1030        check_stats(Statistics::float(Some(1.2), Some(3.4), None, Some(7), true));
1031        check_stats(Statistics::float(
1032            Some(1.2),
1033            Some(3.4),
1034            None,
1035            Some(0),
1036            false,
1037        ));
1038        check_stats(Statistics::float(None, None, None, Some(7), true));
1039
1040        check_stats(Statistics::double(
1041            Some(1.2),
1042            Some(3.4),
1043            None,
1044            Some(7),
1045            true,
1046        ));
1047        check_stats(Statistics::double(
1048            Some(1.2),
1049            Some(3.4),
1050            None,
1051            Some(0),
1052            false,
1053        ));
1054        check_stats(Statistics::double(None, None, None, Some(7), true));
1055
1056        check_stats(Statistics::byte_array(
1057            Some(ByteArray::from(vec![1, 2, 3])),
1058            Some(ByteArray::from(vec![3, 4, 5])),
1059            None,
1060            Some(7),
1061            true,
1062        ));
1063        check_stats(Statistics::byte_array(None, None, None, Some(7), true));
1064
1065        check_stats(Statistics::fixed_len_byte_array(
1066            Some(ByteArray::from(vec![1, 2, 3]).into()),
1067            Some(ByteArray::from(vec![3, 4, 5]).into()),
1068            None,
1069            Some(7),
1070            true,
1071        ));
1072        check_stats(Statistics::fixed_len_byte_array(
1073            None,
1074            None,
1075            None,
1076            Some(7),
1077            true,
1078        ));
1079    }
1080
1081    #[test]
1082    fn test_count_encoding() {
1083        statistics_count_test(None, None);
1084        statistics_count_test(Some(0), Some(0));
1085        statistics_count_test(Some(100), Some(2000));
1086        statistics_count_test(Some(1), None);
1087        statistics_count_test(None, Some(1));
1088    }
1089
1090    #[test]
1091    fn test_count_encoding_distinct_too_large() {
1092        // statistics are stored using i64, so test trying to store larger values
1093        let statistics = make_bool_stats(Some(u64::MAX), Some(100));
1094        let thrift_stats = page_stats_to_thrift(Some(&statistics)).unwrap();
1095        assert_eq!(thrift_stats.distinct_count, None); // can't store u64 max --> null
1096        assert_eq!(thrift_stats.null_count, Some(100));
1097    }
1098
1099    #[test]
1100    fn test_count_encoding_null_too_large() {
1101        // statistics are stored using i64, so test trying to store larger values
1102        let statistics = make_bool_stats(Some(100), Some(u64::MAX));
1103        let thrift_stats = page_stats_to_thrift(Some(&statistics)).unwrap();
1104        assert_eq!(thrift_stats.distinct_count, Some(100));
1105        assert_eq!(thrift_stats.null_count, None); // can' store u64 max --> null
1106    }
1107
1108    #[test]
1109    fn test_count_decoding_null_invalid() {
1110        let tstatistics = PageStatistics {
1111            null_count: Some(-42),
1112            max: None,
1113            min: None,
1114            distinct_count: None,
1115            max_value: None,
1116            min_value: None,
1117            is_max_value_exact: None,
1118            is_min_value_exact: None,
1119            nan_count: None,
1120        };
1121        let err = from_thrift_page_stats(Type::BOOLEAN, Some(tstatistics)).unwrap_err();
1122        assert_eq!(
1123            err.to_string(),
1124            "Parquet error: Statistics null count is negative -42"
1125        );
1126    }
1127
1128    /// Writes statistics to thrift and reads them back and ensures:
1129    /// - The statistics are the same
1130    /// - The statistics written to thrift are the same as the original statistics
1131    fn statistics_count_test(distinct_count: Option<u64>, null_count: Option<u64>) {
1132        let statistics = make_bool_stats(distinct_count, null_count);
1133
1134        let thrift_stats = page_stats_to_thrift(Some(&statistics)).unwrap();
1135        assert_eq!(thrift_stats.null_count.map(|c| c as u64), null_count);
1136        assert_eq!(
1137            thrift_stats.distinct_count.map(|c| c as u64),
1138            distinct_count
1139        );
1140
1141        let round_tripped = from_thrift_page_stats(Type::BOOLEAN, Some(thrift_stats))
1142            .unwrap()
1143            .unwrap();
1144        assert_eq!(round_tripped, statistics);
1145    }
1146
1147    fn make_bool_stats(distinct_count: Option<u64>, null_count: Option<u64>) -> Statistics {
1148        let min = Some(true);
1149        let max = Some(false);
1150        let is_min_max_deprecated = false;
1151
1152        // test is about the counts, so we aren't really testing the min/max values
1153        Statistics::Boolean(ValueStatistics::new(
1154            min,
1155            max,
1156            distinct_count,
1157            null_count,
1158            is_min_max_deprecated,
1159        ))
1160    }
1161
1162    #[test]
1163    fn test_int96_invalid_statistics() {
1164        let mut thrift_stats = PageStatistics {
1165            max: None,
1166            min: Some((0..13).collect()),
1167            null_count: Some(0),
1168            distinct_count: None,
1169            max_value: None,
1170            min_value: None,
1171            is_max_value_exact: None,
1172            is_min_value_exact: None,
1173            nan_count: None,
1174        };
1175
1176        let err = from_thrift_page_stats(Type::INT96, Some(thrift_stats.clone())).unwrap_err();
1177        assert_eq!(
1178            err.to_string(),
1179            "Parquet error: Incorrect Int96 min statistics"
1180        );
1181
1182        thrift_stats.min = None;
1183        thrift_stats.max = Some((0..13).collect());
1184        let err = from_thrift_page_stats(Type::INT96, Some(thrift_stats)).unwrap_err();
1185        assert_eq!(
1186            err.to_string(),
1187            "Parquet error: Incorrect Int96 max statistics"
1188        );
1189    }
1190
1191    // Ensures that we can call ValueStatistics::min_opt from a
1192    // generic function without reyling on a bound to a private trait.
1193    fn generic_statistics_handler<T: std::fmt::Display>(stats: ValueStatistics<T>) -> String {
1194        match stats.min_opt() {
1195            Some(s) => format!("min: {s}"),
1196            None => "min: NA".to_string(),
1197        }
1198    }
1199
1200    #[test]
1201    fn test_generic_access() {
1202        let stats = Statistics::int32(Some(12), Some(45), None, Some(11), false);
1203
1204        match stats {
1205            Statistics::Int32(v) => {
1206                let stats_string = generic_statistics_handler(v);
1207                assert_eq!(&stats_string, "min: 12");
1208            }
1209            _ => unreachable!(),
1210        }
1211    }
1212
1213    #[test]
1214    fn test_nan_count_float() {
1215        // Test NaN count for f32
1216        let stats = Statistics::Float(
1217            ValueStatistics::new(Some(1.0_f32), Some(5.0_f32), None, Some(0), false)
1218                .with_nan_count(Some(3)),
1219        );
1220
1221        assert_eq!(stats.nan_count_opt(), Some(3));
1222
1223        // Verify round-trip through thrift
1224        let thrift_stats = page_stats_to_thrift(Some(&stats)).unwrap();
1225        assert_eq!(thrift_stats.nan_count, Some(3));
1226
1227        let round_tripped = from_thrift_page_stats(Type::FLOAT, Some(thrift_stats))
1228            .unwrap()
1229            .unwrap();
1230        assert_eq!(round_tripped.nan_count_opt(), Some(3));
1231    }
1232
1233    #[test]
1234    fn test_nan_count_double() {
1235        // Test NaN count for f64
1236        let stats = Statistics::Double(
1237            ValueStatistics::new(Some(1.0_f64), Some(5.0_f64), None, Some(0), false)
1238                .with_nan_count(Some(5)),
1239        );
1240
1241        assert_eq!(stats.nan_count_opt(), Some(5));
1242
1243        // Verify round-trip through thrift
1244        let thrift_stats = page_stats_to_thrift(Some(&stats)).unwrap();
1245        assert_eq!(thrift_stats.nan_count, Some(5));
1246
1247        let round_tripped = from_thrift_page_stats(Type::DOUBLE, Some(thrift_stats))
1248            .unwrap()
1249            .unwrap();
1250        assert_eq!(round_tripped.nan_count_opt(), Some(5));
1251    }
1252
1253    #[test]
1254    fn test_nan_count_none_for_non_float() {
1255        // NaN count should not be set for non-floating point types
1256        let stats = Statistics::int32(Some(1), Some(100), None, Some(0), false);
1257        assert_eq!(stats.nan_count_opt(), None);
1258
1259        let thrift_stats = page_stats_to_thrift(Some(&stats)).unwrap();
1260        assert_eq!(thrift_stats.nan_count, None);
1261    }
1262
1263    #[test]
1264    fn test_nan_count_backwards_compatible() {
1265        // Test that missing nan_count field is handled correctly
1266        let thrift_stats = PageStatistics {
1267            min: None,
1268            max: None,
1269            min_value: Some(vec![0, 0, 0, 0]), // 0.0_f32 in bytes
1270            max_value: Some(vec![0, 0, 128, 63]), // 1.0_f32 in bytes
1271            null_count: Some(0),
1272            distinct_count: None,
1273            nan_count: None, // Not set
1274            is_min_value_exact: None,
1275            is_max_value_exact: None,
1276        };
1277
1278        let stats = from_thrift_page_stats(Type::FLOAT, Some(thrift_stats))
1279            .unwrap()
1280            .unwrap();
1281
1282        // nan_count should be None when not provided
1283        assert_eq!(stats.nan_count_opt(), None);
1284    }
1285
1286    #[test]
1287    fn test_statistics_with_nan_min_max() {
1288        // Test that when there are only NaN values, min/max are NaN
1289        let stats = Statistics::Float(
1290            ValueStatistics::new(
1291                Some(f32::NAN), // min and max should have NaN values
1292                Some(f32::NAN),
1293                None,
1294                Some(0),
1295                false,
1296            )
1297            .with_nan_count(Some(10)), // All values are NaN
1298        );
1299
1300        assert_eq!(stats.min_bytes_opt(), Some(f32::NAN.as_bytes()));
1301        assert_eq!(stats.max_bytes_opt(), Some(f32::NAN.as_bytes()));
1302        assert_eq!(stats.nan_count_opt(), Some(10));
1303
1304        // Verify serialization handles this case
1305        let thrift_stats = page_stats_to_thrift(Some(&stats)).unwrap();
1306        assert_eq!(thrift_stats.min_value, Some(f32::NAN.as_bytes().to_vec()));
1307        assert_eq!(thrift_stats.max_value, Some(f32::NAN.as_bytes().to_vec()));
1308        assert_eq!(thrift_stats.nan_count, Some(10));
1309    }
1310
1311    #[test]
1312    fn test_nan_count_too_large() {
1313        // Test that nan_count larger than i64::MAX is not serialized
1314        let stats = Statistics::Float(
1315            ValueStatistics::new(Some(1.0_f32), Some(2.0_f32), None, Some(0), false)
1316                .with_nan_count(Some(u64::MAX)),
1317        );
1318
1319        let thrift_stats = page_stats_to_thrift(Some(&stats)).unwrap();
1320        // u64::MAX can't fit in i64, so it should be None
1321        assert_eq!(thrift_stats.nan_count, None);
1322    }
1323
1324    #[test]
1325    fn test_nan_counts_in_column_index() {
1326        // Test that nan_counts are properly collected in page index
1327        use crate::file::metadata::ColumnIndexBuilder;
1328
1329        // Test for floating-point column - all pages must have Some(n)
1330        let mut float_builder = ColumnIndexBuilder::new(Type::FLOAT);
1331        float_builder.append(false, vec![0u8; 4], vec![255u8; 4], 0, Some(5));
1332        float_builder.append(false, vec![0u8; 4], vec![255u8; 4], 2, Some(3));
1333        float_builder.append(false, vec![0u8; 4], vec![255u8; 4], 0, Some(0)); // No NaN but still Some(0)
1334
1335        let float_column_index = float_builder.build().unwrap();
1336        // Verify nan_counts field is properly set for float column
1337        assert_eq!(float_column_index.nan_counts(), Some(&vec![5, 3, 0]));
1338
1339        // Test for non-floating-point column - all pages must have None
1340        let mut int_builder = ColumnIndexBuilder::new(Type::INT32);
1341        int_builder.append(false, vec![0u8; 4], vec![255u8; 4], 0, None);
1342        int_builder.append(false, vec![0u8; 4], vec![255u8; 4], 2, None);
1343        int_builder.append(false, vec![0u8; 4], vec![255u8; 4], 0, None);
1344
1345        let int_column_index = int_builder.build().unwrap();
1346        // Verify nan_counts field is None for non-float column
1347        assert_eq!(int_column_index.nan_counts(), None);
1348    }
1349}
parquet/file/statistics.rs

parquet/file/
statistics.rs
