1use std::fmt;
43
44use crate::basic::Type;
45use crate::data_type::private::ParquetValueType;
46use crate::data_type::*;
47use crate::errors::{ParquetError, Result};
48use crate::file::metadata::thrift::PageStatistics;
49use crate::util::bit_util::FromBytes;
50
51pub(crate) mod private {
52 use super::*;
53
54 pub trait MakeStatistics {
55 fn make_statistics(statistics: ValueStatistics<Self>) -> Statistics
56 where
57 Self: Sized;
58 }
59
60 macro_rules! gen_make_statistics {
61 ($value_ty:ty, $stat:ident) => {
62 impl MakeStatistics for $value_ty {
63 fn make_statistics(statistics: ValueStatistics<Self>) -> Statistics
64 where
65 Self: Sized,
66 {
67 Statistics::$stat(statistics)
68 }
69 }
70 };
71 }
72
73 gen_make_statistics!(bool, Boolean);
74 gen_make_statistics!(i32, Int32);
75 gen_make_statistics!(i64, Int64);
76 gen_make_statistics!(Int96, Int96);
77 gen_make_statistics!(f32, Float);
78 gen_make_statistics!(f64, Double);
79 gen_make_statistics!(ByteArray, ByteArray);
80 gen_make_statistics!(FixedLenByteArray, FixedLenByteArray);
81}
82
83macro_rules! statistics_new_func {
85 ($func:ident, $vtype:ty, $stat:ident) => {
86 #[doc = concat!("Creates new statistics for `", stringify!($stat), "` column type.")]
87 pub fn $func(
88 min: $vtype,
89 max: $vtype,
90 distinct: Option<u64>,
91 nulls: Option<u64>,
92 is_deprecated: bool,
93 ) -> Self {
94 Statistics::$stat(ValueStatistics::new(
95 min,
96 max,
97 distinct,
98 nulls,
99 is_deprecated,
100 ))
101 }
102 };
103}
104
105macro_rules! statistics_enum_func {
107 ($self:ident, $func:ident) => {{
108 match *$self {
109 Statistics::Boolean(ref typed) => typed.$func(),
110 Statistics::Int32(ref typed) => typed.$func(),
111 Statistics::Int64(ref typed) => typed.$func(),
112 Statistics::Int96(ref typed) => typed.$func(),
113 Statistics::Float(ref typed) => typed.$func(),
114 Statistics::Double(ref typed) => typed.$func(),
115 Statistics::ByteArray(ref typed) => typed.$func(),
116 Statistics::FixedLenByteArray(ref typed) => typed.$func(),
117 }
118 }};
119}
120
121pub(crate) fn from_thrift_page_stats(
123 physical_type: Type,
124 thrift_stats: Option<PageStatistics>,
125) -> Result<Option<Statistics>> {
126 Ok(match thrift_stats {
127 Some(stats) => {
128 let null_count = stats.null_count.unwrap_or(0);
132
133 if null_count < 0 {
134 return Err(ParquetError::General(format!(
135 "Statistics null count is negative {null_count}",
136 )));
137 }
138
139 let null_count = Some(null_count as u64);
141 let distinct_count = stats.distinct_count.map(|value| value as u64);
143 let old_format = stats.min_value.is_none() && stats.max_value.is_none();
145 let min = if old_format {
147 stats.min
148 } else {
149 stats.min_value
150 };
151 let max = if old_format {
153 stats.max
154 } else {
155 stats.max_value
156 };
157
158 fn check_len(min: &Option<Vec<u8>>, max: &Option<Vec<u8>>, len: usize) -> Result<()> {
159 if let Some(min) = min {
160 if min.len() < len {
161 return Err(ParquetError::General(
162 "Insufficient bytes to parse min statistic".to_string(),
163 ));
164 }
165 }
166 if let Some(max) = max {
167 if max.len() < len {
168 return Err(ParquetError::General(
169 "Insufficient bytes to parse max statistic".to_string(),
170 ));
171 }
172 }
173 Ok(())
174 }
175
176 match physical_type {
177 Type::BOOLEAN => check_len(&min, &max, 1),
178 Type::INT32 | Type::FLOAT => check_len(&min, &max, 4),
179 Type::INT64 | Type::DOUBLE => check_len(&min, &max, 8),
180 Type::INT96 => check_len(&min, &max, 12),
181 _ => Ok(()),
182 }?;
183
184 let res = match physical_type {
189 Type::BOOLEAN => Statistics::boolean(
190 min.map(|data| data[0] != 0),
191 max.map(|data| data[0] != 0),
192 distinct_count,
193 null_count,
194 old_format,
195 ),
196 Type::INT32 => Statistics::int32(
197 min.map(|data| i32::from_le_bytes(data[..4].try_into().unwrap())),
198 max.map(|data| i32::from_le_bytes(data[..4].try_into().unwrap())),
199 distinct_count,
200 null_count,
201 old_format,
202 ),
203 Type::INT64 => Statistics::int64(
204 min.map(|data| i64::from_le_bytes(data[..8].try_into().unwrap())),
205 max.map(|data| i64::from_le_bytes(data[..8].try_into().unwrap())),
206 distinct_count,
207 null_count,
208 old_format,
209 ),
210 Type::INT96 => {
211 let min = if let Some(data) = min {
213 if data.len() != 12 {
214 return Err(ParquetError::General(
215 "Incorrect Int96 min statistics".to_string(),
216 ));
217 }
218 Some(Int96::try_from_le_slice(&data)?)
219 } else {
220 None
221 };
222 let max = if let Some(data) = max {
223 if data.len() != 12 {
224 return Err(ParquetError::General(
225 "Incorrect Int96 max statistics".to_string(),
226 ));
227 }
228 Some(Int96::try_from_le_slice(&data)?)
229 } else {
230 None
231 };
232 Statistics::int96(min, max, distinct_count, null_count, old_format)
233 }
234 Type::FLOAT => Statistics::float(
235 min.map(|data| f32::from_le_bytes(data[..4].try_into().unwrap())),
236 max.map(|data| f32::from_le_bytes(data[..4].try_into().unwrap())),
237 distinct_count,
238 null_count,
239 old_format,
240 ),
241 Type::DOUBLE => Statistics::double(
242 min.map(|data| f64::from_le_bytes(data[..8].try_into().unwrap())),
243 max.map(|data| f64::from_le_bytes(data[..8].try_into().unwrap())),
244 distinct_count,
245 null_count,
246 old_format,
247 ),
248 Type::BYTE_ARRAY => Statistics::ByteArray(
249 ValueStatistics::new(
250 min.map(ByteArray::from),
251 max.map(ByteArray::from),
252 distinct_count,
253 null_count,
254 old_format,
255 )
256 .with_max_is_exact(stats.is_max_value_exact.unwrap_or(false))
257 .with_min_is_exact(stats.is_min_value_exact.unwrap_or(false)),
258 ),
259 Type::FIXED_LEN_BYTE_ARRAY => Statistics::FixedLenByteArray(
260 ValueStatistics::new(
261 min.map(ByteArray::from).map(FixedLenByteArray::from),
262 max.map(ByteArray::from).map(FixedLenByteArray::from),
263 distinct_count,
264 null_count,
265 old_format,
266 )
267 .with_max_is_exact(stats.is_max_value_exact.unwrap_or(false))
268 .with_min_is_exact(stats.is_min_value_exact.unwrap_or(false)),
269 ),
270 };
271
272 Some(res)
273 }
274 None => None,
275 })
276}
277
278pub(crate) fn page_stats_to_thrift(stats: Option<&Statistics>) -> Option<PageStatistics> {
280 let stats = stats?;
281
282 let null_count = stats
284 .null_count_opt()
285 .and_then(|value| i64::try_from(value).ok());
286
287 let distinct_count = stats
289 .distinct_count_opt()
290 .and_then(|value| i64::try_from(value).ok());
291
292 let mut thrift_stats = PageStatistics {
293 max: None,
294 min: None,
295 null_count,
296 distinct_count,
297 max_value: None,
298 min_value: None,
299 is_max_value_exact: None,
300 is_min_value_exact: None,
301 };
302
303 let (min, max, min_exact, max_exact) = (
305 stats.min_bytes_opt().map(|x| x.to_vec()),
306 stats.max_bytes_opt().map(|x| x.to_vec()),
307 Some(stats.min_is_exact()),
308 Some(stats.max_is_exact()),
309 );
310 if stats.is_min_max_backwards_compatible() {
311 thrift_stats.min.clone_from(&min);
313 thrift_stats.max.clone_from(&max);
314 }
315
316 if !stats.is_min_max_deprecated() {
317 thrift_stats.min_value = min;
318 thrift_stats.max_value = max;
319 }
320
321 thrift_stats.is_min_value_exact = min_exact;
322 thrift_stats.is_max_value_exact = max_exact;
323
324 Some(thrift_stats)
325}
326
327#[derive(Debug, Clone, PartialEq)]
338pub enum Statistics {
339 Boolean(ValueStatistics<bool>),
341 Int32(ValueStatistics<i32>),
343 Int64(ValueStatistics<i64>),
345 Int96(ValueStatistics<Int96>),
347 Float(ValueStatistics<f32>),
349 Double(ValueStatistics<f64>),
351 ByteArray(ValueStatistics<ByteArray>),
353 FixedLenByteArray(ValueStatistics<FixedLenByteArray>),
355}
356
357impl<T: ParquetValueType> From<ValueStatistics<T>> for Statistics {
358 fn from(t: ValueStatistics<T>) -> Self {
359 T::make_statistics(t)
360 }
361}
362
363impl Statistics {
364 pub fn new<T: ParquetValueType>(
366 min: Option<T>,
367 max: Option<T>,
368 distinct_count: Option<u64>,
369 null_count: Option<u64>,
370 is_deprecated: bool,
371 ) -> Self {
372 Self::from(ValueStatistics::new(
373 min,
374 max,
375 distinct_count,
376 null_count,
377 is_deprecated,
378 ))
379 }
380
381 statistics_new_func![boolean, Option<bool>, Boolean];
382
383 statistics_new_func![int32, Option<i32>, Int32];
384
385 statistics_new_func![int64, Option<i64>, Int64];
386
387 statistics_new_func![int96, Option<Int96>, Int96];
388
389 statistics_new_func![float, Option<f32>, Float];
390
391 statistics_new_func![double, Option<f64>, Double];
392
393 statistics_new_func![byte_array, Option<ByteArray>, ByteArray];
394
395 statistics_new_func![
396 fixed_len_byte_array,
397 Option<FixedLenByteArray>,
398 FixedLenByteArray
399 ];
400
401 pub fn is_min_max_deprecated(&self) -> bool {
408 statistics_enum_func![self, is_min_max_deprecated]
409 }
410
411 pub fn is_min_max_backwards_compatible(&self) -> bool {
422 statistics_enum_func![self, is_min_max_backwards_compatible]
423 }
424
425 pub fn distinct_count_opt(&self) -> Option<u64> {
428 statistics_enum_func![self, distinct_count]
429 }
430
431 pub fn null_count_opt(&self) -> Option<u64> {
438 statistics_enum_func![self, null_count_opt]
439 }
440
441 pub fn min_is_exact(&self) -> bool {
443 statistics_enum_func![self, min_is_exact]
444 }
445
446 pub fn max_is_exact(&self) -> bool {
448 statistics_enum_func![self, max_is_exact]
449 }
450
451 pub fn min_bytes_opt(&self) -> Option<&[u8]> {
453 statistics_enum_func![self, min_bytes_opt]
454 }
455
456 pub fn max_bytes_opt(&self) -> Option<&[u8]> {
458 statistics_enum_func![self, max_bytes_opt]
459 }
460
461 pub fn physical_type(&self) -> Type {
463 match self {
464 Statistics::Boolean(_) => Type::BOOLEAN,
465 Statistics::Int32(_) => Type::INT32,
466 Statistics::Int64(_) => Type::INT64,
467 Statistics::Int96(_) => Type::INT96,
468 Statistics::Float(_) => Type::FLOAT,
469 Statistics::Double(_) => Type::DOUBLE,
470 Statistics::ByteArray(_) => Type::BYTE_ARRAY,
471 Statistics::FixedLenByteArray(_) => Type::FIXED_LEN_BYTE_ARRAY,
472 }
473 }
474}
475
476impl fmt::Display for Statistics {
477 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
478 match self {
479 Statistics::Boolean(typed) => write!(f, "{typed}"),
480 Statistics::Int32(typed) => write!(f, "{typed}"),
481 Statistics::Int64(typed) => write!(f, "{typed}"),
482 Statistics::Int96(typed) => write!(f, "{typed}"),
483 Statistics::Float(typed) => write!(f, "{typed}"),
484 Statistics::Double(typed) => write!(f, "{typed}"),
485 Statistics::ByteArray(typed) => write!(f, "{typed}"),
486 Statistics::FixedLenByteArray(typed) => write!(f, "{typed}"),
487 }
488 }
489}
490
491pub type TypedStatistics<T> = ValueStatistics<<T as DataType>::T>;
493
494#[derive(Clone, Eq, PartialEq)]
498pub struct ValueStatistics<T> {
499 min: Option<T>,
500 max: Option<T>,
501 distinct_count: Option<u64>,
503 null_count: Option<u64>,
504
505 is_max_value_exact: bool,
507 is_min_value_exact: bool,
508
509 is_min_max_deprecated: bool,
512
513 is_min_max_backwards_compatible: bool,
516}
517
518impl<T: ParquetValueType> ValueStatistics<T> {
519 pub fn new(
521 min: Option<T>,
522 max: Option<T>,
523 distinct_count: Option<u64>,
524 null_count: Option<u64>,
525 is_min_max_deprecated: bool,
526 ) -> Self {
527 Self {
528 is_max_value_exact: max.is_some(),
529 is_min_value_exact: min.is_some(),
530 min,
531 max,
532 distinct_count,
533 null_count,
534 is_min_max_deprecated,
535 is_min_max_backwards_compatible: is_min_max_deprecated,
536 }
537 }
538
539 pub fn with_min_is_exact(self, is_min_value_exact: bool) -> Self {
544 Self {
545 is_min_value_exact,
546 ..self
547 }
548 }
549
550 pub fn with_max_is_exact(self, is_max_value_exact: bool) -> Self {
555 Self {
556 is_max_value_exact,
557 ..self
558 }
559 }
560
561 pub fn with_backwards_compatible_min_max(self, backwards_compatible: bool) -> Self {
567 Self {
568 is_min_max_backwards_compatible: backwards_compatible,
569 ..self
570 }
571 }
572
573 pub fn min_opt(&self) -> Option<&T> {
575 self.min.as_ref()
576 }
577
578 pub fn max_opt(&self) -> Option<&T> {
580 self.max.as_ref()
581 }
582
583 pub fn min_bytes_opt(&self) -> Option<&[u8]> {
585 self.min_opt().map(AsBytes::as_bytes)
586 }
587
588 pub fn max_bytes_opt(&self) -> Option<&[u8]> {
590 self.max_opt().map(AsBytes::as_bytes)
591 }
592
593 pub(crate) fn _internal_has_min_max_set(&self) -> bool {
596 self.min.is_some() && self.max.is_some()
597 }
598
599 pub fn max_is_exact(&self) -> bool {
601 self.max.is_some() && self.is_max_value_exact
602 }
603
604 pub fn min_is_exact(&self) -> bool {
606 self.min.is_some() && self.is_min_value_exact
607 }
608
609 pub fn distinct_count(&self) -> Option<u64> {
611 self.distinct_count
612 }
613
614 pub fn null_count_opt(&self) -> Option<u64> {
616 self.null_count
617 }
618
619 fn is_min_max_deprecated(&self) -> bool {
621 self.is_min_max_deprecated
622 }
623
624 pub fn is_min_max_backwards_compatible(&self) -> bool {
635 self.is_min_max_backwards_compatible
636 }
637}
638
639impl<T: ParquetValueType> fmt::Display for ValueStatistics<T> {
640 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
641 write!(f, "{{")?;
642 write!(f, "min: ")?;
643 match self.min {
644 Some(ref value) => write!(f, "{value}")?,
645 None => write!(f, "N/A")?,
646 }
647 write!(f, ", max: ")?;
648 match self.max {
649 Some(ref value) => write!(f, "{value}")?,
650 None => write!(f, "N/A")?,
651 }
652 write!(f, ", distinct_count: ")?;
653 match self.distinct_count {
654 Some(value) => write!(f, "{value}")?,
655 None => write!(f, "N/A")?,
656 }
657 write!(f, ", null_count: ")?;
658 match self.null_count {
659 Some(value) => write!(f, "{value}")?,
660 None => write!(f, "N/A")?,
661 }
662 write!(f, ", min_max_deprecated: {}", self.is_min_max_deprecated)?;
663 write!(f, ", max_value_exact: {}", self.is_max_value_exact)?;
664 write!(f, ", min_value_exact: {}", self.is_min_value_exact)?;
665 write!(f, "}}")
666 }
667}
668
669impl<T: ParquetValueType> fmt::Debug for ValueStatistics<T> {
670 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
671 write!(
672 f,
673 "{{min: {:?}, max: {:?}, distinct_count: {:?}, null_count: {:?}, \
674 min_max_deprecated: {}, min_max_backwards_compatible: {}, max_value_exact: {}, min_value_exact: {}}}",
675 self.min,
676 self.max,
677 self.distinct_count,
678 self.null_count,
679 self.is_min_max_deprecated,
680 self.is_min_max_backwards_compatible,
681 self.is_max_value_exact,
682 self.is_min_value_exact
683 )
684 }
685}
686
687#[cfg(test)]
688mod tests {
689 use super::*;
690
691 #[test]
692 fn test_statistics_min_max_bytes() {
693 let stats = Statistics::int32(Some(-123), Some(234), None, Some(1), false);
694 assert_eq!(stats.min_bytes_opt(), Some((-123).as_bytes()));
695 assert_eq!(stats.max_bytes_opt(), Some(234.as_bytes()));
696
697 let stats = Statistics::byte_array(
698 Some(ByteArray::from(vec![1, 2, 3])),
699 Some(ByteArray::from(vec![3, 4, 5])),
700 None,
701 Some(1),
702 true,
703 );
704 assert_eq!(stats.min_bytes_opt().unwrap(), &[1, 2, 3]);
705 assert_eq!(stats.max_bytes_opt().unwrap(), &[3, 4, 5]);
706 }
707
708 #[test]
709 #[should_panic(expected = "General(\"Statistics null count is negative -10\")")]
710 fn test_statistics_negative_null_count() {
711 let thrift_stats = PageStatistics {
712 max: None,
713 min: None,
714 null_count: Some(-10),
715 distinct_count: None,
716 max_value: None,
717 min_value: None,
718 is_max_value_exact: None,
719 is_min_value_exact: None,
720 };
721
722 from_thrift_page_stats(Type::INT32, Some(thrift_stats)).unwrap();
723 }
724
725 #[test]
726 fn test_statistics_thrift_none() {
727 assert_eq!(from_thrift_page_stats(Type::INT32, None).unwrap(), None);
728 assert_eq!(
729 from_thrift_page_stats(Type::BYTE_ARRAY, None).unwrap(),
730 None
731 );
732 }
733
734 #[test]
735 fn test_statistics_debug() {
736 let stats = Statistics::int32(Some(1), Some(12), None, Some(12), true);
737 assert_eq!(
738 format!("{stats:?}"),
739 "Int32({min: Some(1), max: Some(12), distinct_count: None, null_count: Some(12), \
740 min_max_deprecated: true, min_max_backwards_compatible: true, max_value_exact: true, min_value_exact: true})"
741 );
742
743 let stats = Statistics::int32(None, None, None, Some(7), false);
744 assert_eq!(
745 format!("{stats:?}"),
746 "Int32({min: None, max: None, distinct_count: None, null_count: Some(7), \
747 min_max_deprecated: false, min_max_backwards_compatible: false, max_value_exact: false, min_value_exact: false})"
748 )
749 }
750
751 #[test]
752 fn test_statistics_display() {
753 let stats = Statistics::int32(Some(1), Some(12), None, Some(12), true);
754 assert_eq!(
755 format!("{stats}"),
756 "{min: 1, max: 12, distinct_count: N/A, null_count: 12, min_max_deprecated: true, max_value_exact: true, min_value_exact: true}"
757 );
758
759 let stats = Statistics::int64(None, None, None, Some(7), false);
760 assert_eq!(
761 format!("{stats}"),
762 "{min: N/A, max: N/A, distinct_count: N/A, null_count: 7, min_max_deprecated: \
763 false, max_value_exact: false, min_value_exact: false}"
764 );
765
766 let stats = Statistics::int96(
767 Some(Int96::from(vec![1, 0, 0])),
768 Some(Int96::from(vec![2, 3, 4])),
769 None,
770 Some(3),
771 true,
772 );
773 assert_eq!(
774 format!("{stats}"),
775 "{min: [1, 0, 0], max: [2, 3, 4], distinct_count: N/A, null_count: 3, \
776 min_max_deprecated: true, max_value_exact: true, min_value_exact: true}"
777 );
778
779 let stats = Statistics::ByteArray(
780 ValueStatistics::new(
781 Some(ByteArray::from(vec![1u8])),
782 Some(ByteArray::from(vec![2u8])),
783 Some(5),
784 Some(7),
785 false,
786 )
787 .with_max_is_exact(false)
788 .with_min_is_exact(false),
789 );
790 assert_eq!(
791 format!("{stats}"),
792 "{min: [1], max: [2], distinct_count: 5, null_count: 7, min_max_deprecated: false, max_value_exact: false, min_value_exact: false}"
793 );
794 }
795
796 #[test]
797 fn test_statistics_partial_eq() {
798 let expected = Statistics::int32(Some(12), Some(45), None, Some(11), true);
799
800 assert!(Statistics::int32(Some(12), Some(45), None, Some(11), true) == expected);
801 assert!(Statistics::int32(Some(11), Some(45), None, Some(11), true) != expected);
802 assert!(Statistics::int32(Some(12), Some(44), None, Some(11), true) != expected);
803 assert!(Statistics::int32(Some(12), Some(45), None, Some(23), true) != expected);
804 assert!(Statistics::int32(Some(12), Some(45), None, Some(11), false) != expected);
805
806 assert!(
807 Statistics::int32(Some(12), Some(45), None, Some(11), false)
808 != Statistics::int64(Some(12), Some(45), None, Some(11), false)
809 );
810
811 assert!(
812 Statistics::boolean(Some(false), Some(true), None, None, true)
813 != Statistics::double(Some(1.2), Some(4.5), None, None, true)
814 );
815
816 assert!(
817 Statistics::byte_array(
818 Some(ByteArray::from(vec![1, 2, 3])),
819 Some(ByteArray::from(vec![1, 2, 3])),
820 None,
821 None,
822 true
823 ) != Statistics::fixed_len_byte_array(
824 Some(ByteArray::from(vec![1, 2, 3]).into()),
825 Some(ByteArray::from(vec![1, 2, 3]).into()),
826 None,
827 None,
828 true,
829 )
830 );
831
832 assert!(
833 Statistics::byte_array(
834 Some(ByteArray::from(vec![1, 2, 3])),
835 Some(ByteArray::from(vec![1, 2, 3])),
836 None,
837 None,
838 true,
839 ) != Statistics::ByteArray(
840 ValueStatistics::new(
841 Some(ByteArray::from(vec![1, 2, 3])),
842 Some(ByteArray::from(vec![1, 2, 3])),
843 None,
844 None,
845 true,
846 )
847 .with_max_is_exact(false)
848 )
849 );
850
851 assert!(
852 Statistics::fixed_len_byte_array(
853 Some(FixedLenByteArray::from(vec![1, 2, 3])),
854 Some(FixedLenByteArray::from(vec![1, 2, 3])),
855 None,
856 None,
857 true,
858 ) != Statistics::FixedLenByteArray(
859 ValueStatistics::new(
860 Some(FixedLenByteArray::from(vec![1, 2, 3])),
861 Some(FixedLenByteArray::from(vec![1, 2, 3])),
862 None,
863 None,
864 true,
865 )
866 .with_min_is_exact(false)
867 )
868 );
869 }
870
871 #[test]
872 fn test_statistics_from_thrift() {
873 fn check_stats(stats: Statistics) {
875 let tpe = stats.physical_type();
876 let thrift_stats = page_stats_to_thrift(Some(&stats));
877 assert_eq!(
878 from_thrift_page_stats(tpe, thrift_stats).unwrap(),
879 Some(stats)
880 );
881 }
882
883 check_stats(Statistics::boolean(
884 Some(false),
885 Some(true),
886 None,
887 Some(7),
888 true,
889 ));
890 check_stats(Statistics::boolean(
891 Some(false),
892 Some(true),
893 None,
894 Some(7),
895 true,
896 ));
897 check_stats(Statistics::boolean(
898 Some(false),
899 Some(true),
900 None,
901 Some(0),
902 false,
903 ));
904 check_stats(Statistics::boolean(
905 Some(true),
906 Some(true),
907 None,
908 Some(7),
909 true,
910 ));
911 check_stats(Statistics::boolean(
912 Some(false),
913 Some(false),
914 None,
915 Some(7),
916 true,
917 ));
918 check_stats(Statistics::boolean(None, None, None, Some(7), true));
919
920 check_stats(Statistics::int32(
921 Some(-100),
922 Some(500),
923 None,
924 Some(7),
925 true,
926 ));
927 check_stats(Statistics::int32(
928 Some(-100),
929 Some(500),
930 None,
931 Some(0),
932 false,
933 ));
934 check_stats(Statistics::int32(None, None, None, Some(7), true));
935
936 check_stats(Statistics::int64(
937 Some(-100),
938 Some(200),
939 None,
940 Some(7),
941 true,
942 ));
943 check_stats(Statistics::int64(
944 Some(-100),
945 Some(200),
946 None,
947 Some(0),
948 false,
949 ));
950 check_stats(Statistics::int64(None, None, None, Some(7), true));
951
952 check_stats(Statistics::float(Some(1.2), Some(3.4), None, Some(7), true));
953 check_stats(Statistics::float(
954 Some(1.2),
955 Some(3.4),
956 None,
957 Some(0),
958 false,
959 ));
960 check_stats(Statistics::float(None, None, None, Some(7), true));
961
962 check_stats(Statistics::double(
963 Some(1.2),
964 Some(3.4),
965 None,
966 Some(7),
967 true,
968 ));
969 check_stats(Statistics::double(
970 Some(1.2),
971 Some(3.4),
972 None,
973 Some(0),
974 false,
975 ));
976 check_stats(Statistics::double(None, None, None, Some(7), true));
977
978 check_stats(Statistics::byte_array(
979 Some(ByteArray::from(vec![1, 2, 3])),
980 Some(ByteArray::from(vec![3, 4, 5])),
981 None,
982 Some(7),
983 true,
984 ));
985 check_stats(Statistics::byte_array(None, None, None, Some(7), true));
986
987 check_stats(Statistics::fixed_len_byte_array(
988 Some(ByteArray::from(vec![1, 2, 3]).into()),
989 Some(ByteArray::from(vec![3, 4, 5]).into()),
990 None,
991 Some(7),
992 true,
993 ));
994 check_stats(Statistics::fixed_len_byte_array(
995 None,
996 None,
997 None,
998 Some(7),
999 true,
1000 ));
1001 }
1002
1003 #[test]
1004 fn test_count_encoding() {
1005 statistics_count_test(None, None);
1006 statistics_count_test(Some(0), Some(0));
1007 statistics_count_test(Some(100), Some(2000));
1008 statistics_count_test(Some(1), None);
1009 statistics_count_test(None, Some(1));
1010 }
1011
1012 #[test]
1013 fn test_count_encoding_distinct_too_large() {
1014 let statistics = make_bool_stats(Some(u64::MAX), Some(100));
1016 let thrift_stats = page_stats_to_thrift(Some(&statistics)).unwrap();
1017 assert_eq!(thrift_stats.distinct_count, None); assert_eq!(thrift_stats.null_count, Some(100));
1019 }
1020
1021 #[test]
1022 fn test_count_encoding_null_too_large() {
1023 let statistics = make_bool_stats(Some(100), Some(u64::MAX));
1025 let thrift_stats = page_stats_to_thrift(Some(&statistics)).unwrap();
1026 assert_eq!(thrift_stats.distinct_count, Some(100));
1027 assert_eq!(thrift_stats.null_count, None); }
1029
1030 #[test]
1031 fn test_count_decoding_null_invalid() {
1032 let tstatistics = PageStatistics {
1033 null_count: Some(-42),
1034 max: None,
1035 min: None,
1036 distinct_count: None,
1037 max_value: None,
1038 min_value: None,
1039 is_max_value_exact: None,
1040 is_min_value_exact: None,
1041 };
1042 let err = from_thrift_page_stats(Type::BOOLEAN, Some(tstatistics)).unwrap_err();
1043 assert_eq!(
1044 err.to_string(),
1045 "Parquet error: Statistics null count is negative -42"
1046 );
1047 }
1048
1049 fn statistics_count_test(distinct_count: Option<u64>, null_count: Option<u64>) {
1053 let statistics = make_bool_stats(distinct_count, null_count);
1054
1055 let thrift_stats = page_stats_to_thrift(Some(&statistics)).unwrap();
1056 assert_eq!(thrift_stats.null_count.map(|c| c as u64), null_count);
1057 assert_eq!(
1058 thrift_stats.distinct_count.map(|c| c as u64),
1059 distinct_count
1060 );
1061
1062 let round_tripped = from_thrift_page_stats(Type::BOOLEAN, Some(thrift_stats))
1063 .unwrap()
1064 .unwrap();
1065 if null_count.is_none() {
1068 assert_ne!(round_tripped, statistics);
1069 assert!(round_tripped.null_count_opt().is_some());
1070 assert_eq!(round_tripped.null_count_opt(), Some(0));
1071 assert_eq!(round_tripped.min_bytes_opt(), statistics.min_bytes_opt());
1072 assert_eq!(round_tripped.max_bytes_opt(), statistics.max_bytes_opt());
1073 assert_eq!(
1074 round_tripped.distinct_count_opt(),
1075 statistics.distinct_count_opt()
1076 );
1077 } else {
1078 assert_eq!(round_tripped, statistics);
1079 }
1080 }
1081
1082 fn make_bool_stats(distinct_count: Option<u64>, null_count: Option<u64>) -> Statistics {
1083 let min = Some(true);
1084 let max = Some(false);
1085 let is_min_max_deprecated = false;
1086
1087 Statistics::Boolean(ValueStatistics::new(
1089 min,
1090 max,
1091 distinct_count,
1092 null_count,
1093 is_min_max_deprecated,
1094 ))
1095 }
1096
1097 #[test]
1098 fn test_int96_invalid_statistics() {
1099 let mut thrift_stats = PageStatistics {
1100 max: None,
1101 min: Some((0..13).collect()),
1102 null_count: Some(0),
1103 distinct_count: None,
1104 max_value: None,
1105 min_value: None,
1106 is_max_value_exact: None,
1107 is_min_value_exact: None,
1108 };
1109
1110 let err = from_thrift_page_stats(Type::INT96, Some(thrift_stats.clone())).unwrap_err();
1111 assert_eq!(
1112 err.to_string(),
1113 "Parquet error: Incorrect Int96 min statistics"
1114 );
1115
1116 thrift_stats.min = None;
1117 thrift_stats.max = Some((0..13).collect());
1118 let err = from_thrift_page_stats(Type::INT96, Some(thrift_stats)).unwrap_err();
1119 assert_eq!(
1120 err.to_string(),
1121 "Parquet error: Incorrect Int96 max statistics"
1122 );
1123 }
1124}