1use bytes::Bytes;
8use half::f16;
9
10use crate::bloom_filter::Sbbf;
11use crate::file::page_index::column_index::ColumnIndexMetaData;
12use crate::file::page_index::offset_index::OffsetIndexMetaData;
13use std::cmp::Ordering;
14use std::collections::{BTreeSet, VecDeque};
15use std::str;
16
17use crate::basic::{
18 BoundaryOrder, Compression, ConvertedType, Encoding, EncodingMask, LogicalType, PageType,
19 SortOrder, Type,
20};
21use crate::column::page::{CompressedPage, Page, PageWriteSpec, PageWriter};
22use crate::column::writer::encoder::{ColumnValueEncoder, ColumnValueEncoderImpl, ColumnValues};
23use crate::compression::{Codec, CodecOptionsBuilder, create_codec};
24use crate::data_type::private::ParquetValueType;
25use crate::data_type::*;
26use crate::encodings::levels::LevelEncoder;
27#[cfg(feature = "encryption")]
28use crate::encryption::encrypt::get_column_crypto_metadata;
29use crate::errors::{ParquetError, Result};
30use crate::file::metadata::{
31 ColumnChunkMetaData, ColumnChunkMetaDataBuilder, ColumnIndexBuilder, LevelHistogram,
32 OffsetIndexBuilder, PageEncodingStats,
33};
34use crate::file::properties::{
35 EnabledStatistics, ResolvedColumnProperties, WriterProperties, WriterPropertiesPtr,
36 WriterVersion,
37};
38use crate::file::statistics::{Statistics, ValueStatistics};
39use crate::schema::types::{BasicTypeInfo, ColumnDescPtr, ColumnDescriptor};
40
41mod byte_budget_chunker;
42pub(crate) mod encoder;
43pub(crate) mod min_max;
44
45use byte_budget_chunker::{ByteBudgetChunker, SubBatchStrategy};
46
47macro_rules! downcast_writer {
48 ($e:expr, $i:ident, $b:expr) => {
49 match $e {
50 Self::BoolColumnWriter($i) => $b,
51 Self::Int32ColumnWriter($i) => $b,
52 Self::Int64ColumnWriter($i) => $b,
53 Self::Int96ColumnWriter($i) => $b,
54 Self::FloatColumnWriter($i) => $b,
55 Self::DoubleColumnWriter($i) => $b,
56 Self::ByteArrayColumnWriter($i) => $b,
57 Self::FixedLenByteArrayColumnWriter($i) => $b,
58 }
59 };
60}
61
62pub enum ColumnWriter<'a> {
66 BoolColumnWriter(ColumnWriterImpl<'a, BoolType>),
68 Int32ColumnWriter(ColumnWriterImpl<'a, Int32Type>),
70 Int64ColumnWriter(ColumnWriterImpl<'a, Int64Type>),
72 Int96ColumnWriter(ColumnWriterImpl<'a, Int96Type>),
74 FloatColumnWriter(ColumnWriterImpl<'a, FloatType>),
76 DoubleColumnWriter(ColumnWriterImpl<'a, DoubleType>),
78 ByteArrayColumnWriter(ColumnWriterImpl<'a, ByteArrayType>),
80 FixedLenByteArrayColumnWriter(ColumnWriterImpl<'a, FixedLenByteArrayType>),
82}
83
84impl ColumnWriter<'_> {
85 #[cfg(feature = "arrow")]
87 pub(crate) fn memory_size(&self) -> usize {
88 downcast_writer!(self, typed, typed.memory_size())
89 }
90
91 #[cfg(feature = "arrow")]
93 pub(crate) fn get_estimated_total_bytes(&self) -> u64 {
94 downcast_writer!(self, typed, typed.get_estimated_total_bytes())
95 }
96
97 #[cfg(feature = "arrow")]
102 pub(crate) fn add_data_page(&mut self) -> Result<()> {
103 downcast_writer!(self, typed, typed.add_data_page())
104 }
105
106 #[cfg(feature = "arrow")]
110 pub(crate) fn set_distinct_count_override(&mut self, count: u64) {
111 downcast_writer!(self, typed, typed.set_distinct_count_override(count))
112 }
113
114 pub fn close(self) -> Result<ColumnCloseResult> {
116 downcast_writer!(self, typed, typed.close())
117 }
118}
119
120pub fn get_column_writer<'a>(
122 descr: ColumnDescPtr,
123 props: WriterPropertiesPtr,
124 page_writer: Box<dyn PageWriter + 'a>,
125) -> ColumnWriter<'a> {
126 match descr.physical_type() {
127 Type::BOOLEAN => {
128 ColumnWriter::BoolColumnWriter(ColumnWriterImpl::new(descr, props, page_writer))
129 }
130 Type::INT32 => {
131 ColumnWriter::Int32ColumnWriter(ColumnWriterImpl::new(descr, props, page_writer))
132 }
133 Type::INT64 => {
134 ColumnWriter::Int64ColumnWriter(ColumnWriterImpl::new(descr, props, page_writer))
135 }
136 Type::INT96 => {
137 ColumnWriter::Int96ColumnWriter(ColumnWriterImpl::new(descr, props, page_writer))
138 }
139 Type::FLOAT => {
140 ColumnWriter::FloatColumnWriter(ColumnWriterImpl::new(descr, props, page_writer))
141 }
142 Type::DOUBLE => {
143 ColumnWriter::DoubleColumnWriter(ColumnWriterImpl::new(descr, props, page_writer))
144 }
145 Type::BYTE_ARRAY => {
146 ColumnWriter::ByteArrayColumnWriter(ColumnWriterImpl::new(descr, props, page_writer))
147 }
148 Type::FIXED_LEN_BYTE_ARRAY => ColumnWriter::FixedLenByteArrayColumnWriter(
149 ColumnWriterImpl::new(descr, props, page_writer),
150 ),
151 }
152}
153
154pub fn get_typed_column_writer<T: DataType>(col_writer: ColumnWriter) -> ColumnWriterImpl<T> {
161 T::get_column_writer(col_writer).unwrap_or_else(|| {
162 panic!(
163 "Failed to convert column writer into a typed column writer for `{}` type",
164 T::get_physical_type()
165 )
166 })
167}
168
169pub fn get_typed_column_writer_ref<'a, 'b: 'a, T: DataType>(
171 col_writer: &'b ColumnWriter<'a>,
172) -> &'b ColumnWriterImpl<'a, T> {
173 T::get_column_writer_ref(col_writer).unwrap_or_else(|| {
174 panic!(
175 "Failed to convert column writer into a typed column writer for `{}` type",
176 T::get_physical_type()
177 )
178 })
179}
180
181pub fn get_typed_column_writer_mut<'a, 'b: 'a, T: DataType>(
183 col_writer: &'a mut ColumnWriter<'b>,
184) -> &'a mut ColumnWriterImpl<'b, T> {
185 T::get_column_writer_mut(col_writer).unwrap_or_else(|| {
186 panic!(
187 "Failed to convert column writer into a typed column writer for `{}` type",
188 T::get_physical_type()
189 )
190 })
191}
192
193#[derive(Debug, Clone)]
197pub struct ColumnCloseResult {
198 pub bytes_written: u64,
200 pub rows_written: u64,
202 pub metadata: ColumnChunkMetaData,
204 pub bloom_filter: Option<Sbbf>,
206 pub column_index: Option<ColumnIndexMetaData>,
208 pub offset_index: Option<OffsetIndexMetaData>,
210}
211
212impl ColumnCloseResult {
213 pub fn update_dictionary_location(mut self, dictionary_len: usize) -> Result<Self> {
224 if dictionary_len > 0 {
225 self.metadata = self
226 .metadata
227 .into_builder()
228 .set_dictionary_page_offset(Some(0))
229 .set_data_page_offset(dictionary_len as i64)
230 .build()?;
231 if let Some(offset_index) = self.offset_index.as_mut() {
232 let mut offset = dictionary_len as i64;
233 for location in &mut offset_index.page_locations {
234 location.offset = offset;
235 offset += location.compressed_page_size as i64;
236 }
237 }
238 }
239 Ok(self)
240 }
241}
242
243#[derive(Default)]
245struct PageMetrics {
246 num_buffered_values: u32,
247 num_buffered_rows: u32,
248 page_size_exemption: usize,
254 num_page_nulls: u64,
255 num_page_nans: Option<u64>,
256 repetition_level_histogram: Option<LevelHistogram>,
257 definition_level_histogram: Option<LevelHistogram>,
258}
259
260impl PageMetrics {
261 fn new() -> Self {
262 Default::default()
263 }
264
265 fn with_repetition_level_histogram(mut self, max_level: i16) -> Self {
267 self.repetition_level_histogram = LevelHistogram::try_new(max_level);
268 self
269 }
270
271 fn with_definition_level_histogram(mut self, max_level: i16) -> Self {
273 self.definition_level_histogram = LevelHistogram::try_new(max_level);
274 self
275 }
276
277 fn new_page(&mut self) {
280 self.num_buffered_values = 0;
281 self.num_buffered_rows = 0;
282 self.page_size_exemption = 0;
283 self.num_page_nulls = 0;
284 self.num_page_nans = None;
285 self.repetition_level_histogram
286 .as_mut()
287 .map(LevelHistogram::reset);
288 self.definition_level_histogram
289 .as_mut()
290 .map(LevelHistogram::reset);
291 }
292}
293
294#[derive(Default)]
296struct ColumnMetrics<T: Default> {
297 total_bytes_written: u64,
298 total_rows_written: u64,
299 total_uncompressed_size: u64,
300 total_compressed_size: u64,
301 total_num_values: u64,
302 dictionary_page_offset: Option<u64>,
303 data_page_offset: Option<u64>,
304 min_column_value: Option<T>,
305 max_column_value: Option<T>,
306 num_column_nulls: u64,
307 num_column_nans: Option<u64>,
308 column_distinct_count: Option<u64>,
309 variable_length_bytes: Option<i64>,
310 repetition_level_histogram: Option<LevelHistogram>,
311 definition_level_histogram: Option<LevelHistogram>,
312}
313
314impl<T: Default> ColumnMetrics<T> {
315 fn new() -> Self {
316 Default::default()
317 }
318
319 fn with_repetition_level_histogram(mut self, max_level: i16) -> Self {
321 self.repetition_level_histogram = LevelHistogram::try_new(max_level);
322 self
323 }
324
325 fn with_definition_level_histogram(mut self, max_level: i16) -> Self {
327 self.definition_level_histogram = LevelHistogram::try_new(max_level);
328 self
329 }
330
331 fn update_histogram(
333 chunk_histogram: &mut Option<LevelHistogram>,
334 page_histogram: Option<&LevelHistogram>,
335 ) {
336 if let (Some(page_hist), Some(chunk_hist)) = (page_histogram, chunk_histogram) {
337 chunk_hist.add(page_hist);
338 }
339 }
340
341 fn update_from_page_metrics(&mut self, page_metrics: &PageMetrics) {
344 ColumnMetrics::<T>::update_histogram(
345 &mut self.definition_level_histogram,
346 page_metrics.definition_level_histogram.as_ref(),
347 );
348 ColumnMetrics::<T>::update_histogram(
349 &mut self.repetition_level_histogram,
350 page_metrics.repetition_level_histogram.as_ref(),
351 );
352 }
353
354 fn update_variable_length_bytes(&mut self, variable_length_bytes: Option<i64>) {
356 if let Some(var_bytes) = variable_length_bytes {
357 *self.variable_length_bytes.get_or_insert(0) += var_bytes;
358 }
359 }
360}
361
362#[derive(Debug, Clone, Copy)]
370pub(crate) enum LevelDataRef<'a> {
371 Absent,
372 Materialized(&'a [i16]),
373 Uniform { value: i16, count: usize },
374}
375
376impl<'a> From<&'a [i16]> for LevelDataRef<'a> {
377 fn from(levels: &'a [i16]) -> Self {
378 Self::Materialized(levels)
379 }
380}
381
382impl<'a> From<Option<&'a [i16]>> for LevelDataRef<'a> {
383 fn from(levels: Option<&'a [i16]>) -> Self {
384 levels.map_or(Self::Absent, Self::from)
385 }
386}
387
388impl LevelDataRef<'_> {
389 pub(crate) fn len(self) -> usize {
390 match self {
391 Self::Absent => 0,
392 Self::Materialized(values) => values.len(),
393 Self::Uniform { count, .. } => count,
394 }
395 }
396
397 pub(crate) fn first(self) -> Option<i16> {
398 match self {
399 Self::Absent => None,
400 Self::Materialized(values) => values.first().copied(),
401 Self::Uniform { value, count } => (count > 0).then_some(value),
402 }
403 }
404
405 #[cfg(feature = "arrow")]
406 pub(crate) fn value_at(self, idx: usize) -> Option<i16> {
407 match self {
408 Self::Absent => None,
409 Self::Materialized(values) => values.get(idx).copied(),
410 Self::Uniform { value, count } => (idx < count).then_some(value),
411 }
412 }
413
414 pub(crate) fn slice(self, offset: usize, len: usize) -> Self {
415 match self {
416 Self::Absent => Self::Absent,
417 Self::Materialized(values) => Self::Materialized(&values[offset..offset + len]),
418 Self::Uniform { value, .. } => Self::Uniform { value, count: len },
419 }
420 }
421
422 pub(crate) fn value_count(self, total: usize, max_def: i16) -> usize {
427 match self {
428 Self::Absent => total,
429 Self::Materialized(values) => values.iter().filter(|&&d| d == max_def).count(),
430 Self::Uniform { value, count } => {
431 if value == max_def {
432 count
433 } else {
434 0
435 }
436 }
437 }
438 }
439}
440
441pub type ColumnWriterImpl<'a, T> = GenericColumnWriter<'a, ColumnValueEncoderImpl<T>>;
443
444pub struct GenericColumnWriter<'a, E: ColumnValueEncoder> {
446 descr: ColumnDescPtr,
448 props: WriterPropertiesPtr,
449 column_props: ResolvedColumnProperties,
453
454 page_writer: Box<dyn PageWriter + 'a>,
455 codec: Compression,
456 compressor: Option<Box<dyn Codec>>,
457 encoder: E,
458
459 page_metrics: PageMetrics,
460 column_metrics: ColumnMetrics<E::T>,
462
463 distinct_count_override: Option<u64>,
466
467 encodings: BTreeSet<Encoding>,
470 encoding_stats: Vec<PageEncodingStats>,
471 def_levels_encoder: LevelEncoder,
473 rep_levels_encoder: LevelEncoder,
474 data_pages: VecDeque<CompressedPage>,
475 column_index_builder: ColumnIndexBuilder,
477 offset_index_builder: Option<OffsetIndexBuilder>,
478
479 data_page_boundary_ascending: bool,
482 data_page_boundary_descending: bool,
483 last_non_null_data_page_min_max: Option<(E::T, E::T)>,
485}
486
487impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> {
488 pub fn new(
490 descr: ColumnDescPtr,
491 props: WriterPropertiesPtr,
492 page_writer: Box<dyn PageWriter + 'a>,
493 ) -> Self {
494 let column_props = props.resolve_column_properties(descr.path());
495 let codec = column_props.compression;
496 let codec_options = CodecOptionsBuilder::default().build();
497 let compressor = create_codec(codec, &codec_options).unwrap();
498 let encoder = E::try_new(&descr, props.as_ref(), &column_props).unwrap();
499
500 let statistics_enabled = column_props.statistics_enabled;
501
502 let encodings = BTreeSet::new();
503
504 let mut page_metrics = PageMetrics::new();
505 let mut column_metrics = ColumnMetrics::<E::T>::new();
506
507 if statistics_enabled != EnabledStatistics::None {
509 page_metrics = page_metrics
510 .with_repetition_level_histogram(descr.max_rep_level())
511 .with_definition_level_histogram(descr.max_def_level());
512 column_metrics = column_metrics
513 .with_repetition_level_histogram(descr.max_rep_level())
514 .with_definition_level_histogram(descr.max_def_level())
515 }
516
517 let mut column_index_builder = ColumnIndexBuilder::new(descr.physical_type());
519 if statistics_enabled != EnabledStatistics::Page {
520 column_index_builder.to_invalid()
521 }
522
523 let offset_index_builder = match props.offset_index_disabled() {
525 false => Some(OffsetIndexBuilder::new()),
526 _ => None,
527 };
528
529 Self {
530 def_levels_encoder: Self::create_level_encoder(descr.max_def_level(), &props),
531 rep_levels_encoder: Self::create_level_encoder(descr.max_rep_level(), &props),
532 descr,
533 props,
534 column_props,
535 page_writer,
536 codec,
537 compressor,
538 encoder,
539 data_pages: VecDeque::new(),
540 page_metrics,
541 column_metrics,
542 distinct_count_override: None,
543 column_index_builder,
544 offset_index_builder,
545 encodings,
546 encoding_stats: vec![],
547 data_page_boundary_ascending: true,
548 data_page_boundary_descending: true,
549 last_non_null_data_page_min_max: None,
550 }
551 }
552
553 #[cfg(feature = "arrow")]
559 pub(crate) fn set_distinct_count_override(&mut self, count: u64) {
560 self.distinct_count_override = Some(count);
561 }
562
563 #[expect(clippy::too_many_arguments)]
564 pub(crate) fn write_batch_internal(
565 &mut self,
566 values: &E::Values,
567 value_indices: Option<&[usize]>,
568 def_levels: LevelDataRef<'_>,
569 rep_levels: LevelDataRef<'_>,
570 min: Option<&E::T>,
571 max: Option<&E::T>,
572 distinct_count: Option<u64>,
573 ) -> Result<usize> {
574 if def_levels.len() != 0 && rep_levels.len() != 0 && def_levels.len() != rep_levels.len() {
576 return Err(general_err!(
577 "Inconsistent length of definition and repetition levels: {} != {}",
578 def_levels.len(),
579 rep_levels.len()
580 ));
581 }
582
583 let num_levels = def_levels.len().max(rep_levels.len());
593 let num_levels = if num_levels > 0 {
594 num_levels
595 } else {
596 value_indices.map_or_else(|| values.len(), |i| i.len())
597 };
598
599 if let Some(min) = min {
600 update_min(&self.descr, min, &mut self.column_metrics.min_column_value);
601 }
602 if let Some(max) = max {
603 update_max(&self.descr, max, &mut self.column_metrics.max_column_value);
604 }
605
606 let has_prior_data = self.column_metrics.total_rows_written != 0
608 || self.page_metrics.num_buffered_values != 0;
609 self.column_metrics.column_distinct_count =
610 if has_prior_data { None } else { distinct_count };
611
612 let mut values_offset = 0;
613 let mut levels_offset = 0;
614 let both_levels_compact = !matches!(def_levels, LevelDataRef::Materialized(_))
615 && !matches!(rep_levels, LevelDataRef::Materialized(_));
616 let has_levels = !matches!(def_levels, LevelDataRef::Absent)
617 || !matches!(rep_levels, LevelDataRef::Absent);
618
619 let base_batch_size = if both_levels_compact && has_levels {
623 self.column_props.data_page_row_count_limit
624 } else {
625 self.props.write_batch_size()
626 };
627 debug_assert!(base_batch_size > 0);
628
629 let chunker = ByteBudgetChunker::new(&self.descr, &self.column_props, base_batch_size);
630 while levels_offset < num_levels {
631 let mut end_offset = num_levels.min(levels_offset + base_batch_size);
632
633 if let LevelDataRef::Materialized(levels) = rep_levels {
635 while end_offset < levels.len() && levels[end_offset] != 0 {
636 end_offset += 1;
637 }
638 }
639
640 let chunk_size = end_offset - levels_offset;
641 let chunk_def = def_levels.slice(levels_offset, chunk_size);
642 let chunk_rep = rep_levels.slice(levels_offset, chunk_size);
643
644 let sub_batch = chunker.pick_sub_batch(
651 &self.encoder,
652 values,
653 value_indices,
654 chunk_def,
655 values_offset,
656 chunk_size,
657 );
658
659 match sub_batch {
660 None => {
661 values_offset += self.write_mini_batch(
662 values,
663 values_offset,
664 value_indices,
665 chunk_size,
666 chunk_def,
667 chunk_rep,
668 )?;
669 }
670 Some(sub_batch) => {
671 values_offset += self.write_granular_chunk(
672 values,
673 values_offset,
674 value_indices,
675 chunk_size,
676 chunk_def,
677 chunk_rep,
678 sub_batch,
679 )?;
680 }
681 }
682 levels_offset = end_offset;
683 }
684
685 Ok(values_offset)
687 }
688
689 pub fn write_batch(
702 &mut self,
703 values: &E::Values,
704 def_levels: Option<&[i16]>,
705 rep_levels: Option<&[i16]>,
706 ) -> Result<usize> {
707 self.write_batch_internal(
708 values,
709 None,
710 LevelDataRef::from(def_levels),
711 LevelDataRef::from(rep_levels),
712 None,
713 None,
714 None,
715 )
716 }
717
718 pub fn write_batch_with_statistics(
726 &mut self,
727 values: &E::Values,
728 def_levels: Option<&[i16]>,
729 rep_levels: Option<&[i16]>,
730 min: Option<&E::T>,
731 max: Option<&E::T>,
732 distinct_count: Option<u64>,
733 ) -> Result<usize> {
734 self.write_batch_internal(
735 values,
736 None,
737 LevelDataRef::from(def_levels),
738 LevelDataRef::from(rep_levels),
739 min,
740 max,
741 distinct_count,
742 )
743 }
744
745 #[cfg(feature = "arrow")]
750 pub(crate) fn memory_size(&self) -> usize {
751 self.encoder.estimated_memory_size()
757 + self
758 .data_pages
759 .iter()
760 .map(|page| page.memory_usage())
761 .sum::<usize>()
762 + self.page_writer.buffered_memory_size()
763 }
764
765 pub fn get_total_bytes_written(&self) -> u64 {
771 self.column_metrics.total_bytes_written
772 }
773
774 #[cfg(feature = "arrow")]
780 pub(crate) fn get_estimated_total_bytes(&self) -> u64 {
781 self.data_pages
782 .iter()
783 .map(|page| page.data().len() as u64)
784 .sum::<u64>()
785 + self.column_metrics.total_bytes_written
786 + self.encoder.estimated_data_page_size() as u64
787 + self.encoder.estimated_dict_page_size().unwrap_or_default() as u64
788 }
789
790 pub fn get_total_rows_written(&self) -> u64 {
793 self.column_metrics.total_rows_written
794 }
795
796 pub fn get_descriptor(&self) -> &ColumnDescPtr {
798 &self.descr
799 }
800
801 pub fn close(mut self) -> Result<ColumnCloseResult> {
804 if self.page_metrics.num_buffered_values > 0 {
805 self.add_data_page()?;
806 }
807 if self.encoder.has_dictionary() && self.column_metrics.total_rows_written > 0 {
808 self.write_dictionary_page()?;
809 }
810 self.flush_data_pages()?;
811 let metadata = self.build_column_metadata()?;
812 self.page_writer.close()?;
813
814 let write_bloom_filter = self.props.bloom_filter_for_dictionary_encoded_chunks()
815 || self.has_non_dictionary_data_page();
816 let bloom_filter = self
817 .encoder
818 .flush_bloom_filter()
819 .filter(|_| write_bloom_filter);
820
821 let boundary_order = match (
822 self.data_page_boundary_ascending,
823 self.data_page_boundary_descending,
824 ) {
825 (true, _) => BoundaryOrder::ASCENDING,
828 (false, true) => BoundaryOrder::DESCENDING,
829 (false, false) => BoundaryOrder::UNORDERED,
830 };
831 self.column_index_builder.set_boundary_order(boundary_order);
832
833 let column_index = match self.column_index_builder.valid() {
834 true => Some(self.column_index_builder.build()?),
835 false => None,
836 };
837
838 let offset_index = self.offset_index_builder.map(|b| b.build());
839
840 Ok(ColumnCloseResult {
841 bytes_written: self.column_metrics.total_bytes_written,
842 rows_written: self.column_metrics.total_rows_written,
843 bloom_filter,
844 metadata,
845 column_index,
846 offset_index,
847 })
848 }
849
850 #[expect(clippy::too_many_arguments)]
871 #[inline(never)]
872 fn write_granular_chunk(
873 &mut self,
874 values: &E::Values,
875 values_offset: usize,
876 value_indices: Option<&[usize]>,
877 chunk_size: usize,
878 chunk_def: LevelDataRef<'_>,
879 chunk_rep: LevelDataRef<'_>,
880 sub_batch: SubBatchStrategy,
881 ) -> Result<usize> {
882 debug_assert!(
886 matches!(
887 sub_batch,
888 SubBatchStrategy::Values(n) | SubBatchStrategy::Levels { levels: n, max_values: _ }
889 if n >= 1
890 ),
891 "chunker must size at least one value or level"
892 );
893 let max_def_level = self.descr.max_def_level();
894 let mut values_consumed = 0;
895 let mut sub_start = 0;
896 while sub_start < chunk_size {
897 let window_end = match sub_batch {
898 SubBatchStrategy::Values(n) => {
899 Self::window_end_for_values(chunk_def, chunk_size, max_def_level, sub_start, n)
900 }
901 SubBatchStrategy::Levels { levels, max_values } => {
902 let level_end = (sub_start + levels).min(chunk_size);
905 Self::window_end_for_values(
906 chunk_def,
907 level_end,
908 max_def_level,
909 sub_start,
910 max_values,
911 )
912 }
913 };
914 let sub_end = match chunk_rep {
915 LevelDataRef::Materialized(levels) => {
916 let mut e = window_end;
923 while e < chunk_size && levels[e] != 0 {
924 e += 1;
925 }
926 e
927 }
928 _ => window_end,
929 };
930 let sub_len = sub_end - sub_start;
931 let written = self.write_mini_batch(
932 values,
933 values_offset + values_consumed,
934 value_indices,
935 sub_len,
936 chunk_def.slice(sub_start, sub_len),
937 chunk_rep.slice(sub_start, sub_len),
938 )?;
939 values_consumed += written;
940 sub_start = sub_end;
941 }
942 Ok(values_consumed)
943 }
944
945 fn has_non_dictionary_data_page(&self) -> bool {
946 self.encoding_stats.iter().any(|stats| {
947 matches!(
948 stats.page_type,
949 PageType::DATA_PAGE | PageType::DATA_PAGE_V2
950 ) && !matches!(
951 stats.encoding,
952 Encoding::PLAIN_DICTIONARY | Encoding::RLE_DICTIONARY
953 )
954 })
955 }
956
957 fn window_end_for_values(
972 chunk_def: LevelDataRef<'_>,
973 chunk_size: usize,
974 max_def_level: i16,
975 start: usize,
976 max_values: usize,
977 ) -> usize {
978 match chunk_def {
979 LevelDataRef::Absent => (start + max_values).min(chunk_size),
981 LevelDataRef::Uniform { value, .. } => {
982 if value == max_def_level {
983 (start + max_values).min(chunk_size)
984 } else {
985 chunk_size
989 }
990 }
991 LevelDataRef::Materialized(levels) => {
992 let mut seen = 0;
993 let mut end = start;
994 while end < chunk_size && seen < max_values {
995 if levels[end] == max_def_level {
996 seen += 1;
997 }
998 end += 1;
999 }
1000 end
1001 }
1002 }
1003 }
1004
1005 fn create_level_encoder(max_level: i16, props: &WriterProperties) -> LevelEncoder {
1007 match props.writer_version() {
1008 WriterVersion::PARQUET_1_0 => LevelEncoder::v1_streaming(max_level),
1009 WriterVersion::PARQUET_2_0 => LevelEncoder::v2_streaming(max_level),
1010 }
1011 }
1012
1013 fn write_mini_batch(
1017 &mut self,
1018 values: &E::Values,
1019 values_offset: usize,
1020 value_indices: Option<&[usize]>,
1021 num_levels: usize,
1022 def_levels: LevelDataRef<'_>,
1023 rep_levels: LevelDataRef<'_>,
1024 ) -> Result<usize> {
1025 let values_to_write = if self.descr.max_def_level() > 0 {
1027 let max_def = self.descr.max_def_level();
1028 match def_levels {
1029 LevelDataRef::Absent => {
1030 return Err(general_err!(
1031 "Definition levels are required, because max definition level = {}",
1032 self.descr.max_def_level()
1033 ));
1034 }
1035 LevelDataRef::Materialized(levels) => {
1036 let mut values_to_write = 0usize;
1039 let encoder = &mut self.def_levels_encoder;
1040 match self.page_metrics.definition_level_histogram.as_mut() {
1041 Some(histogram) => encoder.put_with_observer(levels, |level, count| {
1042 values_to_write += count * (level == max_def) as usize;
1043 histogram.increment_by(level, count as i64);
1044 }),
1045 None => encoder.put_with_observer(levels, |level, count| {
1046 values_to_write += count * (level == max_def) as usize;
1047 }),
1048 };
1049 self.page_metrics.num_page_nulls += (levels.len() - values_to_write) as u64;
1050 values_to_write
1051 }
1052 LevelDataRef::Uniform { value, count } => {
1053 let encoder = &mut self.def_levels_encoder;
1056 match self.page_metrics.definition_level_histogram.as_mut() {
1057 Some(histogram) => {
1058 encoder.put_n_with_observer(value, count, |level, run_len| {
1059 histogram.increment_by(level, run_len as i64);
1060 })
1061 }
1062 None => encoder.put_n_with_observer(value, count, |_, _| {}),
1063 }
1064 let values_to_write = count * (value == max_def) as usize;
1065 self.page_metrics.num_page_nulls += (count - values_to_write) as u64;
1066 values_to_write
1067 }
1068 }
1069 } else {
1070 num_levels
1071 };
1072
1073 if self.descr.max_rep_level() > 0 {
1075 let first_level = rep_levels.first().ok_or_else(|| {
1077 general_err!(
1078 "Repetition levels are required, because max repetition level = {}",
1079 self.descr.max_rep_level()
1080 )
1081 })?;
1082
1083 if first_level != 0 {
1084 return Err(general_err!(
1085 "Write must start at a record boundary, got non-zero repetition level of {}",
1086 first_level
1087 ));
1088 }
1089
1090 let mut new_rows = 0u32;
1091 match rep_levels {
1092 LevelDataRef::Absent => unreachable!(),
1093 LevelDataRef::Materialized(levels) => {
1094 let encoder = &mut self.rep_levels_encoder;
1095 match self.page_metrics.repetition_level_histogram.as_mut() {
1096 Some(histogram) => encoder.put_with_observer(levels, |level, count| {
1097 new_rows += (count as u32) * (level == 0) as u32;
1098 histogram.increment_by(level, count as i64);
1099 }),
1100 None => encoder.put_with_observer(levels, |level, count| {
1101 new_rows += (count as u32) * (level == 0) as u32;
1102 }),
1103 };
1104 }
1105 LevelDataRef::Uniform { value, count } => {
1106 let encoder = &mut self.rep_levels_encoder;
1107 match self.page_metrics.repetition_level_histogram.as_mut() {
1108 Some(histogram) => {
1109 encoder.put_n_with_observer(value, count, |level, run_len| {
1110 new_rows += (run_len as u32) * (level == 0) as u32;
1111 histogram.increment_by(level, run_len as i64);
1112 })
1113 }
1114 None => encoder.put_n_with_observer(value, count, |level, run_len| {
1115 new_rows += (run_len as u32) * (level == 0) as u32;
1116 }),
1117 }
1118 }
1119 }
1120 self.page_metrics.num_buffered_rows += new_rows;
1121 } else {
1122 self.page_metrics.num_buffered_rows += num_levels as u32;
1125 }
1126
1127 match value_indices {
1128 Some(all_indices) => {
1129 let indices = &all_indices[values_offset..values_offset + values_to_write];
1130 self.encoder
1131 .write_gather(values, indices, all_indices.len())?;
1132 }
1133 None => self.encoder.write(values, values_offset, values_to_write)?,
1134 }
1135
1136 let page_was_empty = self.page_metrics.num_buffered_values == 0;
1137 self.page_metrics.num_buffered_values += num_levels as u32;
1138
1139 if page_was_empty && values_to_write == 1 {
1140 self.set_page_size_exemption();
1141 }
1142
1143 if self.should_add_data_page() {
1144 self.add_data_page()?;
1145 }
1146
1147 if self.should_dict_fallback() {
1148 self.dict_fallback()?;
1149 }
1150
1151 Ok(values_to_write)
1152 }
1153
1154 #[inline]
1159 fn should_dict_fallback(&self) -> bool {
1160 match self.encoder.estimated_dict_page_size() {
1161 Some(size) => size >= self.column_props.dictionary_page_size_limit,
1162 None => false,
1163 }
1164 }
1165
1166 #[cold]
1194 fn set_page_size_exemption(&mut self) {
1195 if !self.encoder.compresses_against_previous_value() {
1196 return;
1197 }
1198 let size = self.encoder.estimated_data_page_size();
1199 if size >= self.column_props.data_page_size_limit {
1200 self.page_metrics.page_size_exemption = size;
1201 }
1202 }
1203
1204 #[inline]
1206 fn should_add_data_page(&self) -> bool {
1207 if self.page_metrics.num_buffered_values == 0 {
1212 return false;
1213 }
1214
1215 self.page_metrics.num_buffered_rows as usize >= self.column_props.data_page_row_count_limit
1216 || self
1217 .encoder
1218 .estimated_data_page_size()
1219 .saturating_sub(self.page_metrics.page_size_exemption)
1220 >= self.column_props.data_page_size_limit
1221 }
1222
1223 fn dict_fallback(&mut self) -> Result<()> {
1226 if self.page_metrics.num_buffered_values > 0 {
1228 self.add_data_page()?;
1229 }
1230 self.write_dictionary_page()?;
1231 self.flush_data_pages()?;
1232 Ok(())
1233 }
1234
1235 fn get_nan_count<T: ParquetValueType>(&self) -> Option<i64> {
1238 let nan_count = || {
1239 let nan_count = self.page_metrics.num_page_nans.unwrap_or(0);
1240 match i64::try_from(nan_count) {
1241 Ok(count) => Some(count),
1242 _ => Some(i64::MAX),
1243 }
1244 };
1245 match T::PHYSICAL_TYPE {
1246 Type::FLOAT | Type::DOUBLE => nan_count(),
1247 Type::FIXED_LEN_BYTE_ARRAY
1248 if matches!(self.descr.logical_type_ref(), Some(LogicalType::Float16)) =>
1249 {
1250 nan_count()
1251 }
1252 _ => None,
1253 }
1254 }
1255
1256 fn update_column_offset_index(
1258 &mut self,
1259 page_statistics: Option<&ValueStatistics<E::T>>,
1260 page_variable_length_bytes: Option<i64>,
1261 ) {
1262 let null_page =
1266 (self.page_metrics.num_buffered_values as u64) == self.page_metrics.num_page_nulls;
1267 if null_page && self.column_index_builder.valid() {
1270 self.column_index_builder.append(
1271 null_page,
1272 vec![],
1273 vec![],
1274 self.page_metrics.num_page_nulls as i64,
1275 self.get_nan_count::<E::T>(),
1276 );
1277 } else if self.column_index_builder.valid() {
1278 match &page_statistics {
1281 None => {
1282 self.column_index_builder.to_invalid();
1283 }
1284 Some(stat) => {
1285 let new_min = stat.min_opt().unwrap();
1287 let new_max = stat.max_opt().unwrap();
1288 if let Some((last_min, last_max)) = &self.last_non_null_data_page_min_max {
1289 let basic_info = self.descr.get_basic_info();
1290 if self.data_page_boundary_ascending {
1291 let not_ascending = compare_greater(basic_info, last_min, new_min)
1293 || compare_greater(basic_info, last_max, new_max);
1294 if not_ascending {
1295 self.data_page_boundary_ascending = false;
1296 }
1297 }
1298
1299 if self.data_page_boundary_descending {
1300 let not_descending = compare_greater(basic_info, new_min, last_min)
1302 || compare_greater(basic_info, new_max, last_max);
1303 if not_descending {
1304 self.data_page_boundary_descending = false;
1305 }
1306 }
1307 }
1308 self.last_non_null_data_page_min_max = Some((new_min.clone(), new_max.clone()));
1309
1310 if self.can_truncate_value() {
1311 self.column_index_builder.append(
1312 null_page,
1313 self.truncate_min_value(
1314 self.props.column_index_truncate_length(),
1315 stat.min_bytes_opt().unwrap(),
1316 )
1317 .0,
1318 self.truncate_max_value(
1319 self.props.column_index_truncate_length(),
1320 stat.max_bytes_opt().unwrap(),
1321 )
1322 .0,
1323 self.page_metrics.num_page_nulls as i64,
1324 self.get_nan_count::<E::T>(),
1325 );
1326 } else {
1327 self.column_index_builder.append(
1328 null_page,
1329 stat.min_bytes_opt().unwrap().to_vec(),
1330 stat.max_bytes_opt().unwrap().to_vec(),
1331 self.page_metrics.num_page_nulls as i64,
1332 self.get_nan_count::<E::T>(),
1333 );
1334 }
1335 }
1336 }
1337 }
1338
1339 self.column_index_builder.append_histograms(
1341 &self.page_metrics.repetition_level_histogram,
1342 &self.page_metrics.definition_level_histogram,
1343 );
1344
1345 if let Some(builder) = self.offset_index_builder.as_mut() {
1347 builder.append_row_count(self.page_metrics.num_buffered_rows as i64);
1348 builder.append_unencoded_byte_array_data_bytes(page_variable_length_bytes);
1349 }
1350 }
1351
1352 fn can_truncate_value(&self) -> bool {
1354 match self.descr.physical_type() {
1355 Type::FIXED_LEN_BYTE_ARRAY
1359 if !is_decimal_descr(self.descr.get_basic_info())
1360 && !matches!(self.descr.logical_type_ref(), Some(&LogicalType::Float16)) =>
1361 {
1362 true
1363 }
1364 Type::BYTE_ARRAY if !is_decimal_descr(self.descr.get_basic_info()) => true,
1366 _ => false,
1368 }
1369 }
1370
1371 fn is_utf8(&self) -> bool {
1373 self.get_descriptor().logical_type_ref() == Some(&LogicalType::String)
1374 || self.get_descriptor().converted_type() == ConvertedType::UTF8
1375 }
1376
1377 fn truncate_min_value(&self, truncation_length: Option<usize>, data: &[u8]) -> (Vec<u8>, bool) {
1388 truncation_length
1389 .filter(|l| data.len() > *l)
1390 .and_then(|l|
1391 if self.is_utf8() {
1393 match str::from_utf8(data) {
1394 Ok(str_data) => truncate_utf8(str_data, l),
1395 Err(_) => Some(data[..l].to_vec()),
1396 }
1397 } else {
1398 Some(data[..l].to_vec())
1399 }
1400 )
1401 .map(|truncated| (truncated, true))
1402 .unwrap_or_else(|| (data.to_vec(), false))
1403 }
1404
1405 fn truncate_max_value(&self, truncation_length: Option<usize>, data: &[u8]) -> (Vec<u8>, bool) {
1419 truncation_length
1420 .filter(|l| data.len() > *l)
1421 .and_then(|l|
1422 if self.is_utf8() {
1424 match str::from_utf8(data) {
1425 Ok(str_data) => truncate_and_increment_utf8(str_data, l),
1426 Err(_) => increment(data[..l].to_vec()),
1427 }
1428 } else {
1429 increment(data[..l].to_vec())
1430 }
1431 )
1432 .map(|truncated| (truncated, true))
1433 .unwrap_or_else(|| (data.to_vec(), false))
1434 }
1435
1436 fn truncate_statistics(&self, statistics: Statistics) -> Statistics {
1444 let truncation_length = self
1445 .can_truncate_value()
1446 .then(|| self.props.statistics_truncate_length())
1447 .flatten();
1448 match statistics {
1449 Statistics::ByteArray(stats) if stats._internal_has_min_max_set() => {
1450 Statistics::ByteArray(self.copy_and_truncate_min_max(stats, truncation_length))
1451 }
1452 Statistics::FixedLenByteArray(stats) if stats._internal_has_min_max_set() => {
1453 Statistics::FixedLenByteArray(
1454 self.copy_and_truncate_min_max(stats, truncation_length),
1455 )
1456 }
1457 stats => stats,
1458 }
1459 }
1460
1461 fn copy_and_truncate_min_max<T>(
1469 &self,
1470 stats: ValueStatistics<T>,
1471 truncation_length: Option<usize>,
1472 ) -> ValueStatistics<T>
1473 where
1474 T: ParquetValueType + From<Vec<u8>>,
1475 {
1476 let (min, did_truncate_min) =
1477 self.truncate_min_value(truncation_length, stats.min_bytes_opt().unwrap());
1478 let (max, did_truncate_max) =
1479 self.truncate_max_value(truncation_length, stats.max_bytes_opt().unwrap());
1480 let min_is_exact = stats.min_is_exact() && !did_truncate_min;
1481 let max_is_exact = stats.max_is_exact() && !did_truncate_max;
1482 stats
1483 .with_min_max(Some(min.into()), Some(max.into()))
1484 .with_min_is_exact(min_is_exact)
1485 .with_max_is_exact(max_is_exact)
1486 }
1487
1488 pub(crate) fn add_data_page(&mut self) -> Result<()> {
1494 if self.page_metrics.num_buffered_values == 0 {
1495 return Ok(());
1496 }
1497
1498 let values_data = self.encoder.flush_data_page()?;
1500
1501 let max_def_level = self.descr.max_def_level();
1502 let max_rep_level = self.descr.max_rep_level();
1503
1504 self.column_metrics.num_column_nulls += self.page_metrics.num_page_nulls;
1505
1506 if let Some(nan_count) = values_data.nan_count {
1507 *self.column_metrics.num_column_nans.get_or_insert(0) += nan_count;
1508 self.page_metrics.num_page_nans = Some(nan_count);
1509 }
1510
1511 let page_statistics = match (values_data.min_value, values_data.max_value) {
1512 (Some(min), Some(max)) => {
1513 update_min(&self.descr, &min, &mut self.column_metrics.min_column_value);
1515 update_max(&self.descr, &max, &mut self.column_metrics.max_column_value);
1516
1517 (self.column_props.statistics_enabled == EnabledStatistics::Page).then_some(
1518 ValueStatistics::new(
1519 Some(min),
1520 Some(max),
1521 None,
1522 Some(self.page_metrics.num_page_nulls),
1523 false,
1524 )
1525 .with_nan_count(values_data.nan_count),
1526 )
1527 }
1528 _ => None,
1529 };
1530
1531 self.update_column_offset_index(
1533 page_statistics.as_ref(),
1534 values_data.variable_length_bytes,
1535 );
1536
1537 self.column_metrics
1539 .update_from_page_metrics(&self.page_metrics);
1540 self.column_metrics
1541 .update_variable_length_bytes(values_data.variable_length_bytes);
1542
1543 let page_statistics = page_statistics
1545 .filter(|_| self.column_props.write_page_header_statistics)
1546 .map(|stats| self.truncate_statistics(Statistics::from(stats)));
1547
1548 let compressed_page = match self.props.writer_version() {
1549 WriterVersion::PARQUET_1_0 => {
1550 let mut buffer = vec![];
1551
1552 if max_rep_level > 0 {
1553 self.rep_levels_encoder
1554 .flush_to(|data| buffer.extend_from_slice(data));
1555 }
1556
1557 if max_def_level > 0 {
1558 self.def_levels_encoder
1559 .flush_to(|data| buffer.extend_from_slice(data));
1560 }
1561
1562 buffer.extend_from_slice(&values_data.buf);
1563 let uncompressed_size = buffer.len();
1564
1565 if let Some(ref mut cmpr) = self.compressor {
1566 let mut compressed_buf = Vec::with_capacity(uncompressed_size);
1567 cmpr.compress(&buffer[..], &mut compressed_buf)?;
1568 compressed_buf.shrink_to_fit();
1569 buffer = compressed_buf;
1570 }
1571
1572 let data_page = Page::DataPage {
1573 buf: buffer.into(),
1574 num_values: self.page_metrics.num_buffered_values,
1575 encoding: values_data.encoding,
1576 def_level_encoding: Encoding::RLE,
1577 rep_level_encoding: Encoding::RLE,
1578 statistics: page_statistics,
1579 };
1580
1581 CompressedPage::new(data_page, uncompressed_size)
1582 }
1583 WriterVersion::PARQUET_2_0 => {
1584 let mut rep_levels_byte_len = 0;
1585 let mut def_levels_byte_len = 0;
1586 let mut buffer = vec![];
1587
1588 if max_rep_level > 0 {
1589 self.rep_levels_encoder
1590 .flush_to(|data| buffer.extend_from_slice(data));
1591 rep_levels_byte_len = buffer.len();
1592 }
1593
1594 if max_def_level > 0 {
1595 self.def_levels_encoder
1596 .flush_to(|data| buffer.extend_from_slice(data));
1597 def_levels_byte_len = buffer.len() - rep_levels_byte_len;
1598 }
1599
1600 let uncompressed_size =
1601 rep_levels_byte_len + def_levels_byte_len + values_data.buf.len();
1602
1603 let is_compressed = match self.compressor {
1605 Some(ref mut cmpr) => {
1606 let buffer_len = buffer.len();
1607 cmpr.compress(&values_data.buf, &mut buffer)?;
1608 let compressed_values_size = buffer.len() - buffer_len;
1609 let threshold = self.column_props.data_page_v2_compression_ratio_threshold;
1610 if (compressed_values_size as f64) >= (uncompressed_size as f64) * threshold
1611 {
1612 buffer.truncate(buffer_len);
1613 buffer.extend_from_slice(&values_data.buf);
1614 false
1615 } else {
1616 true
1617 }
1618 }
1619 None => {
1620 buffer.extend_from_slice(&values_data.buf);
1621 false
1622 }
1623 };
1624
1625 let data_page = Page::DataPageV2 {
1626 buf: buffer.into(),
1627 num_values: self.page_metrics.num_buffered_values,
1628 encoding: values_data.encoding,
1629 num_nulls: self.page_metrics.num_page_nulls as u32,
1630 num_rows: self.page_metrics.num_buffered_rows,
1631 def_levels_byte_len: def_levels_byte_len as u32,
1632 rep_levels_byte_len: rep_levels_byte_len as u32,
1633 is_compressed,
1634 statistics: page_statistics,
1635 };
1636
1637 CompressedPage::new(data_page, uncompressed_size)
1638 }
1639 };
1640
1641 if self.encoder.has_dictionary() && !self.page_writer.defers_dictionary_ordering() {
1650 self.data_pages.push_back(compressed_page);
1651 } else {
1652 self.write_data_page(compressed_page)?;
1653 }
1654
1655 self.column_metrics.total_rows_written += self.page_metrics.num_buffered_rows as u64;
1657 self.page_metrics.new_page();
1658
1659 Ok(())
1660 }
1661
1662 #[inline]
1665 fn flush_data_pages(&mut self) -> Result<()> {
1666 if self.page_metrics.num_buffered_values > 0 {
1668 self.add_data_page()?;
1669 }
1670
1671 while let Some(page) = self.data_pages.pop_front() {
1672 self.write_data_page(page)?;
1673 }
1674
1675 Ok(())
1676 }
1677
1678 fn build_column_metadata(&mut self) -> Result<ColumnChunkMetaData> {
1680 let total_compressed_size = self.column_metrics.total_compressed_size as i64;
1681 let total_uncompressed_size = self.column_metrics.total_uncompressed_size as i64;
1682 let num_values = self.column_metrics.total_num_values as i64;
1683 let dict_page_offset = self.column_metrics.dictionary_page_offset.map(|v| v as i64);
1684 let data_page_offset = self.column_metrics.data_page_offset.unwrap_or(0) as i64;
1686
1687 let mut builder = ColumnChunkMetaData::builder(self.descr.clone())
1688 .set_compression(self.codec)
1689 .set_encodings_mask(EncodingMask::new_from_encodings(self.encodings.iter()))
1690 .set_page_encoding_stats(self.encoding_stats.clone())
1691 .set_total_compressed_size(total_compressed_size)
1692 .set_total_uncompressed_size(total_uncompressed_size)
1693 .set_num_values(num_values)
1694 .set_data_page_offset(data_page_offset)
1695 .set_dictionary_page_offset(dict_page_offset);
1696
1697 if self.column_props.statistics_enabled != EnabledStatistics::None {
1698 let backwards_compatible_min_max = self.descr.sort_order().is_signed();
1699
1700 let distinct_count = self
1701 .distinct_count_override
1702 .or(self.column_metrics.column_distinct_count);
1703 let statistics = ValueStatistics::<E::T>::new(
1704 self.column_metrics.min_column_value.clone(),
1705 self.column_metrics.max_column_value.clone(),
1706 distinct_count,
1707 Some(self.column_metrics.num_column_nulls),
1708 false,
1709 )
1710 .with_nan_count(self.column_metrics.num_column_nans)
1711 .with_backwards_compatible_min_max(backwards_compatible_min_max)
1712 .into();
1713
1714 let statistics = self.truncate_statistics(statistics);
1715
1716 builder = builder
1717 .set_statistics(statistics)
1718 .set_unencoded_byte_array_data_bytes(self.column_metrics.variable_length_bytes)
1719 .set_repetition_level_histogram(
1720 self.column_metrics.repetition_level_histogram.take(),
1721 )
1722 .set_definition_level_histogram(
1723 self.column_metrics.definition_level_histogram.take(),
1724 );
1725
1726 if let Some(geo_stats) = self.encoder.flush_geospatial_statistics() {
1727 builder = builder.set_geo_statistics(geo_stats);
1728 }
1729 }
1730
1731 builder = self.set_column_chunk_encryption_properties(builder);
1732
1733 let metadata = builder.build()?;
1734 Ok(metadata)
1735 }
1736
1737 #[inline]
1739 fn write_data_page(&mut self, page: CompressedPage) -> Result<()> {
1740 self.encodings.insert(page.encoding());
1741 if self.descr.max_def_level() > 0 || self.descr.max_rep_level() > 0 {
1742 self.encodings.insert(Encoding::RLE);
1743 }
1744 match self.encoding_stats.last_mut() {
1745 Some(encoding_stats)
1746 if encoding_stats.page_type == page.page_type()
1747 && encoding_stats.encoding == page.encoding() =>
1748 {
1749 encoding_stats.count += 1;
1750 }
1751 _ => {
1752 self.encoding_stats.push(PageEncodingStats {
1755 page_type: page.page_type(),
1756 encoding: page.encoding(),
1757 count: 1,
1758 });
1759 }
1760 }
1761 let page_spec = self.page_writer.write_page(page)?;
1762 if let Some(builder) = self.offset_index_builder.as_mut() {
1765 builder
1766 .append_offset_and_size(page_spec.offset as i64, page_spec.compressed_size as i32)
1767 }
1768 self.update_metrics_for_page(page_spec);
1769 Ok(())
1770 }
1771
1772 #[inline]
1774 fn write_dictionary_page(&mut self) -> Result<()> {
1775 let compressed_page = {
1776 let mut page = self
1777 .encoder
1778 .flush_dict_page()?
1779 .ok_or_else(|| general_err!("Dictionary encoder is not set"))?;
1780
1781 let uncompressed_size = page.buf.len();
1782
1783 if let Some(ref mut cmpr) = self.compressor {
1784 let mut output_buf = Vec::with_capacity(uncompressed_size);
1785 cmpr.compress(&page.buf, &mut output_buf)?;
1786 page.buf = Bytes::from(output_buf);
1787 }
1788
1789 let dict_page = Page::DictionaryPage {
1790 buf: page.buf,
1791 num_values: page.num_values as u32,
1792 encoding: self.props.dictionary_page_encoding(),
1793 is_sorted: page.is_sorted,
1794 };
1795 CompressedPage::new(dict_page, uncompressed_size)
1796 };
1797
1798 self.encodings.insert(compressed_page.encoding());
1799 self.encoding_stats.insert(
1804 0,
1805 PageEncodingStats {
1806 page_type: PageType::DICTIONARY_PAGE,
1807 encoding: compressed_page.encoding(),
1808 count: 1,
1809 },
1810 );
1811 let page_spec = self.page_writer.write_page(compressed_page)?;
1812 self.update_metrics_for_page(page_spec);
1813 Ok(())
1815 }
1816
1817 #[inline]
1819 fn update_metrics_for_page(&mut self, page_spec: PageWriteSpec) {
1820 self.column_metrics.total_uncompressed_size += page_spec.uncompressed_size as u64;
1821 self.column_metrics.total_compressed_size += page_spec.compressed_size as u64;
1822 self.column_metrics.total_bytes_written += page_spec.bytes_written;
1823
1824 match page_spec.page_type {
1825 PageType::DATA_PAGE | PageType::DATA_PAGE_V2 => {
1826 self.column_metrics.total_num_values += page_spec.num_values as u64;
1827 if self.column_metrics.data_page_offset.is_none() {
1828 self.column_metrics.data_page_offset = Some(page_spec.offset);
1829 }
1830 }
1831 PageType::DICTIONARY_PAGE => {
1832 assert!(
1833 self.column_metrics.dictionary_page_offset.is_none(),
1834 "Dictionary offset is already set"
1835 );
1836 self.column_metrics.dictionary_page_offset = Some(page_spec.offset);
1837 }
1838 PageType::INDEX_PAGE => {}
1839 }
1840 }
1841
1842 #[inline]
1843 #[cfg(feature = "encryption")]
1844 fn set_column_chunk_encryption_properties(
1845 &self,
1846 builder: ColumnChunkMetaDataBuilder,
1847 ) -> ColumnChunkMetaDataBuilder {
1848 if let Some(encryption_properties) = self.props.file_encryption_properties.as_ref() {
1849 builder.set_column_crypto_metadata(get_column_crypto_metadata(
1850 encryption_properties,
1851 &self.descr,
1852 ))
1853 } else {
1854 builder
1855 }
1856 }
1857
1858 #[inline]
1859 #[cfg(not(feature = "encryption"))]
1860 fn set_column_chunk_encryption_properties(
1861 &self,
1862 builder: ColumnChunkMetaDataBuilder,
1863 ) -> ColumnChunkMetaDataBuilder {
1864 builder
1865 }
1866}
1867
1868fn update_min<T: ParquetValueType>(descr: &ColumnDescriptor, val: &T, min: &mut Option<T>) {
1869 match min {
1870 None => *min = Some(val.clone()),
1871 Some(min) => {
1872 let basic_type_info = descr.get_basic_info();
1873 let is_min_nan = is_nan(basic_type_info, min);
1874 let is_val_nan = is_nan(basic_type_info, val);
1875 match (is_min_nan, is_val_nan) {
1876 (false, true) => {}
1878 (true, false) => *min = val.clone(),
1880 _ => {
1882 update_stat::<T, _>(val, min, |cur| compare_greater(basic_type_info, cur, val))
1883 }
1884 }
1885 }
1886 }
1887}
1888
1889fn update_max<T: ParquetValueType>(descr: &ColumnDescriptor, val: &T, max: &mut Option<T>) {
1890 match max {
1891 None => *max = Some(val.clone()),
1892 Some(max) => {
1893 let basic_type_info = descr.get_basic_info();
1894 let is_max_nan = is_nan(basic_type_info, max);
1895 let is_val_nan = is_nan(basic_type_info, val);
1896 match (is_max_nan, is_val_nan) {
1897 (false, true) => {}
1899 (true, false) => *max = val.clone(),
1901 _ => {
1903 update_stat::<T, _>(val, max, |cur| compare_greater(basic_type_info, val, cur))
1904 }
1905 }
1906 }
1907 }
1908}
1909
1910#[inline]
1911#[expect(clippy::eq_op)]
1912fn is_nan<T: ParquetValueType>(basic_type_info: &BasicTypeInfo, val: &T) -> bool {
1913 match T::PHYSICAL_TYPE {
1914 Type::FLOAT | Type::DOUBLE => val != val,
1915 Type::FIXED_LEN_BYTE_ARRAY
1916 if matches!(basic_type_info.sort_order(), SortOrder::TOTAL_ORDER) =>
1917 {
1918 let val = val.as_bytes();
1920 let uval = ((val[1] as u16) << 8) | val[0] as u16;
1922 uval & 0x7FFFu16 > 0x7C00u16
1923 }
1924 _ => false,
1925 }
1926}
1927
1928fn update_stat<T: ParquetValueType, F>(val: &T, cur: &mut T, should_update: F)
1933where
1934 F: Fn(&T) -> bool,
1935{
1936 if should_update(cur) {
1937 *cur = val.clone();
1938 }
1939}
1940
1941fn compare_greater<T: ParquetValueType>(basic_type_info: &BasicTypeInfo, a: &T, b: &T) -> bool {
1943 match T::PHYSICAL_TYPE {
1944 Type::FLOAT => {
1945 let a = f32::from_le_bytes(a.as_bytes().try_into().unwrap());
1946 let b = f32::from_le_bytes(b.as_bytes().try_into().unwrap());
1947 return a.total_cmp(&b) == Ordering::Greater;
1948 }
1949 Type::DOUBLE => {
1950 let a = f64::from_le_bytes(a.as_bytes().try_into().unwrap());
1951 let b = f64::from_le_bytes(b.as_bytes().try_into().unwrap());
1952 return a.total_cmp(&b) == Ordering::Greater;
1953 }
1954 Type::INT32 | Type::INT64
1955 if matches!(basic_type_info.sort_order(), SortOrder::UNSIGNED) =>
1956 {
1957 return compare_greater_unsigned_int(a, b);
1958 }
1959 Type::FIXED_LEN_BYTE_ARRAY
1960 if matches!(basic_type_info.sort_order(), SortOrder::TOTAL_ORDER) =>
1961 {
1962 return compare_greater_f16(a.as_bytes(), b.as_bytes());
1963 }
1964 Type::FIXED_LEN_BYTE_ARRAY | Type::BYTE_ARRAY if is_decimal_descr(basic_type_info) => {
1965 return compare_greater_byte_array_decimals(a.as_bytes(), b.as_bytes());
1966 }
1967
1968 _ => {}
1969 }
1970
1971 a > b
1973}
1974
1975fn fallback_encoding(kind: Type, props: &WriterProperties) -> Encoding {
1983 match (kind, props.writer_version()) {
1984 (Type::BOOLEAN, WriterVersion::PARQUET_2_0) => Encoding::RLE,
1985 (Type::INT32, WriterVersion::PARQUET_2_0) => Encoding::DELTA_BINARY_PACKED,
1986 (Type::INT64, WriterVersion::PARQUET_2_0) => Encoding::DELTA_BINARY_PACKED,
1987 (Type::BYTE_ARRAY, WriterVersion::PARQUET_2_0) => Encoding::DELTA_BYTE_ARRAY,
1988 (Type::FIXED_LEN_BYTE_ARRAY, WriterVersion::PARQUET_2_0) => Encoding::DELTA_BYTE_ARRAY,
1989 _ => Encoding::PLAIN,
1990 }
1991}
1992
1993fn has_dictionary_support(kind: Type) -> bool {
1995 match kind {
1996 Type::BOOLEAN => false,
1998 _ => true,
1999 }
2000}
2001
2002#[inline]
2003fn compare_greater_unsigned_int<T: ParquetValueType>(a: &T, b: &T) -> bool {
2004 a.as_u64().unwrap() > b.as_u64().unwrap()
2005}
2006
2007#[inline]
2008fn compare_greater_f16(a: &[u8], b: &[u8]) -> bool {
2009 let a = f16::from_le_bytes(a.try_into().unwrap());
2010 let b = f16::from_le_bytes(b.try_into().unwrap());
2011 a.total_cmp(&b) == Ordering::Greater
2012}
2013
2014#[inline]
2016pub(crate) fn is_decimal_descr(basic_type_info: &BasicTypeInfo) -> bool {
2017 basic_type_info.converted_type() == ConvertedType::DECIMAL
2018 || matches!(
2019 basic_type_info.logical_type_ref(),
2020 Some(LogicalType::Decimal(_))
2021 )
2022}
2023
2024pub(crate) fn compare_greater_byte_array_decimals(a: &[u8], b: &[u8]) -> bool {
2026 let a_length = a.len();
2027 let b_length = b.len();
2028
2029 if a_length == 0 || b_length == 0 {
2030 return a_length > 0;
2031 }
2032
2033 let first_a: u8 = a[0];
2034 let first_b: u8 = b[0];
2035
2036 if (0x80 & first_a) != (0x80 & first_b) || (a_length == b_length && first_a != first_b) {
2041 return (first_a as i8) > (first_b as i8);
2042 }
2043
2044 let extension: u8 = if (first_a as i8) < 0 { 0xFF } else { 0 };
2051
2052 if a_length != b_length {
2053 let not_equal = if a_length > b_length {
2054 let lead_length = a_length - b_length;
2055 a[0..lead_length].iter().any(|&x| x != extension)
2056 } else {
2057 let lead_length = b_length - a_length;
2058 b[0..lead_length].iter().any(|&x| x != extension)
2059 };
2060
2061 if not_equal {
2062 let negative_values: bool = (first_a as i8) < 0;
2063 let a_longer: bool = a_length > b_length;
2064 return if negative_values { !a_longer } else { a_longer };
2065 }
2066 }
2067
2068 let tail_length = a_length.min(b_length);
2069 (a[a_length - tail_length..]) > (b[b_length - tail_length..])
2070}
2071
2072fn truncate_utf8(data: &str, length: usize) -> Option<Vec<u8>> {
2078 let split = (1..=length).rfind(|x| data.is_char_boundary(*x))?;
2079 Some(data.as_bytes()[..split].to_vec())
2080}
2081
2082fn truncate_and_increment_utf8(data: &str, length: usize) -> Option<Vec<u8>> {
2088 let lower_bound = length.saturating_sub(3);
2090 let split = (lower_bound..=length).rfind(|x| data.is_char_boundary(*x))?;
2091 increment_utf8(data.get(..split)?)
2092}
2093
2094fn increment_utf8(data: &str) -> Option<Vec<u8>> {
2101 for (idx, original_char) in data.char_indices().rev() {
2102 let original_len = original_char.len_utf8();
2103 if let Some(next_char) = char::from_u32(original_char as u32 + 1) {
2104 if next_char.len_utf8() == original_len {
2106 let mut result = data.as_bytes()[..idx + original_len].to_vec();
2107 next_char.encode_utf8(&mut result[idx..]);
2108 return Some(result);
2109 }
2110 }
2111 }
2112
2113 None
2114}
2115
2116fn increment(mut data: Vec<u8>) -> Option<Vec<u8>> {
2120 for byte in data.iter_mut().rev() {
2121 let (incremented, overflow) = byte.overflowing_add(1);
2122 *byte = incremented;
2123
2124 if !overflow {
2125 return Some(data);
2126 }
2127 }
2128
2129 None
2130}
2131
2132#[cfg(test)]
2133mod tests {
2134 use crate::{
2135 file::{properties::DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH, writer::SerializedFileWriter},
2136 schema::parser::parse_message_type,
2137 };
2138 use core::str;
2139 use rand::distr::uniform::SampleUniform;
2140 use std::{fs::File, sync::Arc};
2141
2142 use crate::column::{
2143 page::PageReader,
2144 reader::{ColumnReaderImpl, get_column_reader, get_typed_column_reader},
2145 };
2146 use crate::file::writer::TrackedWrite;
2147 use crate::file::{
2148 properties::ReaderProperties, reader::SerializedPageReader, writer::SerializedPageWriter,
2149 };
2150 use crate::schema::types::{ColumnPath, Type as SchemaType};
2151 use crate::util::test_common::rand_gen::random_numbers_range;
2152
2153 use super::*;
2154
2155 #[test]
2156 fn test_column_writer_inconsistent_def_rep_length() {
2157 let page_writer = get_test_page_writer();
2158 let props = Default::default();
2159 let mut writer = get_test_column_writer::<Int32Type>(page_writer, 1, 1, props);
2160 let res = writer.write_batch(&[1, 2, 3, 4], Some(&[1, 1, 1]), Some(&[0, 0]));
2161 assert!(res.is_err());
2162 if let Err(err) = res {
2163 assert_eq!(
2164 format!("{err}"),
2165 "Parquet error: Inconsistent length of definition and repetition levels: 3 != 2"
2166 );
2167 }
2168 }
2169
2170 #[test]
2171 fn test_column_writer_invalid_def_levels() {
2172 let page_writer = get_test_page_writer();
2173 let props = Default::default();
2174 let mut writer = get_test_column_writer::<Int32Type>(page_writer, 1, 0, props);
2175 let res = writer.write_batch(&[1, 2, 3, 4], None, None);
2176 assert!(res.is_err());
2177 if let Err(err) = res {
2178 assert_eq!(
2179 format!("{err}"),
2180 "Parquet error: Definition levels are required, because max definition level = 1"
2181 );
2182 }
2183 }
2184
2185 #[test]
2186 fn test_column_writer_invalid_rep_levels() {
2187 let page_writer = get_test_page_writer();
2188 let props = Default::default();
2189 let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 1, props);
2190 let res = writer.write_batch(&[1, 2, 3, 4], None, None);
2191 assert!(res.is_err());
2192 if let Err(err) = res {
2193 assert_eq!(
2194 format!("{err}"),
2195 "Parquet error: Repetition levels are required, because max repetition level = 1"
2196 );
2197 }
2198 }
2199
2200 #[test]
2201 fn test_column_writer_not_enough_values_to_write() {
2202 let page_writer = get_test_page_writer();
2203 let props = Default::default();
2204 let mut writer = get_test_column_writer::<Int32Type>(page_writer, 1, 0, props);
2205 let res = writer.write_batch(&[1, 2], Some(&[1, 1, 1, 1]), None);
2206 assert!(res.is_err());
2207 if let Err(err) = res {
2208 assert_eq!(
2209 format!("{err}"),
2210 "Parquet error: Expected to write 4 values, but have only 2"
2211 );
2212 }
2213 }
2214
2215 #[test]
2216 fn test_column_writer_write_only_one_dictionary_page() {
2217 let page_writer = get_test_page_writer();
2218 let props = Default::default();
2219 let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
2220 writer.write_batch(&[1, 2, 3, 4], None, None).unwrap();
2221 writer.add_data_page().unwrap();
2223 writer.write_dictionary_page().unwrap();
2224 let err = writer.write_dictionary_page().unwrap_err().to_string();
2225 assert_eq!(err, "Parquet error: Dictionary encoder is not set");
2226 }
2227
2228 #[test]
2229 fn test_column_writer_error_when_writing_disabled_dictionary() {
2230 let page_writer = get_test_page_writer();
2231 let props = Arc::new(
2232 WriterProperties::builder()
2233 .set_dictionary_enabled(false)
2234 .build(),
2235 );
2236 let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
2237 writer.write_batch(&[1, 2, 3, 4], None, None).unwrap();
2238 let err = writer.write_dictionary_page().unwrap_err().to_string();
2239 assert_eq!(err, "Parquet error: Dictionary encoder is not set");
2240 }
2241
2242 #[test]
2243 fn test_column_writer_boolean_type_does_not_support_dictionary() {
2244 let page_writer = get_test_page_writer();
2245 let props = Arc::new(
2246 WriterProperties::builder()
2247 .set_dictionary_enabled(true)
2248 .build(),
2249 );
2250 let mut writer = get_test_column_writer::<BoolType>(page_writer, 0, 0, props);
2251 writer
2252 .write_batch(&[true, false, true, false], None, None)
2253 .unwrap();
2254
2255 let r = writer.close().unwrap();
2256 assert_eq!(r.bytes_written, 1);
2259 assert_eq!(r.rows_written, 4);
2260
2261 let metadata = r.metadata;
2262 assert_eq!(
2263 metadata.encodings().collect::<Vec<_>>(),
2264 vec![Encoding::PLAIN]
2265 );
2266 assert_eq!(metadata.num_values(), 4); assert_eq!(metadata.dictionary_page_offset(), None);
2268 }
2269
2270 #[test]
2271 fn test_bloom_filter_for_dictionary_encoded_chunks() {
2272 fn bloom_filter_written(dictionary_enabled: bool, for_dictionary_chunks: bool) -> bool {
2273 let props = Arc::new(
2274 WriterProperties::builder()
2275 .set_dictionary_enabled(dictionary_enabled)
2276 .set_bloom_filter_enabled(true)
2277 .set_bloom_filter_for_dictionary_encoded_chunks(for_dictionary_chunks)
2278 .build(),
2279 );
2280 let mut writer =
2281 get_test_column_writer::<Int32Type>(get_test_page_writer(), 0, 0, props);
2282 writer.write_batch(&[1, 2, 1, 2], None, None).unwrap();
2283 writer.close().unwrap().bloom_filter.is_some()
2284 }
2285
2286 assert!(bloom_filter_written(true, true));
2287 assert!(bloom_filter_written(false, true));
2288 assert!(bloom_filter_written(false, false));
2289 assert!(!bloom_filter_written(true, false));
2290 }
2291
2292 #[test]
2293 fn test_column_writer_default_encoding_support_bool() {
2294 check_encoding_write_support::<BoolType>(
2295 WriterVersion::PARQUET_1_0,
2296 true,
2297 &[true, false],
2298 None,
2299 &[Encoding::PLAIN],
2300 &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)],
2301 );
2302 check_encoding_write_support::<BoolType>(
2303 WriterVersion::PARQUET_1_0,
2304 false,
2305 &[true, false],
2306 None,
2307 &[Encoding::PLAIN],
2308 &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)],
2309 );
2310 check_encoding_write_support::<BoolType>(
2311 WriterVersion::PARQUET_2_0,
2312 true,
2313 &[true, false],
2314 None,
2315 &[Encoding::RLE],
2316 &[encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE, 1)],
2317 );
2318 check_encoding_write_support::<BoolType>(
2319 WriterVersion::PARQUET_2_0,
2320 false,
2321 &[true, false],
2322 None,
2323 &[Encoding::RLE],
2324 &[encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE, 1)],
2325 );
2326 }
2327
2328 #[test]
2329 fn test_column_writer_default_encoding_support_int32() {
2330 check_encoding_write_support::<Int32Type>(
2331 WriterVersion::PARQUET_1_0,
2332 true,
2333 &[1, 2],
2334 Some(0),
2335 &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2336 &[
2337 encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2338 encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1),
2339 ],
2340 );
2341 check_encoding_write_support::<Int32Type>(
2342 WriterVersion::PARQUET_1_0,
2343 false,
2344 &[1, 2],
2345 None,
2346 &[Encoding::PLAIN],
2347 &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)],
2348 );
2349 check_encoding_write_support::<Int32Type>(
2350 WriterVersion::PARQUET_2_0,
2351 true,
2352 &[1, 2],
2353 Some(0),
2354 &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2355 &[
2356 encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2357 encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1),
2358 ],
2359 );
2360 check_encoding_write_support::<Int32Type>(
2361 WriterVersion::PARQUET_2_0,
2362 false,
2363 &[1, 2],
2364 None,
2365 &[Encoding::DELTA_BINARY_PACKED],
2366 &[encoding_stats(
2367 PageType::DATA_PAGE_V2,
2368 Encoding::DELTA_BINARY_PACKED,
2369 1,
2370 )],
2371 );
2372 }
2373
2374 #[test]
2375 fn test_column_writer_default_encoding_support_int64() {
2376 check_encoding_write_support::<Int64Type>(
2377 WriterVersion::PARQUET_1_0,
2378 true,
2379 &[1, 2],
2380 Some(0),
2381 &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2382 &[
2383 encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2384 encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1),
2385 ],
2386 );
2387 check_encoding_write_support::<Int64Type>(
2388 WriterVersion::PARQUET_1_0,
2389 false,
2390 &[1, 2],
2391 None,
2392 &[Encoding::PLAIN],
2393 &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)],
2394 );
2395 check_encoding_write_support::<Int64Type>(
2396 WriterVersion::PARQUET_2_0,
2397 true,
2398 &[1, 2],
2399 Some(0),
2400 &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2401 &[
2402 encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2403 encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1),
2404 ],
2405 );
2406 check_encoding_write_support::<Int64Type>(
2407 WriterVersion::PARQUET_2_0,
2408 false,
2409 &[1, 2],
2410 None,
2411 &[Encoding::DELTA_BINARY_PACKED],
2412 &[encoding_stats(
2413 PageType::DATA_PAGE_V2,
2414 Encoding::DELTA_BINARY_PACKED,
2415 1,
2416 )],
2417 );
2418 }
2419
2420 #[test]
2421 fn test_column_writer_default_encoding_support_int96() {
2422 check_encoding_write_support::<Int96Type>(
2423 WriterVersion::PARQUET_1_0,
2424 true,
2425 &[Int96::from(vec![1, 2, 3])],
2426 Some(0),
2427 &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2428 &[
2429 encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2430 encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1),
2431 ],
2432 );
2433 check_encoding_write_support::<Int96Type>(
2434 WriterVersion::PARQUET_1_0,
2435 false,
2436 &[Int96::from(vec![1, 2, 3])],
2437 None,
2438 &[Encoding::PLAIN],
2439 &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)],
2440 );
2441 check_encoding_write_support::<Int96Type>(
2442 WriterVersion::PARQUET_2_0,
2443 true,
2444 &[Int96::from(vec![1, 2, 3])],
2445 Some(0),
2446 &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2447 &[
2448 encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2449 encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1),
2450 ],
2451 );
2452 check_encoding_write_support::<Int96Type>(
2453 WriterVersion::PARQUET_2_0,
2454 false,
2455 &[Int96::from(vec![1, 2, 3])],
2456 None,
2457 &[Encoding::PLAIN],
2458 &[encoding_stats(PageType::DATA_PAGE_V2, Encoding::PLAIN, 1)],
2459 );
2460 }
2461
2462 #[test]
2463 fn test_column_writer_default_encoding_support_float() {
2464 check_encoding_write_support::<FloatType>(
2465 WriterVersion::PARQUET_1_0,
2466 true,
2467 &[1.0, 2.0],
2468 Some(0),
2469 &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2470 &[
2471 encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2472 encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1),
2473 ],
2474 );
2475 check_encoding_write_support::<FloatType>(
2476 WriterVersion::PARQUET_1_0,
2477 false,
2478 &[1.0, 2.0],
2479 None,
2480 &[Encoding::PLAIN],
2481 &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)],
2482 );
2483 check_encoding_write_support::<FloatType>(
2484 WriterVersion::PARQUET_2_0,
2485 true,
2486 &[1.0, 2.0],
2487 Some(0),
2488 &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2489 &[
2490 encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2491 encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1),
2492 ],
2493 );
2494 check_encoding_write_support::<FloatType>(
2495 WriterVersion::PARQUET_2_0,
2496 false,
2497 &[1.0, 2.0],
2498 None,
2499 &[Encoding::PLAIN],
2500 &[encoding_stats(PageType::DATA_PAGE_V2, Encoding::PLAIN, 1)],
2501 );
2502 }
2503
2504 #[test]
2505 fn test_column_writer_default_encoding_support_double() {
2506 check_encoding_write_support::<DoubleType>(
2507 WriterVersion::PARQUET_1_0,
2508 true,
2509 &[1.0, 2.0],
2510 Some(0),
2511 &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2512 &[
2513 encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2514 encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1),
2515 ],
2516 );
2517 check_encoding_write_support::<DoubleType>(
2518 WriterVersion::PARQUET_1_0,
2519 false,
2520 &[1.0, 2.0],
2521 None,
2522 &[Encoding::PLAIN],
2523 &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)],
2524 );
2525 check_encoding_write_support::<DoubleType>(
2526 WriterVersion::PARQUET_2_0,
2527 true,
2528 &[1.0, 2.0],
2529 Some(0),
2530 &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2531 &[
2532 encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2533 encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1),
2534 ],
2535 );
2536 check_encoding_write_support::<DoubleType>(
2537 WriterVersion::PARQUET_2_0,
2538 false,
2539 &[1.0, 2.0],
2540 None,
2541 &[Encoding::PLAIN],
2542 &[encoding_stats(PageType::DATA_PAGE_V2, Encoding::PLAIN, 1)],
2543 );
2544 }
2545
2546 #[test]
2547 fn test_column_writer_default_encoding_support_byte_array() {
2548 check_encoding_write_support::<ByteArrayType>(
2549 WriterVersion::PARQUET_1_0,
2550 true,
2551 &[ByteArray::from(vec![1u8])],
2552 Some(0),
2553 &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2554 &[
2555 encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2556 encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1),
2557 ],
2558 );
2559 check_encoding_write_support::<ByteArrayType>(
2560 WriterVersion::PARQUET_1_0,
2561 false,
2562 &[ByteArray::from(vec![1u8])],
2563 None,
2564 &[Encoding::PLAIN],
2565 &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)],
2566 );
2567 check_encoding_write_support::<ByteArrayType>(
2568 WriterVersion::PARQUET_2_0,
2569 true,
2570 &[ByteArray::from(vec![1u8])],
2571 Some(0),
2572 &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2573 &[
2574 encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2575 encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1),
2576 ],
2577 );
2578 check_encoding_write_support::<ByteArrayType>(
2579 WriterVersion::PARQUET_2_0,
2580 false,
2581 &[ByteArray::from(vec![1u8])],
2582 None,
2583 &[Encoding::DELTA_BYTE_ARRAY],
2584 &[encoding_stats(
2585 PageType::DATA_PAGE_V2,
2586 Encoding::DELTA_BYTE_ARRAY,
2587 1,
2588 )],
2589 );
2590 }
2591
2592 #[test]
2593 fn test_column_writer_default_encoding_support_fixed_len_byte_array() {
2594 for version in [WriterVersion::PARQUET_1_0, WriterVersion::PARQUET_2_0] {
2595 let default_props = WriterProperties::builder()
2596 .set_writer_version(version)
2597 .build();
2598 let meta = column_write_and_get_metadata::<FixedLenByteArrayType>(
2599 default_props,
2600 &[ByteArray::from(vec![1u8]).into()],
2601 );
2602 assert_eq!(meta.dictionary_page_offset(), Some(0));
2603 }
2604
2605 check_encoding_write_support::<FixedLenByteArrayType>(
2606 WriterVersion::PARQUET_1_0,
2607 true,
2608 &[ByteArray::from(vec![1u8]).into()],
2609 Some(0),
2610 &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2611 &[
2612 encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2613 encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1),
2614 ],
2615 );
2616 let column_props = WriterProperties::builder()
2617 .set_writer_version(WriterVersion::PARQUET_1_0)
2618 .set_dictionary_enabled(false)
2619 .set_column_dictionary_enabled(ColumnPath::from("col"), true)
2620 .build();
2621 let meta = column_write_and_get_metadata::<FixedLenByteArrayType>(
2622 column_props,
2623 &[ByteArray::from(vec![1u8]).into()],
2624 );
2625 assert_eq!(meta.dictionary_page_offset(), Some(0));
2626 check_encoding_write_support::<FixedLenByteArrayType>(
2627 WriterVersion::PARQUET_1_0,
2628 false,
2629 &[ByteArray::from(vec![1u8]).into()],
2630 None,
2631 &[Encoding::PLAIN],
2632 &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)],
2633 );
2634 check_encoding_write_support::<FixedLenByteArrayType>(
2635 WriterVersion::PARQUET_2_0,
2636 true,
2637 &[ByteArray::from(vec![1u8]).into()],
2638 Some(0),
2639 &[Encoding::PLAIN, Encoding::RLE_DICTIONARY],
2640 &[
2641 encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2642 encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1),
2643 ],
2644 );
2645 check_encoding_write_support::<FixedLenByteArrayType>(
2646 WriterVersion::PARQUET_2_0,
2647 false,
2648 &[ByteArray::from(vec![1u8]).into()],
2649 None,
2650 &[Encoding::DELTA_BYTE_ARRAY],
2651 &[encoding_stats(
2652 PageType::DATA_PAGE_V2,
2653 Encoding::DELTA_BYTE_ARRAY,
2654 1,
2655 )],
2656 );
2657 }
2658
2659 #[test]
2660 fn test_column_writer_no_dictionary_page_for_empty_required() {
2661 let props = WriterProperties::builder()
2662 .set_writer_version(WriterVersion::PARQUET_1_0)
2663 .set_dictionary_enabled(true)
2664 .build();
2665 let meta = column_write_and_get_metadata_with_def_levels::<Int64Type>(props, &[], &[], 0);
2666 assert_eq!(meta.dictionary_page_offset(), None);
2667 assert_eq!(meta.data_page_offset(), 0);
2668 assert_eq!(meta.uncompressed_size(), 0);
2669 assert_eq!(meta.compressed_size(), 0);
2670 assert_eq!(meta.encodings().collect::<Vec<_>>(), &[]);
2671 assert_eq!(meta.page_encoding_stats().unwrap(), &[]);
2672 }
2673
2674 #[test]
2675 fn test_column_writer_no_dictionary_page_for_empty_optional() {
2676 let props = WriterProperties::builder()
2677 .set_writer_version(WriterVersion::PARQUET_1_0)
2678 .set_dictionary_enabled(true)
2679 .build();
2680 let meta = column_write_and_get_metadata_with_def_levels::<Int64Type>(props, &[], &[], 1);
2681 assert_eq!(meta.dictionary_page_offset(), None);
2682 assert_eq!(meta.data_page_offset(), 0);
2683 assert_eq!(meta.uncompressed_size(), 0);
2684 assert_eq!(meta.compressed_size(), 0);
2685 assert_eq!(meta.encodings().collect::<Vec<_>>(), &[]);
2686 assert_eq!(meta.page_encoding_stats().unwrap(), &[]);
2687 }
2688
2689 #[test]
2690 fn test_column_writer_encoding_includes_rle_for_all_null_data() {
2691 let props = WriterProperties::builder()
2692 .set_writer_version(WriterVersion::PARQUET_1_0)
2693 .set_dictionary_enabled(true)
2694 .build();
2695 let meta = column_write_and_get_metadata_with_def_levels::<Int64Type>(props, &[], &[0], 1);
2696 assert_eq!(meta.dictionary_page_offset(), Some(0));
2698 assert_eq!(meta.data_page_offset(), 0);
2699 assert_eq!(meta.uncompressed_size(), 7);
2700 assert_eq!(meta.compressed_size(), 7);
2701 assert_eq!(
2702 meta.encodings().collect::<Vec<_>>(),
2703 &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY]
2704 );
2705 assert_eq!(
2706 meta.page_encoding_stats().unwrap(),
2707 &[
2708 encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2709 encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1)
2710 ]
2711 );
2712 }
2713
2714 #[test]
2715 fn test_column_writer_encoding_includes_rle_for_mixed_null_data() {
2716 let props = WriterProperties::builder()
2717 .set_writer_version(WriterVersion::PARQUET_1_0)
2718 .set_dictionary_enabled(true)
2719 .build();
2720 let meta = column_write_and_get_metadata_with_def_levels::<Int64Type>(
2721 props,
2722 &[42_i64],
2723 &[1, 0],
2724 1,
2725 );
2726 assert_eq!(meta.dictionary_page_offset(), Some(0));
2727 assert_eq!(meta.data_page_offset(), 8);
2728 assert_eq!(meta.uncompressed_size(), 16);
2729 assert_eq!(meta.compressed_size(), 16);
2730 assert_eq!(
2731 meta.encodings().collect::<Vec<_>>(),
2732 &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY]
2733 );
2734 assert_eq!(
2735 meta.page_encoding_stats().unwrap(),
2736 &[
2737 encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1),
2738 encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1)
2739 ]
2740 );
2741 }
2742
2743 #[test]
2744 fn test_column_writer_check_metadata() {
2745 let page_writer = get_test_page_writer();
2746 let props = Default::default();
2747 let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
2748 writer.write_batch(&[1, 2, 3, 4], None, None).unwrap();
2749
2750 let r = writer.close().unwrap();
2751 assert_eq!(r.bytes_written, 20);
2752 assert_eq!(r.rows_written, 4);
2753
2754 let metadata = r.metadata;
2755 assert_eq!(
2756 metadata.encodings().collect::<Vec<_>>(),
2757 vec![Encoding::PLAIN, Encoding::RLE_DICTIONARY]
2758 );
2759 assert_eq!(metadata.num_values(), 4);
2760 assert_eq!(metadata.compressed_size(), 20);
2761 assert_eq!(metadata.uncompressed_size(), 20);
2762 assert_eq!(metadata.data_page_offset(), 16);
2763 assert_eq!(metadata.dictionary_page_offset(), Some(0));
2764 if let Some(stats) = metadata.statistics() {
2765 assert_eq!(stats.null_count_opt(), Some(0));
2766 assert_eq!(stats.distinct_count_opt(), None);
2767 if let Statistics::Int32(stats) = stats {
2768 assert_eq!(stats.min_opt().unwrap(), &1);
2769 assert_eq!(stats.max_opt().unwrap(), &4);
2770 } else {
2771 panic!("expecting Statistics::Int32");
2772 }
2773 } else {
2774 panic!("metadata missing statistics");
2775 }
2776 }
2777
2778 #[test]
2779 fn test_column_writer_check_byte_array_min_max() {
2780 let page_writer = get_test_page_writer();
2781 let props = Default::default();
2782 let mut writer = get_test_decimals_column_writer::<ByteArrayType>(page_writer, 0, 0, props);
2783 writer
2784 .write_batch(
2785 &[
2786 ByteArray::from(vec![
2787 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 179u8, 172u8, 19u8,
2788 35u8, 231u8, 90u8, 0u8, 0u8,
2789 ]),
2790 ByteArray::from(vec![
2791 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 228u8, 62u8, 146u8,
2792 152u8, 177u8, 56u8, 0u8, 0u8,
2793 ]),
2794 ByteArray::from(vec![
2795 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
2796 0u8,
2797 ]),
2798 ByteArray::from(vec![
2799 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 41u8, 162u8, 36u8, 26u8, 246u8,
2800 44u8, 0u8, 0u8,
2801 ]),
2802 ],
2803 None,
2804 None,
2805 )
2806 .unwrap();
2807 let metadata = writer.close().unwrap().metadata;
2808 if let Some(stats) = metadata.statistics() {
2809 if let Statistics::ByteArray(stats) = stats {
2810 assert_eq!(
2811 stats.min_opt().unwrap(),
2812 &ByteArray::from(vec![
2813 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 179u8, 172u8, 19u8,
2814 35u8, 231u8, 90u8, 0u8, 0u8,
2815 ])
2816 );
2817 assert_eq!(
2818 stats.max_opt().unwrap(),
2819 &ByteArray::from(vec![
2820 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 41u8, 162u8, 36u8, 26u8, 246u8,
2821 44u8, 0u8, 0u8,
2822 ])
2823 );
2824 } else {
2825 panic!("expecting Statistics::ByteArray");
2826 }
2827 } else {
2828 panic!("metadata missing statistics");
2829 }
2830 }
2831
2832 #[test]
2833 fn test_column_writer_byte_array_min_max_unequal_lengths() {
2834 let page_writer = get_test_page_writer();
2837 let props = Default::default();
2838 let mut writer = get_test_decimals_column_writer::<ByteArrayType>(page_writer, 0, 0, props);
2839 writer
2840 .write_batch(
2841 &[
2842 ByteArray::from(vec![0u8, 255u8]), ByteArray::from(vec![0u8, 128u8, 0u8]), ByteArray::from(vec![255u8, 127u8]), ByteArray::from(vec![128u8]), ],
2847 None,
2848 None,
2849 )
2850 .unwrap();
2851 let metadata = writer.close().unwrap().metadata;
2852 let stats = metadata.statistics().expect("metadata missing statistics");
2853 let Statistics::ByteArray(stats) = stats else {
2854 panic!("expecting Statistics::ByteArray");
2855 };
2856 assert_eq!(
2858 stats.min_opt().unwrap(),
2859 &ByteArray::from(vec![255u8, 127u8])
2860 );
2861 assert_eq!(
2863 stats.max_opt().unwrap(),
2864 &ByteArray::from(vec![0u8, 128u8, 0u8])
2865 );
2866 }
2867
2868 #[test]
2869 fn test_column_writer_uint32_converted_type_min_max() {
2870 let page_writer = get_test_page_writer();
2871 let props = Default::default();
2872 let mut writer = get_test_unsigned_int_given_as_converted_column_writer::<Int32Type>(
2873 page_writer,
2874 0,
2875 0,
2876 props,
2877 );
2878 writer.write_batch(&[0, 1, 2, 3, 4, 5], None, None).unwrap();
2879 let metadata = writer.close().unwrap().metadata;
2880 if let Some(stats) = metadata.statistics() {
2881 if let Statistics::Int32(stats) = stats {
2882 assert_eq!(stats.min_opt().unwrap(), &0,);
2883 assert_eq!(stats.max_opt().unwrap(), &5,);
2884 } else {
2885 panic!("expecting Statistics::Int32");
2886 }
2887 } else {
2888 panic!("metadata missing statistics");
2889 }
2890 }
2891
2892 #[test]
2893 fn test_column_writer_precalculated_statistics() {
2894 let page_writer = get_test_page_writer();
2895 let props = Arc::new(
2896 WriterProperties::builder()
2897 .set_statistics_enabled(EnabledStatistics::Chunk)
2898 .build(),
2899 );
2900 let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
2901 writer
2902 .write_batch_with_statistics(
2903 &[1, 2, 3, 4],
2904 None,
2905 None,
2906 Some(&-17),
2907 Some(&9000),
2908 Some(55),
2909 )
2910 .unwrap();
2911
2912 let r = writer.close().unwrap();
2913 assert_eq!(r.bytes_written, 20);
2914 assert_eq!(r.rows_written, 4);
2915
2916 let metadata = r.metadata;
2917 assert_eq!(
2918 metadata.encodings().collect::<Vec<_>>(),
2919 vec![Encoding::PLAIN, Encoding::RLE_DICTIONARY]
2920 );
2921 assert_eq!(metadata.num_values(), 4);
2922 assert_eq!(metadata.compressed_size(), 20);
2923 assert_eq!(metadata.uncompressed_size(), 20);
2924 assert_eq!(metadata.data_page_offset(), 16);
2925 assert_eq!(metadata.dictionary_page_offset(), Some(0));
2926 if let Some(stats) = metadata.statistics() {
2927 assert_eq!(stats.null_count_opt(), Some(0));
2928 assert_eq!(stats.distinct_count_opt().unwrap_or(0), 55);
2929 if let Statistics::Int32(stats) = stats {
2930 assert_eq!(stats.min_opt().unwrap(), &-17);
2931 assert_eq!(stats.max_opt().unwrap(), &9000);
2932 } else {
2933 panic!("expecting Statistics::Int32");
2934 }
2935 } else {
2936 panic!("metadata missing statistics");
2937 }
2938 }
2939
2940 #[test]
2941 fn test_mixed_precomputed_statistics() {
2942 let mut buf = Vec::with_capacity(100);
2943 let mut write = TrackedWrite::new(&mut buf);
2944 let page_writer = Box::new(SerializedPageWriter::new(&mut write));
2945 let props = Arc::new(
2946 WriterProperties::builder()
2947 .set_write_page_header_statistics(true)
2948 .set_data_page_row_count_limit(4)
2949 .build(),
2950 );
2951 let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
2952
2953 writer.write_batch(&[1, 2, 3, 4], None, None).unwrap();
2954 writer
2955 .write_batch_with_statistics(&[5, 6, 7], None, None, Some(&5), Some(&7), Some(3))
2956 .unwrap();
2957
2958 let r = writer.close().unwrap();
2959
2960 let stats = r.metadata.statistics().unwrap();
2961 assert_eq!(stats.min_bytes_opt().unwrap(), 1_i32.to_le_bytes());
2962 assert_eq!(stats.max_bytes_opt().unwrap(), 7_i32.to_le_bytes());
2963 assert_eq!(stats.null_count_opt(), Some(0));
2964 assert!(stats.distinct_count_opt().is_none());
2965
2966 drop(write);
2967
2968 let props = ReaderProperties::builder()
2969 .set_backward_compatible_lz4(false)
2970 .set_read_page_statistics(true)
2971 .build();
2972 let reader = SerializedPageReader::new_with_properties(
2973 Arc::new(Bytes::from(buf)),
2974 &r.metadata,
2975 r.rows_written as usize,
2976 None,
2977 Arc::new(props),
2978 )
2979 .unwrap();
2980
2981 let pages = reader.collect::<Result<Vec<_>>>().unwrap();
2982 assert_eq!(pages.len(), 3);
2983
2984 assert_eq!(pages[0].page_type(), PageType::DICTIONARY_PAGE);
2985 assert_eq!(pages[1].page_type(), PageType::DATA_PAGE);
2986 assert_eq!(pages[2].page_type(), PageType::DATA_PAGE);
2987 for (page, min, max) in [(&pages[1], 1_i32, 4_i32), (&pages[2], 5_i32, 7_i32)] {
2988 let stats = page.statistics().unwrap();
2989 assert_eq!(stats.min_bytes_opt().unwrap(), min.to_le_bytes());
2990 assert_eq!(stats.max_bytes_opt().unwrap(), max.to_le_bytes());
2991 assert_eq!(stats.null_count_opt(), Some(0));
2992 assert!(stats.distinct_count_opt().is_none());
2993 }
2994 }
2995
2996 #[test]
2997 fn test_disabled_statistics() {
2998 let mut buf = Vec::with_capacity(100);
2999 let mut write = TrackedWrite::new(&mut buf);
3000 let page_writer = Box::new(SerializedPageWriter::new(&mut write));
3001 let props = WriterProperties::builder()
3002 .set_statistics_enabled(EnabledStatistics::None)
3003 .set_writer_version(WriterVersion::PARQUET_2_0)
3004 .build();
3005 let props = Arc::new(props);
3006
3007 let mut writer = get_test_column_writer::<Int32Type>(page_writer, 1, 0, props);
3008 writer
3009 .write_batch(&[1, 2, 3, 4], Some(&[1, 0, 0, 1, 1, 1]), None)
3010 .unwrap();
3011
3012 let r = writer.close().unwrap();
3013 assert!(r.metadata.statistics().is_none());
3014
3015 drop(write);
3016
3017 let props = ReaderProperties::builder()
3018 .set_backward_compatible_lz4(false)
3019 .build();
3020 let reader = SerializedPageReader::new_with_properties(
3021 Arc::new(Bytes::from(buf)),
3022 &r.metadata,
3023 r.rows_written as usize,
3024 None,
3025 Arc::new(props),
3026 )
3027 .unwrap();
3028
3029 let pages = reader.collect::<Result<Vec<_>>>().unwrap();
3030 assert_eq!(pages.len(), 2);
3031
3032 assert_eq!(pages[0].page_type(), PageType::DICTIONARY_PAGE);
3033 assert_eq!(pages[1].page_type(), PageType::DATA_PAGE_V2);
3034
3035 match &pages[1] {
3036 Page::DataPageV2 {
3037 num_values,
3038 num_nulls,
3039 num_rows,
3040 statistics,
3041 ..
3042 } => {
3043 assert_eq!(*num_values, 6);
3044 assert_eq!(*num_nulls, 2);
3045 assert_eq!(*num_rows, 6);
3046 assert!(statistics.is_none());
3047 }
3048 _ => unreachable!(),
3049 }
3050 }
3051
3052 #[test]
3053 fn test_column_writer_empty_column_roundtrip() {
3054 let props = Default::default();
3055 column_roundtrip::<Int32Type>(props, &[], None, None);
3056 }
3057
3058 #[test]
3059 fn test_column_writer_non_nullable_values_roundtrip() {
3060 let props = Default::default();
3061 column_roundtrip_random::<Int32Type>(props, 1024, i32::MIN, i32::MAX, 0, 0);
3062 }
3063
3064 #[test]
3065 fn test_column_writer_nullable_non_repeated_values_roundtrip() {
3066 let props = Default::default();
3067 column_roundtrip_random::<Int32Type>(props, 1024, i32::MIN, i32::MAX, 10, 0);
3068 }
3069
3070 #[test]
3071 fn test_column_writer_nullable_repeated_values_roundtrip() {
3072 let props = Default::default();
3073 column_roundtrip_random::<Int32Type>(props, 1024, i32::MIN, i32::MAX, 10, 10);
3074 }
3075
3076 #[test]
3077 fn test_column_writer_dictionary_fallback_small_data_page() {
3078 let props = WriterProperties::builder()
3079 .set_dictionary_page_size_limit(32)
3080 .set_data_page_size_limit(32)
3081 .build();
3082 column_roundtrip_random::<Int32Type>(props, 1024, i32::MIN, i32::MAX, 10, 10);
3083 }
3084
3085 #[test]
3086 #[cfg_attr(miri, ignore)] fn test_column_writer_small_write_batch_size() {
3088 for i in &[1usize, 2, 5, 10, 11, 1023] {
3089 let props = WriterProperties::builder().set_write_batch_size(*i).build();
3090
3091 column_roundtrip_random::<Int32Type>(props, 1024, i32::MIN, i32::MAX, 10, 10);
3092 }
3093 }
3094
3095 #[test]
3096 fn test_column_writer_dictionary_disabled_v1() {
3097 let props = WriterProperties::builder()
3098 .set_writer_version(WriterVersion::PARQUET_1_0)
3099 .set_dictionary_enabled(false)
3100 .build();
3101 column_roundtrip_random::<Int32Type>(props, 1024, i32::MIN, i32::MAX, 10, 10);
3102 }
3103
3104 #[test]
3105 fn test_column_writer_dictionary_disabled_v2() {
3106 let props = WriterProperties::builder()
3107 .set_writer_version(WriterVersion::PARQUET_2_0)
3108 .set_dictionary_enabled(false)
3109 .build();
3110 column_roundtrip_random::<Int32Type>(props, 1024, i32::MIN, i32::MAX, 10, 10);
3111 }
3112
3113 #[test]
3114 fn test_column_writer_compression_v1() {
3115 let props = WriterProperties::builder()
3116 .set_writer_version(WriterVersion::PARQUET_1_0)
3117 .set_compression(Compression::SNAPPY)
3118 .build();
3119 column_roundtrip_random::<Int32Type>(props, 2048, i32::MIN, i32::MAX, 10, 10);
3120 }
3121
3122 #[test]
3123 fn test_column_writer_compression_v2() {
3124 let props = WriterProperties::builder()
3125 .set_writer_version(WriterVersion::PARQUET_2_0)
3126 .set_compression(Compression::SNAPPY)
3127 .build();
3128 column_roundtrip_random::<Int32Type>(props, 2048, i32::MIN, i32::MAX, 10, 10);
3129 }
3130
3131 #[test]
3132 fn test_column_writer_v2_compression_ratio_threshold() {
3133 fn write_v2_page(threshold: f64) -> bool {
3134 let mut buf = Vec::with_capacity(4096);
3135 let mut write = TrackedWrite::new(&mut buf);
3136 let page_writer = Box::new(SerializedPageWriter::new(&mut write));
3137 let props = Arc::new(
3138 WriterProperties::builder()
3139 .set_writer_version(WriterVersion::PARQUET_2_0)
3140 .set_compression(Compression::SNAPPY)
3141 .set_dictionary_enabled(false)
3142 .set_data_page_v2_compression_ratio_threshold(threshold)
3143 .build(),
3144 );
3145
3146 let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
3147 let values: Vec<i32> = vec![42; 4096];
3148 writer.write_batch(&values, None, None).unwrap();
3149 let r = writer.close().unwrap();
3150 drop(write);
3151
3152 let reader_props = ReaderProperties::builder()
3153 .set_backward_compatible_lz4(false)
3154 .build();
3155 let reader = SerializedPageReader::new_with_properties(
3156 Arc::new(Bytes::from(buf)),
3157 &r.metadata,
3158 r.rows_written as usize,
3159 None,
3160 Arc::new(reader_props),
3161 )
3162 .unwrap();
3163 let pages = reader.collect::<Result<Vec<_>>>().unwrap();
3164 let data_page = pages
3165 .iter()
3166 .find(|p| p.page_type() == PageType::DATA_PAGE_V2)
3167 .expect("expected a v2 data page");
3168 match data_page {
3169 Page::DataPageV2 { is_compressed, .. } => *is_compressed,
3170 _ => unreachable!(),
3171 }
3172 }
3173
3174 assert!(write_v2_page(1.0));
3176 assert!(!write_v2_page(0.001));
3178 }
3179
3180 #[test]
3181 fn test_column_writer_add_data_pages_with_dict() {
3182 let mut file = tempfile::tempfile().unwrap();
3185 let mut write = TrackedWrite::new(&mut file);
3186 let page_writer = Box::new(SerializedPageWriter::new(&mut write));
3187 let props = Arc::new(
3188 WriterProperties::builder()
3189 .set_data_page_size_limit(10)
3190 .set_write_batch_size(3) .build(),
3192 );
3193 let data = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
3194 let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
3195 writer.write_batch(data, None, None).unwrap();
3196 let r = writer.close().unwrap();
3197
3198 drop(write);
3199
3200 let props = ReaderProperties::builder()
3202 .set_backward_compatible_lz4(false)
3203 .build();
3204 let mut page_reader = Box::new(
3205 SerializedPageReader::new_with_properties(
3206 Arc::new(file),
3207 &r.metadata,
3208 r.rows_written as usize,
3209 None,
3210 Arc::new(props),
3211 )
3212 .unwrap(),
3213 );
3214 let mut res = Vec::new();
3215 while let Some(page) = page_reader.get_next_page().unwrap() {
3216 res.push((page.page_type(), page.num_values(), page.buffer().len()));
3217 }
3218 assert_eq!(
3219 res,
3220 vec![
3221 (PageType::DICTIONARY_PAGE, 10, 40),
3222 (PageType::DATA_PAGE, 9, 10),
3223 (PageType::DATA_PAGE, 1, 3),
3224 ]
3225 );
3226 assert_eq!(
3227 r.metadata.page_encoding_stats(),
3228 Some(&vec![
3229 PageEncodingStats {
3230 page_type: PageType::DICTIONARY_PAGE,
3231 encoding: Encoding::PLAIN,
3232 count: 1
3233 },
3234 PageEncodingStats {
3235 page_type: PageType::DATA_PAGE,
3236 encoding: Encoding::RLE_DICTIONARY,
3237 count: 2,
3238 }
3239 ])
3240 );
3241 }
3242
3243 #[test]
3244 fn test_column_writer_column_data_page_size_limit() {
3245 let props = Arc::new(
3246 WriterProperties::builder()
3247 .set_writer_version(WriterVersion::PARQUET_1_0)
3248 .set_dictionary_enabled(false)
3249 .set_data_page_size_limit(1000)
3250 .set_column_data_page_size_limit(ColumnPath::from("col"), 10)
3251 .set_write_batch_size(3)
3252 .build(),
3253 );
3254 let data = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
3255
3256 let col_values =
3257 write_and_collect_page_values(ColumnPath::from("col"), Arc::clone(&props), data);
3258 let other_values = write_and_collect_page_values(ColumnPath::from("other"), props, data);
3259
3260 assert_eq!(col_values, vec![3, 3, 3, 1]);
3261 assert_eq!(other_values, vec![10]);
3262 }
3263
3264 #[test]
3265 fn test_column_writer_column_data_page_row_count_limit() {
3266 let props = Arc::new(
3267 WriterProperties::builder()
3268 .set_writer_version(WriterVersion::PARQUET_1_0)
3269 .set_dictionary_enabled(false)
3270 .set_data_page_row_count_limit(100)
3271 .set_column_data_page_row_count_limit(ColumnPath::from("col"), 3)
3272 .set_write_batch_size(1)
3273 .build(),
3274 );
3275 let data = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
3276
3277 let col_values =
3278 write_and_collect_page_values(ColumnPath::from("col"), Arc::clone(&props), data);
3279 let other_values = write_and_collect_page_values(ColumnPath::from("other"), props, data);
3280
3281 assert_eq!(col_values, vec![3, 3, 3, 1]);
3282 assert_eq!(other_values, vec![10]);
3283 }
3284
3285 #[test]
3286 fn test_column_writer_caps_page_size_for_large_byte_array_values() {
3287 let value_size = 64 * 1024; let page_byte_limit = 16 * 1024; let num_rows = 64;
3295
3296 let props = WriterProperties::builder()
3297 .set_writer_version(WriterVersion::PARQUET_1_0)
3298 .set_dictionary_enabled(false)
3299 .set_encoding(Encoding::PLAIN)
3300 .set_data_page_size_limit(page_byte_limit)
3301 .build();
3304
3305 let data: Vec<_> = (0..num_rows)
3306 .map(|i| ByteArray::from(vec![i as u8; value_size]))
3307 .collect();
3308 let pages = write_and_collect_pages::<ByteArrayType>(props, 0, 0, &data, None, None);
3309
3310 let total_values: u32 = pages.data_pages.iter().map(|(_, n)| n).sum();
3312 assert_eq!(total_values as usize, num_rows);
3313 assert!(
3317 pages.data_pages.len() >= num_rows / 2,
3318 "expected pages to be cut close to one per value, got {:?}",
3319 pages.data_pages,
3320 );
3321 for (size, _) in &pages.data_pages {
3325 assert!(
3326 *size <= value_size + 64,
3327 "page size {size} exceeds one-value bound ({}B) — pages {:?}",
3328 value_size + 64,
3329 pages.data_pages,
3330 );
3331 }
3332 }
3333
3334 #[test]
3335 fn test_column_writer_delta_byte_array_dedups_large_shared_prefix_values() {
3336 let value_size = 64 * 1024; let page_byte_limit = 16 * 1024;
3342 let num_rows = 16;
3343
3344 let props = WriterProperties::builder()
3345 .set_writer_version(WriterVersion::PARQUET_1_0)
3346 .set_dictionary_enabled(false)
3347 .set_encoding(Encoding::DELTA_BYTE_ARRAY)
3348 .set_data_page_size_limit(page_byte_limit)
3349 .set_statistics_enabled(EnabledStatistics::None)
3350 .build();
3351
3352 let data: Vec<_> = (0..num_rows)
3355 .map(|_| ByteArray::from(vec![b'a'; value_size]))
3356 .collect();
3357 let pages = write_and_collect_pages::<ByteArrayType>(props, 0, 0, &data, None, None);
3358
3359 let total_values: u32 = pages.data_pages.iter().map(|(_, n)| n).sum();
3361 assert_eq!(total_values as usize, num_rows);
3362
3363 let total_bytes: usize = pages.data_pages.iter().map(|(size, _)| size).sum();
3366 assert!(
3367 total_bytes < 2 * value_size,
3368 "expected under 2x a single value ({}B) for {num_rows} identical \
3369 values, got {total_bytes}B across pages {:?}",
3370 2 * value_size,
3371 pages.data_pages,
3372 );
3373 }
3374
3375 #[test]
3376 fn test_column_writer_delta_byte_array_bounds_pages_without_shared_prefix() {
3377 let value_size = 64 * 1024;
3382 let page_byte_limit = 16 * 1024;
3383 let num_rows = 16;
3384
3385 let props = WriterProperties::builder()
3386 .set_writer_version(WriterVersion::PARQUET_1_0)
3387 .set_dictionary_enabled(false)
3388 .set_encoding(Encoding::DELTA_BYTE_ARRAY)
3389 .set_data_page_size_limit(page_byte_limit)
3390 .set_statistics_enabled(EnabledStatistics::None)
3391 .build();
3392
3393 let data: Vec<_> = (0..num_rows)
3395 .map(|i| ByteArray::from(vec![i as u8; value_size]))
3396 .collect();
3397 let pages = write_and_collect_pages::<ByteArrayType>(props, 0, 0, &data, None, None);
3398
3399 let total_values: u32 = pages.data_pages.iter().map(|(_, n)| n).sum();
3400 assert_eq!(total_values as usize, num_rows);
3401
3402 let upper_bound = 2 * value_size + 64;
3405 for (size, n_values) in &pages.data_pages {
3406 assert!(
3407 *size <= upper_bound,
3408 "page size {size} exceeds two-value bound ({upper_bound}B); pages {:?}",
3409 pages.data_pages,
3410 );
3411 assert!(
3412 *n_values <= 2,
3413 "page holds {n_values} values, expected at most 2; pages {:?}",
3414 pages.data_pages,
3415 );
3416 }
3417 }
3418
3419 #[test]
3420 fn test_column_writer_caps_page_size_with_sparse_nulls() {
3421 let value_size = 64 * 1024;
3436 let page_byte_limit = 16 * 1024;
3437 let num_values = 16;
3438
3439 let props = WriterProperties::builder()
3440 .set_dictionary_enabled(false)
3441 .set_encoding(Encoding::PLAIN)
3442 .set_data_page_size_limit(page_byte_limit)
3443 .set_statistics_enabled(EnabledStatistics::None)
3444 .build();
3445
3446 let data: Vec<_> = (0..num_values)
3447 .map(|_| ByteArray::from(vec![b'a'; value_size]))
3448 .collect();
3449 let def_levels: Vec<i16> = (0..num_values as i16 + 1)
3451 .map(|i| i16::from(i != 8))
3452 .collect();
3453 let pages =
3454 write_and_collect_pages::<ByteArrayType>(props, 1, 0, &data, Some(&def_levels), None);
3455
3456 let upper_bound = 2 * value_size + 64;
3459 for (size, n_levels) in &pages.data_pages {
3460 assert!(
3461 *size <= upper_bound,
3462 "page size {size} exceeds two-value bound ({upper_bound}B); pages {:?}",
3463 pages.data_pages,
3464 );
3465 assert!(
3466 *n_levels <= 3,
3467 "page holds {n_levels} levels, expected at most 3 (two values + a null); \
3468 pages {:?}",
3469 pages.data_pages,
3470 );
3471 }
3472 let num_levels: usize = num_values + 1;
3475 assert_eq!(pages.data_pages.len(), num_levels.div_ceil(2));
3476 }
3477
3478 #[test]
3479 fn test_column_writer_delta_byte_array_nullable_shared_prefix_dedup() {
3480 let value_size = 64 * 1024;
3492 let page_byte_limit = 16 * 1024;
3493 let num_values = 16;
3494
3495 let props = WriterProperties::builder()
3496 .set_writer_version(WriterVersion::PARQUET_1_0)
3497 .set_dictionary_enabled(false)
3498 .set_encoding(Encoding::DELTA_BYTE_ARRAY)
3499 .set_data_page_size_limit(page_byte_limit)
3500 .set_statistics_enabled(EnabledStatistics::None)
3501 .build();
3502
3503 let data: Vec<_> = (0..num_values)
3504 .map(|_| ByteArray::from(vec![b'a'; value_size]))
3505 .collect();
3506 let def_levels: Vec<i16> = (0..num_values as i16 + 1)
3508 .map(|i| i16::from(i != 8))
3509 .collect();
3510 let pages =
3511 write_and_collect_pages::<ByteArrayType>(props, 1, 0, &data, Some(&def_levels), None);
3512
3513 let per_page_values: Vec<u32> = pages.data_pages.iter().map(|(_, n)| *n).collect();
3516 assert_eq!(per_page_values, vec![num_values as u32 + 1]);
3517
3518 let total_bytes: usize = pages.data_pages.iter().map(|(size, _)| size).sum();
3519 assert!(
3520 total_bytes < 2 * value_size,
3521 "expected ~one full value's worth of bytes (full dedup), \
3522 got {total_bytes}B across pages {:?}",
3523 pages.data_pages,
3524 );
3525 }
3526
3527 #[test]
3528 fn test_column_writer_caps_page_size_for_large_values_in_list() {
3529 let value_size = 32 * 1024;
3540 let page_byte_limit = 16 * 1024;
3541 let values_per_record = 3;
3542 let num_records = 3;
3543 let num_values = values_per_record * num_records;
3544
3545 let mut rep_levels = Vec::with_capacity(num_values);
3547 for _ in 0..num_records {
3548 rep_levels.push(0i16);
3549 rep_levels.extend(std::iter::repeat_n(1i16, values_per_record - 1));
3550 }
3551 let def_levels = vec![1i16; num_values];
3552
3553 let props = WriterProperties::builder()
3554 .set_writer_version(WriterVersion::PARQUET_1_0)
3555 .set_dictionary_enabled(false)
3556 .set_encoding(Encoding::PLAIN)
3557 .set_data_page_size_limit(page_byte_limit)
3558 .build();
3559
3560 let data: Vec<_> = (0..num_values)
3561 .map(|i| ByteArray::from(vec![i as u8; value_size]))
3562 .collect();
3563 let pages = write_and_collect_pages::<ByteArrayType>(
3564 props,
3565 1,
3566 1,
3567 &data,
3568 Some(&def_levels),
3569 Some(&rep_levels),
3570 );
3571 let data_pages = pages.data_pages;
3572
3573 assert_eq!(
3577 data_pages.len(),
3578 num_records,
3579 "expected one data page per record, got {data_pages:?}"
3580 );
3581 for (bytes, n_values) in &data_pages {
3582 assert_eq!(
3583 *n_values as usize, values_per_record,
3584 "each page must hold a whole record's leaves, got {data_pages:?}"
3585 );
3586 let upper_bound = values_per_record * (value_size + 16);
3590 assert!(
3591 *bytes <= upper_bound,
3592 "page size {bytes} exceeds whole-record bound ({upper_bound}); pages {data_pages:?}"
3593 );
3594 }
3595 }
3596
3597 #[test]
3598 fn test_column_writer_caps_page_size_with_nullable_large_values() {
3599 let value_size = 32 * 1024;
3605 let page_byte_limit = 16 * 1024;
3606 let num_levels = 32;
3607
3608 let def_levels: Vec<i16> = (0..num_levels as i16).map(|i| i % 2).collect();
3610 let num_values = def_levels.iter().filter(|&&d| d == 1).count();
3611
3612 let props = WriterProperties::builder()
3613 .set_writer_version(WriterVersion::PARQUET_1_0)
3614 .set_dictionary_enabled(false)
3615 .set_encoding(Encoding::PLAIN)
3616 .set_data_page_size_limit(page_byte_limit)
3617 .build();
3618
3619 let data: Vec<_> = (0..num_values)
3620 .map(|i| ByteArray::from(vec![i as u8; value_size]))
3621 .collect();
3622 let pages =
3623 write_and_collect_pages::<ByteArrayType>(props, 1, 0, &data, Some(&def_levels), None);
3624 let data_pages: Vec<_> = pages.data_pages.iter().map(|(size, _)| *size).collect();
3625
3626 assert!(
3631 data_pages.len() >= num_values / 2,
3632 "expected at least {} pages for {num_values} large values, got {} pages: {data_pages:?}",
3633 num_values / 2,
3634 data_pages.len(),
3635 );
3636 for size in &data_pages {
3638 assert!(
3639 *size <= value_size + 64,
3640 "page size {size} exceeds one-value bound; pages {data_pages:?}"
3641 );
3642 }
3643 }
3644
3645 #[test]
3646 fn test_column_writer_dict_enabled_large_values_post_spill() {
3647 let value_size = 64 * 1024;
3657 let page_byte_limit = 16 * 1024;
3658 let num_rows = 32;
3659
3660 let props = WriterProperties::builder()
3661 .set_writer_version(WriterVersion::PARQUET_1_0)
3662 .set_dictionary_enabled(true)
3663 .set_dictionary_page_size_limit(1024)
3666 .set_data_page_size_limit(page_byte_limit)
3667 .set_write_batch_size(4)
3672 .build();
3673
3674 let data: Vec<_> = (0..num_rows)
3675 .map(|i| ByteArray::from(vec![i as u8; value_size]))
3676 .collect();
3677 let pages = write_and_collect_pages::<ByteArrayType>(props, 0, 0, &data, None, None);
3678 let data_pages: Vec<_> = pages.data_pages.iter().map(|(size, _)| *size).collect();
3679
3680 assert!(
3684 data_pages.len() >= num_rows / 2,
3685 "expected >= {} data pages after dict spill, got {} ({data_pages:?})",
3686 num_rows / 2,
3687 data_pages.len(),
3688 );
3689 for size in &data_pages {
3690 assert!(
3691 *size <= value_size + 64,
3692 "page size {size} exceeds one-value bound; pages {data_pages:?}"
3693 );
3694 }
3695 }
3696
3697 #[test]
3698 fn test_column_writer_caps_dictionary_page_size() {
3699 let value_size = 8 * 1024;
3707 let dict_page_limit = 64 * 1024;
3708 let num_rows = 2048;
3709
3710 let props = WriterProperties::builder()
3711 .set_writer_version(WriterVersion::PARQUET_1_0)
3712 .set_dictionary_enabled(true)
3713 .set_dictionary_page_size_limit(dict_page_limit)
3714 .build();
3715
3716 let data: Vec<_> = (0..num_rows)
3717 .map(|i| {
3718 let mut v = vec![0u8; value_size];
3720 v[..8].copy_from_slice(&(i as u64).to_le_bytes());
3721 ByteArray::from(v)
3722 })
3723 .collect();
3724 let pages = write_and_collect_pages::<ByteArrayType>(props, 0, 0, &data, None, None);
3725 let dict_page_size = pages.dict_page_size;
3726
3727 assert!(
3728 dict_page_size > 0,
3729 "expected the column to dictionary-encode"
3730 );
3731 assert!(
3735 dict_page_size <= 3 * dict_page_limit,
3736 "dictionary page {dict_page_size} exceeds 3x the {dict_page_limit} limit",
3737 );
3738 }
3739
3740 #[test]
3741 fn test_column_writer_caps_dictionary_page_size_with_clustered_nulls() {
3742 let value_size = 64 * 1024;
3748 let num_values = 28;
3749 let num_nulls = 996;
3750
3751 let props = WriterProperties::builder()
3752 .set_writer_version(WriterVersion::PARQUET_1_0)
3753 .set_dictionary_enabled(true)
3754 .set_dictionary_page_size_limit(value_size)
3755 .build();
3756
3757 let data: Vec<_> = (0..num_values)
3758 .map(|i| {
3759 let mut v = vec![0u8; value_size];
3760 v[..8].copy_from_slice(&(i as u64).to_le_bytes());
3761 ByteArray::from(v)
3762 })
3763 .collect();
3764 let mut def_levels = vec![1i16; num_values];
3765 def_levels.extend(std::iter::repeat_n(0i16, num_nulls));
3766
3767 let pages =
3768 write_and_collect_pages::<ByteArrayType>(props, 1, 0, &data, Some(&def_levels), None);
3769 assert!(
3770 pages.dict_page_size > 0,
3771 "expected the column to dictionary-encode"
3772 );
3773 assert!(
3776 pages.dict_page_size <= 3 * value_size,
3777 "dictionary page {} exceeds three values ({}B)",
3778 pages.dict_page_size,
3779 3 * value_size,
3780 );
3781 }
3782
3783 #[test]
3784 fn test_column_writer_caps_page_size_for_fixed_len_byte_array() {
3785 let page_byte_limit = 4;
3791 let num_values = 128;
3792
3793 let props = WriterProperties::builder()
3794 .set_writer_version(WriterVersion::PARQUET_1_0)
3795 .set_dictionary_enabled(false)
3796 .set_encoding(Encoding::PLAIN)
3797 .set_data_page_size_limit(page_byte_limit)
3798 .build();
3799
3800 let data: Vec<_> = (0..num_values)
3801 .map(|i| {
3802 let mut fla = FixedLenByteArray::default();
3803 fla.set_data(Bytes::from(vec![i as u8]));
3804 fla
3805 })
3806 .collect();
3807 let pages =
3808 write_and_collect_pages::<FixedLenByteArrayType>(props, 0, 0, &data, None, None);
3809 let data_pages: Vec<_> = pages.data_pages.iter().map(|(size, _)| *size).collect();
3810
3811 assert!(
3814 data_pages.len() >= num_values / 8,
3815 "expected pages capped by byte budget, got {data_pages:?}"
3816 );
3817 for size in &data_pages {
3818 assert!(
3819 *size <= page_byte_limit * 4,
3820 "page size {size} larger than expected; pages {data_pages:?}"
3821 );
3822 }
3823 }
3824
3825 #[test]
3826 fn test_bool_statistics() {
3827 let stats = statistics_roundtrip::<BoolType>(&[true, false, false, true]);
3828 assert!(!stats.is_min_max_backwards_compatible());
3831 if let Statistics::Boolean(stats) = stats {
3832 assert_eq!(stats.min_opt().unwrap(), &false);
3833 assert_eq!(stats.max_opt().unwrap(), &true);
3834 } else {
3835 panic!("expecting Statistics::Boolean, got {stats:?}");
3836 }
3837 }
3838
3839 #[test]
3840 fn test_int32_statistics() {
3841 let stats = statistics_roundtrip::<Int32Type>(&[-1, 3, -2, 2]);
3842 assert!(stats.is_min_max_backwards_compatible());
3843 if let Statistics::Int32(stats) = stats {
3844 assert_eq!(stats.min_opt().unwrap(), &-2);
3845 assert_eq!(stats.max_opt().unwrap(), &3);
3846 } else {
3847 panic!("expecting Statistics::Int32, got {stats:?}");
3848 }
3849 }
3850
3851 #[test]
3852 fn test_int64_statistics() {
3853 let stats = statistics_roundtrip::<Int64Type>(&[-1, 3, -2, 2]);
3854 assert!(stats.is_min_max_backwards_compatible());
3855 if let Statistics::Int64(stats) = stats {
3856 assert_eq!(stats.min_opt().unwrap(), &-2);
3857 assert_eq!(stats.max_opt().unwrap(), &3);
3858 } else {
3859 panic!("expecting Statistics::Int64, got {stats:?}");
3860 }
3861 }
3862
3863 #[test]
3864 fn test_int96_statistics() {
3865 let input = vec![
3866 Int96::from(vec![1, 20, 30]),
3867 Int96::from(vec![3, 20, 10]),
3868 Int96::from(vec![0, 20, 30]),
3869 Int96::from(vec![2, 20, 30]),
3870 ]
3871 .into_iter()
3872 .collect::<Vec<Int96>>();
3873
3874 let stats = statistics_roundtrip::<Int96Type>(&input);
3875 assert!(!stats.is_min_max_backwards_compatible());
3876 if let Statistics::Int96(stats) = stats {
3877 assert_eq!(stats.min_opt().unwrap(), &Int96::from(vec![3, 20, 10]));
3878 assert_eq!(stats.max_opt().unwrap(), &Int96::from(vec![2, 20, 30]));
3879 } else {
3880 panic!("expecting Statistics::Int96, got {stats:?}");
3881 }
3882 }
3883
3884 #[test]
3885 fn test_float_statistics() {
3886 let stats = statistics_roundtrip::<FloatType>(&[-1.0, 3.0, -2.0, 2.0]);
3887 assert!(!stats.is_min_max_backwards_compatible());
3888 if let Statistics::Float(stats) = stats {
3889 assert_eq!(stats.min_opt().unwrap(), &-2.0);
3890 assert_eq!(stats.max_opt().unwrap(), &3.0);
3891 } else {
3892 panic!("expecting Statistics::Float, got {stats:?}");
3893 }
3894 }
3895
3896 #[test]
3897 fn test_double_statistics() {
3898 let stats = statistics_roundtrip::<DoubleType>(&[-1.0, 3.0, -2.0, 2.0]);
3899 assert!(!stats.is_min_max_backwards_compatible());
3900 if let Statistics::Double(stats) = stats {
3901 assert_eq!(stats.min_opt().unwrap(), &-2.0);
3902 assert_eq!(stats.max_opt().unwrap(), &3.0);
3903 } else {
3904 panic!("expecting Statistics::Double, got {stats:?}");
3905 }
3906 }
3907
3908 #[test]
3909 fn test_byte_array_statistics() {
3910 let input = ["aawaa", "zz", "aaw", "m", "qrs"]
3911 .iter()
3912 .map(|&s| s.into())
3913 .collect::<Vec<_>>();
3914
3915 let stats = statistics_roundtrip::<ByteArrayType>(&input);
3916 assert!(!stats.is_min_max_backwards_compatible());
3917 if let Statistics::ByteArray(stats) = stats {
3918 assert_eq!(stats.min_opt().unwrap(), &ByteArray::from("aaw"));
3919 assert_eq!(stats.max_opt().unwrap(), &ByteArray::from("zz"));
3920 } else {
3921 panic!("expecting Statistics::ByteArray, got {stats:?}");
3922 }
3923 }
3924
3925 #[test]
3926 fn test_fixed_len_byte_array_statistics() {
3927 let input = ["aawaa", "zz ", "aaw ", "m ", "qrs "]
3928 .iter()
3929 .map(|&s| ByteArray::from(s).into())
3930 .collect::<Vec<_>>();
3931
3932 let stats = statistics_roundtrip::<FixedLenByteArrayType>(&input);
3933 assert!(!stats.is_min_max_backwards_compatible());
3934 if let Statistics::FixedLenByteArray(stats) = stats {
3935 let expected_min: FixedLenByteArray = ByteArray::from("aaw ").into();
3936 assert_eq!(stats.min_opt().unwrap(), &expected_min);
3937 let expected_max: FixedLenByteArray = ByteArray::from("zz ").into();
3938 assert_eq!(stats.max_opt().unwrap(), &expected_max);
3939 } else {
3940 panic!("expecting Statistics::FixedLenByteArray, got {stats:?}");
3941 }
3942 }
3943
3944 #[test]
3945 fn test_ieee754_total_order_float() {
3946 let neg_nan = f32::from_bits(0xffc00000); let neg_inf = f32::NEG_INFINITY;
3950 let neg_one = -1.0_f32;
3951 let neg_zero = -0.0_f32;
3952 let pos_zero = 0.0_f32;
3953 let pos_one = 1.0_f32;
3954 let pos_inf = f32::INFINITY;
3955 let pos_nan = f32::from_bits(0x7fc00000); let values = vec![
3958 pos_nan, neg_zero, pos_inf, neg_one, neg_nan, pos_one, neg_inf, pos_zero,
3959 ];
3960
3961 let stats = statistics_roundtrip::<FloatType>(&values);
3962 if let Statistics::Float(stats) = stats {
3963 assert_eq!(stats.min_opt().unwrap(), &neg_inf);
3966 assert_eq!(stats.max_opt().unwrap(), &pos_inf);
3967 assert_eq!(stats.nan_count_opt(), Some(2)); } else {
3969 panic!("Expected float statistics");
3970 }
3971 }
3972
3973 #[test]
3974 fn test_ieee754_total_order_float_only_nan() {
3975 let neg_nan1 = f32::from_bits(0xffc00000); let neg_nan2 = f32::from_bits(0xffc00001); let neg_nan3 = f32::from_bits(0xffc00002); let pos_nan1 = f32::from_bits(0x7fc00000); let pos_nan2 = f32::from_bits(0x7fc00001); let pos_nan3 = f32::from_bits(0x7fc00002); let values = vec![neg_nan1, neg_nan2, neg_nan3, pos_nan1, pos_nan2, pos_nan3];
3985
3986 let stats = statistics_roundtrip::<FloatType>(&values);
3987 if let Statistics::Float(stats) = stats {
3988 assert_eq!(
3990 stats.min_opt().unwrap().total_cmp(&neg_nan3),
3991 Ordering::Equal
3992 );
3993 assert_eq!(
3994 stats.max_opt().unwrap().total_cmp(&pos_nan3),
3995 Ordering::Equal
3996 );
3997 assert_eq!(stats.nan_count_opt(), Some(6));
3998 } else {
3999 panic!("Expected float statistics");
4000 }
4001 }
4002
4003 #[test]
4004 fn test_ieee754_total_order_double() {
4005 let neg_nan = f64::from_bits(0xfff8000000000000);
4007 let neg_inf = f64::NEG_INFINITY;
4008 let neg_one = -1.0_f64;
4009 let neg_zero = -0.0_f64;
4010 let pos_zero = 0.0_f64;
4011 let pos_one = 1.0_f64;
4012 let pos_inf = f64::INFINITY;
4013 let pos_nan = f64::from_bits(0x7ff8000000000000);
4014
4015 let values = vec![
4016 pos_nan, neg_zero, pos_inf, neg_one, neg_nan, pos_one, neg_inf, pos_zero,
4017 ];
4018
4019 let stats = statistics_roundtrip::<DoubleType>(&values);
4020 if let Statistics::Double(stats) = stats {
4021 assert_eq!(stats.min_opt().unwrap(), &neg_inf);
4023 assert_eq!(stats.max_opt().unwrap(), &pos_inf);
4024 assert_eq!(stats.nan_count_opt(), Some(2));
4025 } else {
4026 panic!("Expected double statistics");
4027 }
4028 }
4029
4030 #[test]
4031 fn test_ieee754_total_order_double_only_nan() {
4032 let neg_nan1 = f64::from_bits(0xfff8000000000000);
4035 let neg_nan2 = f64::from_bits(0xfff8000000000001);
4036 let neg_nan3 = f64::from_bits(0xfff8000000000002);
4037 let pos_nan1 = f64::from_bits(0x7ff8000000000000);
4038 let pos_nan2 = f64::from_bits(0x7ff8000000000001);
4039 let pos_nan3 = f64::from_bits(0x7ff8000000000002);
4040
4041 let values = vec![neg_nan1, neg_nan2, neg_nan3, pos_nan1, pos_nan2, pos_nan3];
4042
4043 let stats = statistics_roundtrip::<DoubleType>(&values);
4044 if let Statistics::Double(stats) = stats {
4045 assert_eq!(
4047 stats.min_opt().unwrap().total_cmp(&neg_nan3),
4048 Ordering::Equal
4049 );
4050 assert_eq!(
4051 stats.max_opt().unwrap().total_cmp(&pos_nan3),
4052 Ordering::Equal
4053 );
4054 assert_eq!(stats.nan_count_opt(), Some(6));
4055 } else {
4056 panic!("Expected float statistics");
4057 }
4058 }
4059
4060 #[test]
4061 fn test_ieee754_total_order_zeros() {
4062 let values = vec![-0.0_f32, 0.0_f32, -0.0_f32, 0.0_f32];
4064
4065 let stats = statistics_roundtrip::<FloatType>(&values);
4066 if let Statistics::Float(stats) = stats {
4067 assert_eq!(stats.min_opt().unwrap().to_bits(), (-0.0_f32).to_bits());
4069 assert_eq!(stats.max_opt().unwrap().to_bits(), 0.0_f32.to_bits());
4070 } else {
4071 panic!("Expected float statistics");
4072 }
4073 }
4074
4075 #[test]
4076 #[cfg_attr(miri, ignore)] fn test_column_writer_check_float16_min_max() {
4078 let input = [
4079 -f16::ONE,
4080 f16::from_f32(3.0),
4081 -f16::from_f32(2.0),
4082 f16::from_f32(2.0),
4083 ]
4084 .into_iter()
4085 .map(|s| ByteArray::from(s).into())
4086 .collect::<Vec<_>>();
4087
4088 let stats = float16_statistics_roundtrip(&input);
4089 assert!(!stats.is_min_max_backwards_compatible());
4090 assert_eq!(
4091 stats.min_opt().unwrap(),
4092 &ByteArray::from(-f16::from_f32(2.0))
4093 );
4094 assert_eq!(
4095 stats.max_opt().unwrap(),
4096 &ByteArray::from(f16::from_f32(3.0))
4097 );
4098 }
4099
4100 #[test]
4101 #[cfg_attr(miri, ignore)] fn test_column_writer_check_float16_nan_middle() {
4103 let input = [f16::ONE, f16::NAN, f16::ONE + f16::ONE]
4104 .into_iter()
4105 .map(|s| ByteArray::from(s).into())
4106 .collect::<Vec<_>>();
4107
4108 let stats = float16_statistics_roundtrip(&input);
4109 assert!(!stats.is_min_max_backwards_compatible());
4110 assert_eq!(stats.min_opt().unwrap(), &ByteArray::from(f16::ONE));
4111 assert_eq!(
4112 stats.max_opt().unwrap(),
4113 &ByteArray::from(f16::ONE + f16::ONE)
4114 );
4115 assert_eq!(stats.nan_count_opt(), Some(1));
4116 }
4117
4118 #[test]
4119 #[cfg_attr(miri, ignore)] fn test_float16_statistics_nan_middle() {
4121 let input = [f16::ONE, f16::NAN, f16::ONE + f16::ONE]
4122 .into_iter()
4123 .map(|s| ByteArray::from(s).into())
4124 .collect::<Vec<_>>();
4125
4126 let stats = float16_statistics_roundtrip(&input);
4127 assert!(!stats.is_min_max_backwards_compatible());
4128 assert_eq!(stats.min_opt().unwrap(), &ByteArray::from(f16::ONE));
4129 assert_eq!(
4130 stats.max_opt().unwrap(),
4131 &ByteArray::from(f16::ONE + f16::ONE)
4132 );
4133 assert_eq!(stats.nan_count_opt(), Some(1));
4134 }
4135
4136 #[test]
4137 #[cfg_attr(miri, ignore)] fn test_float16_statistics_nan_start() {
4139 let input = [f16::NAN, f16::ONE, f16::ONE + f16::ONE]
4140 .into_iter()
4141 .map(|s| ByteArray::from(s).into())
4142 .collect::<Vec<_>>();
4143
4144 let stats = float16_statistics_roundtrip(&input);
4145 assert!(!stats.is_min_max_backwards_compatible());
4146 assert_eq!(stats.min_opt().unwrap(), &ByteArray::from(f16::ONE));
4147 assert_eq!(
4148 stats.max_opt().unwrap(),
4149 &ByteArray::from(f16::ONE + f16::ONE)
4150 );
4151 assert_eq!(stats.nan_count_opt(), Some(1));
4152 }
4153
4154 #[test]
4155 fn test_float16_statistics_nan_only() {
4156 let input = [f16::NAN, f16::NAN]
4157 .into_iter()
4158 .map(|s| ByteArray::from(s).into())
4159 .collect::<Vec<_>>();
4160
4161 let stats = float16_statistics_roundtrip(&input);
4162 assert_eq!(
4163 stats.min_bytes_opt(),
4164 Some(ByteArray::from(f16::NAN).as_bytes())
4165 );
4166 assert_eq!(
4167 stats.max_bytes_opt(),
4168 Some(ByteArray::from(f16::NAN).as_bytes())
4169 );
4170 assert!(!stats.is_min_max_backwards_compatible());
4171 assert_eq!(stats.nan_count_opt(), Some(2));
4172 }
4173
4174 #[test]
4175 fn test_float16_statistics_zero_only() {
4176 let input = std::iter::once(f16::ZERO)
4177 .map(|s| ByteArray::from(s).into())
4178 .collect::<Vec<_>>();
4179
4180 let stats = float16_statistics_roundtrip(&input);
4181 assert!(!stats.is_min_max_backwards_compatible());
4182 assert_eq!(stats.min_opt().unwrap(), &ByteArray::from(f16::ZERO));
4183 assert_eq!(stats.max_opt().unwrap(), &ByteArray::from(f16::ZERO));
4184 }
4185
4186 #[test]
4187 fn test_float16_statistics_neg_zero_only() {
4188 let input = std::iter::once(f16::NEG_ZERO)
4189 .map(|s| ByteArray::from(s).into())
4190 .collect::<Vec<_>>();
4191
4192 let stats = float16_statistics_roundtrip(&input);
4193 assert!(!stats.is_min_max_backwards_compatible());
4194 assert_eq!(stats.min_opt().unwrap(), &ByteArray::from(f16::NEG_ZERO));
4195 assert_eq!(stats.max_opt().unwrap(), &ByteArray::from(f16::NEG_ZERO));
4196 }
4197
4198 #[test]
4199 fn test_float16_statistics_zero_min() {
4200 let input = [f16::ZERO, f16::ONE, f16::NAN, f16::PI]
4201 .into_iter()
4202 .map(|s| ByteArray::from(s).into())
4203 .collect::<Vec<_>>();
4204
4205 let stats = float16_statistics_roundtrip(&input);
4206 assert!(!stats.is_min_max_backwards_compatible());
4207 assert_eq!(stats.min_opt().unwrap(), &ByteArray::from(f16::ZERO));
4208 assert_eq!(stats.max_opt().unwrap(), &ByteArray::from(f16::PI));
4209 }
4210
4211 #[test]
4212 fn test_float16_statistics_neg_zero_max() {
4213 let input = [f16::NEG_ZERO, f16::NEG_ONE, f16::NAN, -f16::PI]
4214 .into_iter()
4215 .map(|s| ByteArray::from(s).into())
4216 .collect::<Vec<_>>();
4217
4218 let stats = float16_statistics_roundtrip(&input);
4219 assert!(!stats.is_min_max_backwards_compatible());
4220 assert_eq!(stats.min_opt().unwrap(), &ByteArray::from(-f16::PI));
4221 assert_eq!(stats.max_opt().unwrap(), &ByteArray::from(f16::NEG_ZERO));
4222 }
4223
4224 #[test]
4225 fn test_float_statistics_nan_middle() {
4226 let stats = statistics_roundtrip::<FloatType>(&[1.0, f32::NAN, 2.0]);
4227 assert!(!stats.is_min_max_backwards_compatible());
4228 if let Statistics::Float(stats) = stats {
4229 assert_eq!(stats.min_opt().unwrap(), &1.0);
4230 assert_eq!(stats.max_opt().unwrap(), &2.0);
4231 assert_eq!(stats.nan_count_opt(), Some(1))
4232 } else {
4233 panic!("expecting Statistics::Float");
4234 }
4235 }
4236
4237 #[test]
4238 fn test_float_statistics_nan_start() {
4239 let stats = statistics_roundtrip::<FloatType>(&[f32::NAN, 1.0, 2.0]);
4240 assert!(!stats.is_min_max_backwards_compatible());
4241 if let Statistics::Float(stats) = stats {
4242 assert_eq!(stats.min_opt().unwrap(), &1.0);
4243 assert_eq!(stats.max_opt().unwrap(), &2.0);
4244 assert_eq!(stats.nan_count_opt(), Some(1))
4245 } else {
4246 panic!("expecting Statistics::Float");
4247 }
4248 }
4249
4250 #[test]
4251 fn test_float_statistics_nan_only() {
4252 let stats = statistics_roundtrip::<FloatType>(&[f32::NAN, f32::NAN]);
4253 assert_eq!(stats.min_bytes_opt(), Some(f32::NAN.as_bytes()));
4254 assert_eq!(stats.max_bytes_opt(), Some(f32::NAN.as_bytes()));
4255 assert_eq!(stats.nan_count_opt(), Some(2));
4256 assert!(!stats.is_min_max_backwards_compatible());
4257 assert!(matches!(stats, Statistics::Float(_)));
4258 }
4259
4260 #[test]
4261 fn test_float_statistics_zero_only() {
4262 let stats = statistics_roundtrip::<FloatType>(&[0.0]);
4263 assert!(!stats.is_min_max_backwards_compatible());
4264 if let Statistics::Float(stats) = stats {
4265 assert_eq!(stats.min_opt().unwrap(), &0.0);
4266 assert!(stats.min_opt().unwrap().is_sign_positive());
4267 assert_eq!(stats.max_opt().unwrap(), &0.0);
4268 assert!(stats.max_opt().unwrap().is_sign_positive());
4269 } else {
4270 panic!("expecting Statistics::Float");
4271 }
4272 }
4273
4274 #[test]
4275 fn test_float_statistics_neg_zero_only() {
4276 let stats = statistics_roundtrip::<FloatType>(&[-0.0]);
4277 assert!(!stats.is_min_max_backwards_compatible());
4278 if let Statistics::Float(stats) = stats {
4279 assert_eq!(stats.min_opt().unwrap(), &-0.0);
4280 assert!(stats.min_opt().unwrap().is_sign_negative());
4281 assert_eq!(stats.max_opt().unwrap(), &-0.0);
4282 assert!(stats.max_opt().unwrap().is_sign_negative());
4283 } else {
4284 panic!("expecting Statistics::Float");
4285 }
4286 }
4287
4288 #[test]
4289 fn test_float_statistics_zero_min() {
4290 let stats = statistics_roundtrip::<FloatType>(&[0.0, 1.0, f32::NAN, 2.0]);
4291 assert!(!stats.is_min_max_backwards_compatible());
4292 if let Statistics::Float(stats) = stats {
4293 assert_eq!(stats.min_opt().unwrap(), &0.0);
4294 assert!(stats.min_opt().unwrap().is_sign_positive());
4295 assert_eq!(stats.max_opt().unwrap(), &2.0);
4296 } else {
4297 panic!("expecting Statistics::Float");
4298 }
4299 }
4300
4301 #[test]
4302 fn test_float_statistics_neg_zero_max() {
4303 let stats = statistics_roundtrip::<FloatType>(&[-0.0, -1.0, f32::NAN, -2.0]);
4304 assert!(!stats.is_min_max_backwards_compatible());
4305 if let Statistics::Float(stats) = stats {
4306 assert_eq!(stats.min_opt().unwrap(), &-2.0);
4307 assert_eq!(stats.max_opt().unwrap(), &-0.0);
4308 assert!(stats.max_opt().unwrap().is_sign_negative());
4309 } else {
4310 panic!("expecting Statistics::Float");
4311 }
4312 }
4313
4314 #[test]
4315 fn test_double_statistics_nan_middle() {
4316 let stats = statistics_roundtrip::<DoubleType>(&[1.0, f64::NAN, 2.0]);
4317 assert!(!stats.is_min_max_backwards_compatible());
4318 if let Statistics::Double(stats) = stats {
4319 assert_eq!(stats.min_opt().unwrap(), &1.0);
4320 assert_eq!(stats.max_opt().unwrap(), &2.0);
4321 assert_eq!(stats.nan_count_opt(), Some(1))
4322 } else {
4323 panic!("expecting Statistics::Double");
4324 }
4325 }
4326
4327 #[test]
4328 fn test_double_statistics_nan_start() {
4329 let stats = statistics_roundtrip::<DoubleType>(&[f64::NAN, 1.0, 2.0]);
4330 assert!(!stats.is_min_max_backwards_compatible());
4331 if let Statistics::Double(stats) = stats {
4332 assert_eq!(stats.min_opt().unwrap(), &1.0);
4333 assert_eq!(stats.max_opt().unwrap(), &2.0);
4334 assert_eq!(stats.nan_count_opt(), Some(1))
4335 } else {
4336 panic!("expecting Statistics::Double");
4337 }
4338 }
4339
4340 #[test]
4341 fn test_double_statistics_nan_only() {
4342 let stats = statistics_roundtrip::<DoubleType>(&[f64::NAN, f64::NAN]);
4343 assert_eq!(stats.min_bytes_opt(), Some(f64::NAN.as_bytes()));
4344 assert_eq!(stats.max_bytes_opt(), Some(f64::NAN.as_bytes()));
4345 assert_eq!(stats.nan_count_opt(), Some(2));
4346 assert!(matches!(stats, Statistics::Double(_)));
4347 assert!(!stats.is_min_max_backwards_compatible());
4348 }
4349
4350 #[test]
4351 fn test_double_statistics_zero_only() {
4352 let stats = statistics_roundtrip::<DoubleType>(&[0.0]);
4353 assert!(!stats.is_min_max_backwards_compatible());
4354 if let Statistics::Double(stats) = stats {
4355 assert_eq!(stats.min_opt().unwrap(), &0.0);
4356 assert!(stats.min_opt().unwrap().is_sign_positive());
4357 assert_eq!(stats.max_opt().unwrap(), &0.0);
4358 assert!(stats.max_opt().unwrap().is_sign_positive());
4359 } else {
4360 panic!("expecting Statistics::Double");
4361 }
4362 }
4363
4364 #[test]
4365 fn test_double_statistics_neg_zero_only() {
4366 let stats = statistics_roundtrip::<DoubleType>(&[-0.0]);
4367 assert!(!stats.is_min_max_backwards_compatible());
4368 if let Statistics::Double(stats) = stats {
4369 assert_eq!(stats.min_opt().unwrap(), &-0.0);
4370 assert!(stats.min_opt().unwrap().is_sign_negative());
4371 assert_eq!(stats.max_opt().unwrap(), &-0.0);
4372 assert!(stats.max_opt().unwrap().is_sign_negative());
4373 } else {
4374 panic!("expecting Statistics::Double");
4375 }
4376 }
4377
4378 #[test]
4379 fn test_double_statistics_zero_min() {
4380 let stats = statistics_roundtrip::<DoubleType>(&[0.0, 1.0, f64::NAN, 2.0]);
4381 assert!(!stats.is_min_max_backwards_compatible());
4382 if let Statistics::Double(stats) = stats {
4383 assert_eq!(stats.min_opt().unwrap(), &0.0);
4384 assert!(stats.min_opt().unwrap().is_sign_positive());
4385 assert_eq!(stats.max_opt().unwrap(), &2.0);
4386 } else {
4387 panic!("expecting Statistics::Double");
4388 }
4389 }
4390
4391 #[test]
4392 fn test_double_statistics_neg_zero_max() {
4393 let stats = statistics_roundtrip::<DoubleType>(&[-0.0, -1.0, f64::NAN, -2.0]);
4394 assert!(!stats.is_min_max_backwards_compatible());
4395 if let Statistics::Double(stats) = stats {
4396 assert_eq!(stats.min_opt().unwrap(), &-2.0);
4397 assert_eq!(stats.max_opt().unwrap(), &-0.0);
4398 assert!(stats.max_opt().unwrap().is_sign_negative());
4399 } else {
4400 panic!("expecting Statistics::Double");
4401 }
4402 }
4403
4404 #[test]
4405 fn test_compare_greater_byte_array_decimals() {
4406 assert!(!compare_greater_byte_array_decimals(&[], &[],),);
4407 assert!(compare_greater_byte_array_decimals(&[1u8,], &[],),);
4408 assert!(!compare_greater_byte_array_decimals(&[], &[1u8,],),);
4409 assert!(compare_greater_byte_array_decimals(&[1u8,], &[0u8,],),);
4410 assert!(!compare_greater_byte_array_decimals(&[1u8,], &[1u8,],),);
4411 assert!(compare_greater_byte_array_decimals(&[1u8, 0u8,], &[0u8,],),);
4412 assert!(!compare_greater_byte_array_decimals(
4413 &[0u8, 1u8,],
4414 &[1u8, 0u8,],
4415 ),);
4416 assert!(!compare_greater_byte_array_decimals(
4417 &[255u8, 35u8, 0u8, 0u8,],
4418 &[0u8,],
4419 ),);
4420 assert!(compare_greater_byte_array_decimals(
4421 &[0u8,],
4422 &[255u8, 35u8, 0u8, 0u8,],
4423 ),);
4424
4425 assert!(compare_greater_byte_array_decimals(
4431 &[0u8, 128u8, 0u8,],
4432 &[0u8, 255u8,],
4433 ),);
4434 assert!(!compare_greater_byte_array_decimals(
4435 &[0u8, 255u8,],
4436 &[0u8, 128u8, 0u8,],
4437 ),);
4438 assert!(compare_greater_byte_array_decimals(
4440 &[128u8,],
4441 &[255u8, 127u8,],
4442 ),);
4443 assert!(!compare_greater_byte_array_decimals(
4444 &[255u8, 127u8,],
4445 &[128u8,],
4446 ),);
4447 assert!(compare_greater_byte_array_decimals(
4449 &[128u8,],
4450 &[255u8, 0u8,],
4451 ),);
4452 assert!(compare_greater_byte_array_decimals(&[0u8, 10u8,], &[5u8,],),);
4454 assert!(compare_greater_byte_array_decimals(&[10u8,], &[0u8, 5u8,],),);
4455 assert!(!compare_greater_byte_array_decimals(
4457 &[255u8, 128u8,],
4458 &[128u8,],
4459 ),);
4460 assert!(!compare_greater_byte_array_decimals(
4461 &[128u8,],
4462 &[255u8, 128u8,],
4463 ),);
4464 }
4465
4466 #[test]
4467 fn test_column_index_with_null_pages() {
4468 let page_writer = get_test_page_writer();
4470 let props = Default::default();
4471 let mut writer = get_test_column_writer::<Int32Type>(page_writer, 1, 0, props);
4472 writer.write_batch(&[], Some(&[0, 0, 0, 0]), None).unwrap();
4473
4474 let r = writer.close().unwrap();
4475 assert!(r.column_index.is_some());
4476 let col_idx = r.column_index.unwrap();
4477 let ColumnIndexMetaData::INT32(col_idx) = col_idx else {
4478 panic!("wrong stats type")
4479 };
4480 assert!(col_idx.is_null_page(0));
4482 assert!(col_idx.min_value(0).is_none());
4484 assert!(col_idx.max_value(0).is_none());
4485 assert!(col_idx.null_count(0).is_some());
4487 assert_eq!(col_idx.null_count(0), Some(4));
4488 assert!(col_idx.repetition_level_histogram(0).is_none());
4490 assert!(col_idx.definition_level_histogram(0).is_some());
4492 assert_eq!(col_idx.definition_level_histogram(0).unwrap(), &[4, 0]);
4493 }
4494
4495 #[test]
4496 fn test_column_index_repeated_page_with_values_not_null() {
4497 let page_writer = get_test_page_writer();
4499 let props = Default::default();
4500 let mut writer = get_test_column_writer::<Int32Type>(page_writer, 2, 1, props);
4501 writer
4502 .write_batch(
4503 &[7, 7, 7, 7],
4504 Some(&[1, 2, 1, 2, 1, 2, 1, 2]),
4505 Some(&[0, 1, 0, 1, 0, 1, 0, 1]),
4506 )
4507 .unwrap();
4508
4509 let r = writer.close().unwrap();
4510 let ColumnIndexMetaData::INT32(col_idx) = r.column_index.unwrap() else {
4511 panic!("wrong stats type")
4512 };
4513 assert!(!col_idx.is_null_page(0));
4514 assert_eq!(col_idx.min_value(0), Some(&7));
4515 assert_eq!(col_idx.max_value(0), Some(&7));
4516 assert_eq!(col_idx.null_count(0), Some(4));
4517 }
4518
4519 #[test]
4520 fn test_column_index_repeated_page_of_nulls() {
4521 let page_writer = get_test_page_writer();
4523 let props = Default::default();
4524 let mut writer = get_test_column_writer::<Int32Type>(page_writer, 2, 1, props);
4525 writer
4526 .write_batch(&[], Some(&[1, 1, 1, 1]), Some(&[0, 1, 0, 1]))
4527 .unwrap();
4528
4529 let r = writer.close().unwrap();
4530 let ColumnIndexMetaData::INT32(col_idx) = r.column_index.unwrap() else {
4531 panic!("wrong stats type")
4532 };
4533 assert!(col_idx.is_null_page(0));
4534 assert!(col_idx.min_value(0).is_none());
4535 assert!(col_idx.max_value(0).is_none());
4536 assert_eq!(col_idx.null_count(0), Some(4));
4537 }
4538
4539 #[test]
4540 fn test_column_offset_index_metadata() {
4541 let page_writer = get_test_page_writer();
4544 let props = Default::default();
4545 let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
4546 writer.write_batch(&[1, 2, 3, 4], None, None).unwrap();
4547 writer.flush_data_pages().unwrap();
4549 writer.write_batch(&[4, 8, 2, -5], None, None).unwrap();
4551
4552 let r = writer.close().unwrap();
4553 let column_index = r.column_index.unwrap();
4554 let offset_index = r.offset_index.unwrap();
4555
4556 assert_eq!(8, r.rows_written);
4557
4558 let ColumnIndexMetaData::INT32(column_index) = column_index else {
4560 panic!("wrong stats type")
4561 };
4562 assert_eq!(2, column_index.num_pages());
4563 assert_eq!(2, offset_index.page_locations.len());
4564 assert_eq!(BoundaryOrder::UNORDERED, column_index.boundary_order);
4565 for idx in 0..2 {
4566 assert!(!column_index.is_null_page(idx));
4567 assert_eq!(0, column_index.null_count(0).unwrap());
4568 }
4569
4570 if let Some(stats) = r.metadata.statistics() {
4571 assert_eq!(stats.null_count_opt(), Some(0));
4572 assert_eq!(stats.distinct_count_opt(), None);
4573 if let Statistics::Int32(stats) = stats {
4574 assert_eq!(stats.min_opt(), column_index.min_value(1));
4578 assert_eq!(stats.max_opt(), column_index.max_value(1));
4579 } else {
4580 panic!("expecting Statistics::Int32");
4581 }
4582 } else {
4583 panic!("metadata missing statistics");
4584 }
4585
4586 assert_eq!(0, offset_index.page_locations[0].first_row_index);
4588 assert_eq!(4, offset_index.page_locations[1].first_row_index);
4589 }
4590
4591 #[test]
4593 fn test_column_offset_index_metadata_truncating() {
4594 let page_writer = get_test_page_writer();
4597 let props = WriterProperties::builder()
4598 .set_statistics_truncate_length(None) .build()
4600 .into();
4601 let mut writer = get_test_column_writer::<FixedLenByteArrayType>(page_writer, 0, 0, props);
4602
4603 let mut data = vec![FixedLenByteArray::default(); 3];
4604 data[0].set_data(Bytes::from(vec![97_u8; 200]));
4606 data[1].set_data(Bytes::from(vec![112_u8; 200]));
4608 data[2].set_data(Bytes::from(vec![98_u8; 200]));
4609
4610 writer.write_batch(&data, None, None).unwrap();
4611
4612 writer.flush_data_pages().unwrap();
4613
4614 let r = writer.close().unwrap();
4615 let column_index = r.column_index.unwrap();
4616 let offset_index = r.offset_index.unwrap();
4617
4618 let ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(column_index) = column_index else {
4619 panic!("wrong stats type")
4620 };
4621
4622 assert_eq!(3, r.rows_written);
4623
4624 assert_eq!(1, column_index.num_pages());
4626 assert_eq!(1, offset_index.page_locations.len());
4627 assert_eq!(BoundaryOrder::ASCENDING, column_index.boundary_order);
4628 assert!(!column_index.is_null_page(0));
4629 assert_eq!(Some(0), column_index.null_count(0));
4630
4631 if let Some(stats) = r.metadata.statistics() {
4632 assert_eq!(stats.null_count_opt(), Some(0));
4633 assert_eq!(stats.distinct_count_opt(), None);
4634 if let Statistics::FixedLenByteArray(stats) = stats {
4635 let column_index_min_value = column_index.min_value(0).unwrap();
4636 let column_index_max_value = column_index.max_value(0).unwrap();
4637
4638 assert_ne!(stats.min_bytes_opt().unwrap(), column_index_min_value);
4640 assert_ne!(stats.max_bytes_opt().unwrap(), column_index_max_value);
4641
4642 assert_eq!(
4643 column_index_min_value.len(),
4644 DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH.unwrap()
4645 );
4646 assert_eq!(column_index_min_value, &[97_u8; 64]);
4647 assert_eq!(
4648 column_index_max_value.len(),
4649 DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH.unwrap()
4650 );
4651
4652 assert_eq!(
4654 *column_index_max_value.last().unwrap(),
4655 *column_index_max_value.first().unwrap() + 1
4656 );
4657 } else {
4658 panic!("expecting Statistics::FixedLenByteArray");
4659 }
4660 } else {
4661 panic!("metadata missing statistics");
4662 }
4663 }
4664
4665 #[test]
4666 fn test_column_offset_index_truncating_spec_example() {
4667 let page_writer = get_test_page_writer();
4670
4671 let builder = WriterProperties::builder().set_column_index_truncate_length(Some(1));
4673 let props = Arc::new(builder.build());
4674 let mut writer = get_test_column_writer::<FixedLenByteArrayType>(page_writer, 0, 0, props);
4675
4676 let mut data = vec![FixedLenByteArray::default(); 1];
4677 data[0].set_data(Bytes::from(String::from("Blart Versenwald III")));
4679
4680 writer.write_batch(&data, None, None).unwrap();
4681
4682 writer.flush_data_pages().unwrap();
4683
4684 let r = writer.close().unwrap();
4685 let column_index = r.column_index.unwrap();
4686 let offset_index = r.offset_index.unwrap();
4687
4688 let ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(column_index) = column_index else {
4689 panic!("wrong stats type")
4690 };
4691
4692 assert_eq!(1, r.rows_written);
4693
4694 assert_eq!(1, column_index.num_pages());
4696 assert_eq!(1, offset_index.page_locations.len());
4697 assert_eq!(BoundaryOrder::ASCENDING, column_index.boundary_order);
4698 assert!(!column_index.is_null_page(0));
4699 assert_eq!(Some(0), column_index.null_count(0));
4700
4701 if let Some(stats) = r.metadata.statistics() {
4702 assert_eq!(stats.null_count_opt(), Some(0));
4703 assert_eq!(stats.distinct_count_opt(), None);
4704 if let Statistics::FixedLenByteArray(_stats) = stats {
4705 let column_index_min_value = column_index.min_value(0).unwrap();
4706 let column_index_max_value = column_index.max_value(0).unwrap();
4707
4708 assert_eq!(column_index_min_value.len(), 1);
4709 assert_eq!(column_index_max_value.len(), 1);
4710
4711 assert_eq!(b"B", column_index_min_value);
4712 assert_eq!(b"C", column_index_max_value);
4713
4714 assert_ne!(column_index_min_value, stats.min_bytes_opt().unwrap());
4715 assert_ne!(column_index_max_value, stats.max_bytes_opt().unwrap());
4716 } else {
4717 panic!("expecting Statistics::FixedLenByteArray");
4718 }
4719 } else {
4720 panic!("metadata missing statistics");
4721 }
4722 }
4723
4724 #[test]
4725 fn test_float16_min_max_no_truncation() {
4726 let builder = WriterProperties::builder().set_column_index_truncate_length(Some(1));
4728 let props = Arc::new(builder.build());
4729 let page_writer = get_test_page_writer();
4730 let mut writer = get_test_float16_column_writer(page_writer, props);
4731
4732 let expected_value = f16::PI.to_le_bytes().to_vec();
4733 let data = vec![ByteArray::from(expected_value.clone()).into()];
4734 writer.write_batch(&data, None, None).unwrap();
4735 writer.flush_data_pages().unwrap();
4736
4737 let r = writer.close().unwrap();
4738
4739 let column_index = r.column_index.unwrap();
4742 let ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(column_index) = column_index else {
4743 panic!("wrong stats type")
4744 };
4745 let column_index_min_bytes = column_index.min_value(0).unwrap();
4746 let column_index_max_bytes = column_index.max_value(0).unwrap();
4747 assert_eq!(expected_value, column_index_min_bytes);
4748 assert_eq!(expected_value, column_index_max_bytes);
4749
4750 let stats = r.metadata.statistics().unwrap();
4752 if let Statistics::FixedLenByteArray(stats) = stats {
4753 let stats_min_bytes = stats.min_bytes_opt().unwrap();
4754 let stats_max_bytes = stats.max_bytes_opt().unwrap();
4755 assert_eq!(expected_value, stats_min_bytes);
4756 assert_eq!(expected_value, stats_max_bytes);
4757 } else {
4758 panic!("expecting Statistics::FixedLenByteArray");
4759 }
4760 }
4761
4762 #[test]
4763 fn test_decimal_min_max_no_truncation() {
4764 let builder = WriterProperties::builder().set_column_index_truncate_length(Some(1));
4766 let props = Arc::new(builder.build());
4767 let page_writer = get_test_page_writer();
4768 let mut writer =
4769 get_test_decimals_column_writer::<FixedLenByteArrayType>(page_writer, 0, 0, props);
4770
4771 let expected_value = vec![
4772 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 255u8, 179u8, 172u8, 19u8, 35u8,
4773 231u8, 90u8, 0u8, 0u8,
4774 ];
4775 let data = vec![ByteArray::from(expected_value.clone()).into()];
4776 writer.write_batch(&data, None, None).unwrap();
4777 writer.flush_data_pages().unwrap();
4778
4779 let r = writer.close().unwrap();
4780
4781 let column_index = r.column_index.unwrap();
4784 let ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(column_index) = column_index else {
4785 panic!("wrong stats type")
4786 };
4787 let column_index_min_bytes = column_index.min_value(0).unwrap();
4788 let column_index_max_bytes = column_index.max_value(0).unwrap();
4789 assert_eq!(expected_value, column_index_min_bytes);
4790 assert_eq!(expected_value, column_index_max_bytes);
4791
4792 let stats = r.metadata.statistics().unwrap();
4794 if let Statistics::FixedLenByteArray(stats) = stats {
4795 let stats_min_bytes = stats.min_bytes_opt().unwrap();
4796 let stats_max_bytes = stats.max_bytes_opt().unwrap();
4797 assert_eq!(expected_value, stats_min_bytes);
4798 assert_eq!(expected_value, stats_max_bytes);
4799 } else {
4800 panic!("expecting Statistics::FixedLenByteArray");
4801 }
4802 }
4803
4804 #[test]
4805 fn test_decimal_statistics_do_not_alias_written_values() {
4806 let page_writer = get_test_page_writer();
4813 let mut writer = get_test_decimals_column_writer::<FixedLenByteArrayType>(
4814 page_writer,
4815 0,
4816 0,
4817 Default::default(),
4818 );
4819
4820 let arena = Bytes::from((0..64u8).collect::<Vec<_>>());
4822 let values: Vec<FixedLenByteArray> = (0..4)
4823 .map(|i| ByteArray::from(arena.slice(i * 16..(i + 1) * 16)).into())
4824 .collect();
4825 writer.write_batch(&values, None, None).unwrap();
4826 let r = writer.close().unwrap();
4827 drop(values);
4828
4829 let statistics = r.metadata.statistics().unwrap();
4830 assert!(!statistics.is_min_max_deprecated());
4833 assert!(statistics.is_min_max_backwards_compatible());
4834 let Statistics::FixedLenByteArray(stats) = statistics else {
4835 panic!("expecting Statistics::FixedLenByteArray");
4836 };
4837 let min = stats.min_bytes_opt().unwrap();
4838 let max = stats.max_bytes_opt().unwrap();
4839 assert_eq!(min, &arena[..16]);
4840 assert_eq!(max, &arena[48..]);
4841
4842 let arena_range = arena.as_ptr_range();
4844 assert!(
4845 !arena_range.contains(&min.as_ptr()),
4846 "min aliases the written buffer"
4847 );
4848 assert!(
4849 !arena_range.contains(&max.as_ptr()),
4850 "max aliases the written buffer"
4851 );
4852
4853 assert!(stats.min_is_exact());
4854 assert!(stats.max_is_exact());
4855 }
4856
4857 #[test]
4858 fn test_decimal_byte_array_min_max_no_statistics_truncation() {
4859 let page_writer = get_test_page_writer();
4861
4862 let builder = WriterProperties::builder().set_statistics_truncate_length(Some(1));
4864 let props = Arc::new(builder.build());
4865 let mut writer = get_test_decimals_column_writer::<ByteArrayType>(page_writer, 0, 0, props);
4866
4867 let positive_200 = vec![0x00u8, 0xC8u8];
4873 let negative_200 = vec![0xFFu8, 0x38u8];
4874
4875 let data = vec![
4876 ByteArray::from(positive_200.clone()),
4877 ByteArray::from(negative_200.clone()),
4878 ];
4879 writer.write_batch(&data, None, None).unwrap();
4880 writer.flush_data_pages().unwrap();
4881
4882 let r = writer.close().unwrap();
4883 assert_eq!(2, r.rows_written);
4884
4885 let stats = r.metadata.statistics().expect("statistics");
4886 if let Statistics::ByteArray(stats) = stats {
4887 assert!(stats.min_is_exact());
4890 assert!(stats.max_is_exact());
4891
4892 let min_value = stats.min_bytes_opt().unwrap();
4893 let max_value = stats.max_bytes_opt().unwrap();
4894
4895 assert_eq!(negative_200, min_value);
4897 assert_eq!(positive_200, max_value);
4898 } else {
4899 panic!("expecting Statistics::ByteArray");
4900 }
4901 }
4902
4903 #[test]
4904 fn test_statistics_truncating_byte_array_default() {
4905 let page_writer = get_test_page_writer();
4906
4907 let props = WriterProperties::builder().build().into();
4909 let mut writer = get_test_column_writer::<ByteArrayType>(page_writer, 0, 0, props);
4910
4911 let mut data = vec![ByteArray::default(); 1];
4912 data[0].set_data(Bytes::from(String::from(
4913 "This string is longer than 64 bytes, so it will almost certainly be truncated.",
4914 )));
4915 writer.write_batch(&data, None, None).unwrap();
4916 writer.flush_data_pages().unwrap();
4917
4918 let r = writer.close().unwrap();
4919
4920 assert_eq!(1, r.rows_written);
4921
4922 let stats = r.metadata.statistics().expect("statistics");
4923 if let Statistics::ByteArray(stats) = stats {
4924 let min_value = stats.min_opt().unwrap();
4925 let max_value = stats.max_opt().unwrap();
4926
4927 assert!(!stats.min_is_exact());
4928 assert!(!stats.max_is_exact());
4929
4930 let expected_len = 64;
4931 assert_eq!(min_value.len(), expected_len);
4932 assert_eq!(max_value.len(), expected_len);
4933
4934 let expected_min =
4935 "This string is longer than 64 bytes, so it will almost certainly".as_bytes();
4936 assert_eq!(expected_min, min_value.as_bytes());
4937 let expected_max =
4939 "This string is longer than 64 bytes, so it will almost certainlz".as_bytes();
4940 assert_eq!(expected_max, max_value.as_bytes());
4941 } else {
4942 panic!("expecting Statistics::ByteArray");
4943 }
4944 }
4945
4946 #[test]
4947 fn test_statistics_truncating_byte_array() {
4948 let page_writer = get_test_page_writer();
4949
4950 const TEST_TRUNCATE_LENGTH: usize = 1;
4951
4952 let builder =
4954 WriterProperties::builder().set_statistics_truncate_length(Some(TEST_TRUNCATE_LENGTH));
4955 let props = Arc::new(builder.build());
4956 let mut writer = get_test_column_writer::<ByteArrayType>(page_writer, 0, 0, props);
4957
4958 let mut data = vec![ByteArray::default(); 1];
4959 data[0].set_data(Bytes::from(String::from("Blart Versenwald III")));
4961
4962 writer.write_batch(&data, None, None).unwrap();
4963
4964 writer.flush_data_pages().unwrap();
4965
4966 let r = writer.close().unwrap();
4967
4968 assert_eq!(1, r.rows_written);
4969
4970 let stats = r.metadata.statistics().expect("statistics");
4971 assert_eq!(stats.null_count_opt(), Some(0));
4972 assert_eq!(stats.distinct_count_opt(), None);
4973 if let Statistics::ByteArray(stats) = stats {
4974 let min_value = stats.min_opt().unwrap();
4975 let max_value = stats.max_opt().unwrap();
4976
4977 assert!(!stats.min_is_exact());
4978 assert!(!stats.max_is_exact());
4979
4980 assert_eq!(min_value.len(), TEST_TRUNCATE_LENGTH);
4981 assert_eq!(max_value.len(), TEST_TRUNCATE_LENGTH);
4982
4983 assert_eq!(b"B", min_value.as_bytes());
4984 assert_eq!(b"C", max_value.as_bytes());
4985 } else {
4986 panic!("expecting Statistics::ByteArray");
4987 }
4988 }
4989
4990 #[test]
4991 fn test_statistics_truncating_fixed_len_byte_array() {
4992 let page_writer = get_test_page_writer();
4993
4994 const TEST_TRUNCATE_LENGTH: usize = 1;
4995
4996 let builder =
4998 WriterProperties::builder().set_statistics_truncate_length(Some(TEST_TRUNCATE_LENGTH));
4999 let props = Arc::new(builder.build());
5000 let mut writer = get_test_column_writer::<FixedLenByteArrayType>(page_writer, 0, 0, props);
5001
5002 let mut data = vec![FixedLenByteArray::default(); 1];
5003
5004 const PSEUDO_DECIMAL_VALUE: i128 = 6541894651216648486512564456564654;
5005 const PSEUDO_DECIMAL_BYTES: [u8; 16] = PSEUDO_DECIMAL_VALUE.to_be_bytes();
5006
5007 const EXPECTED_MIN: [u8; TEST_TRUNCATE_LENGTH] = [PSEUDO_DECIMAL_BYTES[0]]; const EXPECTED_MAX: [u8; TEST_TRUNCATE_LENGTH] =
5009 [PSEUDO_DECIMAL_BYTES[0].overflowing_add(1).0];
5010
5011 data[0].set_data(Bytes::from(PSEUDO_DECIMAL_BYTES.as_slice()));
5013
5014 writer.write_batch(&data, None, None).unwrap();
5015
5016 writer.flush_data_pages().unwrap();
5017
5018 let r = writer.close().unwrap();
5019
5020 assert_eq!(1, r.rows_written);
5021
5022 let stats = r.metadata.statistics().expect("statistics");
5023 assert_eq!(stats.null_count_opt(), Some(0));
5024 assert_eq!(stats.distinct_count_opt(), None);
5025 if let Statistics::FixedLenByteArray(stats) = stats {
5026 let min_value = stats.min_opt().unwrap();
5027 let max_value = stats.max_opt().unwrap();
5028
5029 assert!(!stats.min_is_exact());
5030 assert!(!stats.max_is_exact());
5031
5032 assert_eq!(min_value.len(), TEST_TRUNCATE_LENGTH);
5033 assert_eq!(max_value.len(), TEST_TRUNCATE_LENGTH);
5034
5035 assert_eq!(EXPECTED_MIN.as_slice(), min_value.as_bytes());
5036 assert_eq!(EXPECTED_MAX.as_slice(), max_value.as_bytes());
5037
5038 let reconstructed_min = i128::from_be_bytes([
5039 min_value.as_bytes()[0],
5040 0,
5041 0,
5042 0,
5043 0,
5044 0,
5045 0,
5046 0,
5047 0,
5048 0,
5049 0,
5050 0,
5051 0,
5052 0,
5053 0,
5054 0,
5055 ]);
5056
5057 let reconstructed_max = i128::from_be_bytes([
5058 max_value.as_bytes()[0],
5059 0,
5060 0,
5061 0,
5062 0,
5063 0,
5064 0,
5065 0,
5066 0,
5067 0,
5068 0,
5069 0,
5070 0,
5071 0,
5072 0,
5073 0,
5074 ]);
5075
5076 println!("min: {reconstructed_min} {PSEUDO_DECIMAL_VALUE}");
5078 assert!(reconstructed_min <= PSEUDO_DECIMAL_VALUE);
5079 println!("max {reconstructed_max} {PSEUDO_DECIMAL_VALUE}");
5080 assert!(reconstructed_max >= PSEUDO_DECIMAL_VALUE);
5081 } else {
5082 panic!("expecting Statistics::FixedLenByteArray");
5083 }
5084 }
5085
5086 #[test]
5087 fn test_send() {
5088 fn test<T: Send>() {}
5089 test::<ColumnWriterImpl<Int32Type>>();
5090 }
5091
5092 #[test]
5093 fn test_increment() {
5094 let v = increment(vec![0, 0, 0]).unwrap();
5095 assert_eq!(&v, &[0, 0, 1]);
5096
5097 let v = increment(vec![0, 255, 255]).unwrap();
5099 assert_eq!(&v, &[1, 0, 0]);
5100
5101 let v = increment(vec![255, 255, 255]);
5103 assert!(v.is_none());
5104 }
5105
5106 #[test]
5107 fn test_increment_utf8() {
5108 let test_inc = |o: &str, expected: &str| {
5109 if let Ok(v) = String::from_utf8(increment_utf8(o).unwrap()) {
5110 assert_eq!(v, expected);
5112 assert!(*v > *o);
5114 let mut greater = ByteArray::new();
5116 greater.set_data(Bytes::from(v));
5117 let mut original = ByteArray::new();
5118 original.set_data(Bytes::from(o.as_bytes().to_vec()));
5119 assert!(greater > original);
5120 } else {
5121 panic!("Expected incremented UTF8 string to also be valid.");
5122 }
5123 };
5124
5125 test_inc("hello", "hellp");
5127
5128 test_inc("a\u{7f}", "b");
5130
5131 assert!(increment_utf8("\u{7f}\u{7f}").is_none());
5133
5134 test_inc("❤️🧡💛💚💙💜", "❤️🧡💛💚💙💝");
5136
5137 test_inc("éééé", "éééê");
5139
5140 test_inc("\u{ff}\u{ff}", "\u{ff}\u{100}");
5142
5143 test_inc("a\u{7ff}", "b");
5145
5146 assert!(increment_utf8("\u{7ff}\u{7ff}").is_none());
5148
5149 test_inc("ࠀࠀ", "ࠀࠁ");
5152
5153 test_inc("a\u{ffff}", "b");
5155
5156 assert!(increment_utf8("\u{ffff}\u{ffff}").is_none());
5158
5159 test_inc("𐀀𐀀", "𐀀𐀁");
5161
5162 test_inc("a\u{10ffff}", "b");
5164
5165 assert!(increment_utf8("\u{10ffff}\u{10ffff}").is_none());
5167
5168 test_inc("a\u{D7FF}", "b");
5171 }
5172
5173 #[test]
5174 fn test_truncate_utf8() {
5175 let data = "❤️🧡💛💚💙💜";
5177 let r = truncate_utf8(data, data.len()).unwrap();
5178 assert_eq!(r.len(), data.len());
5179 assert_eq!(&r, data.as_bytes());
5180
5181 let r = truncate_utf8(data, 13).unwrap();
5183 assert_eq!(r.len(), 10);
5184 assert_eq!(&r, "❤️🧡".as_bytes());
5185
5186 let r = truncate_utf8("\u{0836}", 1);
5188 assert!(r.is_none());
5189
5190 let r = truncate_and_increment_utf8("yyyyyyyyy", 8).unwrap();
5193 assert_eq!(&r, b"yyyyyyyz");
5194
5195 let r = truncate_and_increment_utf8("ééééé", 7).unwrap();
5197 assert_eq!(&r, "ééê".as_bytes());
5198
5199 let r = truncate_and_increment_utf8("\u{ff}\u{ff}\u{ff}\u{ff}\u{ff}", 8).unwrap();
5201 assert_eq!(&r, "\u{ff}\u{ff}\u{ff}\u{100}".as_bytes());
5202
5203 let r = truncate_and_increment_utf8("߿߿߿߿߿", 8);
5205 assert!(r.is_none());
5206
5207 let r = truncate_and_increment_utf8("ࠀࠀࠀࠀ", 8).unwrap();
5210 assert_eq!(&r, "ࠀࠁ".as_bytes());
5211
5212 let r = truncate_and_increment_utf8("\u{ffff}\u{ffff}\u{ffff}", 8);
5214 assert!(r.is_none());
5215
5216 let r = truncate_and_increment_utf8("𐀀𐀀𐀀𐀀", 9).unwrap();
5218 assert_eq!(&r, "𐀀𐀁".as_bytes());
5219
5220 let r = truncate_and_increment_utf8("\u{10ffff}\u{10ffff}", 8);
5222 assert!(r.is_none());
5223 }
5224
5225 #[test]
5226 fn test_byte_array_truncate_invalid_utf8_statistics() {
5229 let message_type = "
5230 message test_schema {
5231 OPTIONAL BYTE_ARRAY a (UTF8);
5232 }
5233 ";
5234 let schema = Arc::new(parse_message_type(message_type).unwrap());
5235
5236 let data = vec![ByteArray::from(vec![128u8; 32]); 7];
5238 let def_levels = [1, 1, 1, 1, 0, 1, 0, 1, 0, 1];
5239 let file: File = tempfile::tempfile().unwrap();
5240 let props = Arc::new(
5241 WriterProperties::builder()
5242 .set_statistics_enabled(EnabledStatistics::Chunk)
5243 .set_statistics_truncate_length(Some(8))
5244 .build(),
5245 );
5246
5247 let mut writer = SerializedFileWriter::new(&file, schema, props).unwrap();
5248 let mut row_group_writer = writer.next_row_group().unwrap();
5249
5250 let mut col_writer = row_group_writer.next_column().unwrap().unwrap();
5251 col_writer
5252 .typed::<ByteArrayType>()
5253 .write_batch(&data, Some(&def_levels), None)
5254 .unwrap();
5255 col_writer.close().unwrap();
5256 row_group_writer.close().unwrap();
5257 let file_metadata = writer.close().unwrap();
5258 let stats = file_metadata.row_group(0).column(0).statistics().unwrap();
5259 assert!(!stats.max_is_exact());
5260 assert_eq!(
5263 stats.max_bytes_opt().map(|v| v.to_vec()),
5264 Some([128, 128, 128, 128, 128, 128, 128, 129].to_vec())
5265 );
5266 }
5267
5268 #[test]
5269 fn test_increment_max_binary_chars() {
5270 let r = increment(vec![0xFF, 0xFE, 0xFD, 0xFF, 0xFF]);
5271 assert_eq!(&r.unwrap(), &[0xFF, 0xFE, 0xFE, 0x00, 0x00]);
5272
5273 let incremented = increment(vec![0xFF, 0xFF, 0xFF]);
5274 assert!(incremented.is_none())
5275 }
5276
5277 #[test]
5278 fn test_no_column_index_when_stats_disabled() {
5279 let descr = Arc::new(get_test_column_descr::<Int32Type>(1, 0));
5283 let props = Arc::new(
5284 WriterProperties::builder()
5285 .set_statistics_enabled(EnabledStatistics::None)
5286 .build(),
5287 );
5288 let column_writer = get_column_writer(descr, props, get_test_page_writer());
5289 let mut writer = get_typed_column_writer::<Int32Type>(column_writer);
5290
5291 let data = Vec::new();
5292 let def_levels = vec![0; 10];
5293 writer.write_batch(&data, Some(&def_levels), None).unwrap();
5294 writer.flush_data_pages().unwrap();
5295
5296 let column_close_result = writer.close().unwrap();
5297 assert!(column_close_result.offset_index.is_some());
5298 assert!(column_close_result.column_index.is_none());
5299 }
5300
5301 #[test]
5302 fn test_no_offset_index_when_disabled() {
5303 let descr = Arc::new(get_test_column_descr::<Int32Type>(1, 0));
5305 let props = Arc::new(
5306 WriterProperties::builder()
5307 .set_statistics_enabled(EnabledStatistics::None)
5308 .set_offset_index_disabled(true)
5309 .build(),
5310 );
5311 let column_writer = get_column_writer(descr, props, get_test_page_writer());
5312 let mut writer = get_typed_column_writer::<Int32Type>(column_writer);
5313
5314 let data = Vec::new();
5315 let def_levels = vec![0; 10];
5316 writer.write_batch(&data, Some(&def_levels), None).unwrap();
5317 writer.flush_data_pages().unwrap();
5318
5319 let column_close_result = writer.close().unwrap();
5320 assert!(column_close_result.offset_index.is_none());
5321 assert!(column_close_result.column_index.is_none());
5322 }
5323
5324 #[test]
5325 fn test_offset_index_overridden() {
5326 let descr = Arc::new(get_test_column_descr::<Int32Type>(1, 0));
5328 let props = Arc::new(
5329 WriterProperties::builder()
5330 .set_statistics_enabled(EnabledStatistics::Page)
5331 .set_offset_index_disabled(true)
5332 .build(),
5333 );
5334 let column_writer = get_column_writer(descr, props, get_test_page_writer());
5335 let mut writer = get_typed_column_writer::<Int32Type>(column_writer);
5336
5337 let data = Vec::new();
5338 let def_levels = vec![0; 10];
5339 writer.write_batch(&data, Some(&def_levels), None).unwrap();
5340 writer.flush_data_pages().unwrap();
5341
5342 let column_close_result = writer.close().unwrap();
5343 assert!(column_close_result.offset_index.is_some());
5344 assert!(column_close_result.column_index.is_some());
5345 }
5346
5347 #[test]
5348 fn test_boundary_order() -> Result<()> {
5349 let descr = Arc::new(get_test_column_descr::<Int32Type>(1, 0));
5350 let column_close_result = write_multiple_pages::<Int32Type>(
5352 &descr,
5353 &[
5354 &[Some(-10), Some(10)],
5355 &[Some(-5), Some(11)],
5356 &[None],
5357 &[Some(-5), Some(11)],
5358 ],
5359 )?;
5360 let boundary_order = column_close_result
5361 .column_index
5362 .unwrap()
5363 .get_boundary_order();
5364 assert_eq!(boundary_order, Some(BoundaryOrder::ASCENDING));
5365
5366 let column_close_result = write_multiple_pages::<Int32Type>(
5368 &descr,
5369 &[
5370 &[Some(10), Some(11)],
5371 &[Some(5), Some(11)],
5372 &[None],
5373 &[Some(-5), Some(0)],
5374 ],
5375 )?;
5376 let boundary_order = column_close_result
5377 .column_index
5378 .unwrap()
5379 .get_boundary_order();
5380 assert_eq!(boundary_order, Some(BoundaryOrder::DESCENDING));
5381
5382 let column_close_result = write_multiple_pages::<Int32Type>(
5384 &descr,
5385 &[&[Some(10), Some(11)], &[None], &[Some(10), Some(11)]],
5386 )?;
5387 let boundary_order = column_close_result
5388 .column_index
5389 .unwrap()
5390 .get_boundary_order();
5391 assert_eq!(boundary_order, Some(BoundaryOrder::ASCENDING));
5392
5393 let column_close_result =
5395 write_multiple_pages::<Int32Type>(&descr, &[&[None], &[None], &[None]])?;
5396 let boundary_order = column_close_result
5397 .column_index
5398 .unwrap()
5399 .get_boundary_order();
5400 assert_eq!(boundary_order, Some(BoundaryOrder::ASCENDING));
5401
5402 let column_close_result =
5404 write_multiple_pages::<Int32Type>(&descr, &[&[Some(-10), Some(10)]])?;
5405 let boundary_order = column_close_result
5406 .column_index
5407 .unwrap()
5408 .get_boundary_order();
5409 assert_eq!(boundary_order, Some(BoundaryOrder::ASCENDING));
5410
5411 let column_close_result =
5413 write_multiple_pages::<Int32Type>(&descr, &[&[Some(-10), Some(10)], &[None]])?;
5414 let boundary_order = column_close_result
5415 .column_index
5416 .unwrap()
5417 .get_boundary_order();
5418 assert_eq!(boundary_order, Some(BoundaryOrder::ASCENDING));
5419
5420 let column_close_result = write_multiple_pages::<Int32Type>(
5422 &descr,
5423 &[
5424 &[Some(10), Some(11)],
5425 &[Some(11), Some(16)],
5426 &[None],
5427 &[Some(-5), Some(0)],
5428 ],
5429 )?;
5430 let boundary_order = column_close_result
5431 .column_index
5432 .unwrap()
5433 .get_boundary_order();
5434 assert_eq!(boundary_order, Some(BoundaryOrder::UNORDERED));
5435
5436 let column_close_result = write_multiple_pages::<Int32Type>(
5438 &descr,
5439 &[
5440 &[Some(1), Some(9)],
5441 &[Some(2), Some(8)],
5442 &[None],
5443 &[Some(3), Some(7)],
5444 ],
5445 )?;
5446 let boundary_order = column_close_result
5447 .column_index
5448 .unwrap()
5449 .get_boundary_order();
5450 assert_eq!(boundary_order, Some(BoundaryOrder::UNORDERED));
5451
5452 Ok(())
5453 }
5454
5455 #[test]
5456 fn test_boundary_order_logical_type() -> Result<()> {
5457 let f16_descr = Arc::new(get_test_float16_column_descr(1, 0));
5460 let fba_descr = {
5461 let type_ = SchemaType::primitive_type_builder(
5462 "col",
5463 FixedLenByteArrayType::get_physical_type(),
5464 )
5465 .with_length(2)
5466 .build()?;
5467 Arc::new(ColumnDescriptor::new(
5468 Arc::new(type_),
5469 1,
5470 0,
5471 ColumnPath::from("col"),
5472 ))
5473 };
5474
5475 let values: &[&[Option<FixedLenByteArray>]] = &[
5476 &[Some(FixedLenByteArray::from(ByteArray::from(f16::ONE)))],
5477 &[Some(FixedLenByteArray::from(ByteArray::from(f16::ZERO)))],
5478 &[Some(FixedLenByteArray::from(ByteArray::from(
5479 f16::NEG_ZERO,
5480 )))],
5481 &[Some(FixedLenByteArray::from(ByteArray::from(f16::NEG_ONE)))],
5482 ];
5483
5484 let column_close_result =
5486 write_multiple_pages::<FixedLenByteArrayType>(&f16_descr, values)?;
5487 let boundary_order = column_close_result
5488 .column_index
5489 .unwrap()
5490 .get_boundary_order();
5491 assert_eq!(boundary_order, Some(BoundaryOrder::DESCENDING));
5492
5493 let column_close_result =
5495 write_multiple_pages::<FixedLenByteArrayType>(&fba_descr, values)?;
5496 let boundary_order = column_close_result
5497 .column_index
5498 .unwrap()
5499 .get_boundary_order();
5500 assert_eq!(boundary_order, Some(BoundaryOrder::UNORDERED));
5501
5502 Ok(())
5503 }
5504
5505 #[test]
5506 fn test_interval_stats_should_not_have_min_max() {
5507 let input = [
5508 vec![0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
5509 vec![0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
5510 vec![0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2],
5511 ]
5512 .into_iter()
5513 .map(|s| ByteArray::from(s).into())
5514 .collect::<Vec<_>>();
5515
5516 let page_writer = get_test_page_writer();
5517 let mut writer = get_test_interval_column_writer(page_writer);
5518 writer.write_batch(&input, None, None).unwrap();
5519
5520 let metadata = writer.close().unwrap().metadata;
5521 let stats = if let Some(Statistics::FixedLenByteArray(stats)) = metadata.statistics() {
5522 stats.clone()
5523 } else {
5524 panic!("metadata missing statistics");
5525 };
5526 assert!(stats.min_bytes_opt().is_none());
5527 assert!(stats.max_bytes_opt().is_none());
5528 }
5529
5530 #[test]
5531 #[cfg(feature = "arrow")]
5532 fn test_column_writer_get_estimated_total_bytes() {
5533 let page_writer = get_test_page_writer();
5534 let props = Default::default();
5535 let mut writer = get_test_column_writer::<Int32Type>(page_writer, 0, 0, props);
5536 assert_eq!(writer.get_estimated_total_bytes(), 0);
5537
5538 writer.write_batch(&[1, 2, 3, 4], None, None).unwrap();
5539 writer.add_data_page().unwrap();
5540 let size_with_one_page = writer.get_estimated_total_bytes();
5541 assert_eq!(size_with_one_page, 20);
5542
5543 writer.write_batch(&[5, 6, 7, 8], None, None).unwrap();
5544 writer.add_data_page().unwrap();
5545 let size_with_two_pages = writer.get_estimated_total_bytes();
5546 assert_eq!(size_with_two_pages, 20 + 21);
5548 }
5549
5550 fn write_multiple_pages<T: DataType>(
5551 column_descr: &Arc<ColumnDescriptor>,
5552 pages: &[&[Option<T::T>]],
5553 ) -> Result<ColumnCloseResult> {
5554 let column_writer = get_column_writer(
5555 column_descr.clone(),
5556 Default::default(),
5557 get_test_page_writer(),
5558 );
5559 let mut writer = get_typed_column_writer::<T>(column_writer);
5560
5561 for &page in pages {
5562 let values = page.iter().filter_map(Clone::clone).collect::<Vec<_>>();
5563 let def_levels = page
5564 .iter()
5565 .map(|maybe_value| i16::from(maybe_value.is_some()))
5566 .collect::<Vec<_>>();
5567 writer.write_batch(&values, Some(&def_levels), None)?;
5568 writer.flush_data_pages()?;
5569 }
5570
5571 writer.close()
5572 }
5573
5574 fn column_roundtrip_random<T: DataType>(
5578 props: WriterProperties,
5579 max_size: usize,
5580 min_value: T::T,
5581 max_value: T::T,
5582 max_def_level: i16,
5583 max_rep_level: i16,
5584 ) where
5585 T::T: PartialOrd + SampleUniform + Copy,
5586 {
5587 let mut num_values: usize = 0;
5588
5589 let mut buf: Vec<i16> = Vec::new();
5590 let def_levels = if max_def_level > 0 {
5591 random_numbers_range(max_size, 0, max_def_level + 1, &mut buf);
5592 for &dl in &buf[..] {
5593 if dl == max_def_level {
5594 num_values += 1;
5595 }
5596 }
5597 Some(&buf[..])
5598 } else {
5599 num_values = max_size;
5600 None
5601 };
5602
5603 let mut buf: Vec<i16> = Vec::new();
5604 let rep_levels = if max_rep_level > 0 {
5605 random_numbers_range(max_size, 0, max_rep_level + 1, &mut buf);
5606 buf[0] = 0; Some(&buf[..])
5608 } else {
5609 None
5610 };
5611
5612 let mut values: Vec<T::T> = Vec::new();
5613 random_numbers_range(num_values, min_value, max_value, &mut values);
5614
5615 column_roundtrip::<T>(props, &values[..], def_levels, rep_levels);
5616 }
5617
5618 fn column_roundtrip<T: DataType>(
5620 props: WriterProperties,
5621 values: &[T::T],
5622 def_levels: Option<&[i16]>,
5623 rep_levels: Option<&[i16]>,
5624 ) {
5625 let mut file = tempfile::tempfile().unwrap();
5626 let mut write = TrackedWrite::new(&mut file);
5627 let page_writer = Box::new(SerializedPageWriter::new(&mut write));
5628
5629 let max_def_level = match def_levels {
5630 Some(buf) => *buf.iter().max().unwrap_or(&0i16),
5631 None => 0i16,
5632 };
5633
5634 let max_rep_level = match rep_levels {
5635 Some(buf) => *buf.iter().max().unwrap_or(&0i16),
5636 None => 0i16,
5637 };
5638
5639 let mut max_batch_size = values.len();
5640 if let Some(levels) = def_levels {
5641 max_batch_size = max_batch_size.max(levels.len());
5642 }
5643 if let Some(levels) = rep_levels {
5644 max_batch_size = max_batch_size.max(levels.len());
5645 }
5646
5647 let mut writer =
5648 get_test_column_writer::<T>(page_writer, max_def_level, max_rep_level, Arc::new(props));
5649
5650 let values_written = writer.write_batch(values, def_levels, rep_levels).unwrap();
5651 assert_eq!(values_written, values.len());
5652 let result = writer.close().unwrap();
5653
5654 drop(write);
5655
5656 let props = ReaderProperties::builder()
5657 .set_backward_compatible_lz4(false)
5658 .build();
5659 let page_reader = Box::new(
5660 SerializedPageReader::new_with_properties(
5661 Arc::new(file),
5662 &result.metadata,
5663 result.rows_written as usize,
5664 None,
5665 Arc::new(props),
5666 )
5667 .unwrap(),
5668 );
5669 let mut reader = get_test_column_reader::<T>(page_reader, max_def_level, max_rep_level);
5670
5671 let mut actual_values = Vec::with_capacity(max_batch_size);
5672 let mut actual_def_levels = def_levels.map(|_| Vec::with_capacity(max_batch_size));
5673 let mut actual_rep_levels = rep_levels.map(|_| Vec::with_capacity(max_batch_size));
5674
5675 let (_, values_read, levels_read) = reader
5676 .read_records(
5677 max_batch_size,
5678 actual_def_levels.as_mut(),
5679 actual_rep_levels.as_mut(),
5680 &mut actual_values,
5681 )
5682 .unwrap();
5683
5684 assert_eq!(&actual_values[..values_read], values);
5687 match actual_def_levels {
5688 Some(ref vec) => assert_eq!(Some(&vec[..levels_read]), def_levels),
5689 None => assert_eq!(None, def_levels),
5690 }
5691 match actual_rep_levels {
5692 Some(ref vec) => assert_eq!(Some(&vec[..levels_read]), rep_levels),
5693 None => assert_eq!(None, rep_levels),
5694 }
5695
5696 if let Some(levels) = actual_rep_levels {
5699 let mut actual_rows_written = 0;
5700 for l in levels {
5701 if l == 0 {
5702 actual_rows_written += 1;
5703 }
5704 }
5705 assert_eq!(actual_rows_written, result.rows_written);
5706 } else if actual_def_levels.is_some() {
5707 assert_eq!(levels_read as u64, result.rows_written);
5708 } else {
5709 assert_eq!(values_read as u64, result.rows_written);
5710 }
5711 }
5712
5713 fn column_write_and_get_metadata<T: DataType>(
5716 props: WriterProperties,
5717 values: &[T::T],
5718 ) -> ColumnChunkMetaData {
5719 let page_writer = get_test_page_writer();
5720 let props = Arc::new(props);
5721 let mut writer = get_test_column_writer::<T>(page_writer, 0, 0, props);
5722 writer.write_batch(values, None, None).unwrap();
5723 writer.close().unwrap().metadata
5724 }
5725
5726 fn column_write_and_get_metadata_with_def_levels<T: DataType>(
5727 props: WriterProperties,
5728 values: &[T::T],
5729 def_levels: &[i16],
5730 max_def_level: i16,
5731 ) -> ColumnChunkMetaData {
5732 let page_writer = get_test_page_writer();
5733 let props = Arc::new(props);
5734 let mut writer = get_test_column_writer::<T>(page_writer, max_def_level, 0, props);
5735 writer.write_batch(values, Some(def_levels), None).unwrap();
5736 writer.close().unwrap().metadata
5737 }
5738
5739 fn encoding_stats(page_type: PageType, encoding: Encoding, count: i32) -> PageEncodingStats {
5741 PageEncodingStats {
5742 page_type,
5743 encoding,
5744 count,
5745 }
5746 }
5747
5748 fn check_encoding_write_support<T: DataType>(
5752 version: WriterVersion,
5753 dict_enabled: bool,
5754 data: &[T::T],
5755 dictionary_page_offset: Option<i64>,
5756 encodings: &[Encoding],
5757 page_encoding_stats: &[PageEncodingStats],
5758 ) {
5759 let props = WriterProperties::builder()
5760 .set_writer_version(version)
5761 .set_dictionary_enabled(dict_enabled)
5762 .build();
5763 let meta = column_write_and_get_metadata::<T>(props, data);
5764 assert_eq!(meta.dictionary_page_offset(), dictionary_page_offset);
5765 assert_eq!(meta.encodings().collect::<Vec<_>>(), encodings);
5766 assert_eq!(meta.page_encoding_stats().unwrap(), page_encoding_stats);
5767 }
5768
5769 fn get_test_column_writer<'a, T: DataType>(
5771 page_writer: Box<dyn PageWriter + 'a>,
5772 max_def_level: i16,
5773 max_rep_level: i16,
5774 props: WriterPropertiesPtr,
5775 ) -> ColumnWriterImpl<'a, T> {
5776 let descr = Arc::new(get_test_column_descr::<T>(max_def_level, max_rep_level));
5777 let column_writer = get_column_writer(descr, props, page_writer);
5778 get_typed_column_writer::<T>(column_writer)
5779 }
5780
5781 fn get_test_column_writer_with_path<'a, T: DataType>(
5782 page_writer: Box<dyn PageWriter + 'a>,
5783 max_def_level: i16,
5784 max_rep_level: i16,
5785 props: WriterPropertiesPtr,
5786 path: ColumnPath,
5787 ) -> ColumnWriterImpl<'a, T> {
5788 let descr = Arc::new(get_test_column_descr_with_path::<T>(
5789 max_def_level,
5790 max_rep_level,
5791 path,
5792 ));
5793 let column_writer = get_column_writer(descr, props, page_writer);
5794 get_typed_column_writer::<T>(column_writer)
5795 }
5796
5797 struct CollectedPages {
5799 data_pages: Vec<(usize, u32)>,
5801 dict_page_size: usize,
5803 }
5804
5805 fn write_and_collect_pages<T: DataType>(
5810 props: WriterProperties,
5811 max_def_level: i16,
5812 max_rep_level: i16,
5813 data: &[T::T],
5814 def_levels: Option<&[i16]>,
5815 rep_levels: Option<&[i16]>,
5816 ) -> CollectedPages {
5817 let mut file = tempfile::tempfile().unwrap();
5818 let mut write = TrackedWrite::new(&mut file);
5819 let page_writer = Box::new(SerializedPageWriter::new(&mut write));
5820 let mut writer =
5821 get_test_column_writer::<T>(page_writer, max_def_level, max_rep_level, Arc::new(props));
5822 writer.write_batch(data, def_levels, rep_levels).unwrap();
5823 let r = writer.close().unwrap();
5824 drop(write);
5825
5826 let read_props = ReaderProperties::builder()
5827 .set_backward_compatible_lz4(false)
5828 .build();
5829 let mut page_reader = Box::new(
5830 SerializedPageReader::new_with_properties(
5831 Arc::new(file),
5832 &r.metadata,
5833 r.rows_written as usize,
5834 None,
5835 Arc::new(read_props),
5836 )
5837 .unwrap(),
5838 );
5839
5840 let mut collected = CollectedPages {
5841 data_pages: Vec::new(),
5842 dict_page_size: 0,
5843 };
5844 while let Some(page) = page_reader.get_next_page().unwrap() {
5845 match page.page_type() {
5846 PageType::DATA_PAGE | PageType::DATA_PAGE_V2 => {
5847 collected
5848 .data_pages
5849 .push((page.buffer().len(), page.num_values()));
5850 }
5851 PageType::DICTIONARY_PAGE => {
5852 collected.dict_page_size = collected.dict_page_size.max(page.buffer().len());
5853 }
5854 PageType::INDEX_PAGE => {}
5855 }
5856 }
5857 collected
5858 }
5859
5860 fn get_test_column_reader<T: DataType>(
5862 page_reader: Box<dyn PageReader>,
5863 max_def_level: i16,
5864 max_rep_level: i16,
5865 ) -> ColumnReaderImpl<T> {
5866 let descr = Arc::new(get_test_column_descr::<T>(max_def_level, max_rep_level));
5867 let column_reader = get_column_reader(descr, page_reader);
5868 get_typed_column_reader::<T>(column_reader)
5869 }
5870
5871 fn get_test_column_descr<T: DataType>(
5873 max_def_level: i16,
5874 max_rep_level: i16,
5875 ) -> ColumnDescriptor {
5876 let path = ColumnPath::from("col");
5877 let type_ = SchemaType::primitive_type_builder("col", T::get_physical_type())
5878 .with_length(1)
5881 .build()
5882 .unwrap();
5883 ColumnDescriptor::new(Arc::new(type_), max_def_level, max_rep_level, path)
5884 }
5885
5886 fn get_test_column_descr_with_path<T: DataType>(
5887 max_def_level: i16,
5888 max_rep_level: i16,
5889 path: ColumnPath,
5890 ) -> ColumnDescriptor {
5891 let name = path.string();
5892 let type_ = SchemaType::primitive_type_builder(&name, T::get_physical_type())
5893 .with_length(1)
5896 .build()
5897 .unwrap();
5898 ColumnDescriptor::new(Arc::new(type_), max_def_level, max_rep_level, path)
5899 }
5900
5901 fn write_and_collect_page_values(
5902 path: ColumnPath,
5903 props: WriterPropertiesPtr,
5904 data: &[i32],
5905 ) -> Vec<u32> {
5906 let mut file = tempfile::tempfile().unwrap();
5907 let mut write = TrackedWrite::new(&mut file);
5908 let page_writer = Box::new(SerializedPageWriter::new(&mut write));
5909 let mut writer =
5910 get_test_column_writer_with_path::<Int32Type>(page_writer, 0, 0, props, path);
5911 writer.write_batch(data, None, None).unwrap();
5912 let r = writer.close().unwrap();
5913
5914 drop(write);
5915
5916 let props = ReaderProperties::builder()
5917 .set_backward_compatible_lz4(false)
5918 .build();
5919 let mut page_reader = Box::new(
5920 SerializedPageReader::new_with_properties(
5921 Arc::new(file),
5922 &r.metadata,
5923 r.rows_written as usize,
5924 None,
5925 Arc::new(props),
5926 )
5927 .unwrap(),
5928 );
5929
5930 let mut values_per_page = Vec::new();
5931 while let Some(page) = page_reader.get_next_page().unwrap() {
5932 assert_eq!(page.page_type(), PageType::DATA_PAGE);
5933 values_per_page.push(page.num_values());
5934 }
5935
5936 values_per_page
5937 }
5938
5939 fn get_test_page_writer() -> Box<dyn PageWriter> {
5941 Box::new(TestPageWriter::default())
5942 }
5943
5944 #[derive(Default)]
5945 struct TestPageWriter {
5946 bytes_written: u64,
5947 }
5948
5949 impl PageWriter for TestPageWriter {
5950 fn write_page(&mut self, page: CompressedPage) -> Result<PageWriteSpec> {
5951 let mut res = PageWriteSpec::new();
5952 res.page_type = page.page_type();
5953 res.uncompressed_size = page.uncompressed_size();
5954 res.compressed_size = page.compressed_size();
5955 res.num_values = page.num_values();
5956 res.offset = self.bytes_written;
5957 res.bytes_written = page.data().len() as u64;
5958 self.bytes_written += res.bytes_written;
5959 Ok(res)
5960 }
5961
5962 fn close(&mut self) -> Result<()> {
5963 Ok(())
5964 }
5965 }
5966
5967 fn statistics_roundtrip<T: DataType>(values: &[<T as DataType>::T]) -> Statistics {
5969 let page_writer = get_test_page_writer();
5970 let props = Default::default();
5971 let mut writer = get_test_column_writer::<T>(page_writer, 0, 0, props);
5972 writer.write_batch(values, None, None).unwrap();
5973
5974 let metadata = writer.close().unwrap().metadata;
5975 if let Some(stats) = metadata.statistics() {
5976 stats.clone()
5977 } else {
5978 panic!("metadata missing statistics");
5979 }
5980 }
5981
5982 fn get_test_decimals_column_writer<T: DataType>(
5984 page_writer: Box<dyn PageWriter>,
5985 max_def_level: i16,
5986 max_rep_level: i16,
5987 props: WriterPropertiesPtr,
5988 ) -> ColumnWriterImpl<'static, T> {
5989 let descr = Arc::new(get_test_decimals_column_descr::<T>(
5990 max_def_level,
5991 max_rep_level,
5992 ));
5993 let column_writer = get_column_writer(descr, props, page_writer);
5994 get_typed_column_writer::<T>(column_writer)
5995 }
5996
5997 fn get_test_decimals_column_descr<T: DataType>(
5999 max_def_level: i16,
6000 max_rep_level: i16,
6001 ) -> ColumnDescriptor {
6002 let path = ColumnPath::from("col");
6003 let type_ = SchemaType::primitive_type_builder("col", T::get_physical_type())
6004 .with_length(16)
6005 .with_logical_type(Some(LogicalType::decimal(2, 3)))
6006 .with_scale(2)
6007 .with_precision(3)
6008 .build()
6009 .unwrap();
6010 ColumnDescriptor::new(Arc::new(type_), max_def_level, max_rep_level, path)
6011 }
6012
6013 fn float16_statistics_roundtrip(
6014 values: &[FixedLenByteArray],
6015 ) -> ValueStatistics<FixedLenByteArray> {
6016 let page_writer = get_test_page_writer();
6017 let mut writer = get_test_float16_column_writer(page_writer, Default::default());
6018 writer.write_batch(values, None, None).unwrap();
6019
6020 let metadata = writer.close().unwrap().metadata;
6021 if let Some(Statistics::FixedLenByteArray(stats)) = metadata.statistics() {
6022 stats.clone()
6023 } else {
6024 panic!("metadata missing statistics");
6025 }
6026 }
6027
6028 fn get_test_float16_column_writer(
6029 page_writer: Box<dyn PageWriter>,
6030 props: WriterPropertiesPtr,
6031 ) -> ColumnWriterImpl<'static, FixedLenByteArrayType> {
6032 let descr = Arc::new(get_test_float16_column_descr(0, 0));
6033 let column_writer = get_column_writer(descr, props, page_writer);
6034 get_typed_column_writer::<FixedLenByteArrayType>(column_writer)
6035 }
6036
6037 fn get_test_float16_column_descr(max_def_level: i16, max_rep_level: i16) -> ColumnDescriptor {
6038 let path = ColumnPath::from("col");
6039 let type_ =
6040 SchemaType::primitive_type_builder("col", FixedLenByteArrayType::get_physical_type())
6041 .with_length(2)
6042 .with_logical_type(Some(LogicalType::Float16))
6043 .build()
6044 .unwrap();
6045 ColumnDescriptor::new(Arc::new(type_), max_def_level, max_rep_level, path)
6046 }
6047
6048 fn get_test_interval_column_writer(
6049 page_writer: Box<dyn PageWriter>,
6050 ) -> ColumnWriterImpl<'static, FixedLenByteArrayType> {
6051 let descr = Arc::new(get_test_interval_column_descr());
6052 let column_writer = get_column_writer(descr, Default::default(), page_writer);
6053 get_typed_column_writer::<FixedLenByteArrayType>(column_writer)
6054 }
6055
6056 fn get_test_interval_column_descr() -> ColumnDescriptor {
6057 let path = ColumnPath::from("col");
6058 let type_ =
6059 SchemaType::primitive_type_builder("col", FixedLenByteArrayType::get_physical_type())
6060 .with_length(12)
6061 .with_converted_type(ConvertedType::INTERVAL)
6062 .build()
6063 .unwrap();
6064 ColumnDescriptor::new(Arc::new(type_), 0, 0, path)
6065 }
6066
6067 fn get_test_unsigned_int_given_as_converted_column_writer<'a, T: DataType>(
6069 page_writer: Box<dyn PageWriter + 'a>,
6070 max_def_level: i16,
6071 max_rep_level: i16,
6072 props: WriterPropertiesPtr,
6073 ) -> ColumnWriterImpl<'a, T> {
6074 let descr = Arc::new(get_test_converted_type_unsigned_integer_column_descr::<T>(
6075 max_def_level,
6076 max_rep_level,
6077 ));
6078 let column_writer = get_column_writer(descr, props, page_writer);
6079 get_typed_column_writer::<T>(column_writer)
6080 }
6081
6082 fn get_test_converted_type_unsigned_integer_column_descr<T: DataType>(
6084 max_def_level: i16,
6085 max_rep_level: i16,
6086 ) -> ColumnDescriptor {
6087 let path = ColumnPath::from("col");
6088 let type_ = SchemaType::primitive_type_builder("col", T::get_physical_type())
6089 .with_converted_type(ConvertedType::UINT_32)
6090 .build()
6091 .unwrap();
6092 ColumnDescriptor::new(Arc::new(type_), max_def_level, max_rep_level, path)
6093 }
6094
6095 #[test]
6096 fn test_page_v2_snappy_compression_fallback() {
6097 let page_writer = TestPageWriter::default();
6099
6100 let props = WriterProperties::builder()
6102 .set_writer_version(WriterVersion::PARQUET_2_0)
6103 .set_dictionary_enabled(false)
6105 .set_compression(Compression::SNAPPY)
6106 .build();
6107
6108 let mut column_writer =
6109 get_test_column_writer::<ByteArrayType>(Box::new(page_writer), 0, 0, Arc::new(props));
6110
6111 let values = vec![ByteArray::from("a")];
6114
6115 column_writer.write_batch(&values, None, None).unwrap();
6116
6117 let result = column_writer.close().unwrap();
6118 assert_eq!(
6119 result.metadata.uncompressed_size(),
6120 result.metadata.compressed_size()
6121 );
6122 }
6123
6124 struct ColumnRoundTripUniform<'a, T: DataType> {
6125 props: WriterProperties,
6126 values: &'a [T::T],
6127 def_levels: LevelDataRef<'a>,
6128 rep_levels: LevelDataRef<'a>,
6129 max_def_level: i16,
6130 max_rep_level: i16,
6131 expected_values: &'a [T::T],
6132 expected_def_levels: Option<&'a [i16]>,
6133 expected_rep_levels: Option<&'a [i16]>,
6134 }
6135
6136 impl<'a, T: DataType> ColumnRoundTripUniform<'a, T>
6137 where
6138 T::T: PartialEq + std::fmt::Debug,
6139 {
6140 fn new() -> Self {
6141 Self {
6142 props: Default::default(),
6143 values: &[],
6144 def_levels: LevelDataRef::Absent,
6145 rep_levels: LevelDataRef::Absent,
6146 max_def_level: 0,
6147 max_rep_level: 0,
6148 expected_values: &[],
6149 expected_def_levels: None,
6150 expected_rep_levels: None,
6151 }
6152 }
6153
6154 fn with_props(mut self, props: WriterProperties) -> Self {
6155 self.props = props;
6156 self
6157 }
6158
6159 fn with_values(mut self, values: &'a [T::T]) -> Self {
6160 self.values = values;
6161 self
6162 }
6163
6164 fn with_def_levels(mut self, def_levels: LevelDataRef<'a>) -> Self {
6165 self.def_levels = def_levels;
6166 self
6167 }
6168
6169 fn with_rep_levels(mut self, rep_levels: LevelDataRef<'a>) -> Self {
6170 self.rep_levels = rep_levels;
6171 self
6172 }
6173
6174 fn with_max_def_level(mut self, max_def_level: i16) -> Self {
6175 self.max_def_level = max_def_level;
6176 self
6177 }
6178
6179 fn with_max_rep_level(mut self, max_rep_level: i16) -> Self {
6180 self.max_rep_level = max_rep_level;
6181 self
6182 }
6183
6184 fn with_expected_values(mut self, expected_values: &'a [T::T]) -> Self {
6185 self.expected_values = expected_values;
6186 self
6187 }
6188
6189 fn with_expected_def_levels(mut self, expected_def_levels: &'a [i16]) -> Self {
6190 self.expected_def_levels = Some(expected_def_levels);
6191 self
6192 }
6193
6194 fn with_expected_rep_levels(mut self, expected_rep_levels: &'a [i16]) -> Self {
6195 self.expected_rep_levels = Some(expected_rep_levels);
6196 self
6197 }
6198
6199 fn run(self) {
6202 let mut file = tempfile::tempfile().unwrap();
6203 let mut write = TrackedWrite::new(&mut file);
6204 let page_writer = Box::new(SerializedPageWriter::new(&mut write));
6205 let mut writer = get_test_column_writer::<T>(
6206 page_writer,
6207 self.max_def_level,
6208 self.max_rep_level,
6209 Arc::new(self.props),
6210 );
6211
6212 writer
6213 .write_batch_internal(
6214 self.values,
6215 None,
6216 self.def_levels,
6217 self.rep_levels,
6218 None,
6219 None,
6220 None,
6221 )
6222 .unwrap();
6223 let result = writer.close().unwrap();
6224 drop(write);
6225
6226 let props = ReaderProperties::builder()
6227 .set_backward_compatible_lz4(false)
6228 .build();
6229 let page_reader = Box::new(
6230 SerializedPageReader::new_with_properties(
6231 Arc::new(file),
6232 &result.metadata,
6233 result.rows_written as usize,
6234 None,
6235 Arc::new(props),
6236 )
6237 .unwrap(),
6238 );
6239 let mut reader =
6240 get_test_column_reader::<T>(page_reader, self.max_def_level, self.max_rep_level);
6241
6242 let batch_size = self
6243 .expected_def_levels
6244 .map_or(self.expected_values.len(), |l| l.len());
6245 let mut actual_values = Vec::with_capacity(batch_size);
6246 let mut actual_def = self
6247 .expected_def_levels
6248 .map(|_| Vec::with_capacity(batch_size));
6249 let mut actual_rep = self
6250 .expected_rep_levels
6251 .map(|_| Vec::with_capacity(batch_size));
6252
6253 let (_, values_read, levels_read) = reader
6254 .read_records(
6255 batch_size,
6256 actual_def.as_mut(),
6257 actual_rep.as_mut(),
6258 &mut actual_values,
6259 )
6260 .unwrap();
6261
6262 assert_eq!(&actual_values[..values_read], self.expected_values);
6263 if let Some(ref v) = actual_def {
6264 assert_eq!(&v[..levels_read], self.expected_def_levels.unwrap());
6265 }
6266 if let Some(ref v) = actual_rep {
6267 assert_eq!(&v[..levels_read], self.expected_rep_levels.unwrap());
6268 }
6269 }
6270 }
6271
6272 #[test]
6273 fn test_level_data_ref_value_count() {
6274 let max_def = 2;
6279 assert_eq!(LevelDataRef::Absent.value_count(64, max_def), 64);
6282 assert_eq!(
6284 LevelDataRef::Uniform {
6285 value: max_def,
6286 count: 40
6287 }
6288 .value_count(40, max_def),
6289 40
6290 );
6291 assert_eq!(
6292 LevelDataRef::Uniform {
6293 value: max_def - 1,
6294 count: 40
6295 }
6296 .value_count(40, max_def),
6297 0
6298 );
6299 let levels = [2i16, 0, 2, 1, 2, 2, 0];
6302 assert_eq!(
6303 LevelDataRef::Materialized(&levels).value_count(levels.len(), max_def),
6304 4
6305 );
6306 }
6307
6308 #[test]
6309 fn test_uniform_def_levels_all_null() {
6310 let max_def_level = 1;
6312 let count = 100;
6313 let expected_def_levels = vec![0i16; count];
6314 ColumnRoundTripUniform::<Int32Type>::new()
6315 .with_def_levels(LevelDataRef::Uniform { value: 0, count })
6316 .with_max_def_level(max_def_level)
6317 .with_expected_def_levels(&expected_def_levels)
6318 .run();
6319 }
6320
6321 #[test]
6322 fn test_uniform_def_levels_all_valid() {
6323 let max_def_level = 1;
6325 let values: Vec<i32> = (0..50).collect();
6326 let expected_def_levels = vec![max_def_level; values.len()];
6327 ColumnRoundTripUniform::<Int32Type>::new()
6328 .with_values(&values)
6329 .with_def_levels(LevelDataRef::Uniform {
6330 value: max_def_level,
6331 count: values.len(),
6332 })
6333 .with_max_def_level(max_def_level)
6334 .with_expected_values(&values)
6335 .with_expected_def_levels(&expected_def_levels)
6336 .run();
6337 }
6338
6339 #[test]
6340 fn test_uniform_def_and_rep_levels() {
6341 let max_def_level = 2;
6344 let max_rep_level = 1;
6345 let count = 200;
6346 let expected_def_levels = vec![0i16; count];
6347 let expected_rep_levels = vec![0i16; count];
6348 ColumnRoundTripUniform::<Int32Type>::new()
6349 .with_def_levels(LevelDataRef::Uniform { value: 0, count })
6350 .with_rep_levels(LevelDataRef::Uniform { value: 0, count })
6351 .with_max_def_level(max_def_level)
6352 .with_max_rep_level(max_rep_level)
6353 .with_expected_def_levels(&expected_def_levels)
6354 .with_expected_rep_levels(&expected_rep_levels)
6355 .run();
6356 }
6357
6358 #[test]
6359 fn test_uniform_levels_v1_and_v2() {
6360 for version in [WriterVersion::PARQUET_1_0, WriterVersion::PARQUET_2_0] {
6362 let props = WriterProperties::builder()
6363 .set_writer_version(version)
6364 .build();
6365 let max_def = 1;
6366 let count = 100;
6367 let expected_def_levels = vec![0i16; count];
6368 ColumnRoundTripUniform::<Int32Type>::new()
6369 .with_props(props)
6370 .with_def_levels(LevelDataRef::Uniform { value: 0, count })
6371 .with_max_def_level(max_def)
6372 .with_expected_def_levels(&expected_def_levels)
6373 .run();
6374 }
6375 }
6376}