1mod column_chunk;
41#[cfg(feature = "arrow")]
42mod dictionary;
43mod footer_tail;
44mod memory;
45mod options;
46pub mod page_index;
47mod parser;
48mod push_decoder;
49pub(crate) mod reader;
50pub(crate) mod thrift;
51mod writer;
52
53use crate::basic::{
54 BoundaryOrder, ColumnOrder, Compression, CompressionCodec, Encoding, EncodingMask, PageType,
55 Type,
56};
57#[cfg(feature = "encryption")]
58use crate::encryption::decrypt::FileDecryptor;
59use crate::errors::{ParquetError, Result};
60#[cfg(feature = "encryption")]
61use crate::file::column_crypto_metadata::ColumnCryptoMetaData;
62pub(crate) use crate::file::metadata::memory::HeapSize;
63use crate::file::metadata::page_index::{PageIndex, PageIndexProvider, RowGroupPageIndex};
64#[cfg(feature = "encryption")]
65use crate::file::metadata::thrift::encryption::EncryptionAlgorithm;
66use crate::file::page_index::column_index::{ByteArrayColumnIndex, PrimitiveColumnIndex};
67use crate::file::page_index::{column_index::ColumnIndexMetaData, offset_index::PageLocation};
68use crate::file::statistics::Statistics;
69use crate::geospatial::statistics as geo_statistics;
70use crate::parquet_thrift::{
71 ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol, ThriftCompactOutputProtocol,
72 WriteThrift, WriteThriftField,
73};
74use crate::schema::types::{
75 ColumnDescPtr, ColumnDescriptor, ColumnPath, SchemaDescPtr, SchemaDescriptor,
76 Type as SchemaType,
77};
78use crate::thrift_struct;
79use crate::{
80 data_type::private::ParquetValueType, file::page_index::offset_index::OffsetIndexMetaData,
81};
82
83pub use footer_tail::FooterTail;
84pub use options::{ParquetMetaDataOptions, ParquetStatisticsPolicy};
85pub use push_decoder::ParquetMetaDataPushDecoder;
86pub use reader::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader};
87use std::io::Write;
88use std::ops::Range;
89use std::sync::Arc;
90pub use writer::ParquetMetaDataWriter;
91pub(crate) use writer::ThriftMetadataWriter;
92
93#[derive(Debug, Clone)]
116pub struct ParquetMetaData {
117 file_metadata: Arc<FileMetaData>,
119 row_groups: Arc<[RowGroupMetaData]>,
121 page_index: Option<Arc<dyn PageIndexProvider>>,
123 #[cfg(feature = "encryption")]
125 file_decryptor: Option<Arc<FileDecryptor>>,
126}
127
128impl ParquetMetaData {
129 pub fn new(file_metadata: FileMetaData, row_groups: Vec<RowGroupMetaData>) -> Self {
132 ParquetMetaData {
133 file_metadata: Arc::new(file_metadata),
134 row_groups: row_groups.into(),
135 page_index: None,
136 #[cfg(feature = "encryption")]
137 file_decryptor: None,
138 }
139 }
140
141 pub fn into_builder(self) -> ParquetMetaDataBuilder {
146 self.into()
147 }
148
149 pub fn file_metadata(&self) -> &FileMetaData {
151 &self.file_metadata
152 }
153
154 #[cfg(feature = "encryption")]
156 pub(crate) fn file_decryptor(&self) -> Option<&FileDecryptor> {
157 self.file_decryptor.as_deref()
158 }
159
160 pub fn num_row_groups(&self) -> usize {
162 self.row_groups.len()
163 }
164
165 pub fn row_group(&self, i: usize) -> &RowGroupMetaData {
168 &self.row_groups[i]
169 }
170
171 pub fn row_groups(&self) -> &[RowGroupMetaData] {
173 &self.row_groups
174 }
175
176 pub fn column_chunk(
180 &self,
181 row_group_idx: usize,
182 column_idx: usize,
183 ) -> Result<&ColumnChunkMetaData> {
184 let row_group = self.row_groups().get(row_group_idx).ok_or_else(|| {
185 general_err!(
186 "Row group index {row_group_idx} out of bounds for file with {} row groups",
187 self.num_row_groups()
188 )
189 })?;
190 row_group.columns().get(column_idx).ok_or_else(|| {
191 general_err!(
192 "Column index {column_idx} out of bounds for row group {row_group_idx} with {} columns",
193 row_group.num_columns()
194 )
195 })
196 }
197
198 pub fn row_group_num_rows(&self, row_group_idx: usize) -> Result<usize> {
203 self.row_groups
204 .get(row_group_idx)
205 .ok_or_else(|| {
206 ParquetError::General(format!(
207 "Row group index {row_group_idx} out of bounds for file with {} row groups",
208 self.num_row_groups()
209 ))
210 })?
211 .num_rows()
212 .try_into()
213 .map_err(|e| ParquetError::General(format!("Row count overflow: {e}")))
214 }
215
216 pub fn page_index(&self) -> Option<&Arc<dyn PageIndexProvider>> {
223 self.page_index.as_ref()
224 }
225
226 pub fn page_index_for_row_group(&self, row_group_idx: usize) -> RowGroupPageIndex {
228 RowGroupPageIndex::new(row_group_idx, self.page_index.clone())
229 }
230
231 pub fn memory_size(&self) -> usize {
248 #[cfg(feature = "encryption")]
249 let encryption_size = self.file_decryptor.heap_size();
250 #[cfg(not(feature = "encryption"))]
251 let encryption_size = 0usize;
252
253 let page_index_size = if let Some(page_index) = self.page_index.as_ref() {
258 if let Some(page_index) = page_index.as_any().downcast_ref::<PageIndex>() {
259 2 * std::mem::size_of::<usize>() + std::mem::size_of::<PageIndex>() + page_index.heap_size()
263 } else {
264 0
265 }
266 } else {
267 0
268 };
269
270 std::mem::size_of::<Self>()
271 + self.file_metadata.heap_size()
272 + self.row_groups.heap_size()
273 + page_index_size
274 + encryption_size
275 }
276
277 pub(crate) fn set_page_index(&mut self, index: Option<Arc<dyn PageIndexProvider>>) {
279 self.page_index = index;
280 }
281}
282
283impl PartialEq for ParquetMetaData {
284 fn eq(&self, other: &Self) -> bool {
285 if self.file_metadata != other.file_metadata || self.row_groups != other.row_groups {
287 return false;
288 }
289
290 #[cfg(feature = "encryption")]
291 if self.file_decryptor != other.file_decryptor {
292 return false;
293 }
294
295 match (&self.page_index, &other.page_index) {
297 (None, None) => true,
298 (Some(a), Some(b)) => {
299 let a_page_index = a.as_any().downcast_ref::<PageIndex>();
301 let b_page_index = b.as_any().downcast_ref::<PageIndex>();
302 match (a_page_index, b_page_index) {
303 (Some(a_idx), Some(b_idx)) => a_idx == b_idx,
304 _ => Arc::ptr_eq(a, b), }
306 }
307 _ => false,
308 }
309 }
310}
311
312enum RowGroups {
314 Owned(Vec<RowGroupMetaData>),
316 Shared(Arc<[RowGroupMetaData]>),
318}
319
320impl RowGroups {
321 fn as_slice(&self) -> &[RowGroupMetaData] {
323 match self {
324 RowGroups::Owned(v) => v.as_slice(),
325 RowGroups::Shared(arc) => arc.as_ref(),
326 }
327 }
328
329 fn into_vec(self) -> Vec<RowGroupMetaData> {
331 match self {
332 RowGroups::Owned(v) => v,
333 RowGroups::Shared(arc) => arc.to_vec(),
334 }
335 }
336
337 fn convert_to_owned(&mut self) {
339 if matches!(self, RowGroups::Shared(_)) {
340 let old = std::mem::replace(self, RowGroups::Owned(Vec::new()));
341 *self = RowGroups::Owned(old.into_vec());
342 }
343 }
344
345 fn as_mut_vec(&mut self) -> &mut Vec<RowGroupMetaData> {
347 self.convert_to_owned();
348 match self {
349 RowGroups::Owned(v) => v,
350 RowGroups::Shared(_) => unreachable!(),
351 }
352 }
353}
354
355pub struct ParquetMetaDataBuilder {
393 file_metadata: Arc<FileMetaData>,
394 row_groups: RowGroups,
395 page_index: Option<Arc<dyn PageIndexProvider>>,
396 #[cfg(feature = "encryption")]
397 file_decryptor: Option<Arc<FileDecryptor>>,
398}
399
400impl ParquetMetaDataBuilder {
401 pub fn new(file_meta_data: FileMetaData) -> Self {
403 Self {
404 file_metadata: Arc::new(file_meta_data),
405 row_groups: RowGroups::Owned(Vec::new()),
406 page_index: None,
407 #[cfg(feature = "encryption")]
408 file_decryptor: None,
409 }
410 }
411
412 pub fn new_from_metadata(metadata: ParquetMetaData) -> Self {
414 Self {
415 file_metadata: metadata.file_metadata,
416 row_groups: RowGroups::Shared(metadata.row_groups),
417 page_index: metadata.page_index,
418 #[cfg(feature = "encryption")]
419 file_decryptor: metadata.file_decryptor,
420 }
421 }
422
423 pub fn set_key_value_metadata(mut self, key_value_metadata: Option<Vec<KeyValue>>) -> Self {
431 Arc::make_mut(&mut self.file_metadata).key_value_metadata = key_value_metadata;
432 self
433 }
434
435 pub fn add_row_group(mut self, row_group: RowGroupMetaData) -> Self {
440 self.row_groups.as_mut_vec().push(row_group);
441 self
442 }
443
444 pub fn set_row_groups(mut self, row_groups: Vec<RowGroupMetaData>) -> Self {
446 self.row_groups = RowGroups::Owned(row_groups);
447 self
448 }
449
450 pub fn take_row_groups(&mut self) -> Vec<RowGroupMetaData> {
459 std::mem::replace(&mut self.row_groups, RowGroups::Owned(Vec::new())).into_vec()
460 }
461
462 pub fn row_groups(&self) -> &[RowGroupMetaData] {
464 self.row_groups.as_slice()
465 }
466
467 pub fn set_page_index(mut self, page_index: Option<Arc<dyn PageIndexProvider>>) -> Self {
476 self.page_index = page_index;
477 self
478 }
479
480 pub fn take_page_index(&mut self) -> Option<Arc<dyn PageIndexProvider>> {
482 std::mem::take(&mut self.page_index)
483 }
484
485 pub fn page_index(&self) -> Option<&dyn PageIndexProvider> {
487 self.page_index.as_ref().map(|arc| arc.as_ref())
488 }
489
490 #[cfg(feature = "encryption")]
492 pub(crate) fn set_file_decryptor(mut self, file_decryptor: Option<FileDecryptor>) -> Self {
493 self.file_decryptor = file_decryptor.map(Arc::new);
494 self
495 }
496
497 pub fn build(self) -> ParquetMetaData {
499 ParquetMetaData {
500 file_metadata: self.file_metadata,
501 row_groups: match self.row_groups {
502 RowGroups::Owned(v) => v.into(),
503 RowGroups::Shared(arc) => arc,
504 },
505 page_index: self.page_index,
506 #[cfg(feature = "encryption")]
507 file_decryptor: self.file_decryptor,
508 }
509 }
510}
511
512impl From<ParquetMetaData> for ParquetMetaDataBuilder {
513 fn from(meta_data: ParquetMetaData) -> Self {
514 Self::new_from_metadata(meta_data)
515 }
516}
517
518thrift_struct!(
519pub struct KeyValue {
521 1: required string key
522 2: optional string value
523}
524);
525
526impl KeyValue {
527 pub fn new<F2>(key: String, value: F2) -> KeyValue
529 where
530 F2: Into<Option<String>>,
531 {
532 KeyValue {
533 key,
534 value: value.into(),
535 }
536 }
537}
538
539thrift_struct!(
540pub struct PageEncodingStats {
542 1: required PageType page_type;
543 2: required Encoding encoding;
544 3: required i32 count;
545}
546);
547
548#[derive(Debug, Clone, PartialEq)]
551enum ParquetPageEncodingStats {
552 Full(Vec<PageEncodingStats>),
554 Mask(EncodingMask),
556}
557
558pub type FileMetaDataPtr = Arc<FileMetaData>;
560
561#[derive(Debug, Clone, PartialEq)]
565pub struct FileMetaData {
566 version: i32,
567 num_rows: i64,
568 created_by: Option<String>,
569 key_value_metadata: Option<Vec<KeyValue>>,
570 schema_descr: SchemaDescPtr,
571 column_orders: Option<Vec<ColumnOrder>>,
572 #[cfg(feature = "encryption")]
573 encryption_algorithm: Option<Box<EncryptionAlgorithm>>,
574 #[cfg(feature = "encryption")]
575 footer_signing_key_metadata: Option<Vec<u8>>,
576}
577
578impl FileMetaData {
579 pub fn new(
581 version: i32,
582 num_rows: i64,
583 created_by: Option<String>,
584 key_value_metadata: Option<Vec<KeyValue>>,
585 schema_descr: SchemaDescPtr,
586 column_orders: Option<Vec<ColumnOrder>>,
587 ) -> Self {
588 FileMetaData {
589 version,
590 num_rows,
591 created_by,
592 key_value_metadata,
593 schema_descr,
594 column_orders,
595 #[cfg(feature = "encryption")]
596 encryption_algorithm: None,
597 #[cfg(feature = "encryption")]
598 footer_signing_key_metadata: None,
599 }
600 }
601
602 #[cfg(feature = "encryption")]
603 pub(crate) fn with_encryption_algorithm(
604 mut self,
605 encryption_algorithm: Option<EncryptionAlgorithm>,
606 ) -> Self {
607 self.encryption_algorithm = encryption_algorithm.map(Box::new);
608 self
609 }
610
611 #[cfg(feature = "encryption")]
612 pub(crate) fn with_footer_signing_key_metadata(
613 mut self,
614 footer_signing_key_metadata: Option<Vec<u8>>,
615 ) -> Self {
616 self.footer_signing_key_metadata = footer_signing_key_metadata;
617 self
618 }
619
620 pub fn version(&self) -> i32 {
622 self.version
623 }
624
625 pub fn num_rows(&self) -> i64 {
627 self.num_rows
628 }
629
630 pub fn created_by(&self) -> Option<&str> {
639 self.created_by.as_deref()
640 }
641
642 pub fn key_value_metadata(&self) -> Option<&Vec<KeyValue>> {
644 self.key_value_metadata.as_ref()
645 }
646
647 pub fn schema(&self) -> &SchemaType {
651 self.schema_descr.root_schema()
652 }
653
654 pub fn schema_descr(&self) -> &SchemaDescriptor {
656 &self.schema_descr
657 }
658
659 pub fn schema_descr_ptr(&self) -> SchemaDescPtr {
661 self.schema_descr.clone()
662 }
663
664 pub fn column_orders(&self) -> Option<&Vec<ColumnOrder>> {
672 self.column_orders.as_ref()
673 }
674
675 pub fn column_order(&self, i: usize) -> ColumnOrder {
678 self.column_orders
679 .as_ref()
680 .map(|data| data[i])
681 .unwrap_or(ColumnOrder::UNDEFINED)
682 }
683}
684
685thrift_struct!(
686pub struct SortingColumn {
688 1: required i32 column_idx
690
691 2: required bool descending
693
694 3: required bool nulls_first
697}
698);
699
700pub type RowGroupMetaDataPtr = Arc<RowGroupMetaData>;
702
703#[derive(Debug, Clone, PartialEq)]
708pub struct RowGroupMetaData {
709 columns: Vec<ColumnChunkMetaData>,
710 num_rows: i64,
711 sorting_columns: Option<Vec<SortingColumn>>,
712 total_byte_size: i64,
713 schema_descr: SchemaDescPtr,
714 file_offset: Option<i64>,
716 ordinal: Option<i32>,
718}
719
720impl RowGroupMetaData {
721 pub fn builder(schema_descr: SchemaDescPtr) -> RowGroupMetaDataBuilder {
723 RowGroupMetaDataBuilder::new(schema_descr)
724 }
725
726 pub fn num_columns(&self) -> usize {
728 self.columns.len()
729 }
730
731 pub fn column(&self, i: usize) -> &ColumnChunkMetaData {
733 &self.columns[i]
734 }
735
736 pub fn columns(&self) -> &[ColumnChunkMetaData] {
738 &self.columns
739 }
740
741 pub fn columns_mut(&mut self) -> &mut [ColumnChunkMetaData] {
743 &mut self.columns
744 }
745
746 pub fn num_rows(&self) -> i64 {
748 self.num_rows
749 }
750
751 pub fn sorting_columns(&self) -> Option<&Vec<SortingColumn>> {
753 self.sorting_columns.as_ref()
754 }
755
756 pub fn total_byte_size(&self) -> i64 {
758 self.total_byte_size
759 }
760
761 pub fn compressed_size(&self) -> i64 {
763 self.columns.iter().map(|c| c.total_compressed_size).sum()
764 }
765
766 pub fn schema_descr(&self) -> &SchemaDescriptor {
768 self.schema_descr.as_ref()
769 }
770
771 pub fn schema_descr_ptr(&self) -> SchemaDescPtr {
773 self.schema_descr.clone()
774 }
775
776 #[inline(always)]
781 pub fn ordinal(&self) -> Option<i32> {
782 self.ordinal
783 }
784
785 #[inline(always)]
787 pub fn file_offset(&self) -> Option<i64> {
788 self.file_offset
789 }
790
791 pub fn into_builder(self) -> RowGroupMetaDataBuilder {
793 RowGroupMetaDataBuilder(self)
794 }
795}
796
797impl HeapSize for Arc<[RowGroupMetaData]> {
798 fn heap_size(&self) -> usize {
799 let rg_size = self.iter().map(|t| t.heap_size()).sum::<usize>();
801 2 * std::mem::size_of::<usize>() + std::mem::size_of_val(self.as_ref()) + rg_size
804 }
805}
806
807pub struct RowGroupMetaDataBuilder(RowGroupMetaData);
809
810impl RowGroupMetaDataBuilder {
811 fn new(schema_descr: SchemaDescPtr) -> Self {
813 Self(RowGroupMetaData {
814 columns: Vec::with_capacity(schema_descr.num_columns()),
815 schema_descr,
816 file_offset: None,
817 num_rows: 0,
818 sorting_columns: None,
819 total_byte_size: 0,
820 ordinal: None,
821 })
822 }
823
824 pub fn set_num_rows(mut self, value: i64) -> Self {
826 self.0.num_rows = value;
827 self
828 }
829
830 pub fn set_sorting_columns(mut self, value: Option<Vec<SortingColumn>>) -> Self {
832 self.0.sorting_columns = value;
833 self
834 }
835
836 pub fn set_total_byte_size(mut self, value: i64) -> Self {
838 self.0.total_byte_size = value;
839 self
840 }
841
842 pub fn take_columns(&mut self) -> Vec<ColumnChunkMetaData> {
848 std::mem::take(&mut self.0.columns)
849 }
850
851 pub fn set_column_metadata(mut self, value: Vec<ColumnChunkMetaData>) -> Self {
853 self.0.columns = value;
854 self
855 }
856
857 pub fn add_column_metadata(mut self, value: ColumnChunkMetaData) -> Self {
859 self.0.columns.push(value);
860 self
861 }
862
863 pub fn set_ordinal(mut self, value: i32) -> Self {
865 self.0.ordinal = Some(value);
866 self
867 }
868
869 pub fn set_file_offset(mut self, value: i64) -> Self {
871 self.0.file_offset = Some(value);
872 self
873 }
874
875 pub fn build(self) -> Result<RowGroupMetaData> {
877 if self.0.schema_descr.num_columns() != self.0.columns.len() {
878 return Err(general_err!(
879 "Column length mismatch: {} != {}",
880 self.0.schema_descr.num_columns(),
881 self.0.columns.len()
882 ));
883 }
884
885 Ok(self.0)
886 }
887
888 pub(super) fn build_unchecked(self) -> RowGroupMetaData {
890 self.0
891 }
892}
893
894#[derive(Debug, Clone, PartialEq)]
896pub struct ColumnChunkMetaData {
897 column_descr: ColumnDescPtr,
898 encodings: EncodingMask,
899 file_path: Option<String>,
900 file_offset: i64,
901 num_values: i64,
902 compression: CompressionCodec,
903 total_compressed_size: i64,
904 total_uncompressed_size: i64,
905 data_page_offset: i64,
906 index_page_offset: Option<i64>,
907 dictionary_page_offset: Option<i64>,
908 statistics: Option<Statistics>,
909 geo_statistics: Option<Box<geo_statistics::GeospatialStatistics>>,
910 encoding_stats: Option<ParquetPageEncodingStats>,
911 bloom_filter_offset: Option<i64>,
912 bloom_filter_length: Option<i32>,
913 offset_index_offset: Option<i64>,
914 offset_index_length: Option<i32>,
915 column_index_offset: Option<i64>,
916 column_index_length: Option<i32>,
917 unencoded_byte_array_data_bytes: Option<i64>,
918 repetition_level_histogram: Option<LevelHistogram>,
919 definition_level_histogram: Option<LevelHistogram>,
920 #[cfg(feature = "encryption")]
921 column_crypto_metadata: Option<Box<ColumnCryptoMetaData>>,
922 #[cfg(feature = "encryption")]
923 encrypted_column_metadata: Option<Vec<u8>>,
924 #[cfg(feature = "encryption")]
928 plaintext_footer_mode: bool,
929}
930
931#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)]
940pub struct LevelHistogram {
941 inner: Vec<i64>,
942}
943
944impl LevelHistogram {
945 pub fn try_new(max_level: i16) -> Option<Self> {
951 if max_level > 0 {
952 Some(Self {
953 inner: vec![0; max_level as usize + 1],
954 })
955 } else {
956 None
957 }
958 }
959 pub fn values(&self) -> &[i64] {
961 &self.inner
962 }
963
964 pub fn into_inner(self) -> Vec<i64> {
966 self.inner
967 }
968
969 pub fn get(&self, index: usize) -> Option<i64> {
976 self.inner.get(index).copied()
977 }
978
979 pub fn add(&mut self, other: &Self) {
984 assert_eq!(self.len(), other.len());
985 for (dst, src) in self.inner.iter_mut().zip(other.inner.iter()) {
986 *dst += src;
987 }
988 }
989
990 pub fn len(&self) -> usize {
992 self.inner.len()
993 }
994
995 pub fn is_empty(&self) -> bool {
997 self.inner.is_empty()
998 }
999
1000 pub fn reset(&mut self) {
1002 self.inner.fill(0);
1003 }
1004
1005 #[inline]
1007 pub fn increment_by(&mut self, level: i16, count: i64) {
1008 self.inner[level as usize] += count;
1009 }
1010
1011 #[deprecated(since = "58.2.0", note = "Use `increment_by` instead")]
1017 pub fn update_from_levels(&mut self, levels: &[i16]) {
1018 for &level in levels {
1019 self.increment_by(level, 1);
1020 }
1021 }
1022}
1023
1024impl From<Vec<i64>> for LevelHistogram {
1025 fn from(inner: Vec<i64>) -> Self {
1026 Self { inner }
1027 }
1028}
1029
1030impl From<LevelHistogram> for Vec<i64> {
1031 fn from(value: LevelHistogram) -> Self {
1032 value.into_inner()
1033 }
1034}
1035
1036impl HeapSize for LevelHistogram {
1037 fn heap_size(&self) -> usize {
1038 self.inner.heap_size()
1039 }
1040}
1041
1042impl ColumnChunkMetaData {
1044 pub fn builder(column_descr: ColumnDescPtr) -> ColumnChunkMetaDataBuilder {
1046 ColumnChunkMetaDataBuilder::new(column_descr)
1047 }
1048
1049 pub fn file_path(&self) -> Option<&str> {
1054 self.file_path.as_deref()
1055 }
1056
1057 pub fn file_offset(&self) -> i64 {
1064 self.file_offset
1065 }
1066
1067 pub fn column_type(&self) -> Type {
1069 self.column_descr.physical_type()
1070 }
1071
1072 pub fn column_path(&self) -> &ColumnPath {
1074 self.column_descr.path()
1075 }
1076
1077 pub fn column_descr(&self) -> &ColumnDescriptor {
1079 self.column_descr.as_ref()
1080 }
1081
1082 pub fn column_descr_ptr(&self) -> ColumnDescPtr {
1084 self.column_descr.clone()
1085 }
1086
1087 pub fn encodings(&self) -> impl Iterator<Item = Encoding> {
1089 self.encodings.encodings()
1090 }
1091
1092 pub fn encodings_mask(&self) -> &EncodingMask {
1094 &self.encodings
1095 }
1096
1097 pub fn num_values(&self) -> i64 {
1099 self.num_values
1100 }
1101
1102 pub fn compression(&self) -> Compression {
1109 self.compression.into()
1110 }
1111
1112 pub fn compression_codec(&self) -> CompressionCodec {
1114 self.compression
1115 }
1116
1117 pub fn compressed_size(&self) -> i64 {
1119 self.total_compressed_size
1120 }
1121
1122 pub fn uncompressed_size(&self) -> i64 {
1124 self.total_uncompressed_size
1125 }
1126
1127 pub fn data_page_offset(&self) -> i64 {
1129 self.data_page_offset
1130 }
1131
1132 pub fn index_page_offset(&self) -> Option<i64> {
1134 self.index_page_offset
1135 }
1136
1137 pub fn dictionary_page_offset(&self) -> Option<i64> {
1139 self.dictionary_page_offset
1140 }
1141
1142 pub fn byte_range(&self) -> (u64, u64) {
1148 let col_start = match self.dictionary_page_offset() {
1149 Some(dictionary_page_offset) => dictionary_page_offset,
1150 None => self.data_page_offset(),
1151 };
1152 let col_len = self.compressed_size();
1153 assert!(
1154 col_start >= 0 && col_len >= 0,
1155 "column start and length should not be negative"
1156 );
1157 (col_start as u64, col_len as u64)
1158 }
1159
1160 pub fn statistics(&self) -> Option<&Statistics> {
1163 self.statistics.as_ref()
1164 }
1165
1166 pub fn geo_statistics(&self) -> Option<&geo_statistics::GeospatialStatistics> {
1169 self.geo_statistics.as_deref()
1170 }
1171
1172 pub fn page_encoding_stats(&self) -> Option<&Vec<PageEncodingStats>> {
1179 match self.encoding_stats.as_ref() {
1180 Some(ParquetPageEncodingStats::Full(stats)) => Some(stats),
1181 _ => None,
1182 }
1183 }
1184
1185 pub fn page_encoding_stats_mask(&self) -> Option<&EncodingMask> {
1215 match self.encoding_stats.as_ref() {
1216 Some(ParquetPageEncodingStats::Mask(stats)) => Some(stats),
1217 _ => None,
1218 }
1219 }
1220
1221 pub fn bloom_filter_offset(&self) -> Option<i64> {
1223 self.bloom_filter_offset
1224 }
1225
1226 pub fn bloom_filter_length(&self) -> Option<i32> {
1228 self.bloom_filter_length
1229 }
1230
1231 pub fn column_index_offset(&self) -> Option<i64> {
1233 self.column_index_offset
1234 }
1235
1236 pub fn column_index_length(&self) -> Option<i32> {
1238 self.column_index_length
1239 }
1240
1241 pub fn column_index_range(&self) -> Option<Range<u64>> {
1247 let offset = u64::try_from(self.column_index_offset?).ok()?;
1248 let length = u64::try_from(self.column_index_length?).ok()?;
1249 Some(offset..(offset + length))
1250 }
1251
1252 pub fn offset_index_offset(&self) -> Option<i64> {
1254 self.offset_index_offset
1255 }
1256
1257 pub fn offset_index_length(&self) -> Option<i32> {
1259 self.offset_index_length
1260 }
1261
1262 pub fn offset_index_range(&self) -> Option<Range<u64>> {
1268 let offset = u64::try_from(self.offset_index_offset?).ok()?;
1269 let length = u64::try_from(self.offset_index_length?).ok()?;
1270 Some(offset..(offset + length))
1271 }
1272
1273 pub fn unencoded_byte_array_data_bytes(&self) -> Option<i64> {
1278 self.unencoded_byte_array_data_bytes
1279 }
1280
1281 pub fn repetition_level_histogram(&self) -> Option<&LevelHistogram> {
1287 self.repetition_level_histogram.as_ref()
1288 }
1289
1290 pub fn definition_level_histogram(&self) -> Option<&LevelHistogram> {
1296 self.definition_level_histogram.as_ref()
1297 }
1298
1299 #[cfg(feature = "encryption")]
1301 pub fn crypto_metadata(&self) -> Option<&ColumnCryptoMetaData> {
1302 self.column_crypto_metadata.as_deref()
1303 }
1304
1305 pub fn into_builder(self) -> ColumnChunkMetaDataBuilder {
1307 ColumnChunkMetaDataBuilder::from(self)
1308 }
1309}
1310
1311pub struct ColumnChunkMetaDataBuilder(ColumnChunkMetaData);
1330
1331impl ColumnChunkMetaDataBuilder {
1332 fn new(column_descr: ColumnDescPtr) -> Self {
1336 Self(ColumnChunkMetaData {
1337 column_descr,
1338 encodings: Default::default(),
1339 file_path: None,
1340 file_offset: 0,
1341 num_values: 0,
1342 compression: CompressionCodec::UNCOMPRESSED,
1343 total_compressed_size: 0,
1344 total_uncompressed_size: 0,
1345 data_page_offset: 0,
1346 index_page_offset: None,
1347 dictionary_page_offset: None,
1348 statistics: None,
1349 geo_statistics: None,
1350 encoding_stats: None,
1351 bloom_filter_offset: None,
1352 bloom_filter_length: None,
1353 offset_index_offset: None,
1354 offset_index_length: None,
1355 column_index_offset: None,
1356 column_index_length: None,
1357 unencoded_byte_array_data_bytes: None,
1358 repetition_level_histogram: None,
1359 definition_level_histogram: None,
1360 #[cfg(feature = "encryption")]
1361 column_crypto_metadata: None,
1362 #[cfg(feature = "encryption")]
1363 encrypted_column_metadata: None,
1364 #[cfg(feature = "encryption")]
1365 plaintext_footer_mode: false,
1366 })
1367 }
1368
1369 pub fn set_encodings(mut self, encodings: Vec<Encoding>) -> Self {
1371 self.0.encodings = EncodingMask::new_from_encodings(encodings.iter());
1372 self
1373 }
1374
1375 pub fn set_encodings_mask(mut self, encodings: EncodingMask) -> Self {
1377 self.0.encodings = encodings;
1378 self
1379 }
1380
1381 pub fn set_file_path(mut self, value: String) -> Self {
1383 self.0.file_path = Some(value);
1384 self
1385 }
1386
1387 pub fn set_num_values(mut self, value: i64) -> Self {
1389 self.0.num_values = value;
1390 self
1391 }
1392
1393 pub fn set_compression(mut self, value: Compression) -> Self {
1395 self.0.compression = value.into();
1396 self
1397 }
1398
1399 pub fn set_compression_codec(mut self, value: CompressionCodec) -> Self {
1401 self.0.compression = value;
1402 self
1403 }
1404
1405 pub fn set_total_compressed_size(mut self, value: i64) -> Self {
1407 self.0.total_compressed_size = value;
1408 self
1409 }
1410
1411 pub fn set_total_uncompressed_size(mut self, value: i64) -> Self {
1413 self.0.total_uncompressed_size = value;
1414 self
1415 }
1416
1417 pub fn set_data_page_offset(mut self, value: i64) -> Self {
1419 self.0.data_page_offset = value;
1420 self
1421 }
1422
1423 pub fn set_dictionary_page_offset(mut self, value: Option<i64>) -> Self {
1425 self.0.dictionary_page_offset = value;
1426 self
1427 }
1428
1429 pub fn set_index_page_offset(mut self, value: Option<i64>) -> Self {
1431 self.0.index_page_offset = value;
1432 self
1433 }
1434
1435 pub fn set_statistics(mut self, value: Statistics) -> Self {
1437 self.0.statistics = Some(value);
1438 self
1439 }
1440
1441 pub fn set_geo_statistics(mut self, value: Box<geo_statistics::GeospatialStatistics>) -> Self {
1443 self.0.geo_statistics = Some(value);
1444 self
1445 }
1446
1447 pub fn clear_statistics(mut self) -> Self {
1449 self.0.statistics = None;
1450 self
1451 }
1452
1453 pub fn set_page_encoding_stats(mut self, value: Vec<PageEncodingStats>) -> Self {
1457 self.0.encoding_stats = Some(ParquetPageEncodingStats::Full(value));
1458 self
1459 }
1460
1461 pub fn set_page_encoding_stats_mask(mut self, value: EncodingMask) -> Self {
1465 self.0.encoding_stats = Some(ParquetPageEncodingStats::Mask(value));
1466 self
1467 }
1468
1469 pub fn clear_page_encoding_stats(mut self) -> Self {
1471 self.0.encoding_stats = None;
1472 self
1473 }
1474
1475 pub fn set_bloom_filter_offset(mut self, value: Option<i64>) -> Self {
1477 self.0.bloom_filter_offset = value;
1478 self
1479 }
1480
1481 pub fn set_bloom_filter_length(mut self, value: Option<i32>) -> Self {
1483 self.0.bloom_filter_length = value;
1484 self
1485 }
1486
1487 pub fn set_offset_index_offset(mut self, value: Option<i64>) -> Self {
1489 self.0.offset_index_offset = value;
1490 self
1491 }
1492
1493 pub fn set_offset_index_length(mut self, value: Option<i32>) -> Self {
1495 self.0.offset_index_length = value;
1496 self
1497 }
1498
1499 pub fn set_column_index_offset(mut self, value: Option<i64>) -> Self {
1501 self.0.column_index_offset = value;
1502 self
1503 }
1504
1505 pub fn set_column_index_length(mut self, value: Option<i32>) -> Self {
1507 self.0.column_index_length = value;
1508 self
1509 }
1510
1511 pub fn set_unencoded_byte_array_data_bytes(mut self, value: Option<i64>) -> Self {
1513 self.0.unencoded_byte_array_data_bytes = value;
1514 self
1515 }
1516
1517 pub fn set_repetition_level_histogram(mut self, value: Option<LevelHistogram>) -> Self {
1519 self.0.repetition_level_histogram = value;
1520 self
1521 }
1522
1523 pub fn set_definition_level_histogram(mut self, value: Option<LevelHistogram>) -> Self {
1525 self.0.definition_level_histogram = value;
1526 self
1527 }
1528
1529 #[cfg(feature = "encryption")]
1530 pub fn set_column_crypto_metadata(mut self, value: Option<ColumnCryptoMetaData>) -> Self {
1532 self.0.column_crypto_metadata = value.map(Box::new);
1533 self
1534 }
1535
1536 #[cfg(feature = "encryption")]
1537 pub fn set_encrypted_column_metadata(mut self, value: Option<Vec<u8>>) -> Self {
1539 self.0.encrypted_column_metadata = value;
1540 self
1541 }
1542
1543 pub fn build(self) -> Result<ColumnChunkMetaData> {
1545 Ok(self.0)
1546 }
1547}
1548
1549pub struct ColumnIndexBuilder {
1554 column_type: Type,
1555 null_pages: Vec<bool>,
1556 min_values: Vec<Vec<u8>>,
1557 max_values: Vec<Vec<u8>>,
1558 null_counts: Vec<i64>,
1559 nan_counts: Vec<Option<i64>>,
1560 boundary_order: BoundaryOrder,
1561 repetition_level_histograms: Option<Vec<i64>>,
1563 definition_level_histograms: Option<Vec<i64>>,
1565 valid: bool,
1573}
1574
1575impl ColumnIndexBuilder {
1576 pub fn new(column_type: Type) -> Self {
1578 ColumnIndexBuilder {
1579 column_type,
1580 null_pages: Vec::new(),
1581 min_values: Vec::new(),
1582 max_values: Vec::new(),
1583 null_counts: Vec::new(),
1584 nan_counts: Vec::new(),
1585 boundary_order: BoundaryOrder::UNORDERED,
1586 repetition_level_histograms: None,
1587 definition_level_histograms: None,
1588 valid: true,
1589 }
1590 }
1591
1592 pub fn append(
1599 &mut self,
1600 null_page: bool,
1601 min_value: Vec<u8>,
1602 max_value: Vec<u8>,
1603 null_count: i64,
1604 nan_count: Option<i64>,
1605 ) {
1606 self.null_pages.push(null_page);
1607 self.min_values.push(min_value);
1608 self.max_values.push(max_value);
1609 self.null_counts.push(null_count);
1610 self.nan_counts.push(nan_count);
1611 }
1612
1613 pub fn append_histograms(
1618 &mut self,
1619 repetition_level_histogram: &Option<LevelHistogram>,
1620 definition_level_histogram: &Option<LevelHistogram>,
1621 ) {
1622 if !self.valid {
1623 return;
1624 }
1625 if let Some(rep_lvl_hist) = repetition_level_histogram {
1626 let hist = self.repetition_level_histograms.get_or_insert(Vec::new());
1627 hist.reserve(rep_lvl_hist.len());
1628 hist.extend(rep_lvl_hist.values());
1629 }
1630 if let Some(def_lvl_hist) = definition_level_histogram {
1631 let hist = self.definition_level_histograms.get_or_insert(Vec::new());
1632 hist.reserve(def_lvl_hist.len());
1633 hist.extend(def_lvl_hist.values());
1634 }
1635 }
1636
1637 pub fn set_boundary_order(&mut self, boundary_order: BoundaryOrder) {
1639 self.boundary_order = boundary_order;
1640 }
1641
1642 pub fn to_invalid(&mut self) {
1644 self.valid = false;
1645 }
1646
1647 pub fn valid(&self) -> bool {
1649 self.valid
1650 }
1651
1652 pub fn build(self) -> Result<ColumnIndexMetaData> {
1656 Ok(match self.column_type {
1657 Type::BOOLEAN => {
1658 let index = self.build_page_index(false)?;
1659 ColumnIndexMetaData::BOOLEAN(index)
1660 }
1661 Type::INT32 => {
1662 let index = self.build_page_index(false)?;
1663 ColumnIndexMetaData::INT32(index)
1664 }
1665 Type::INT64 => {
1666 let index = self.build_page_index(false)?;
1667 ColumnIndexMetaData::INT64(index)
1668 }
1669 Type::INT96 => {
1670 let index = self.build_page_index(false)?;
1671 ColumnIndexMetaData::INT96(index)
1672 }
1673 Type::FLOAT => {
1674 let index = self.build_page_index(true)?;
1675 ColumnIndexMetaData::FLOAT(index)
1676 }
1677 Type::DOUBLE => {
1678 let index = self.build_page_index(true)?;
1679 ColumnIndexMetaData::DOUBLE(index)
1680 }
1681 Type::BYTE_ARRAY => {
1682 let index = self.build_byte_array_index(false)?;
1683 ColumnIndexMetaData::BYTE_ARRAY(index)
1684 }
1685 Type::FIXED_LEN_BYTE_ARRAY => {
1686 let index = self.build_byte_array_index(true)?;
1687 ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(index)
1688 }
1689 })
1690 }
1691
1692 fn build_nan_counts(nan_counts: &[Option<i64>]) -> Option<Vec<i64>> {
1693 let has_some = nan_counts.iter().any(|x| x.is_some());
1694 let has_none = nan_counts.iter().any(|x| x.is_none());
1695
1696 if has_some && !has_none {
1697 Some(nan_counts.iter().map(|x| x.unwrap()).collect())
1698 } else if !has_some && has_none {
1699 None
1700 } else {
1701 debug_assert!(
1702 false,
1703 "Mixed Some/None in nan_counts - caller should provide consistent values"
1704 );
1705 Some(nan_counts.iter().map(|x| x.unwrap_or(0)).collect())
1706 }
1707 }
1708
1709 fn build_page_index<T>(self, may_have_nan: bool) -> Result<PrimitiveColumnIndex<T>>
1710 where
1711 T: ParquetValueType,
1712 {
1713 let min_values: Vec<&[u8]> = self.min_values.iter().map(|v| v.as_slice()).collect();
1714 let max_values: Vec<&[u8]> = self.max_values.iter().map(|v| v.as_slice()).collect();
1715
1716 let nan_counts = if may_have_nan && !self.nan_counts.is_empty() {
1721 Self::build_nan_counts(&self.nan_counts)
1722 } else {
1723 None
1724 };
1725
1726 PrimitiveColumnIndex::try_new(
1727 self.null_pages,
1728 self.boundary_order,
1729 Some(self.null_counts),
1730 nan_counts,
1731 self.repetition_level_histograms,
1732 self.definition_level_histograms,
1733 min_values,
1734 max_values,
1735 )
1736 }
1737
1738 fn build_byte_array_index(self, may_have_nan: bool) -> Result<ByteArrayColumnIndex> {
1739 let min_values: Vec<&[u8]> = self.min_values.iter().map(|v| v.as_slice()).collect();
1740 let max_values: Vec<&[u8]> = self.max_values.iter().map(|v| v.as_slice()).collect();
1741
1742 let nan_counts = if may_have_nan && !self.nan_counts.is_empty() {
1747 Self::build_nan_counts(&self.nan_counts)
1748 } else {
1749 None
1750 };
1751
1752 ByteArrayColumnIndex::try_new(
1753 self.null_pages,
1754 self.boundary_order,
1755 Some(self.null_counts),
1756 nan_counts,
1757 self.repetition_level_histograms,
1758 self.definition_level_histograms,
1759 min_values,
1760 max_values,
1761 )
1762 }
1763}
1764
1765impl From<ColumnChunkMetaData> for ColumnChunkMetaDataBuilder {
1766 fn from(value: ColumnChunkMetaData) -> Self {
1767 ColumnChunkMetaDataBuilder(value)
1768 }
1769}
1770
1771pub struct OffsetIndexBuilder {
1775 offset_array: Vec<i64>,
1776 compressed_page_size_array: Vec<i32>,
1777 first_row_index_array: Vec<i64>,
1778 unencoded_byte_array_data_bytes_array: Option<Vec<i64>>,
1779 current_first_row_index: i64,
1780}
1781
1782impl Default for OffsetIndexBuilder {
1783 fn default() -> Self {
1784 Self::new()
1785 }
1786}
1787
1788impl OffsetIndexBuilder {
1789 pub fn new() -> Self {
1791 OffsetIndexBuilder {
1792 offset_array: Vec::new(),
1793 compressed_page_size_array: Vec::new(),
1794 first_row_index_array: Vec::new(),
1795 unencoded_byte_array_data_bytes_array: None,
1796 current_first_row_index: 0,
1797 }
1798 }
1799
1800 pub fn append_row_count(&mut self, row_count: i64) {
1802 let current_page_row_index = self.current_first_row_index;
1803 self.first_row_index_array.push(current_page_row_index);
1804 self.current_first_row_index += row_count;
1805 }
1806
1807 pub fn append_offset_and_size(&mut self, offset: i64, compressed_page_size: i32) {
1809 self.offset_array.push(offset);
1810 self.compressed_page_size_array.push(compressed_page_size);
1811 }
1812
1813 pub fn append_unencoded_byte_array_data_bytes(
1815 &mut self,
1816 unencoded_byte_array_data_bytes: Option<i64>,
1817 ) {
1818 if let Some(val) = unencoded_byte_array_data_bytes {
1819 self.unencoded_byte_array_data_bytes_array
1820 .get_or_insert(Vec::new())
1821 .push(val);
1822 }
1823 }
1824
1825 pub fn build(self) -> OffsetIndexMetaData {
1827 let locations = self
1828 .offset_array
1829 .iter()
1830 .zip(self.compressed_page_size_array.iter())
1831 .zip(self.first_row_index_array.iter())
1832 .map(|((offset, size), row_index)| PageLocation {
1833 offset: *offset,
1834 compressed_page_size: *size,
1835 first_row_index: *row_index,
1836 })
1837 .collect::<Vec<_>>();
1838 OffsetIndexMetaData {
1839 page_locations: locations,
1840 unencoded_byte_array_data_bytes: self.unencoded_byte_array_data_bytes_array,
1841 }
1842 }
1843}
1844
1845#[cfg(test)]
1846mod tests {
1847 use super::*;
1848 use crate::basic::{PageType, SortOrder};
1849 use crate::file::metadata::page_index::PageIndexBuilder;
1850 use crate::file::metadata::thrift::tests::{
1851 read_column_chunk, read_column_chunk_with_options, read_row_group,
1852 };
1853
1854 #[test]
1855 #[expect(deprecated)]
1856 fn test_level_histogram_update_from_levels_compat() {
1857 let mut histogram = LevelHistogram::try_new(2).unwrap();
1858 histogram.update_from_levels(&[0, 2, 1, 2, 2]);
1859 assert_eq!(histogram.values(), &[1, 1, 3]);
1860 }
1861
1862 #[test]
1863 fn test_column_chunk() {
1864 let schema = get_test_schema_descr();
1865 let row_groups = (0..2)
1866 .map(|row| {
1867 let columns = schema
1868 .columns()
1869 .iter()
1870 .enumerate()
1871 .map(|(column, descriptor)| {
1872 ColumnChunkMetaData::builder(descriptor.clone())
1873 .set_data_page_offset((row * 2 + column) as i64)
1874 .build()
1875 .unwrap()
1876 })
1877 .collect();
1878 RowGroupMetaData::builder(schema.clone())
1879 .set_column_metadata(columns)
1880 .build()
1881 .unwrap()
1882 })
1883 .collect();
1884 let metadata = ParquetMetaData::new(
1885 FileMetaData::new(1, 0, None, None, schema, None),
1886 row_groups,
1887 );
1888 for row in 0..2 {
1889 for column in 0..2 {
1890 assert_eq!(
1891 metadata
1892 .column_chunk(row, column)
1893 .unwrap()
1894 .data_page_offset(),
1895 (row * 2 + column) as i64
1896 );
1897 }
1898 }
1899 for index in [2, usize::MAX] {
1900 assert_eq!(
1901 metadata
1902 .column_chunk(index, usize::MAX)
1903 .unwrap_err()
1904 .to_string(),
1905 format!(
1906 "Parquet error: Row group index {index} out of bounds for file with 2 row groups"
1907 )
1908 );
1909 assert_eq!(
1910 metadata.column_chunk(1, index).unwrap_err().to_string(),
1911 format!(
1912 "Parquet error: Column index {index} out of bounds for row group 1 with 2 columns"
1913 )
1914 );
1915 }
1916 }
1917
1918 #[test]
1919 fn test_row_group_metadata_thrift_conversion() {
1920 let schema_descr = get_test_schema_descr();
1921
1922 let mut columns = vec![];
1923 for ptr in schema_descr.columns() {
1924 let column = ColumnChunkMetaData::builder(ptr.clone()).build().unwrap();
1925 columns.push(column);
1926 }
1927 let row_group_meta = RowGroupMetaData::builder(schema_descr.clone())
1928 .set_num_rows(1000)
1929 .set_total_byte_size(2000)
1930 .set_column_metadata(columns)
1931 .set_ordinal(1)
1932 .build()
1933 .unwrap();
1934
1935 let mut buf = Vec::new();
1936 let mut writer = ThriftCompactOutputProtocol::new(&mut buf);
1937 row_group_meta.write_thrift(&mut writer).unwrap();
1938
1939 let row_group_res = read_row_group(&buf, schema_descr).unwrap();
1940
1941 assert_eq!(row_group_res, row_group_meta);
1942 }
1943
1944 #[test]
1945 fn test_row_group_metadata_thrift_conversion_empty() {
1946 let schema_descr = get_test_schema_descr();
1947
1948 let row_group_meta = RowGroupMetaData::builder(schema_descr).build();
1949
1950 assert!(row_group_meta.is_err());
1951 if let Err(e) = row_group_meta {
1952 assert_eq!(
1953 format!("{e}"),
1954 "Parquet error: Column length mismatch: 2 != 0"
1955 );
1956 }
1957 }
1958
1959 #[test]
1961 fn test_row_group_metadata_thrift_corrupted() {
1962 let schema_descr_2cols = Arc::new(SchemaDescriptor::new(Arc::new(
1963 SchemaType::group_type_builder("schema")
1964 .with_fields(vec![
1965 Arc::new(
1966 SchemaType::primitive_type_builder("a", Type::INT32)
1967 .build()
1968 .unwrap(),
1969 ),
1970 Arc::new(
1971 SchemaType::primitive_type_builder("b", Type::INT32)
1972 .build()
1973 .unwrap(),
1974 ),
1975 ])
1976 .build()
1977 .unwrap(),
1978 )));
1979
1980 let schema_descr_3cols = Arc::new(SchemaDescriptor::new(Arc::new(
1981 SchemaType::group_type_builder("schema")
1982 .with_fields(vec![
1983 Arc::new(
1984 SchemaType::primitive_type_builder("a", Type::INT32)
1985 .build()
1986 .unwrap(),
1987 ),
1988 Arc::new(
1989 SchemaType::primitive_type_builder("b", Type::INT32)
1990 .build()
1991 .unwrap(),
1992 ),
1993 Arc::new(
1994 SchemaType::primitive_type_builder("c", Type::INT32)
1995 .build()
1996 .unwrap(),
1997 ),
1998 ])
1999 .build()
2000 .unwrap(),
2001 )));
2002
2003 let row_group_meta_2cols = RowGroupMetaData::builder(schema_descr_2cols.clone())
2004 .set_num_rows(1000)
2005 .set_total_byte_size(2000)
2006 .set_column_metadata(vec![
2007 ColumnChunkMetaData::builder(schema_descr_2cols.column(0))
2008 .build()
2009 .unwrap(),
2010 ColumnChunkMetaData::builder(schema_descr_2cols.column(1))
2011 .build()
2012 .unwrap(),
2013 ])
2014 .set_ordinal(1)
2015 .build()
2016 .unwrap();
2017 let mut buf = Vec::new();
2018 let mut writer = ThriftCompactOutputProtocol::new(&mut buf);
2019 row_group_meta_2cols.write_thrift(&mut writer).unwrap();
2020
2021 let err = read_row_group(&buf, schema_descr_3cols)
2022 .unwrap_err()
2023 .to_string();
2024 assert_eq!(
2025 err,
2026 "Parquet error: Column count mismatch. Schema has 3 columns while Row Group has 2"
2027 );
2028 }
2029
2030 #[test]
2031 fn test_column_chunk_metadata_thrift_conversion() {
2032 let column_descr = get_test_schema_descr().column(0);
2033 let col_metadata = ColumnChunkMetaData::builder(column_descr.clone())
2034 .set_encodings_mask(EncodingMask::new_from_encodings(
2035 [Encoding::PLAIN, Encoding::RLE].iter(),
2036 ))
2037 .set_file_path("file_path".to_owned())
2038 .set_num_values(1000)
2039 .set_compression_codec(CompressionCodec::SNAPPY)
2040 .set_total_compressed_size(2000)
2041 .set_total_uncompressed_size(3000)
2042 .set_data_page_offset(4000)
2043 .set_dictionary_page_offset(Some(5000))
2044 .set_page_encoding_stats(vec![
2045 PageEncodingStats {
2046 page_type: PageType::DATA_PAGE,
2047 encoding: Encoding::PLAIN,
2048 count: 3,
2049 },
2050 PageEncodingStats {
2051 page_type: PageType::DATA_PAGE,
2052 encoding: Encoding::RLE,
2053 count: 5,
2054 },
2055 ])
2056 .set_bloom_filter_offset(Some(6000))
2057 .set_bloom_filter_length(Some(25))
2058 .set_offset_index_offset(Some(7000))
2059 .set_offset_index_length(Some(25))
2060 .set_column_index_offset(Some(8000))
2061 .set_column_index_length(Some(25))
2062 .set_unencoded_byte_array_data_bytes(Some(2000))
2063 .set_repetition_level_histogram(Some(LevelHistogram::from(vec![100, 100])))
2064 .set_definition_level_histogram(Some(LevelHistogram::from(vec![0, 200])))
2065 .build()
2066 .unwrap();
2067
2068 let mut buf = Vec::new();
2069 let mut writer = ThriftCompactOutputProtocol::new(&mut buf);
2070 col_metadata.write_thrift(&mut writer).unwrap();
2071 let col_chunk_res = read_column_chunk(&buf, column_descr.clone()).unwrap();
2072
2073 let expected_metadata = ColumnChunkMetaData::builder(column_descr)
2074 .set_encodings_mask(EncodingMask::new_from_encodings(
2075 [Encoding::PLAIN, Encoding::RLE].iter(),
2076 ))
2077 .set_file_path("file_path".to_owned())
2078 .set_num_values(1000)
2079 .set_compression_codec(CompressionCodec::SNAPPY)
2080 .set_total_compressed_size(2000)
2081 .set_total_uncompressed_size(3000)
2082 .set_data_page_offset(4000)
2083 .set_dictionary_page_offset(Some(5000))
2084 .set_page_encoding_stats_mask(EncodingMask::new_from_encodings(
2085 [Encoding::PLAIN, Encoding::RLE].iter(),
2086 ))
2087 .set_bloom_filter_offset(Some(6000))
2088 .set_bloom_filter_length(Some(25))
2089 .set_offset_index_offset(Some(7000))
2090 .set_offset_index_length(Some(25))
2091 .set_column_index_offset(Some(8000))
2092 .set_column_index_length(Some(25))
2093 .set_unencoded_byte_array_data_bytes(Some(2000))
2094 .set_repetition_level_histogram(Some(LevelHistogram::from(vec![100, 100])))
2095 .set_definition_level_histogram(Some(LevelHistogram::from(vec![0, 200])))
2096 .build()
2097 .unwrap();
2098
2099 assert_eq!(col_chunk_res, expected_metadata);
2100 }
2101
2102 #[test]
2103 fn test_column_chunk_metadata_thrift_conversion_full_stats() {
2104 let column_descr = get_test_schema_descr().column(0);
2105 let stats = vec![
2106 PageEncodingStats {
2107 page_type: PageType::DATA_PAGE,
2108 encoding: Encoding::PLAIN,
2109 count: 3,
2110 },
2111 PageEncodingStats {
2112 page_type: PageType::DATA_PAGE,
2113 encoding: Encoding::RLE,
2114 count: 5,
2115 },
2116 ];
2117 let col_metadata = ColumnChunkMetaData::builder(column_descr.clone())
2118 .set_encodings_mask(EncodingMask::new_from_encodings(
2119 [Encoding::PLAIN, Encoding::RLE].iter(),
2120 ))
2121 .set_num_values(1000)
2122 .set_compression_codec(CompressionCodec::SNAPPY)
2123 .set_total_compressed_size(2000)
2124 .set_total_uncompressed_size(3000)
2125 .set_data_page_offset(4000)
2126 .set_page_encoding_stats(stats)
2127 .build()
2128 .unwrap();
2129
2130 let mut buf = Vec::new();
2131 let mut writer = ThriftCompactOutputProtocol::new(&mut buf);
2132 col_metadata.write_thrift(&mut writer).unwrap();
2133
2134 let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false);
2135 let col_chunk_res =
2136 read_column_chunk_with_options(&buf, column_descr, Some(&options)).unwrap();
2137
2138 assert_eq!(col_chunk_res, col_metadata);
2139 }
2140
2141 #[test]
2142 fn test_column_chunk_metadata_thrift_conversion_empty() {
2143 let column_descr = get_test_schema_descr().column(0);
2144
2145 let col_metadata = ColumnChunkMetaData::builder(column_descr.clone())
2146 .build()
2147 .unwrap();
2148
2149 let mut buf = Vec::new();
2150 let mut writer = ThriftCompactOutputProtocol::new(&mut buf);
2151 col_metadata.write_thrift(&mut writer).unwrap();
2152 let col_chunk_res = read_column_chunk(&buf, column_descr).unwrap();
2153
2154 assert_eq!(col_chunk_res, col_metadata);
2155 }
2156
2157 #[test]
2158 fn test_compressed_size() {
2159 let schema_descr = get_test_schema_descr();
2160
2161 let mut columns = vec![];
2162 for column_descr in schema_descr.columns() {
2163 let column = ColumnChunkMetaData::builder(column_descr.clone())
2164 .set_total_compressed_size(500)
2165 .set_total_uncompressed_size(700)
2166 .build()
2167 .unwrap();
2168 columns.push(column);
2169 }
2170 let row_group_meta = RowGroupMetaData::builder(schema_descr)
2171 .set_num_rows(1000)
2172 .set_column_metadata(columns)
2173 .build()
2174 .unwrap();
2175
2176 let compressed_size_res = row_group_meta.compressed_size();
2177 let compressed_size_exp: i64 = 1000;
2178
2179 assert_eq!(compressed_size_res, compressed_size_exp);
2180 }
2181
2182 #[test]
2183 fn test_memory_size() {
2184 let schema_descr = get_test_schema_descr();
2185
2186 let columns = schema_descr
2187 .columns()
2188 .iter()
2189 .map(|column_descr| {
2190 ColumnChunkMetaData::builder(column_descr.clone())
2191 .set_statistics(Statistics::new::<i32>(None, None, None, None, false))
2192 .build()
2193 })
2194 .collect::<Result<Vec<_>>>()
2195 .unwrap();
2196 let row_group_meta = RowGroupMetaData::builder(schema_descr.clone())
2197 .set_num_rows(1000)
2198 .set_column_metadata(columns)
2199 .build()
2200 .unwrap();
2201 let row_group_meta = vec![row_group_meta];
2202
2203 let version = 2;
2204 let num_rows = 1000;
2205 let created_by = Some(String::from("test harness"));
2206 let key_value_metadata = Some(vec![KeyValue::new(
2207 String::from("Foo"),
2208 Some(String::from("bar")),
2209 )]);
2210 let column_orders = Some(vec![
2211 ColumnOrder::UNDEFINED,
2212 ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::UNSIGNED),
2213 ]);
2214 let file_metadata = FileMetaData::new(
2215 version,
2216 num_rows,
2217 created_by,
2218 key_value_metadata,
2219 schema_descr.clone(),
2220 column_orders,
2221 );
2222
2223 let columns_with_stats = schema_descr
2225 .columns()
2226 .iter()
2227 .map(|column_descr| {
2228 ColumnChunkMetaData::builder(column_descr.clone())
2229 .set_statistics(Statistics::new::<i32>(
2230 Some(0),
2231 Some(100),
2232 None,
2233 None,
2234 false,
2235 ))
2236 .build()
2237 })
2238 .collect::<Result<Vec<_>>>()
2239 .unwrap();
2240
2241 let row_group_meta_with_stats = RowGroupMetaData::builder(schema_descr)
2242 .set_num_rows(1000)
2243 .set_column_metadata(columns_with_stats)
2244 .build()
2245 .unwrap();
2246 let row_group_meta_with_stats = vec![row_group_meta_with_stats];
2247
2248 let parquet_meta = ParquetMetaDataBuilder::new(file_metadata.clone())
2249 .set_row_groups(row_group_meta_with_stats)
2250 .build();
2251
2252 #[cfg(not(feature = "encryption"))]
2254 let base_expected_size = 2798;
2255 #[cfg(feature = "encryption")]
2256 let base_expected_size = 2966;
2257
2258 assert_eq!(parquet_meta.memory_size(), base_expected_size);
2259
2260 let mut page_index = PageIndexBuilder::new(1, 1);
2261
2262 let mut column_index = ColumnIndexBuilder::new(Type::BOOLEAN);
2263 column_index.append(false, vec![1u8], vec![2u8, 3u8], 4, None);
2264 let column_index = column_index.build().unwrap();
2265 {
2266 let ColumnIndexMetaData::BOOLEAN(_) = column_index else {
2267 panic!("wrong type of column index")
2268 };
2269 }
2270
2271 page_index.try_put_column_index(column_index, 0, 0).unwrap();
2272
2273 let mut offset_index = OffsetIndexBuilder::new();
2275 offset_index.append_row_count(1);
2276 offset_index.append_offset_and_size(2, 3);
2277 offset_index.append_unencoded_byte_array_data_bytes(Some(10));
2278 offset_index.append_row_count(1);
2279 offset_index.append_offset_and_size(2, 3);
2280 offset_index.append_unencoded_byte_array_data_bytes(Some(10));
2281 let offset_index = offset_index.build();
2282 page_index.try_put_offset_index(offset_index, 0, 0).unwrap();
2283
2284 let parquet_meta = ParquetMetaDataBuilder::new(file_metadata)
2285 .set_row_groups(row_group_meta)
2286 .set_page_index(Some(Arc::new(page_index.build())))
2287 .build();
2288
2289 #[cfg(not(feature = "encryption"))]
2291 let bigger_expected_size = 3296;
2292 #[cfg(feature = "encryption")]
2293 let bigger_expected_size = 3464;
2294
2295 assert!(bigger_expected_size > base_expected_size);
2297 assert_eq!(parquet_meta.memory_size(), bigger_expected_size);
2298 }
2299
2300 #[test]
2301 #[cfg(feature = "encryption")]
2302 fn test_memory_size_with_decryptor() {
2303 use crate::encryption::decrypt::FileDecryptionProperties;
2304 use crate::file::metadata::thrift::encryption::AesGcmV1;
2305
2306 let schema_descr = get_test_schema_descr();
2307
2308 let columns = schema_descr
2309 .columns()
2310 .iter()
2311 .map(|column_descr| ColumnChunkMetaData::builder(column_descr.clone()).build())
2312 .collect::<Result<Vec<_>>>()
2313 .unwrap();
2314 let row_group_meta = RowGroupMetaData::builder(schema_descr.clone())
2315 .set_num_rows(1000)
2316 .set_column_metadata(columns)
2317 .build()
2318 .unwrap();
2319 let row_group_meta = vec![row_group_meta];
2320
2321 let version = 2;
2322 let num_rows = 1000;
2323 let aad_file_unique = vec![1u8; 8];
2324 let aad_prefix = vec![2u8; 8];
2325 let encryption_algorithm = EncryptionAlgorithm::AES_GCM_V1(AesGcmV1 {
2326 aad_prefix: Some(aad_prefix.clone()),
2327 aad_file_unique: Some(aad_file_unique.clone()),
2328 supply_aad_prefix: Some(true),
2329 });
2330 let footer_key_metadata = Some(vec![3u8; 8]);
2331 let file_metadata =
2332 FileMetaData::new(version, num_rows, None, None, schema_descr.clone(), None)
2333 .with_encryption_algorithm(Some(encryption_algorithm))
2334 .with_footer_signing_key_metadata(footer_key_metadata.clone());
2335
2336 let parquet_meta_data = ParquetMetaDataBuilder::new(file_metadata.clone())
2337 .set_row_groups(row_group_meta.clone())
2338 .build();
2339
2340 let base_expected_size = 2074;
2341 assert_eq!(parquet_meta_data.memory_size(), base_expected_size);
2342
2343 let footer_key = b"0123456789012345";
2344 let column_key = b"1234567890123450";
2345 let mut decryption_properties_builder =
2346 FileDecryptionProperties::builder(footer_key.to_vec())
2347 .with_aad_prefix(aad_prefix.clone());
2348 for column in schema_descr.columns() {
2349 decryption_properties_builder = decryption_properties_builder
2350 .with_column_key(&column.path().string(), column_key.to_vec());
2351 }
2352 let decryption_properties = decryption_properties_builder.build().unwrap();
2353 let decryptor = FileDecryptor::new(
2354 &decryption_properties,
2355 footer_key_metadata.as_deref(),
2356 aad_file_unique,
2357 aad_prefix,
2358 )
2359 .unwrap();
2360
2361 let parquet_meta_data = ParquetMetaDataBuilder::new(file_metadata.clone())
2362 .set_row_groups(row_group_meta.clone())
2363 .set_file_decryptor(Some(decryptor))
2364 .build();
2365
2366 let expected_size_with_decryptor = 3104;
2367 assert!(expected_size_with_decryptor > base_expected_size);
2368
2369 assert_eq!(
2370 parquet_meta_data.memory_size(),
2371 expected_size_with_decryptor
2372 );
2373 }
2374
2375 fn get_test_schema_descr() -> SchemaDescPtr {
2377 let schema = SchemaType::group_type_builder("schema")
2378 .with_fields(vec![
2379 Arc::new(
2380 SchemaType::primitive_type_builder("a", Type::INT32)
2381 .build()
2382 .unwrap(),
2383 ),
2384 Arc::new(
2385 SchemaType::primitive_type_builder("b", Type::INT32)
2386 .build()
2387 .unwrap(),
2388 ),
2389 ])
2390 .build()
2391 .unwrap();
2392
2393 Arc::new(SchemaDescriptor::new(Arc::new(schema)))
2394 }
2395
2396 #[test]
2397 fn test_page_index_builder_partial_population() {
2398 let mut builder = PageIndexBuilder::new(1, 1);
2399
2400 let mut offset_idx_builder = OffsetIndexBuilder::new();
2402 offset_idx_builder.append_row_count(50);
2403 offset_idx_builder.append_offset_and_size(1000, 500);
2404 let offset_idx = offset_idx_builder.build();
2405 builder.try_put_offset_index(offset_idx, 0, 0).unwrap();
2406
2407 let page_index = builder.build();
2408
2409 assert!(!page_index.has_column_indexes());
2411 assert!(page_index.has_offset_indexes());
2413 }
2414
2415 #[test]
2416 fn test_page_index_builder_empty_to_none() {
2417 let builder = PageIndexBuilder::new(2, 2);
2419
2420 let page_index = builder.build();
2421
2422 assert!(!page_index.has_column_indexes());
2424 assert!(!page_index.has_offset_indexes());
2425 }
2426
2427 #[test]
2428 fn test_set_key_value_metadata_preserves_shared_metadata() {
2429 let original = vec![KeyValue::new("original".to_string(), "value".to_string())];
2430 let replacement = vec![KeyValue::new("replacement".to_string(), "new".to_string())];
2431 for initial in [None, Some(original)] {
2432 let file_metadata = FileMetaData::new(
2433 2,
2434 0,
2435 Some("test writer".to_string()),
2436 initial.clone(),
2437 get_test_schema_descr(),
2438 None,
2439 );
2440 #[cfg(feature = "encryption")]
2441 let file_metadata = file_metadata
2442 .with_encryption_algorithm(Some(EncryptionAlgorithm::AES_GCM_V1(
2443 crate::file::metadata::thrift::encryption::AesGcmV1 {
2444 aad_prefix: Some(vec![1; 8]),
2445 aad_file_unique: Some(vec![2; 8]),
2446 supply_aad_prefix: Some(true),
2447 },
2448 )))
2449 .with_footer_signing_key_metadata(Some(vec![3; 8]));
2450 let cached = ParquetMetaDataBuilder::new(file_metadata)
2451 .set_page_index(Some(Arc::new(PageIndexBuilder::new(0, 2).build())))
2452 .build();
2453 for values in [None, Some(vec![]), Some(replacement.clone())] {
2454 let updated = cached
2455 .clone()
2456 .into_builder()
2457 .set_key_value_metadata(values.clone())
2458 .build();
2459 let mut expected = cached.file_metadata().clone();
2460 expected.key_value_metadata = values;
2461 assert_eq!(updated.file_metadata(), &expected);
2462 assert_eq!(
2463 cached.file_metadata().key_value_metadata(),
2464 initial.as_ref()
2465 );
2466 assert!(Arc::ptr_eq(&cached.row_groups, &updated.row_groups));
2467 assert!(Arc::ptr_eq(
2468 cached.page_index().unwrap(),
2469 updated.page_index().unwrap()
2470 ));
2471 }
2472 }
2473 }
2474
2475 #[test]
2476 fn test_replace_page_index_preserves_shared_metadata() {
2477 let schema_descr = get_test_schema_descr();
2478 let file_metadata = FileMetaData::new(1, 0, None, None, schema_descr, None);
2479 let cached = Arc::new(ParquetMetaData::new(file_metadata, vec![]));
2480
2481 let page_index = Arc::new(PageIndexBuilder::new(0, 0).build());
2482 let rebuilt = cached
2483 .as_ref()
2484 .clone()
2485 .into_builder()
2486 .set_page_index(Some(page_index))
2487 .build();
2488
2489 assert!(Arc::ptr_eq(&cached.file_metadata, &rebuilt.file_metadata));
2490 assert!(Arc::ptr_eq(&cached.row_groups, &rebuilt.row_groups));
2491 }
2492
2493 #[test]
2494 fn test_rebuild_page_index() {
2495 let mut builder = PageIndexBuilder::new(1, 2);
2497
2498 let mut col_idx_builder = ColumnIndexBuilder::new(Type::INT32);
2500 col_idx_builder.append(false, vec![1, 0, 0, 0], vec![100, 0, 0, 0], 10, None);
2501 let col_idx = col_idx_builder.build().unwrap();
2502 builder.try_put_column_index(col_idx.clone(), 0, 0).unwrap();
2503
2504 let mut offset_idx_builder = OffsetIndexBuilder::new();
2506 offset_idx_builder.append_row_count(50);
2507 offset_idx_builder.append_offset_and_size(1000, 500);
2508 let offset_idx = offset_idx_builder.build();
2509 builder
2510 .try_put_offset_index(offset_idx.clone(), 0, 0)
2511 .unwrap();
2512
2513 let page_index = builder.build();
2514
2515 assert!(page_index.is_complete());
2517
2518 assert!(page_index.column_index(0, 0).is_some());
2520 assert!(page_index.offset_index(0, 0).is_some());
2521 assert!(page_index.column_index(0, 1).is_none());
2523 assert!(page_index.offset_index(0, 1).is_none());
2524
2525 let mut builder = PageIndexBuilder::new_from(page_index);
2526
2527 builder.try_put_column_index(col_idx, 0, 1).unwrap();
2529 builder.try_put_offset_index(offset_idx, 0, 1).unwrap();
2530 let page_index = builder.build();
2531
2532 assert!(page_index.column_index(0, 0).is_some());
2534 assert!(page_index.offset_index(0, 0).is_some());
2535 assert!(page_index.column_index(0, 1).is_some());
2536 assert!(page_index.offset_index(0, 1).is_some());
2537 }
2538}