Skip to main content

parquet/file/metadata/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Parquet metadata API
6//!
7//! Users should use these structures to interact with Parquet metadata.
8//!
9//! * [`ParquetMetaData`]: Top level metadata container, read from the Parquet
10//!   file footer.
11//!
12//! * [`FileMetaData`]: File level metadata such as schema, row counts and
13//!   version.
14//!
15//! * [`RowGroupMetaData`]: Metadata for each Row Group with a File, such as
16//!   location and number of rows, and column chunks.
17//!
18//! * [`ColumnChunkMetaData`]: Metadata for each column chunk (primitive leaf)
19//!   within a Row Group including encoding and compression information,
20//!   number of values, statistics, etc.
21//!
22//! * [`PageIndex`]: Metadata and statistics used to reduce page-level I/O.
23//!
24//! # APIs for working with Parquet Metadata
25//!
26//! The Parquet readers and writers in this crate handle reading and writing
27//! metadata into Parquet files. To work with metadata directly,
28//! the following APIs are available:
29//!
30//! * [`ParquetMetaDataReader`] for reading metadata from an I/O source (sync and async)
31//! * [`ParquetMetaDataPushDecoder`] for decoding from bytes without I/O
32//! * [`ParquetMetaDataWriter`] for writing.
33//!
34//! # Examples
35//!
36//! Please see [`external_metadata.rs`]
37//!
38//! [`external_metadata.rs`]: https://github.com/apache/arrow-rs/tree/master/parquet/examples/external_metadata.rs
39//!
40mod column_chunk;
41#[cfg(feature = "arrow")]
42mod dictionary;
43mod footer_tail;
44mod memory;
45mod options;
46pub mod page_index;
47mod parser;
48mod push_decoder;
49pub(crate) mod reader;
50pub(crate) mod thrift;
51mod writer;
52
53use crate::basic::{
54    BoundaryOrder, ColumnOrder, Compression, CompressionCodec, Encoding, EncodingMask, PageType,
55    Type,
56};
57#[cfg(feature = "encryption")]
58use crate::encryption::decrypt::FileDecryptor;
59use crate::errors::{ParquetError, Result};
60#[cfg(feature = "encryption")]
61use crate::file::column_crypto_metadata::ColumnCryptoMetaData;
62pub(crate) use crate::file::metadata::memory::HeapSize;
63use crate::file::metadata::page_index::{PageIndex, PageIndexProvider, RowGroupPageIndex};
64#[cfg(feature = "encryption")]
65use crate::file::metadata::thrift::encryption::EncryptionAlgorithm;
66use crate::file::page_index::column_index::{ByteArrayColumnIndex, PrimitiveColumnIndex};
67use crate::file::page_index::{column_index::ColumnIndexMetaData, offset_index::PageLocation};
68use crate::file::statistics::Statistics;
69use crate::geospatial::statistics as geo_statistics;
70use crate::parquet_thrift::{
71    ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol, ThriftCompactOutputProtocol,
72    WriteThrift, WriteThriftField,
73};
74use crate::schema::types::{
75    ColumnDescPtr, ColumnDescriptor, ColumnPath, SchemaDescPtr, SchemaDescriptor,
76    Type as SchemaType,
77};
78use crate::thrift_struct;
79use crate::{
80    data_type::private::ParquetValueType, file::page_index::offset_index::OffsetIndexMetaData,
81};
82
83pub use footer_tail::FooterTail;
84pub use options::{ParquetMetaDataOptions, ParquetStatisticsPolicy};
85pub use push_decoder::ParquetMetaDataPushDecoder;
86pub use reader::{ColumnChunkMask, PageIndexPolicy, ParquetMetaDataReader};
87use std::io::Write;
88use std::ops::Range;
89use std::sync::Arc;
90pub use writer::ParquetMetaDataWriter;
91pub(crate) use writer::ThriftMetadataWriter;
92
93/// Parsed metadata for a single Parquet file
94///
95/// This structure is stored in the footer of Parquet files, in the format
96/// defined by [`parquet.thrift`].
97///
98/// # Overview
99/// The fields of this structure are:
100/// * [`FileMetaData`]: Information about the overall file (such as the schema) (See [`Self::file_metadata`])
101/// * [`RowGroupMetaData`]: Information about each Row Group (see [`Self::row_groups`])
102/// * [`PageIndexProvider`]: Optional Page Index (see [`Self::page_index`])
103///
104/// This structure is read by the various readers in this crate or can be read
105/// directly from a file using the [`ParquetMetaDataReader`] struct.
106///
107/// The individual fields of this structure are stored in a way that make cloning
108/// this structure low-cost.
109///
110/// See [`PageIndexProvider`] for details on customizing page index handling.  
111///
112/// See the [`ParquetMetaDataBuilder`] to create and modify this structure.
113///
114/// [`parquet.thrift`]: https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift
115#[derive(Debug, Clone)]
116pub struct ParquetMetaData {
117    /// File level metadata
118    file_metadata: Arc<FileMetaData>,
119    /// Row group metadata
120    row_groups: Arc<[RowGroupMetaData]>,
121    /// Page level index for each page in each column chunk
122    page_index: Option<Arc<dyn PageIndexProvider>>,
123    /// Optional file decryptor
124    #[cfg(feature = "encryption")]
125    file_decryptor: Option<Arc<FileDecryptor>>,
126}
127
128impl ParquetMetaData {
129    /// Creates Parquet metadata from file metadata and a list of row
130    /// group metadata
131    pub fn new(file_metadata: FileMetaData, row_groups: Vec<RowGroupMetaData>) -> Self {
132        ParquetMetaData {
133            file_metadata: Arc::new(file_metadata),
134            row_groups: row_groups.into(),
135            page_index: None,
136            #[cfg(feature = "encryption")]
137            file_decryptor: None,
138        }
139    }
140
141    /// Convert this ParquetMetaData into a [`ParquetMetaDataBuilder`].
142    ///
143    /// The builder initially shares any reference-counted metadata. Mutating shared
144    /// fields may clone their contents using copy-on-write.
145    pub fn into_builder(self) -> ParquetMetaDataBuilder {
146        self.into()
147    }
148
149    /// Returns file metadata as reference.
150    pub fn file_metadata(&self) -> &FileMetaData {
151        &self.file_metadata
152    }
153
154    /// Returns file decryptor as reference.
155    #[cfg(feature = "encryption")]
156    pub(crate) fn file_decryptor(&self) -> Option<&FileDecryptor> {
157        self.file_decryptor.as_deref()
158    }
159
160    /// Returns number of row groups in this file.
161    pub fn num_row_groups(&self) -> usize {
162        self.row_groups.len()
163    }
164
165    /// Returns row group metadata for `i`th position.
166    /// Position should be less than number of row groups `num_row_groups`.
167    pub fn row_group(&self, i: usize) -> &RowGroupMetaData {
168        &self.row_groups[i]
169    }
170
171    /// Returns slice of row groups in this file.
172    pub fn row_groups(&self) -> &[RowGroupMetaData] {
173        &self.row_groups
174    }
175
176    /// Returns [`ColumnChunkMetaData`] for a row-group position and physical leaf column index.
177    ///
178    /// Returns an error if either index is out of bounds.
179    pub fn column_chunk(
180        &self,
181        row_group_idx: usize,
182        column_idx: usize,
183    ) -> Result<&ColumnChunkMetaData> {
184        let row_group = self.row_groups().get(row_group_idx).ok_or_else(|| {
185            general_err!(
186                "Row group index {row_group_idx} out of bounds for file with {} row groups",
187                self.num_row_groups()
188            )
189        })?;
190        row_group.columns().get(column_idx).ok_or_else(|| {
191            general_err!(
192                "Column index {column_idx} out of bounds for row group {row_group_idx} with {} columns",
193                row_group.num_columns()
194            )
195        })
196    }
197
198    /// Returns the number of rows in `row_group_idx`.
199    ///
200    /// Returns an error if the row group index is out of bounds or its row
201    /// count cannot be represented as a [`usize`].
202    pub fn row_group_num_rows(&self, row_group_idx: usize) -> Result<usize> {
203        self.row_groups
204            .get(row_group_idx)
205            .ok_or_else(|| {
206                ParquetError::General(format!(
207                    "Row group index {row_group_idx} out of bounds for file with {} row groups",
208                    self.num_row_groups()
209                ))
210            })?
211            .num_rows()
212            .try_into()
213            .map_err(|e| ParquetError::General(format!("Row count overflow: {e}")))
214    }
215
216    /// Returns a [`PageIndexProvider`] for this file
217    ///
218    /// Returns `None` if the parquet file lacks page indexes or
219    /// [ArrowReaderOptions::with_page_index_policy] was set to [`PageIndexPolicy::Skip`].
220    ///
221    /// [ArrowReaderOptions::with_page_index_policy]: https://docs.rs/parquet/latest/parquet/arrow/arrow_reader/struct.ArrowReaderOptions.html#method.with_page_index_policy
222    pub fn page_index(&self) -> Option<&Arc<dyn PageIndexProvider>> {
223        self.page_index.as_ref()
224    }
225
226    /// Returns a [`RowGroupPageIndex`] for a specific row group
227    pub fn page_index_for_row_group(&self, row_group_idx: usize) -> RowGroupPageIndex {
228        RowGroupPageIndex::new(row_group_idx, self.page_index.clone())
229    }
230
231    /// Estimate of the bytes allocated to store `ParquetMetadata`
232    ///
233    /// # Notes:
234    ///
235    /// 1. Includes size of self
236    ///
237    /// 2. Includes heap memory for sub fields such as [`FileMetaData`],
238    ///    [`RowGroupMetaData`], and the built-in [`PageIndex`] provider.
239    ///
240    /// 3. Includes memory reachable through shared pointers, such as [`SchemaDescPtr`] and the
241    ///    [`Arc`]-wrapped page index provider. Therefore, the same allocation may be counted by
242    ///    multiple metadata values or when the provider is also retained in an external cache.
243    ///
244    /// 4. Does not include heap memory used by custom [`PageIndexProvider`] implementations.
245    ///
246    /// 5. Does not include any allocator overheads.
247    pub fn memory_size(&self) -> usize {
248        #[cfg(feature = "encryption")]
249        let encryption_size = self.file_decryptor.heap_size();
250        #[cfg(not(feature = "encryption"))]
251        let encryption_size = 0usize;
252
253        // We can only determine the heap size for PageIndex.
254        //
255        // Heap size for custom `PageIndexProvider` tracked by
256        // <https://github.com/apache/arrow-rs/issues/11028>
257        let page_index_size = if let Some(page_index) = self.page_index.as_ref() {
258            if let Some(page_index) = page_index.as_any().downcast_ref::<PageIndex>() {
259                // need to account for the `Arc` wrapper overhead
260                2 * std::mem::size_of::<usize>() // Arc stores weak and strong counts on the heap...
261                    + std::mem::size_of::<PageIndex>() // alongside an instance of PageIndex
262                    + page_index.heap_size()
263            } else {
264                0
265            }
266        } else {
267            0
268        };
269
270        std::mem::size_of::<Self>()
271            + self.file_metadata.heap_size()
272            + self.row_groups.heap_size()
273            + page_index_size
274            + encryption_size
275    }
276
277    /// Override the page index
278    pub(crate) fn set_page_index(&mut self, index: Option<Arc<dyn PageIndexProvider>>) {
279        self.page_index = index;
280    }
281}
282
283impl PartialEq for ParquetMetaData {
284    fn eq(&self, other: &Self) -> bool {
285        // Compare file metadata and row groups
286        if self.file_metadata != other.file_metadata || self.row_groups != other.row_groups {
287            return false;
288        }
289
290        #[cfg(feature = "encryption")]
291        if self.file_decryptor != other.file_decryptor {
292            return false;
293        }
294
295        // Compare page_index by downcasting to PageIndex and comparing
296        match (&self.page_index, &other.page_index) {
297            (None, None) => true,
298            (Some(a), Some(b)) => {
299                // Try to downcast both to PageIndex for comparison
300                let a_page_index = a.as_any().downcast_ref::<PageIndex>();
301                let b_page_index = b.as_any().downcast_ref::<PageIndex>();
302                match (a_page_index, b_page_index) {
303                    (Some(a_idx), Some(b_idx)) => a_idx == b_idx,
304                    _ => Arc::ptr_eq(a, b), // Fall back to pointer equality if not PageIndex
305                }
306            }
307            _ => false,
308        }
309    }
310}
311
312// helper enum for ParquetMetaDataBuilder
313enum RowGroups {
314    /// Owned vector - cheap to mutate
315    Owned(Vec<RowGroupMetaData>),
316    /// Shared Arc slice - cheap to clone
317    Shared(Arc<[RowGroupMetaData]>),
318}
319
320impl RowGroups {
321    /// Get a reference to the row groups
322    fn as_slice(&self) -> &[RowGroupMetaData] {
323        match self {
324            RowGroups::Owned(v) => v.as_slice(),
325            RowGroups::Shared(arc) => arc.as_ref(),
326        }
327    }
328
329    /// Take ownership of the row groups, converting to Vec
330    fn into_vec(self) -> Vec<RowGroupMetaData> {
331        match self {
332            RowGroups::Owned(v) => v,
333            RowGroups::Shared(arc) => arc.to_vec(),
334        }
335    }
336
337    /// Convert to Owned variant, cloning if necessary
338    fn convert_to_owned(&mut self) {
339        if matches!(self, RowGroups::Shared(_)) {
340            let old = std::mem::replace(self, RowGroups::Owned(Vec::new()));
341            *self = RowGroups::Owned(old.into_vec());
342        }
343    }
344
345    /// Get a mutable reference to the Vec, converting from Shared if necessary
346    fn as_mut_vec(&mut self) -> &mut Vec<RowGroupMetaData> {
347        self.convert_to_owned();
348        match self {
349            RowGroups::Owned(v) => v,
350            RowGroups::Shared(_) => unreachable!(),
351        }
352    }
353}
354
355/// A builder for creating / manipulating [`ParquetMetaData`]
356///
357/// # Example creating a new [`ParquetMetaData`]
358///
359///```no_run
360/// # use parquet::file::metadata::{FileMetaData, ParquetMetaData, ParquetMetaDataBuilder, RowGroupMetaData, RowGroupMetaDataBuilder};
361/// # fn get_file_metadata() -> FileMetaData { unimplemented!(); }
362/// // Create a new builder given the file metadata
363/// let file_metadata = get_file_metadata();
364/// // Create a row group
365/// let row_group = RowGroupMetaData::builder(file_metadata.schema_descr_ptr())
366///    .set_num_rows(100)
367///    // ... (A real row group needs more than just the number of rows)
368///    .build()
369///    .unwrap();
370/// // Create the final metadata
371/// let metadata: ParquetMetaData = ParquetMetaDataBuilder::new(file_metadata)
372///   .add_row_group(row_group)
373///   .build();
374/// ```
375///
376/// # Example modifying an existing [`ParquetMetaData`]
377/// ```no_run
378/// # use parquet::file::metadata::ParquetMetaData;
379/// # fn load_metadata() -> ParquetMetaData { unimplemented!(); }
380/// // Modify the metadata so only the last RowGroup remains
381/// let metadata: ParquetMetaData = load_metadata();
382/// let mut builder = metadata.into_builder();
383///
384/// // Take existing row groups to modify
385/// let mut row_groups = builder.take_row_groups();
386/// let last_row_group = row_groups.pop().unwrap();
387///
388/// let metadata = builder
389///   .add_row_group(last_row_group)
390///   .build();
391/// ```
392pub struct ParquetMetaDataBuilder {
393    file_metadata: Arc<FileMetaData>,
394    row_groups: RowGroups,
395    page_index: Option<Arc<dyn PageIndexProvider>>,
396    #[cfg(feature = "encryption")]
397    file_decryptor: Option<Arc<FileDecryptor>>,
398}
399
400impl ParquetMetaDataBuilder {
401    /// Create a new builder from a file metadata, with no row groups
402    pub fn new(file_meta_data: FileMetaData) -> Self {
403        Self {
404            file_metadata: Arc::new(file_meta_data),
405            row_groups: RowGroups::Owned(Vec::new()),
406            page_index: None,
407            #[cfg(feature = "encryption")]
408            file_decryptor: None,
409        }
410    }
411
412    /// Create a new builder from an existing ParquetMetaData
413    pub fn new_from_metadata(metadata: ParquetMetaData) -> Self {
414        Self {
415            file_metadata: metadata.file_metadata,
416            row_groups: RowGroups::Shared(metadata.row_groups),
417            page_index: metadata.page_index,
418            #[cfg(feature = "encryption")]
419            file_decryptor: metadata.file_decryptor,
420        }
421    }
422
423    /// Replaces the file-level key-value metadata.
424    ///
425    /// Pass `None` to remove all entries. This only updates the in-memory metadata;
426    /// it does not modify the Parquet file.
427    ///
428    /// Other metadata, including row groups, page indexes, and decryption state,
429    /// is preserved. Shared file metadata is cloned before modification.
430    pub fn set_key_value_metadata(mut self, key_value_metadata: Option<Vec<KeyValue>>) -> Self {
431        Arc::make_mut(&mut self.file_metadata).key_value_metadata = key_value_metadata;
432        self
433    }
434
435    /// Adds a row group to the metadata
436    ///
437    /// If the row groups are shared with another `ParquetMetaData`, this clones
438    /// the existing row groups before appending.
439    pub fn add_row_group(mut self, row_group: RowGroupMetaData) -> Self {
440        self.row_groups.as_mut_vec().push(row_group);
441        self
442    }
443
444    /// Sets all the row groups to the specified list
445    pub fn set_row_groups(mut self, row_groups: Vec<RowGroupMetaData>) -> Self {
446        self.row_groups = RowGroups::Owned(row_groups);
447        self
448    }
449
450    /// Takes ownership of the row groups in this builder, and clears the list
451    /// of row groups.
452    ///
453    /// This can be used for more efficient creation of a new ParquetMetaData
454    /// from an existing one.
455    ///
456    /// Returns the existing allocation when uniquely owned; otherwise clones the
457    /// shared row groups.
458    pub fn take_row_groups(&mut self) -> Vec<RowGroupMetaData> {
459        std::mem::replace(&mut self.row_groups, RowGroups::Owned(Vec::new())).into_vec()
460    }
461
462    /// Return a reference to the current row groups
463    pub fn row_groups(&self) -> &[RowGroupMetaData] {
464        self.row_groups.as_slice()
465    }
466
467    /// Sets the [`PageIndexProvider`]
468    ///
469    /// This crate provides [`PageIndex`] as a default implementation. Custom providers
470    /// can implement application-specific behavior such as lazy parsing or filtered access.
471    ///
472    /// For an example see [`custom_page_index.rs`]
473    ///
474    /// [`custom_page_index.rs`]: https://github.com/apache/arrow-rs/blob/main/parquet/examples/custom_page_index.rs
475    pub fn set_page_index(mut self, page_index: Option<Arc<dyn PageIndexProvider>>) -> Self {
476        self.page_index = page_index;
477        self
478    }
479
480    /// Returns the current [`PageIndexProvider`] from the builder, replacing it with `None`
481    pub fn take_page_index(&mut self) -> Option<Arc<dyn PageIndexProvider>> {
482        std::mem::take(&mut self.page_index)
483    }
484
485    /// Return a reference to the current [`PageIndexProvider`], if any
486    pub fn page_index(&self) -> Option<&dyn PageIndexProvider> {
487        self.page_index.as_ref().map(|arc| arc.as_ref())
488    }
489
490    /// Sets the file decryptor needed to decrypt this metadata.
491    #[cfg(feature = "encryption")]
492    pub(crate) fn set_file_decryptor(mut self, file_decryptor: Option<FileDecryptor>) -> Self {
493        self.file_decryptor = file_decryptor.map(Arc::new);
494        self
495    }
496
497    /// Creates a new ParquetMetaData from the builder
498    pub fn build(self) -> ParquetMetaData {
499        ParquetMetaData {
500            file_metadata: self.file_metadata,
501            row_groups: match self.row_groups {
502                RowGroups::Owned(v) => v.into(),
503                RowGroups::Shared(arc) => arc,
504            },
505            page_index: self.page_index,
506            #[cfg(feature = "encryption")]
507            file_decryptor: self.file_decryptor,
508        }
509    }
510}
511
512impl From<ParquetMetaData> for ParquetMetaDataBuilder {
513    fn from(meta_data: ParquetMetaData) -> Self {
514        Self::new_from_metadata(meta_data)
515    }
516}
517
518thrift_struct!(
519/// A key-value pair for [`FileMetaData`].
520pub struct KeyValue {
521  1: required string key
522  2: optional string value
523}
524);
525
526impl KeyValue {
527    /// Create a new key value pair
528    pub fn new<F2>(key: String, value: F2) -> KeyValue
529    where
530        F2: Into<Option<String>>,
531    {
532        KeyValue {
533            key,
534            value: value.into(),
535        }
536    }
537}
538
539thrift_struct!(
540/// PageEncodingStats for a column chunk and data page.
541pub struct PageEncodingStats {
542  1: required PageType page_type;
543  2: required Encoding encoding;
544  3: required i32 count;
545}
546);
547
548/// Internal representation of the page encoding stats in the [`ColumnChunkMetaData`].
549/// This is not publicly exposed, with different getters defined for each variant.
550#[derive(Debug, Clone, PartialEq)]
551enum ParquetPageEncodingStats {
552    /// The full array of stats as defined in the Parquet spec.
553    Full(Vec<PageEncodingStats>),
554    /// A condensed version of only page encodings seen.
555    Mask(EncodingMask),
556}
557
558/// Reference counted pointer for [`FileMetaData`].
559pub type FileMetaDataPtr = Arc<FileMetaData>;
560
561/// File level metadata for a Parquet file.
562///
563/// Includes the version of the file, metadata, number of rows, schema, and column orders
564#[derive(Debug, Clone, PartialEq)]
565pub struct FileMetaData {
566    version: i32,
567    num_rows: i64,
568    created_by: Option<String>,
569    key_value_metadata: Option<Vec<KeyValue>>,
570    schema_descr: SchemaDescPtr,
571    column_orders: Option<Vec<ColumnOrder>>,
572    #[cfg(feature = "encryption")]
573    encryption_algorithm: Option<Box<EncryptionAlgorithm>>,
574    #[cfg(feature = "encryption")]
575    footer_signing_key_metadata: Option<Vec<u8>>,
576}
577
578impl FileMetaData {
579    /// Creates new file metadata.
580    pub fn new(
581        version: i32,
582        num_rows: i64,
583        created_by: Option<String>,
584        key_value_metadata: Option<Vec<KeyValue>>,
585        schema_descr: SchemaDescPtr,
586        column_orders: Option<Vec<ColumnOrder>>,
587    ) -> Self {
588        FileMetaData {
589            version,
590            num_rows,
591            created_by,
592            key_value_metadata,
593            schema_descr,
594            column_orders,
595            #[cfg(feature = "encryption")]
596            encryption_algorithm: None,
597            #[cfg(feature = "encryption")]
598            footer_signing_key_metadata: None,
599        }
600    }
601
602    #[cfg(feature = "encryption")]
603    pub(crate) fn with_encryption_algorithm(
604        mut self,
605        encryption_algorithm: Option<EncryptionAlgorithm>,
606    ) -> Self {
607        self.encryption_algorithm = encryption_algorithm.map(Box::new);
608        self
609    }
610
611    #[cfg(feature = "encryption")]
612    pub(crate) fn with_footer_signing_key_metadata(
613        mut self,
614        footer_signing_key_metadata: Option<Vec<u8>>,
615    ) -> Self {
616        self.footer_signing_key_metadata = footer_signing_key_metadata;
617        self
618    }
619
620    /// Returns version of this file.
621    pub fn version(&self) -> i32 {
622        self.version
623    }
624
625    /// Returns number of rows in the file.
626    pub fn num_rows(&self) -> i64 {
627        self.num_rows
628    }
629
630    /// String message for application that wrote this file.
631    ///
632    /// This should have the following format:
633    /// `<application> version <application version> (build <application build hash>)`.
634    ///
635    /// ```shell
636    /// parquet-mr version 1.8.0 (build 0fda28af84b9746396014ad6a415b90592a98b3b)
637    /// ```
638    pub fn created_by(&self) -> Option<&str> {
639        self.created_by.as_deref()
640    }
641
642    /// Returns key_value_metadata of this file.
643    pub fn key_value_metadata(&self) -> Option<&Vec<KeyValue>> {
644        self.key_value_metadata.as_ref()
645    }
646
647    /// Returns Parquet [`Type`] that describes schema in this file.
648    ///
649    /// [`Type`]: crate::schema::types::Type
650    pub fn schema(&self) -> &SchemaType {
651        self.schema_descr.root_schema()
652    }
653
654    /// Returns a reference to schema descriptor.
655    pub fn schema_descr(&self) -> &SchemaDescriptor {
656        &self.schema_descr
657    }
658
659    /// Returns reference counted clone for schema descriptor.
660    pub fn schema_descr_ptr(&self) -> SchemaDescPtr {
661        self.schema_descr.clone()
662    }
663
664    /// Column (sort) order used for `min` and `max` values of each column in this file.
665    ///
666    /// Each column order corresponds to one column, determined by its position in the
667    /// list, matching the position of the column in the schema.
668    ///
669    /// When `None` is returned, there are no column orders available, and each column
670    /// should be assumed to have undefined (legacy) column order.
671    pub fn column_orders(&self) -> Option<&Vec<ColumnOrder>> {
672        self.column_orders.as_ref()
673    }
674
675    /// Returns column order for `i`th column in this file.
676    /// If column orders are not available, returns undefined (legacy) column order.
677    pub fn column_order(&self, i: usize) -> ColumnOrder {
678        self.column_orders
679            .as_ref()
680            .map(|data| data[i])
681            .unwrap_or(ColumnOrder::UNDEFINED)
682    }
683}
684
685thrift_struct!(
686/// Sort order within a RowGroup of a leaf column
687pub struct SortingColumn {
688  /// The ordinal position of the column (in this row group)
689  1: required i32 column_idx
690
691  /// If true, indicates this column is sorted in descending order.
692  2: required bool descending
693
694  /// If true, nulls will come before non-null values, otherwise,
695  /// nulls go at the end. */
696  3: required bool nulls_first
697}
698);
699
700/// Reference counted pointer for [`RowGroupMetaData`].
701pub type RowGroupMetaDataPtr = Arc<RowGroupMetaData>;
702
703/// Metadata for a row group
704///
705/// Includes [`ColumnChunkMetaData`] for each column in the row group, the number of rows
706/// the total byte size of the row group, and the [`SchemaDescriptor`] for the row group.
707#[derive(Debug, Clone, PartialEq)]
708pub struct RowGroupMetaData {
709    columns: Vec<ColumnChunkMetaData>,
710    num_rows: i64,
711    sorting_columns: Option<Vec<SortingColumn>>,
712    total_byte_size: i64,
713    schema_descr: SchemaDescPtr,
714    /// We can't infer from file offset of first column since there may empty columns in row group.
715    file_offset: Option<i64>,
716    /// Ordinal position of this row group in file
717    ordinal: Option<i32>,
718}
719
720impl RowGroupMetaData {
721    /// Returns builder for row group metadata.
722    pub fn builder(schema_descr: SchemaDescPtr) -> RowGroupMetaDataBuilder {
723        RowGroupMetaDataBuilder::new(schema_descr)
724    }
725
726    /// Number of columns in this row group.
727    pub fn num_columns(&self) -> usize {
728        self.columns.len()
729    }
730
731    /// Returns column chunk metadata for `i`th column.
732    pub fn column(&self, i: usize) -> &ColumnChunkMetaData {
733        &self.columns[i]
734    }
735
736    /// Returns slice of column chunk metadata.
737    pub fn columns(&self) -> &[ColumnChunkMetaData] {
738        &self.columns
739    }
740
741    /// Returns mutable slice of column chunk metadata.
742    pub fn columns_mut(&mut self) -> &mut [ColumnChunkMetaData] {
743        &mut self.columns
744    }
745
746    /// Number of rows in this row group.
747    pub fn num_rows(&self) -> i64 {
748        self.num_rows
749    }
750
751    /// Returns the sort ordering of the rows in this RowGroup if any
752    pub fn sorting_columns(&self) -> Option<&Vec<SortingColumn>> {
753        self.sorting_columns.as_ref()
754    }
755
756    /// Total byte size of all uncompressed column data in this row group.
757    pub fn total_byte_size(&self) -> i64 {
758        self.total_byte_size
759    }
760
761    /// Total size of all compressed column data in this row group.
762    pub fn compressed_size(&self) -> i64 {
763        self.columns.iter().map(|c| c.total_compressed_size).sum()
764    }
765
766    /// Returns reference to a schema descriptor.
767    pub fn schema_descr(&self) -> &SchemaDescriptor {
768        self.schema_descr.as_ref()
769    }
770
771    /// Returns reference counted clone of schema descriptor.
772    pub fn schema_descr_ptr(&self) -> SchemaDescPtr {
773        self.schema_descr.clone()
774    }
775
776    /// Returns ordinal position of this row group in file.
777    ///
778    /// For example if this is the first row group in the file, this will return 0.
779    /// If this is the second row group in the file, this will return 1.
780    #[inline(always)]
781    pub fn ordinal(&self) -> Option<i32> {
782        self.ordinal
783    }
784
785    /// Returns file offset of this row group in file.
786    #[inline(always)]
787    pub fn file_offset(&self) -> Option<i64> {
788        self.file_offset
789    }
790
791    /// Converts this [`RowGroupMetaData`] into a [`RowGroupMetaDataBuilder`]
792    pub fn into_builder(self) -> RowGroupMetaDataBuilder {
793        RowGroupMetaDataBuilder(self)
794    }
795}
796
797impl HeapSize for Arc<[RowGroupMetaData]> {
798    fn heap_size(&self) -> usize {
799        // account for the contents of the slice
800        let rg_size = self.iter().map(|t| t.heap_size()).sum::<usize>();
801        // The heap block contains the strong and weak counts followed by the slice,
802        // The fat pointer itself is stored inline.
803        2 * std::mem::size_of::<usize>() + std::mem::size_of_val(self.as_ref()) + rg_size
804    }
805}
806
807/// Builder for row group metadata.
808pub struct RowGroupMetaDataBuilder(RowGroupMetaData);
809
810impl RowGroupMetaDataBuilder {
811    /// Creates new builder from schema descriptor.
812    fn new(schema_descr: SchemaDescPtr) -> Self {
813        Self(RowGroupMetaData {
814            columns: Vec::with_capacity(schema_descr.num_columns()),
815            schema_descr,
816            file_offset: None,
817            num_rows: 0,
818            sorting_columns: None,
819            total_byte_size: 0,
820            ordinal: None,
821        })
822    }
823
824    /// Sets number of rows in this row group.
825    pub fn set_num_rows(mut self, value: i64) -> Self {
826        self.0.num_rows = value;
827        self
828    }
829
830    /// Sets the sorting order for columns
831    pub fn set_sorting_columns(mut self, value: Option<Vec<SortingColumn>>) -> Self {
832        self.0.sorting_columns = value;
833        self
834    }
835
836    /// Sets total size in bytes for this row group.
837    pub fn set_total_byte_size(mut self, value: i64) -> Self {
838        self.0.total_byte_size = value;
839        self
840    }
841
842    /// Takes ownership of the column metadata in this builder, and clears
843    /// the list of columns.
844    ///
845    /// This can be used for more efficient creation of a new RowGroupMetaData
846    /// from an existing one.
847    pub fn take_columns(&mut self) -> Vec<ColumnChunkMetaData> {
848        std::mem::take(&mut self.0.columns)
849    }
850
851    /// Sets column metadata for this row group.
852    pub fn set_column_metadata(mut self, value: Vec<ColumnChunkMetaData>) -> Self {
853        self.0.columns = value;
854        self
855    }
856
857    /// Adds a column metadata to this row group
858    pub fn add_column_metadata(mut self, value: ColumnChunkMetaData) -> Self {
859        self.0.columns.push(value);
860        self
861    }
862
863    /// Sets ordinal for this row group.
864    pub fn set_ordinal(mut self, value: i32) -> Self {
865        self.0.ordinal = Some(value);
866        self
867    }
868
869    /// Sets file offset for this row group.
870    pub fn set_file_offset(mut self, value: i64) -> Self {
871        self.0.file_offset = Some(value);
872        self
873    }
874
875    /// Builds row group metadata.
876    pub fn build(self) -> Result<RowGroupMetaData> {
877        if self.0.schema_descr.num_columns() != self.0.columns.len() {
878            return Err(general_err!(
879                "Column length mismatch: {} != {}",
880                self.0.schema_descr.num_columns(),
881                self.0.columns.len()
882            ));
883        }
884
885        Ok(self.0)
886    }
887
888    /// Build row group metadata without validation.
889    pub(super) fn build_unchecked(self) -> RowGroupMetaData {
890        self.0
891    }
892}
893
894/// Metadata for a column chunk.
895#[derive(Debug, Clone, PartialEq)]
896pub struct ColumnChunkMetaData {
897    column_descr: ColumnDescPtr,
898    encodings: EncodingMask,
899    file_path: Option<String>,
900    file_offset: i64,
901    num_values: i64,
902    compression: CompressionCodec,
903    total_compressed_size: i64,
904    total_uncompressed_size: i64,
905    data_page_offset: i64,
906    index_page_offset: Option<i64>,
907    dictionary_page_offset: Option<i64>,
908    statistics: Option<Statistics>,
909    geo_statistics: Option<Box<geo_statistics::GeospatialStatistics>>,
910    encoding_stats: Option<ParquetPageEncodingStats>,
911    bloom_filter_offset: Option<i64>,
912    bloom_filter_length: Option<i32>,
913    offset_index_offset: Option<i64>,
914    offset_index_length: Option<i32>,
915    column_index_offset: Option<i64>,
916    column_index_length: Option<i32>,
917    unencoded_byte_array_data_bytes: Option<i64>,
918    repetition_level_histogram: Option<LevelHistogram>,
919    definition_level_histogram: Option<LevelHistogram>,
920    #[cfg(feature = "encryption")]
921    column_crypto_metadata: Option<Box<ColumnCryptoMetaData>>,
922    #[cfg(feature = "encryption")]
923    encrypted_column_metadata: Option<Vec<u8>>,
924    /// When true, indicates the footer is plaintext (not encrypted).
925    /// This affects how column metadata is serialized when `encrypted_column_metadata` is present.
926    /// This field is only used at write time and is not needed when reading metadata.
927    #[cfg(feature = "encryption")]
928    plaintext_footer_mode: bool,
929}
930
931/// Histograms for repetition and definition levels.
932///
933/// Each histogram is a vector of length `max_level + 1`. The value at index `i` is the number of
934/// values at level `i`.
935///
936/// For example, `vec[0]` is the number of rows with level 0, `vec[1]` is the
937/// number of rows with level 1, and so on.
938///
939#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)]
940pub struct LevelHistogram {
941    inner: Vec<i64>,
942}
943
944impl LevelHistogram {
945    /// Creates a new level histogram data.
946    ///
947    /// Length will be `max_level + 1`.
948    ///
949    /// Returns `None` when `max_level == 0` (because histograms are not necessary in this case)
950    pub fn try_new(max_level: i16) -> Option<Self> {
951        if max_level > 0 {
952            Some(Self {
953                inner: vec![0; max_level as usize + 1],
954            })
955        } else {
956            None
957        }
958    }
959    /// Returns a reference to the histogram's values.
960    pub fn values(&self) -> &[i64] {
961        &self.inner
962    }
963
964    /// Return the inner vector, consuming self
965    pub fn into_inner(self) -> Vec<i64> {
966        self.inner
967    }
968
969    /// Returns the histogram value at the given index.
970    ///
971    /// The value of `i` is the number of values with level `i`. For example,
972    /// `get(1)` returns the number of values with level 1.
973    ///
974    /// Returns `None` if the index is out of bounds.
975    pub fn get(&self, index: usize) -> Option<i64> {
976        self.inner.get(index).copied()
977    }
978
979    /// Adds the values from the other histogram to this histogram
980    ///
981    /// # Panics
982    /// If the histograms have different lengths
983    pub fn add(&mut self, other: &Self) {
984        assert_eq!(self.len(), other.len());
985        for (dst, src) in self.inner.iter_mut().zip(other.inner.iter()) {
986            *dst += src;
987        }
988    }
989
990    /// return the length of the histogram
991    pub fn len(&self) -> usize {
992        self.inner.len()
993    }
994
995    /// returns if the histogram is empty
996    pub fn is_empty(&self) -> bool {
997        self.inner.is_empty()
998    }
999
1000    /// Sets the values of all histogram levels to 0.
1001    pub fn reset(&mut self) {
1002        self.inner.fill(0);
1003    }
1004
1005    /// Increments the count for a level value by `count`.
1006    #[inline]
1007    pub fn increment_by(&mut self, level: i16, count: i64) {
1008        self.inner[level as usize] += count;
1009    }
1010
1011    /// Updates histogram values using provided repetition levels
1012    ///
1013    /// # Panics
1014    /// if any of the levels is greater than the length of the histogram (
1015    /// the argument supplied to [`Self::try_new`])
1016    #[deprecated(since = "58.2.0", note = "Use `increment_by` instead")]
1017    pub fn update_from_levels(&mut self, levels: &[i16]) {
1018        for &level in levels {
1019            self.increment_by(level, 1);
1020        }
1021    }
1022}
1023
1024impl From<Vec<i64>> for LevelHistogram {
1025    fn from(inner: Vec<i64>) -> Self {
1026        Self { inner }
1027    }
1028}
1029
1030impl From<LevelHistogram> for Vec<i64> {
1031    fn from(value: LevelHistogram) -> Self {
1032        value.into_inner()
1033    }
1034}
1035
1036impl HeapSize for LevelHistogram {
1037    fn heap_size(&self) -> usize {
1038        self.inner.heap_size()
1039    }
1040}
1041
1042/// Represents common operations for a column chunk.
1043impl ColumnChunkMetaData {
1044    /// Returns builder for column chunk metadata.
1045    pub fn builder(column_descr: ColumnDescPtr) -> ColumnChunkMetaDataBuilder {
1046        ColumnChunkMetaDataBuilder::new(column_descr)
1047    }
1048
1049    /// File where the column chunk is stored.
1050    ///
1051    /// If not set, assumed to belong to the same file as the metadata.
1052    /// This path is relative to the current file.
1053    pub fn file_path(&self) -> Option<&str> {
1054        self.file_path.as_deref()
1055    }
1056
1057    /// Byte offset of `ColumnMetaData` in `file_path()`.
1058    ///
1059    /// Note that the meaning of this field has been inconsistent between implementations
1060    /// so its use has since been deprecated in the Parquet specification. Modern implementations
1061    /// will set this to `0` to indicate that the `ColumnMetaData` is solely contained in the
1062    /// `ColumnChunk` struct.
1063    pub fn file_offset(&self) -> i64 {
1064        self.file_offset
1065    }
1066
1067    /// Type of this column. Must be primitive.
1068    pub fn column_type(&self) -> Type {
1069        self.column_descr.physical_type()
1070    }
1071
1072    /// Path (or identifier) of this column.
1073    pub fn column_path(&self) -> &ColumnPath {
1074        self.column_descr.path()
1075    }
1076
1077    /// Descriptor for this column.
1078    pub fn column_descr(&self) -> &ColumnDescriptor {
1079        self.column_descr.as_ref()
1080    }
1081
1082    /// Reference counted clone of descriptor for this column.
1083    pub fn column_descr_ptr(&self) -> ColumnDescPtr {
1084        self.column_descr.clone()
1085    }
1086
1087    /// All encodings used for this column.
1088    pub fn encodings(&self) -> impl Iterator<Item = Encoding> {
1089        self.encodings.encodings()
1090    }
1091
1092    /// All encodings used for this column, returned as a bitmask.
1093    pub fn encodings_mask(&self) -> &EncodingMask {
1094        &self.encodings
1095    }
1096
1097    /// Total number of values in this column chunk.
1098    pub fn num_values(&self) -> i64 {
1099        self.num_values
1100    }
1101
1102    /// [`Compression`] for this column.
1103    ///
1104    /// This is a default value suitable for passing to [`WriterPropertiesBuilder::set_compression`].
1105    /// It is constructed from the `codec` field of the Parquet `ColumnMetaData`
1106    ///
1107    /// [`WriterPropertiesBuilder::set_compression`]: crate::file::properties::WriterPropertiesBuilder
1108    pub fn compression(&self) -> Compression {
1109        self.compression.into()
1110    }
1111
1112    /// Returns the compression codec used when writing this column.
1113    pub fn compression_codec(&self) -> CompressionCodec {
1114        self.compression
1115    }
1116
1117    /// Returns the total compressed data size of this column chunk.
1118    pub fn compressed_size(&self) -> i64 {
1119        self.total_compressed_size
1120    }
1121
1122    /// Returns the total uncompressed data size of this column chunk.
1123    pub fn uncompressed_size(&self) -> i64 {
1124        self.total_uncompressed_size
1125    }
1126
1127    /// Returns the offset for the column data.
1128    pub fn data_page_offset(&self) -> i64 {
1129        self.data_page_offset
1130    }
1131
1132    /// Returns the offset for the index page.
1133    pub fn index_page_offset(&self) -> Option<i64> {
1134        self.index_page_offset
1135    }
1136
1137    /// Returns the offset for the dictionary page, if any.
1138    pub fn dictionary_page_offset(&self) -> Option<i64> {
1139        self.dictionary_page_offset
1140    }
1141
1142    /// Returns the offset and length in bytes of the column chunk within the file
1143    ///
1144    /// # Panics
1145    ///
1146    /// Panics if the column start offset or the compressed size is negative
1147    pub fn byte_range(&self) -> (u64, u64) {
1148        let col_start = match self.dictionary_page_offset() {
1149            Some(dictionary_page_offset) => dictionary_page_offset,
1150            None => self.data_page_offset(),
1151        };
1152        let col_len = self.compressed_size();
1153        assert!(
1154            col_start >= 0 && col_len >= 0,
1155            "column start and length should not be negative"
1156        );
1157        (col_start as u64, col_len as u64)
1158    }
1159
1160    /// Returns statistics that are set for this column chunk,
1161    /// or `None` if no statistics are available.
1162    pub fn statistics(&self) -> Option<&Statistics> {
1163        self.statistics.as_ref()
1164    }
1165
1166    /// Returns geospatial statistics that are set for this column chunk,
1167    /// or `None` if no geospatial statistics are available.
1168    pub fn geo_statistics(&self) -> Option<&geo_statistics::GeospatialStatistics> {
1169        self.geo_statistics.as_deref()
1170    }
1171
1172    /// Returns the page encoding statistics, or `None` if no page encoding statistics
1173    /// are available (or they were converted to a mask).
1174    ///
1175    /// Note: By default, this crate converts page encoding statistics to a mask for performance
1176    /// reasons. To get the full statistics, you must set [`ParquetMetaDataOptions::with_encoding_stats_as_mask`]
1177    /// to `false`.
1178    pub fn page_encoding_stats(&self) -> Option<&Vec<PageEncodingStats>> {
1179        match self.encoding_stats.as_ref() {
1180            Some(ParquetPageEncodingStats::Full(stats)) => Some(stats),
1181            _ => None,
1182        }
1183    }
1184
1185    /// Returns the page encoding statistics reduced to a bitmask, or `None` if statistics are
1186    /// not available (or they were left in their original form).
1187    ///
1188    /// Note: This is the default behavior for this crate.
1189    ///
1190    /// The [`PageEncodingStats`] struct was added to the Parquet specification specifically to
1191    /// enable fast determination of whether all pages in a column chunk are dictionary encoded
1192    /// (see <https://github.com/apache/parquet-format/pull/16>).
1193    /// Decoding the full page encoding statistics, however, can be very costly, and is not
1194    /// necessary to support the aforementioned use case. As an alternative, this crate can
1195    /// instead distill the list of `PageEncodingStats` down to a bitmask of just the encodings
1196    /// used for data pages
1197    /// (see [`ParquetMetaDataOptions::set_encoding_stats_as_mask`]).
1198    /// To test for an all-dictionary-encoded chunk one could use this bitmask in the following way:
1199    ///
1200    /// ```rust
1201    /// use parquet::basic::Encoding;
1202    /// use parquet::file::metadata::ColumnChunkMetaData;
1203    /// // test if all data pages in the column chunk are dictionary encoded
1204    /// fn is_all_dictionary_encoded(col_meta: &ColumnChunkMetaData) -> bool {
1205    ///     // check that dictionary encoding was used
1206    ///     col_meta.dictionary_page_offset().is_some()
1207    ///         && col_meta.page_encoding_stats_mask().is_some_and(|mask| {
1208    ///             // mask should only have one bit set, either for PLAIN_DICTIONARY or
1209    ///             // RLE_DICTIONARY
1210    ///             mask.is_only(Encoding::PLAIN_DICTIONARY) || mask.is_only(Encoding::RLE_DICTIONARY)
1211    ///         })
1212    /// }
1213    /// ```
1214    pub fn page_encoding_stats_mask(&self) -> Option<&EncodingMask> {
1215        match self.encoding_stats.as_ref() {
1216            Some(ParquetPageEncodingStats::Mask(stats)) => Some(stats),
1217            _ => None,
1218        }
1219    }
1220
1221    /// Returns the offset for the bloom filter.
1222    pub fn bloom_filter_offset(&self) -> Option<i64> {
1223        self.bloom_filter_offset
1224    }
1225
1226    /// Returns the offset for the bloom filter.
1227    pub fn bloom_filter_length(&self) -> Option<i32> {
1228        self.bloom_filter_length
1229    }
1230
1231    /// Returns the offset for the column index.
1232    pub fn column_index_offset(&self) -> Option<i64> {
1233        self.column_index_offset
1234    }
1235
1236    /// Returns the length for the column index.
1237    pub fn column_index_length(&self) -> Option<i32> {
1238        self.column_index_length
1239    }
1240
1241    /// Returns the range for the column index, if any.
1242    ///
1243    /// The range is `[column_index_offset, column_index_offset + column_index_length)`,
1244    /// where the offset is relative to the start of the Parquet file in which the index
1245    /// resides.
1246    pub fn column_index_range(&self) -> Option<Range<u64>> {
1247        let offset = u64::try_from(self.column_index_offset?).ok()?;
1248        let length = u64::try_from(self.column_index_length?).ok()?;
1249        Some(offset..(offset + length))
1250    }
1251
1252    /// Returns the offset for the offset index.
1253    pub fn offset_index_offset(&self) -> Option<i64> {
1254        self.offset_index_offset
1255    }
1256
1257    /// Returns the length for the offset index.
1258    pub fn offset_index_length(&self) -> Option<i32> {
1259        self.offset_index_length
1260    }
1261
1262    /// Returns the range for the offset index, if any.
1263    ///
1264    /// The range is `[offset_index_offset, offset_index_offset + offset_index_length)`,
1265    /// where the offset is relative to the start of the Parquet file in which the index
1266    /// resides.
1267    pub fn offset_index_range(&self) -> Option<Range<u64>> {
1268        let offset = u64::try_from(self.offset_index_offset?).ok()?;
1269        let length = u64::try_from(self.offset_index_length?).ok()?;
1270        Some(offset..(offset + length))
1271    }
1272
1273    /// Returns the number of bytes of variable length data after decoding.
1274    ///
1275    /// Only set for BYTE_ARRAY columns. This field may not be set by older
1276    /// writers.
1277    pub fn unencoded_byte_array_data_bytes(&self) -> Option<i64> {
1278        self.unencoded_byte_array_data_bytes
1279    }
1280
1281    /// Returns the repetition level histogram.
1282    ///
1283    /// The returned value `vec[i]` is how many values are at repetition level `i`. For example,
1284    /// `vec[0]` indicates how many rows the page contains.
1285    /// This field may not be set by older writers.
1286    pub fn repetition_level_histogram(&self) -> Option<&LevelHistogram> {
1287        self.repetition_level_histogram.as_ref()
1288    }
1289
1290    /// Returns the definition level histogram.
1291    ///
1292    /// The returned value `vec[i]` is how many values are at definition level `i`. For example,
1293    /// `vec[max_definition_level]` indicates how many non-null values are present in the page.
1294    /// This field may not be set by older writers.
1295    pub fn definition_level_histogram(&self) -> Option<&LevelHistogram> {
1296        self.definition_level_histogram.as_ref()
1297    }
1298
1299    /// Returns the encryption metadata for this column chunk.
1300    #[cfg(feature = "encryption")]
1301    pub fn crypto_metadata(&self) -> Option<&ColumnCryptoMetaData> {
1302        self.column_crypto_metadata.as_deref()
1303    }
1304
1305    /// Converts this [`ColumnChunkMetaData`] into a [`ColumnChunkMetaDataBuilder`]
1306    pub fn into_builder(self) -> ColumnChunkMetaDataBuilder {
1307        ColumnChunkMetaDataBuilder::from(self)
1308    }
1309}
1310
1311/// Builder for [`ColumnChunkMetaData`]
1312///
1313/// This builder is used to create a new column chunk metadata or modify an
1314/// existing one.
1315///
1316/// # Example
1317/// ```no_run
1318/// # use parquet::file::metadata::{ColumnChunkMetaData, ColumnChunkMetaDataBuilder};
1319/// # fn get_column_chunk_metadata() -> ColumnChunkMetaData { unimplemented!(); }
1320/// let column_chunk_metadata = get_column_chunk_metadata();
1321/// // create a new builder from existing column chunk metadata
1322/// let builder = ColumnChunkMetaDataBuilder::from(column_chunk_metadata);
1323/// // clear the statistics:
1324/// let column_chunk_metadata: ColumnChunkMetaData = builder
1325///   .clear_statistics()
1326///   .build()
1327///   .unwrap();
1328/// ```
1329pub struct ColumnChunkMetaDataBuilder(ColumnChunkMetaData);
1330
1331impl ColumnChunkMetaDataBuilder {
1332    /// Creates new column chunk metadata builder.
1333    ///
1334    /// See also [`ColumnChunkMetaData::builder`]
1335    fn new(column_descr: ColumnDescPtr) -> Self {
1336        Self(ColumnChunkMetaData {
1337            column_descr,
1338            encodings: Default::default(),
1339            file_path: None,
1340            file_offset: 0,
1341            num_values: 0,
1342            compression: CompressionCodec::UNCOMPRESSED,
1343            total_compressed_size: 0,
1344            total_uncompressed_size: 0,
1345            data_page_offset: 0,
1346            index_page_offset: None,
1347            dictionary_page_offset: None,
1348            statistics: None,
1349            geo_statistics: None,
1350            encoding_stats: None,
1351            bloom_filter_offset: None,
1352            bloom_filter_length: None,
1353            offset_index_offset: None,
1354            offset_index_length: None,
1355            column_index_offset: None,
1356            column_index_length: None,
1357            unencoded_byte_array_data_bytes: None,
1358            repetition_level_histogram: None,
1359            definition_level_histogram: None,
1360            #[cfg(feature = "encryption")]
1361            column_crypto_metadata: None,
1362            #[cfg(feature = "encryption")]
1363            encrypted_column_metadata: None,
1364            #[cfg(feature = "encryption")]
1365            plaintext_footer_mode: false,
1366        })
1367    }
1368
1369    /// Sets list of encodings for this column chunk.
1370    pub fn set_encodings(mut self, encodings: Vec<Encoding>) -> Self {
1371        self.0.encodings = EncodingMask::new_from_encodings(encodings.iter());
1372        self
1373    }
1374
1375    /// Sets the encodings mask for this column chunk.
1376    pub fn set_encodings_mask(mut self, encodings: EncodingMask) -> Self {
1377        self.0.encodings = encodings;
1378        self
1379    }
1380
1381    /// Sets optional file path for this column chunk.
1382    pub fn set_file_path(mut self, value: String) -> Self {
1383        self.0.file_path = Some(value);
1384        self
1385    }
1386
1387    /// Sets number of values.
1388    pub fn set_num_values(mut self, value: i64) -> Self {
1389        self.0.num_values = value;
1390        self
1391    }
1392
1393    /// Sets compression codec given a [`Compression`] configuration value.
1394    pub fn set_compression(mut self, value: Compression) -> Self {
1395        self.0.compression = value.into();
1396        self
1397    }
1398
1399    /// Sets compression codec.
1400    pub fn set_compression_codec(mut self, value: CompressionCodec) -> Self {
1401        self.0.compression = value;
1402        self
1403    }
1404
1405    /// Sets total compressed size in bytes.
1406    pub fn set_total_compressed_size(mut self, value: i64) -> Self {
1407        self.0.total_compressed_size = value;
1408        self
1409    }
1410
1411    /// Sets total uncompressed size in bytes.
1412    pub fn set_total_uncompressed_size(mut self, value: i64) -> Self {
1413        self.0.total_uncompressed_size = value;
1414        self
1415    }
1416
1417    /// Sets data page offset in bytes.
1418    pub fn set_data_page_offset(mut self, value: i64) -> Self {
1419        self.0.data_page_offset = value;
1420        self
1421    }
1422
1423    /// Sets optional dictionary page offset in bytes.
1424    pub fn set_dictionary_page_offset(mut self, value: Option<i64>) -> Self {
1425        self.0.dictionary_page_offset = value;
1426        self
1427    }
1428
1429    /// Sets optional index page offset in bytes.
1430    pub fn set_index_page_offset(mut self, value: Option<i64>) -> Self {
1431        self.0.index_page_offset = value;
1432        self
1433    }
1434
1435    /// Sets statistics for this column chunk.
1436    pub fn set_statistics(mut self, value: Statistics) -> Self {
1437        self.0.statistics = Some(value);
1438        self
1439    }
1440
1441    /// Sets geospatial statistics for this column chunk.
1442    pub fn set_geo_statistics(mut self, value: Box<geo_statistics::GeospatialStatistics>) -> Self {
1443        self.0.geo_statistics = Some(value);
1444        self
1445    }
1446
1447    /// Clears the statistics for this column chunk.
1448    pub fn clear_statistics(mut self) -> Self {
1449        self.0.statistics = None;
1450        self
1451    }
1452
1453    /// Sets page encoding stats for this column chunk.
1454    ///
1455    /// This will overwrite any existing stats, either `Vec` based or bitmask.
1456    pub fn set_page_encoding_stats(mut self, value: Vec<PageEncodingStats>) -> Self {
1457        self.0.encoding_stats = Some(ParquetPageEncodingStats::Full(value));
1458        self
1459    }
1460
1461    /// Sets page encoding stats mask for this column chunk.
1462    ///
1463    /// This will overwrite any existing stats, either `Vec` based or bitmask.
1464    pub fn set_page_encoding_stats_mask(mut self, value: EncodingMask) -> Self {
1465        self.0.encoding_stats = Some(ParquetPageEncodingStats::Mask(value));
1466        self
1467    }
1468
1469    /// Clears the page encoding stats for this column chunk.
1470    pub fn clear_page_encoding_stats(mut self) -> Self {
1471        self.0.encoding_stats = None;
1472        self
1473    }
1474
1475    /// Sets optional bloom filter offset in bytes.
1476    pub fn set_bloom_filter_offset(mut self, value: Option<i64>) -> Self {
1477        self.0.bloom_filter_offset = value;
1478        self
1479    }
1480
1481    /// Sets optional bloom filter length in bytes.
1482    pub fn set_bloom_filter_length(mut self, value: Option<i32>) -> Self {
1483        self.0.bloom_filter_length = value;
1484        self
1485    }
1486
1487    /// Sets optional offset index offset in bytes.
1488    pub fn set_offset_index_offset(mut self, value: Option<i64>) -> Self {
1489        self.0.offset_index_offset = value;
1490        self
1491    }
1492
1493    /// Sets optional offset index length in bytes.
1494    pub fn set_offset_index_length(mut self, value: Option<i32>) -> Self {
1495        self.0.offset_index_length = value;
1496        self
1497    }
1498
1499    /// Sets optional column index offset in bytes.
1500    pub fn set_column_index_offset(mut self, value: Option<i64>) -> Self {
1501        self.0.column_index_offset = value;
1502        self
1503    }
1504
1505    /// Sets optional column index length in bytes.
1506    pub fn set_column_index_length(mut self, value: Option<i32>) -> Self {
1507        self.0.column_index_length = value;
1508        self
1509    }
1510
1511    /// Sets optional length of variable length data in bytes.
1512    pub fn set_unencoded_byte_array_data_bytes(mut self, value: Option<i64>) -> Self {
1513        self.0.unencoded_byte_array_data_bytes = value;
1514        self
1515    }
1516
1517    /// Sets optional repetition level histogram
1518    pub fn set_repetition_level_histogram(mut self, value: Option<LevelHistogram>) -> Self {
1519        self.0.repetition_level_histogram = value;
1520        self
1521    }
1522
1523    /// Sets optional repetition level histogram
1524    pub fn set_definition_level_histogram(mut self, value: Option<LevelHistogram>) -> Self {
1525        self.0.definition_level_histogram = value;
1526        self
1527    }
1528
1529    #[cfg(feature = "encryption")]
1530    /// Set the encryption metadata for an encrypted column
1531    pub fn set_column_crypto_metadata(mut self, value: Option<ColumnCryptoMetaData>) -> Self {
1532        self.0.column_crypto_metadata = value.map(Box::new);
1533        self
1534    }
1535
1536    #[cfg(feature = "encryption")]
1537    /// Set the encryption metadata for an encrypted column
1538    pub fn set_encrypted_column_metadata(mut self, value: Option<Vec<u8>>) -> Self {
1539        self.0.encrypted_column_metadata = value;
1540        self
1541    }
1542
1543    /// Builds column chunk metadata.
1544    pub fn build(self) -> Result<ColumnChunkMetaData> {
1545        Ok(self.0)
1546    }
1547}
1548
1549/// Builder for Parquet [`ColumnIndex`], part of the Parquet [PageIndex]
1550///
1551/// [PageIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
1552/// [`ColumnIndex`]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
1553pub struct ColumnIndexBuilder {
1554    column_type: Type,
1555    null_pages: Vec<bool>,
1556    min_values: Vec<Vec<u8>>,
1557    max_values: Vec<Vec<u8>>,
1558    null_counts: Vec<i64>,
1559    nan_counts: Vec<Option<i64>>,
1560    boundary_order: BoundaryOrder,
1561    /// contains the concatenation of the histograms of all pages
1562    repetition_level_histograms: Option<Vec<i64>>,
1563    /// contains the concatenation of the histograms of all pages
1564    definition_level_histograms: Option<Vec<i64>>,
1565    /// Is the information in the builder valid?
1566    ///
1567    /// Set to `false` if any entry in the page doesn't have statistics for
1568    /// some reason, so statistics for that page won't be written to the file.
1569    /// This might happen if the page is entirely null, or
1570    /// is a floating point column without any non-nan values
1571    /// e.g. <https://github.com/apache/parquet-format/pull/196>
1572    valid: bool,
1573}
1574
1575impl ColumnIndexBuilder {
1576    /// Creates a new column index builder.
1577    pub fn new(column_type: Type) -> Self {
1578        ColumnIndexBuilder {
1579            column_type,
1580            null_pages: Vec::new(),
1581            min_values: Vec::new(),
1582            max_values: Vec::new(),
1583            null_counts: Vec::new(),
1584            nan_counts: Vec::new(),
1585            boundary_order: BoundaryOrder::UNORDERED,
1586            repetition_level_histograms: None,
1587            definition_level_histograms: None,
1588            valid: true,
1589        }
1590    }
1591
1592    /// Append statistics for the next page
1593    ///
1594    /// For floating-point columns (FLOAT, DOUBLE, or FLOAT16), `nan_count` must always
1595    /// be `Some(n)`, even if n is 0. For non-floating-point columns, `nan_count` must
1596    /// always be `None`. This requirement ensures correct serialization according to
1597    /// the Parquet specification.
1598    pub fn append(
1599        &mut self,
1600        null_page: bool,
1601        min_value: Vec<u8>,
1602        max_value: Vec<u8>,
1603        null_count: i64,
1604        nan_count: Option<i64>,
1605    ) {
1606        self.null_pages.push(null_page);
1607        self.min_values.push(min_value);
1608        self.max_values.push(max_value);
1609        self.null_counts.push(null_count);
1610        self.nan_counts.push(nan_count);
1611    }
1612
1613    /// Append the given page-level histograms to the [`ColumnIndex`] histograms.
1614    /// Does nothing if the `ColumnIndexBuilder` is not in the `valid` state.
1615    ///
1616    /// [`ColumnIndex`]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
1617    pub fn append_histograms(
1618        &mut self,
1619        repetition_level_histogram: &Option<LevelHistogram>,
1620        definition_level_histogram: &Option<LevelHistogram>,
1621    ) {
1622        if !self.valid {
1623            return;
1624        }
1625        if let Some(rep_lvl_hist) = repetition_level_histogram {
1626            let hist = self.repetition_level_histograms.get_or_insert(Vec::new());
1627            hist.reserve(rep_lvl_hist.len());
1628            hist.extend(rep_lvl_hist.values());
1629        }
1630        if let Some(def_lvl_hist) = definition_level_histogram {
1631            let hist = self.definition_level_histograms.get_or_insert(Vec::new());
1632            hist.reserve(def_lvl_hist.len());
1633            hist.extend(def_lvl_hist.values());
1634        }
1635    }
1636
1637    /// Set the boundary order of the column index
1638    pub fn set_boundary_order(&mut self, boundary_order: BoundaryOrder) {
1639        self.boundary_order = boundary_order;
1640    }
1641
1642    /// Mark this column index as invalid
1643    pub fn to_invalid(&mut self) {
1644        self.valid = false;
1645    }
1646
1647    /// Is the information in the builder valid?
1648    pub fn valid(&self) -> bool {
1649        self.valid
1650    }
1651
1652    /// Build and get the column index
1653    ///
1654    /// Note: callers should check [`Self::valid`] before calling this method
1655    pub fn build(self) -> Result<ColumnIndexMetaData> {
1656        Ok(match self.column_type {
1657            Type::BOOLEAN => {
1658                let index = self.build_page_index(false)?;
1659                ColumnIndexMetaData::BOOLEAN(index)
1660            }
1661            Type::INT32 => {
1662                let index = self.build_page_index(false)?;
1663                ColumnIndexMetaData::INT32(index)
1664            }
1665            Type::INT64 => {
1666                let index = self.build_page_index(false)?;
1667                ColumnIndexMetaData::INT64(index)
1668            }
1669            Type::INT96 => {
1670                let index = self.build_page_index(false)?;
1671                ColumnIndexMetaData::INT96(index)
1672            }
1673            Type::FLOAT => {
1674                let index = self.build_page_index(true)?;
1675                ColumnIndexMetaData::FLOAT(index)
1676            }
1677            Type::DOUBLE => {
1678                let index = self.build_page_index(true)?;
1679                ColumnIndexMetaData::DOUBLE(index)
1680            }
1681            Type::BYTE_ARRAY => {
1682                let index = self.build_byte_array_index(false)?;
1683                ColumnIndexMetaData::BYTE_ARRAY(index)
1684            }
1685            Type::FIXED_LEN_BYTE_ARRAY => {
1686                let index = self.build_byte_array_index(true)?;
1687                ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(index)
1688            }
1689        })
1690    }
1691
1692    fn build_nan_counts(nan_counts: &[Option<i64>]) -> Option<Vec<i64>> {
1693        let has_some = nan_counts.iter().any(|x| x.is_some());
1694        let has_none = nan_counts.iter().any(|x| x.is_none());
1695
1696        if has_some && !has_none {
1697            Some(nan_counts.iter().map(|x| x.unwrap()).collect())
1698        } else if !has_some && has_none {
1699            None
1700        } else {
1701            debug_assert!(
1702                false,
1703                "Mixed Some/None in nan_counts - caller should provide consistent values"
1704            );
1705            Some(nan_counts.iter().map(|x| x.unwrap_or(0)).collect())
1706        }
1707    }
1708
1709    fn build_page_index<T>(self, may_have_nan: bool) -> Result<PrimitiveColumnIndex<T>>
1710    where
1711        T: ParquetValueType,
1712    {
1713        let min_values: Vec<&[u8]> = self.min_values.iter().map(|v| v.as_slice()).collect();
1714        let max_values: Vec<&[u8]> = self.max_values.iter().map(|v| v.as_slice()).collect();
1715
1716        // Parquet spec requires nan_counts to be either present for all pages or absent entirely.
1717        // Callers must ensure consistency:
1718        // - For floating-point columns: all pages must have Some(n)
1719        // - For non-floating-point columns: all pages must have None
1720        let nan_counts = if may_have_nan && !self.nan_counts.is_empty() {
1721            Self::build_nan_counts(&self.nan_counts)
1722        } else {
1723            None
1724        };
1725
1726        PrimitiveColumnIndex::try_new(
1727            self.null_pages,
1728            self.boundary_order,
1729            Some(self.null_counts),
1730            nan_counts,
1731            self.repetition_level_histograms,
1732            self.definition_level_histograms,
1733            min_values,
1734            max_values,
1735        )
1736    }
1737
1738    fn build_byte_array_index(self, may_have_nan: bool) -> Result<ByteArrayColumnIndex> {
1739        let min_values: Vec<&[u8]> = self.min_values.iter().map(|v| v.as_slice()).collect();
1740        let max_values: Vec<&[u8]> = self.max_values.iter().map(|v| v.as_slice()).collect();
1741
1742        // Parquet spec requires nan_counts to be either present for all pages or absent entirely.
1743        // Callers must ensure consistency:
1744        // - For floating-point columns: all pages must have Some(n)
1745        // - For non-floating-point columns: all pages must have None
1746        let nan_counts = if may_have_nan && !self.nan_counts.is_empty() {
1747            Self::build_nan_counts(&self.nan_counts)
1748        } else {
1749            None
1750        };
1751
1752        ByteArrayColumnIndex::try_new(
1753            self.null_pages,
1754            self.boundary_order,
1755            Some(self.null_counts),
1756            nan_counts,
1757            self.repetition_level_histograms,
1758            self.definition_level_histograms,
1759            min_values,
1760            max_values,
1761        )
1762    }
1763}
1764
1765impl From<ColumnChunkMetaData> for ColumnChunkMetaDataBuilder {
1766    fn from(value: ColumnChunkMetaData) -> Self {
1767        ColumnChunkMetaDataBuilder(value)
1768    }
1769}
1770
1771/// Builder for offset index, part of the Parquet [PageIndex].
1772///
1773/// [PageIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
1774pub struct OffsetIndexBuilder {
1775    offset_array: Vec<i64>,
1776    compressed_page_size_array: Vec<i32>,
1777    first_row_index_array: Vec<i64>,
1778    unencoded_byte_array_data_bytes_array: Option<Vec<i64>>,
1779    current_first_row_index: i64,
1780}
1781
1782impl Default for OffsetIndexBuilder {
1783    fn default() -> Self {
1784        Self::new()
1785    }
1786}
1787
1788impl OffsetIndexBuilder {
1789    /// Creates a new offset index builder.
1790    pub fn new() -> Self {
1791        OffsetIndexBuilder {
1792            offset_array: Vec::new(),
1793            compressed_page_size_array: Vec::new(),
1794            first_row_index_array: Vec::new(),
1795            unencoded_byte_array_data_bytes_array: None,
1796            current_first_row_index: 0,
1797        }
1798    }
1799
1800    /// Append the row count of the next page.
1801    pub fn append_row_count(&mut self, row_count: i64) {
1802        let current_page_row_index = self.current_first_row_index;
1803        self.first_row_index_array.push(current_page_row_index);
1804        self.current_first_row_index += row_count;
1805    }
1806
1807    /// Append the offset and size of the next page.
1808    pub fn append_offset_and_size(&mut self, offset: i64, compressed_page_size: i32) {
1809        self.offset_array.push(offset);
1810        self.compressed_page_size_array.push(compressed_page_size);
1811    }
1812
1813    /// Append the unencoded byte array data bytes of the next page.
1814    pub fn append_unencoded_byte_array_data_bytes(
1815        &mut self,
1816        unencoded_byte_array_data_bytes: Option<i64>,
1817    ) {
1818        if let Some(val) = unencoded_byte_array_data_bytes {
1819            self.unencoded_byte_array_data_bytes_array
1820                .get_or_insert(Vec::new())
1821                .push(val);
1822        }
1823    }
1824
1825    /// Build and get the thrift metadata of offset index
1826    pub fn build(self) -> OffsetIndexMetaData {
1827        let locations = self
1828            .offset_array
1829            .iter()
1830            .zip(self.compressed_page_size_array.iter())
1831            .zip(self.first_row_index_array.iter())
1832            .map(|((offset, size), row_index)| PageLocation {
1833                offset: *offset,
1834                compressed_page_size: *size,
1835                first_row_index: *row_index,
1836            })
1837            .collect::<Vec<_>>();
1838        OffsetIndexMetaData {
1839            page_locations: locations,
1840            unencoded_byte_array_data_bytes: self.unencoded_byte_array_data_bytes_array,
1841        }
1842    }
1843}
1844
1845#[cfg(test)]
1846mod tests {
1847    use super::*;
1848    use crate::basic::{PageType, SortOrder};
1849    use crate::file::metadata::page_index::PageIndexBuilder;
1850    use crate::file::metadata::thrift::tests::{
1851        read_column_chunk, read_column_chunk_with_options, read_row_group,
1852    };
1853
1854    #[test]
1855    #[expect(deprecated)]
1856    fn test_level_histogram_update_from_levels_compat() {
1857        let mut histogram = LevelHistogram::try_new(2).unwrap();
1858        histogram.update_from_levels(&[0, 2, 1, 2, 2]);
1859        assert_eq!(histogram.values(), &[1, 1, 3]);
1860    }
1861
1862    #[test]
1863    fn test_column_chunk() {
1864        let schema = get_test_schema_descr();
1865        let row_groups = (0..2)
1866            .map(|row| {
1867                let columns = schema
1868                    .columns()
1869                    .iter()
1870                    .enumerate()
1871                    .map(|(column, descriptor)| {
1872                        ColumnChunkMetaData::builder(descriptor.clone())
1873                            .set_data_page_offset((row * 2 + column) as i64)
1874                            .build()
1875                            .unwrap()
1876                    })
1877                    .collect();
1878                RowGroupMetaData::builder(schema.clone())
1879                    .set_column_metadata(columns)
1880                    .build()
1881                    .unwrap()
1882            })
1883            .collect();
1884        let metadata = ParquetMetaData::new(
1885            FileMetaData::new(1, 0, None, None, schema, None),
1886            row_groups,
1887        );
1888        for row in 0..2 {
1889            for column in 0..2 {
1890                assert_eq!(
1891                    metadata
1892                        .column_chunk(row, column)
1893                        .unwrap()
1894                        .data_page_offset(),
1895                    (row * 2 + column) as i64
1896                );
1897            }
1898        }
1899        for index in [2, usize::MAX] {
1900            assert_eq!(
1901                metadata
1902                    .column_chunk(index, usize::MAX)
1903                    .unwrap_err()
1904                    .to_string(),
1905                format!(
1906                    "Parquet error: Row group index {index} out of bounds for file with 2 row groups"
1907                )
1908            );
1909            assert_eq!(
1910                metadata.column_chunk(1, index).unwrap_err().to_string(),
1911                format!(
1912                    "Parquet error: Column index {index} out of bounds for row group 1 with 2 columns"
1913                )
1914            );
1915        }
1916    }
1917
1918    #[test]
1919    fn test_row_group_metadata_thrift_conversion() {
1920        let schema_descr = get_test_schema_descr();
1921
1922        let mut columns = vec![];
1923        for ptr in schema_descr.columns() {
1924            let column = ColumnChunkMetaData::builder(ptr.clone()).build().unwrap();
1925            columns.push(column);
1926        }
1927        let row_group_meta = RowGroupMetaData::builder(schema_descr.clone())
1928            .set_num_rows(1000)
1929            .set_total_byte_size(2000)
1930            .set_column_metadata(columns)
1931            .set_ordinal(1)
1932            .build()
1933            .unwrap();
1934
1935        let mut buf = Vec::new();
1936        let mut writer = ThriftCompactOutputProtocol::new(&mut buf);
1937        row_group_meta.write_thrift(&mut writer).unwrap();
1938
1939        let row_group_res = read_row_group(&buf, schema_descr).unwrap();
1940
1941        assert_eq!(row_group_res, row_group_meta);
1942    }
1943
1944    #[test]
1945    fn test_row_group_metadata_thrift_conversion_empty() {
1946        let schema_descr = get_test_schema_descr();
1947
1948        let row_group_meta = RowGroupMetaData::builder(schema_descr).build();
1949
1950        assert!(row_group_meta.is_err());
1951        if let Err(e) = row_group_meta {
1952            assert_eq!(
1953                format!("{e}"),
1954                "Parquet error: Column length mismatch: 2 != 0"
1955            );
1956        }
1957    }
1958
1959    /// Test reading a corrupted Parquet file with 3 columns in its schema but only 2 in its row group
1960    #[test]
1961    fn test_row_group_metadata_thrift_corrupted() {
1962        let schema_descr_2cols = Arc::new(SchemaDescriptor::new(Arc::new(
1963            SchemaType::group_type_builder("schema")
1964                .with_fields(vec![
1965                    Arc::new(
1966                        SchemaType::primitive_type_builder("a", Type::INT32)
1967                            .build()
1968                            .unwrap(),
1969                    ),
1970                    Arc::new(
1971                        SchemaType::primitive_type_builder("b", Type::INT32)
1972                            .build()
1973                            .unwrap(),
1974                    ),
1975                ])
1976                .build()
1977                .unwrap(),
1978        )));
1979
1980        let schema_descr_3cols = Arc::new(SchemaDescriptor::new(Arc::new(
1981            SchemaType::group_type_builder("schema")
1982                .with_fields(vec![
1983                    Arc::new(
1984                        SchemaType::primitive_type_builder("a", Type::INT32)
1985                            .build()
1986                            .unwrap(),
1987                    ),
1988                    Arc::new(
1989                        SchemaType::primitive_type_builder("b", Type::INT32)
1990                            .build()
1991                            .unwrap(),
1992                    ),
1993                    Arc::new(
1994                        SchemaType::primitive_type_builder("c", Type::INT32)
1995                            .build()
1996                            .unwrap(),
1997                    ),
1998                ])
1999                .build()
2000                .unwrap(),
2001        )));
2002
2003        let row_group_meta_2cols = RowGroupMetaData::builder(schema_descr_2cols.clone())
2004            .set_num_rows(1000)
2005            .set_total_byte_size(2000)
2006            .set_column_metadata(vec![
2007                ColumnChunkMetaData::builder(schema_descr_2cols.column(0))
2008                    .build()
2009                    .unwrap(),
2010                ColumnChunkMetaData::builder(schema_descr_2cols.column(1))
2011                    .build()
2012                    .unwrap(),
2013            ])
2014            .set_ordinal(1)
2015            .build()
2016            .unwrap();
2017        let mut buf = Vec::new();
2018        let mut writer = ThriftCompactOutputProtocol::new(&mut buf);
2019        row_group_meta_2cols.write_thrift(&mut writer).unwrap();
2020
2021        let err = read_row_group(&buf, schema_descr_3cols)
2022            .unwrap_err()
2023            .to_string();
2024        assert_eq!(
2025            err,
2026            "Parquet error: Column count mismatch. Schema has 3 columns while Row Group has 2"
2027        );
2028    }
2029
2030    #[test]
2031    fn test_column_chunk_metadata_thrift_conversion() {
2032        let column_descr = get_test_schema_descr().column(0);
2033        let col_metadata = ColumnChunkMetaData::builder(column_descr.clone())
2034            .set_encodings_mask(EncodingMask::new_from_encodings(
2035                [Encoding::PLAIN, Encoding::RLE].iter(),
2036            ))
2037            .set_file_path("file_path".to_owned())
2038            .set_num_values(1000)
2039            .set_compression_codec(CompressionCodec::SNAPPY)
2040            .set_total_compressed_size(2000)
2041            .set_total_uncompressed_size(3000)
2042            .set_data_page_offset(4000)
2043            .set_dictionary_page_offset(Some(5000))
2044            .set_page_encoding_stats(vec![
2045                PageEncodingStats {
2046                    page_type: PageType::DATA_PAGE,
2047                    encoding: Encoding::PLAIN,
2048                    count: 3,
2049                },
2050                PageEncodingStats {
2051                    page_type: PageType::DATA_PAGE,
2052                    encoding: Encoding::RLE,
2053                    count: 5,
2054                },
2055            ])
2056            .set_bloom_filter_offset(Some(6000))
2057            .set_bloom_filter_length(Some(25))
2058            .set_offset_index_offset(Some(7000))
2059            .set_offset_index_length(Some(25))
2060            .set_column_index_offset(Some(8000))
2061            .set_column_index_length(Some(25))
2062            .set_unencoded_byte_array_data_bytes(Some(2000))
2063            .set_repetition_level_histogram(Some(LevelHistogram::from(vec![100, 100])))
2064            .set_definition_level_histogram(Some(LevelHistogram::from(vec![0, 200])))
2065            .build()
2066            .unwrap();
2067
2068        let mut buf = Vec::new();
2069        let mut writer = ThriftCompactOutputProtocol::new(&mut buf);
2070        col_metadata.write_thrift(&mut writer).unwrap();
2071        let col_chunk_res = read_column_chunk(&buf, column_descr.clone()).unwrap();
2072
2073        let expected_metadata = ColumnChunkMetaData::builder(column_descr)
2074            .set_encodings_mask(EncodingMask::new_from_encodings(
2075                [Encoding::PLAIN, Encoding::RLE].iter(),
2076            ))
2077            .set_file_path("file_path".to_owned())
2078            .set_num_values(1000)
2079            .set_compression_codec(CompressionCodec::SNAPPY)
2080            .set_total_compressed_size(2000)
2081            .set_total_uncompressed_size(3000)
2082            .set_data_page_offset(4000)
2083            .set_dictionary_page_offset(Some(5000))
2084            .set_page_encoding_stats_mask(EncodingMask::new_from_encodings(
2085                [Encoding::PLAIN, Encoding::RLE].iter(),
2086            ))
2087            .set_bloom_filter_offset(Some(6000))
2088            .set_bloom_filter_length(Some(25))
2089            .set_offset_index_offset(Some(7000))
2090            .set_offset_index_length(Some(25))
2091            .set_column_index_offset(Some(8000))
2092            .set_column_index_length(Some(25))
2093            .set_unencoded_byte_array_data_bytes(Some(2000))
2094            .set_repetition_level_histogram(Some(LevelHistogram::from(vec![100, 100])))
2095            .set_definition_level_histogram(Some(LevelHistogram::from(vec![0, 200])))
2096            .build()
2097            .unwrap();
2098
2099        assert_eq!(col_chunk_res, expected_metadata);
2100    }
2101
2102    #[test]
2103    fn test_column_chunk_metadata_thrift_conversion_full_stats() {
2104        let column_descr = get_test_schema_descr().column(0);
2105        let stats = vec![
2106            PageEncodingStats {
2107                page_type: PageType::DATA_PAGE,
2108                encoding: Encoding::PLAIN,
2109                count: 3,
2110            },
2111            PageEncodingStats {
2112                page_type: PageType::DATA_PAGE,
2113                encoding: Encoding::RLE,
2114                count: 5,
2115            },
2116        ];
2117        let col_metadata = ColumnChunkMetaData::builder(column_descr.clone())
2118            .set_encodings_mask(EncodingMask::new_from_encodings(
2119                [Encoding::PLAIN, Encoding::RLE].iter(),
2120            ))
2121            .set_num_values(1000)
2122            .set_compression_codec(CompressionCodec::SNAPPY)
2123            .set_total_compressed_size(2000)
2124            .set_total_uncompressed_size(3000)
2125            .set_data_page_offset(4000)
2126            .set_page_encoding_stats(stats)
2127            .build()
2128            .unwrap();
2129
2130        let mut buf = Vec::new();
2131        let mut writer = ThriftCompactOutputProtocol::new(&mut buf);
2132        col_metadata.write_thrift(&mut writer).unwrap();
2133
2134        let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false);
2135        let col_chunk_res =
2136            read_column_chunk_with_options(&buf, column_descr, Some(&options)).unwrap();
2137
2138        assert_eq!(col_chunk_res, col_metadata);
2139    }
2140
2141    #[test]
2142    fn test_column_chunk_metadata_thrift_conversion_empty() {
2143        let column_descr = get_test_schema_descr().column(0);
2144
2145        let col_metadata = ColumnChunkMetaData::builder(column_descr.clone())
2146            .build()
2147            .unwrap();
2148
2149        let mut buf = Vec::new();
2150        let mut writer = ThriftCompactOutputProtocol::new(&mut buf);
2151        col_metadata.write_thrift(&mut writer).unwrap();
2152        let col_chunk_res = read_column_chunk(&buf, column_descr).unwrap();
2153
2154        assert_eq!(col_chunk_res, col_metadata);
2155    }
2156
2157    #[test]
2158    fn test_compressed_size() {
2159        let schema_descr = get_test_schema_descr();
2160
2161        let mut columns = vec![];
2162        for column_descr in schema_descr.columns() {
2163            let column = ColumnChunkMetaData::builder(column_descr.clone())
2164                .set_total_compressed_size(500)
2165                .set_total_uncompressed_size(700)
2166                .build()
2167                .unwrap();
2168            columns.push(column);
2169        }
2170        let row_group_meta = RowGroupMetaData::builder(schema_descr)
2171            .set_num_rows(1000)
2172            .set_column_metadata(columns)
2173            .build()
2174            .unwrap();
2175
2176        let compressed_size_res = row_group_meta.compressed_size();
2177        let compressed_size_exp: i64 = 1000;
2178
2179        assert_eq!(compressed_size_res, compressed_size_exp);
2180    }
2181
2182    #[test]
2183    fn test_memory_size() {
2184        let schema_descr = get_test_schema_descr();
2185
2186        let columns = schema_descr
2187            .columns()
2188            .iter()
2189            .map(|column_descr| {
2190                ColumnChunkMetaData::builder(column_descr.clone())
2191                    .set_statistics(Statistics::new::<i32>(None, None, None, None, false))
2192                    .build()
2193            })
2194            .collect::<Result<Vec<_>>>()
2195            .unwrap();
2196        let row_group_meta = RowGroupMetaData::builder(schema_descr.clone())
2197            .set_num_rows(1000)
2198            .set_column_metadata(columns)
2199            .build()
2200            .unwrap();
2201        let row_group_meta = vec![row_group_meta];
2202
2203        let version = 2;
2204        let num_rows = 1000;
2205        let created_by = Some(String::from("test harness"));
2206        let key_value_metadata = Some(vec![KeyValue::new(
2207            String::from("Foo"),
2208            Some(String::from("bar")),
2209        )]);
2210        let column_orders = Some(vec![
2211            ColumnOrder::UNDEFINED,
2212            ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::UNSIGNED),
2213        ]);
2214        let file_metadata = FileMetaData::new(
2215            version,
2216            num_rows,
2217            created_by,
2218            key_value_metadata,
2219            schema_descr.clone(),
2220            column_orders,
2221        );
2222
2223        // Now, add in Exact Statistics
2224        let columns_with_stats = schema_descr
2225            .columns()
2226            .iter()
2227            .map(|column_descr| {
2228                ColumnChunkMetaData::builder(column_descr.clone())
2229                    .set_statistics(Statistics::new::<i32>(
2230                        Some(0),
2231                        Some(100),
2232                        None,
2233                        None,
2234                        false,
2235                    ))
2236                    .build()
2237            })
2238            .collect::<Result<Vec<_>>>()
2239            .unwrap();
2240
2241        let row_group_meta_with_stats = RowGroupMetaData::builder(schema_descr)
2242            .set_num_rows(1000)
2243            .set_column_metadata(columns_with_stats)
2244            .build()
2245            .unwrap();
2246        let row_group_meta_with_stats = vec![row_group_meta_with_stats];
2247
2248        let parquet_meta = ParquetMetaDataBuilder::new(file_metadata.clone())
2249            .set_row_groups(row_group_meta_with_stats)
2250            .build();
2251
2252        // Base size without page index
2253        #[cfg(not(feature = "encryption"))]
2254        let base_expected_size = 2798;
2255        #[cfg(feature = "encryption")]
2256        let base_expected_size = 2966;
2257
2258        assert_eq!(parquet_meta.memory_size(), base_expected_size);
2259
2260        let mut page_index = PageIndexBuilder::new(1, 1);
2261
2262        let mut column_index = ColumnIndexBuilder::new(Type::BOOLEAN);
2263        column_index.append(false, vec![1u8], vec![2u8, 3u8], 4, None);
2264        let column_index = column_index.build().unwrap();
2265        {
2266            let ColumnIndexMetaData::BOOLEAN(_) = column_index else {
2267                panic!("wrong type of column index")
2268            };
2269        }
2270
2271        page_index.try_put_column_index(column_index, 0, 0).unwrap();
2272
2273        // Now, add in OffsetIndex
2274        let mut offset_index = OffsetIndexBuilder::new();
2275        offset_index.append_row_count(1);
2276        offset_index.append_offset_and_size(2, 3);
2277        offset_index.append_unencoded_byte_array_data_bytes(Some(10));
2278        offset_index.append_row_count(1);
2279        offset_index.append_offset_and_size(2, 3);
2280        offset_index.append_unencoded_byte_array_data_bytes(Some(10));
2281        let offset_index = offset_index.build();
2282        page_index.try_put_offset_index(offset_index, 0, 0).unwrap();
2283
2284        let parquet_meta = ParquetMetaDataBuilder::new(file_metadata)
2285            .set_row_groups(row_group_meta)
2286            .set_page_index(Some(Arc::new(page_index.build())))
2287            .build();
2288
2289        // Size with page index (includes Arc overhead plus PageIndex heap size)
2290        #[cfg(not(feature = "encryption"))]
2291        let bigger_expected_size = 3296;
2292        #[cfg(feature = "encryption")]
2293        let bigger_expected_size = 3464;
2294
2295        // more set fields means more memory usage
2296        assert!(bigger_expected_size > base_expected_size);
2297        assert_eq!(parquet_meta.memory_size(), bigger_expected_size);
2298    }
2299
2300    #[test]
2301    #[cfg(feature = "encryption")]
2302    fn test_memory_size_with_decryptor() {
2303        use crate::encryption::decrypt::FileDecryptionProperties;
2304        use crate::file::metadata::thrift::encryption::AesGcmV1;
2305
2306        let schema_descr = get_test_schema_descr();
2307
2308        let columns = schema_descr
2309            .columns()
2310            .iter()
2311            .map(|column_descr| ColumnChunkMetaData::builder(column_descr.clone()).build())
2312            .collect::<Result<Vec<_>>>()
2313            .unwrap();
2314        let row_group_meta = RowGroupMetaData::builder(schema_descr.clone())
2315            .set_num_rows(1000)
2316            .set_column_metadata(columns)
2317            .build()
2318            .unwrap();
2319        let row_group_meta = vec![row_group_meta];
2320
2321        let version = 2;
2322        let num_rows = 1000;
2323        let aad_file_unique = vec![1u8; 8];
2324        let aad_prefix = vec![2u8; 8];
2325        let encryption_algorithm = EncryptionAlgorithm::AES_GCM_V1(AesGcmV1 {
2326            aad_prefix: Some(aad_prefix.clone()),
2327            aad_file_unique: Some(aad_file_unique.clone()),
2328            supply_aad_prefix: Some(true),
2329        });
2330        let footer_key_metadata = Some(vec![3u8; 8]);
2331        let file_metadata =
2332            FileMetaData::new(version, num_rows, None, None, schema_descr.clone(), None)
2333                .with_encryption_algorithm(Some(encryption_algorithm))
2334                .with_footer_signing_key_metadata(footer_key_metadata.clone());
2335
2336        let parquet_meta_data = ParquetMetaDataBuilder::new(file_metadata.clone())
2337            .set_row_groups(row_group_meta.clone())
2338            .build();
2339
2340        let base_expected_size = 2074;
2341        assert_eq!(parquet_meta_data.memory_size(), base_expected_size);
2342
2343        let footer_key = b"0123456789012345";
2344        let column_key = b"1234567890123450";
2345        let mut decryption_properties_builder =
2346            FileDecryptionProperties::builder(footer_key.to_vec())
2347                .with_aad_prefix(aad_prefix.clone());
2348        for column in schema_descr.columns() {
2349            decryption_properties_builder = decryption_properties_builder
2350                .with_column_key(&column.path().string(), column_key.to_vec());
2351        }
2352        let decryption_properties = decryption_properties_builder.build().unwrap();
2353        let decryptor = FileDecryptor::new(
2354            &decryption_properties,
2355            footer_key_metadata.as_deref(),
2356            aad_file_unique,
2357            aad_prefix,
2358        )
2359        .unwrap();
2360
2361        let parquet_meta_data = ParquetMetaDataBuilder::new(file_metadata.clone())
2362            .set_row_groups(row_group_meta.clone())
2363            .set_file_decryptor(Some(decryptor))
2364            .build();
2365
2366        let expected_size_with_decryptor = 3104;
2367        assert!(expected_size_with_decryptor > base_expected_size);
2368
2369        assert_eq!(
2370            parquet_meta_data.memory_size(),
2371            expected_size_with_decryptor
2372        );
2373    }
2374
2375    /// Returns sample schema descriptor so we can create column metadata.
2376    fn get_test_schema_descr() -> SchemaDescPtr {
2377        let schema = SchemaType::group_type_builder("schema")
2378            .with_fields(vec![
2379                Arc::new(
2380                    SchemaType::primitive_type_builder("a", Type::INT32)
2381                        .build()
2382                        .unwrap(),
2383                ),
2384                Arc::new(
2385                    SchemaType::primitive_type_builder("b", Type::INT32)
2386                        .build()
2387                        .unwrap(),
2388                ),
2389            ])
2390            .build()
2391            .unwrap();
2392
2393        Arc::new(SchemaDescriptor::new(Arc::new(schema)))
2394    }
2395
2396    #[test]
2397    fn test_page_index_builder_partial_population() {
2398        let mut builder = PageIndexBuilder::new(1, 1);
2399
2400        // Add an offset index
2401        let mut offset_idx_builder = OffsetIndexBuilder::new();
2402        offset_idx_builder.append_row_count(50);
2403        offset_idx_builder.append_offset_and_size(1000, 500);
2404        let offset_idx = offset_idx_builder.build();
2405        builder.try_put_offset_index(offset_idx, 0, 0).unwrap();
2406
2407        let page_index = builder.build();
2408
2409        // Column indexes should not exist
2410        assert!(!page_index.has_column_indexes());
2411        // Offset indexes should exist
2412        assert!(page_index.has_offset_indexes());
2413    }
2414
2415    #[test]
2416    fn test_page_index_builder_empty_to_none() {
2417        // Create builder but don't populate any indexes
2418        let builder = PageIndexBuilder::new(2, 2);
2419
2420        let page_index = builder.build();
2421
2422        // Both should be None since they were never populated
2423        assert!(!page_index.has_column_indexes());
2424        assert!(!page_index.has_offset_indexes());
2425    }
2426
2427    #[test]
2428    fn test_set_key_value_metadata_preserves_shared_metadata() {
2429        let original = vec![KeyValue::new("original".to_string(), "value".to_string())];
2430        let replacement = vec![KeyValue::new("replacement".to_string(), "new".to_string())];
2431        for initial in [None, Some(original)] {
2432            let file_metadata = FileMetaData::new(
2433                2,
2434                0,
2435                Some("test writer".to_string()),
2436                initial.clone(),
2437                get_test_schema_descr(),
2438                None,
2439            );
2440            #[cfg(feature = "encryption")]
2441            let file_metadata = file_metadata
2442                .with_encryption_algorithm(Some(EncryptionAlgorithm::AES_GCM_V1(
2443                    crate::file::metadata::thrift::encryption::AesGcmV1 {
2444                        aad_prefix: Some(vec![1; 8]),
2445                        aad_file_unique: Some(vec![2; 8]),
2446                        supply_aad_prefix: Some(true),
2447                    },
2448                )))
2449                .with_footer_signing_key_metadata(Some(vec![3; 8]));
2450            let cached = ParquetMetaDataBuilder::new(file_metadata)
2451                .set_page_index(Some(Arc::new(PageIndexBuilder::new(0, 2).build())))
2452                .build();
2453            for values in [None, Some(vec![]), Some(replacement.clone())] {
2454                let updated = cached
2455                    .clone()
2456                    .into_builder()
2457                    .set_key_value_metadata(values.clone())
2458                    .build();
2459                let mut expected = cached.file_metadata().clone();
2460                expected.key_value_metadata = values;
2461                assert_eq!(updated.file_metadata(), &expected);
2462                assert_eq!(
2463                    cached.file_metadata().key_value_metadata(),
2464                    initial.as_ref()
2465                );
2466                assert!(Arc::ptr_eq(&cached.row_groups, &updated.row_groups));
2467                assert!(Arc::ptr_eq(
2468                    cached.page_index().unwrap(),
2469                    updated.page_index().unwrap()
2470                ));
2471            }
2472        }
2473    }
2474
2475    #[test]
2476    fn test_replace_page_index_preserves_shared_metadata() {
2477        let schema_descr = get_test_schema_descr();
2478        let file_metadata = FileMetaData::new(1, 0, None, None, schema_descr, None);
2479        let cached = Arc::new(ParquetMetaData::new(file_metadata, vec![]));
2480
2481        let page_index = Arc::new(PageIndexBuilder::new(0, 0).build());
2482        let rebuilt = cached
2483            .as_ref()
2484            .clone()
2485            .into_builder()
2486            .set_page_index(Some(page_index))
2487            .build();
2488
2489        assert!(Arc::ptr_eq(&cached.file_metadata, &rebuilt.file_metadata));
2490        assert!(Arc::ptr_eq(&cached.row_groups, &rebuilt.row_groups));
2491    }
2492
2493    #[test]
2494    fn test_rebuild_page_index() {
2495        // Create builder with one rowgroup and two columns
2496        let mut builder = PageIndexBuilder::new(1, 2);
2497
2498        // Add column index for first column
2499        let mut col_idx_builder = ColumnIndexBuilder::new(Type::INT32);
2500        col_idx_builder.append(false, vec![1, 0, 0, 0], vec![100, 0, 0, 0], 10, None);
2501        let col_idx = col_idx_builder.build().unwrap();
2502        builder.try_put_column_index(col_idx.clone(), 0, 0).unwrap();
2503
2504        // Add an offset index for first column
2505        let mut offset_idx_builder = OffsetIndexBuilder::new();
2506        offset_idx_builder.append_row_count(50);
2507        offset_idx_builder.append_offset_and_size(1000, 500);
2508        let offset_idx = offset_idx_builder.build();
2509        builder
2510            .try_put_offset_index(offset_idx.clone(), 0, 0)
2511            .unwrap();
2512
2513        let page_index = builder.build();
2514
2515        // Check indexes
2516        assert!(page_index.is_complete());
2517
2518        // first column populated
2519        assert!(page_index.column_index(0, 0).is_some());
2520        assert!(page_index.offset_index(0, 0).is_some());
2521        // second column not populated
2522        assert!(page_index.column_index(0, 1).is_none());
2523        assert!(page_index.offset_index(0, 1).is_none());
2524
2525        let mut builder = PageIndexBuilder::new_from(page_index);
2526
2527        // Add indexes for second column
2528        builder.try_put_column_index(col_idx, 0, 1).unwrap();
2529        builder.try_put_offset_index(offset_idx, 0, 1).unwrap();
2530        let page_index = builder.build();
2531
2532        // now all populated
2533        assert!(page_index.column_index(0, 0).is_some());
2534        assert!(page_index.offset_index(0, 0).is_some());
2535        assert!(page_index.column_index(0, 1).is_some());
2536        assert!(page_index.offset_index(0, 1).is_some());
2537    }
2538}