Skip to main content

parquet/file/
serialized_reader.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Contains implementations of the reader traits FileReader, RowGroupReader and PageReader
6//! Also contains implementations of the ChunkReader for files (with buffering) and byte arrays (RAM)
7
8use crate::basic::{PageType, Type};
9use crate::bloom_filter::Sbbf;
10use crate::column::page::{Page, PageMetadata, PageReader};
11use crate::compression::{Codec, create_codec};
12#[cfg(feature = "encryption")]
13use crate::encryption::decrypt::{CryptoContext, read_and_decrypt};
14use crate::errors::{ParquetError, Result};
15use crate::file::metadata::page_index::RowGroupPageIndex;
16use crate::file::metadata::thrift::PageHeader;
17use crate::file::page_index::offset_index::PageLocation;
18use crate::file::statistics;
19use crate::file::{
20    metadata::*,
21    properties::{ReaderProperties, ReaderPropertiesPtr},
22    reader::*,
23};
24#[cfg(feature = "encryption")]
25use crate::parquet_thrift::ThriftSliceInputProtocol;
26use crate::parquet_thrift::{ReadThrift, ThriftReadInputProtocol};
27use crate::record::Row;
28use crate::record::reader::RowIter;
29use crate::schema::types::{SchemaDescPtr, Type as SchemaType};
30use bytes::Bytes;
31use std::collections::VecDeque;
32use std::{fs::File, io::Read, path::Path, sync::Arc};
33
34impl TryFrom<File> for SerializedFileReader<File> {
35    type Error = ParquetError;
36
37    fn try_from(file: File) -> Result<Self> {
38        Self::new(file)
39    }
40}
41
42impl TryFrom<&Path> for SerializedFileReader<File> {
43    type Error = ParquetError;
44
45    fn try_from(path: &Path) -> Result<Self> {
46        let file = File::open(path)?;
47        Self::try_from(file)
48    }
49}
50
51impl TryFrom<String> for SerializedFileReader<File> {
52    type Error = ParquetError;
53
54    fn try_from(path: String) -> Result<Self> {
55        Self::try_from(Path::new(&path))
56    }
57}
58
59impl TryFrom<&str> for SerializedFileReader<File> {
60    type Error = ParquetError;
61
62    fn try_from(path: &str) -> Result<Self> {
63        Self::try_from(Path::new(&path))
64    }
65}
66
67/// Conversion into a [`RowIter`]
68/// using the full file schema over all row groups.
69impl IntoIterator for SerializedFileReader<File> {
70    type Item = Result<Row>;
71    type IntoIter = RowIter<'static>;
72
73    fn into_iter(self) -> Self::IntoIter {
74        RowIter::from_file_into(Box::new(self))
75    }
76}
77
78// ----------------------------------------------------------------------
79// Implementations of file & row group readers
80
81/// A serialized implementation for Parquet [`FileReader`].
82pub struct SerializedFileReader<R: ChunkReader> {
83    chunk_reader: Arc<R>,
84    metadata: Arc<ParquetMetaData>,
85    props: ReaderPropertiesPtr,
86}
87
88/// A predicate for filtering row groups, invoked with the metadata and index
89/// of each row group in the file. Only row groups for which the predicate
90/// evaluates to `true` will be scanned
91pub type ReadGroupPredicate = Box<dyn FnMut(&RowGroupMetaData, usize) -> bool>;
92
93/// A builder for [`ReadOptions`].
94/// For the predicates that are added to the builder,
95/// they will be chained using 'AND' to filter the row groups.
96#[derive(Default)]
97pub struct ReadOptionsBuilder {
98    predicates: Vec<ReadGroupPredicate>,
99    enable_page_index: bool,
100    props: Option<ReaderProperties>,
101    metadata_options: ParquetMetaDataOptions,
102}
103
104impl ReadOptionsBuilder {
105    /// New builder
106    pub fn new() -> Self {
107        Self::default()
108    }
109
110    /// Add a predicate on row group metadata to the reading option,
111    /// Filter only row groups that match the predicate criteria
112    pub fn with_predicate(mut self, predicate: ReadGroupPredicate) -> Self {
113        self.predicates.push(predicate);
114        self
115    }
116
117    /// Add a range predicate on filtering row groups if their midpoints are within
118    /// the Closed-Open range `[start..end) {x | start <= x < end}`
119    ///
120    /// # Panics
121    ///
122    /// Panics if `end <= start`
123    pub fn with_range(mut self, start: i64, end: i64) -> Self {
124        assert!(start < end);
125        let predicate = move |rg: &RowGroupMetaData, _: usize| {
126            let mid = get_midpoint_offset(rg);
127            mid >= start && mid < end
128        };
129        self.predicates.push(Box::new(predicate));
130        self
131    }
132
133    /// Enable reading the page index structures described in
134    /// "[Column Index] Layout to Support Page Skipping"
135    ///
136    /// [Column Index]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
137    pub fn with_page_index(mut self) -> Self {
138        self.enable_page_index = true;
139        self
140    }
141
142    /// Set the [`ReaderProperties`] configuration.
143    pub fn with_reader_properties(mut self, properties: ReaderProperties) -> Self {
144        self.props = Some(properties);
145        self
146    }
147
148    /// Provide a Parquet schema to use when decoding the metadata. The schema in the Parquet
149    /// footer will be skipped.
150    pub fn with_parquet_schema(mut self, schema: SchemaDescPtr) -> Self {
151        self.metadata_options.set_schema(schema);
152        self
153    }
154
155    /// Set whether to convert the [`encoding_stats`] in the Parquet `ColumnMetaData` to a bitmask
156    /// (defaults to `false`).
157    ///
158    /// See [`ColumnChunkMetaData::page_encoding_stats_mask`] for an explanation of why this
159    /// might be desirable.
160    ///
161    /// [`encoding_stats`]:
162    /// https://github.com/apache/parquet-format/blob/786142e26740487930ddc3ec5e39d780bd930907/src/main/thrift/parquet.thrift#L917
163    pub fn with_encoding_stats_as_mask(mut self, val: bool) -> Self {
164        self.metadata_options.set_encoding_stats_as_mask(val);
165        self
166    }
167
168    /// Sets the decoding policy for [`encoding_stats`] in the Parquet `ColumnMetaData`.
169    ///
170    /// [`encoding_stats`]:
171    /// https://github.com/apache/parquet-format/blob/786142e26740487930ddc3ec5e39d780bd930907/src/main/thrift/parquet.thrift#L917
172    pub fn with_encoding_stats_policy(mut self, policy: ParquetStatisticsPolicy) -> Self {
173        self.metadata_options.set_encoding_stats_policy(policy);
174        self
175    }
176
177    /// Sets the decoding policy for [`statistics`] in the Parquet `ColumnMetaData`.
178    ///
179    /// [`statistics`]:
180    /// https://github.com/apache/parquet-format/blob/786142e26740487930ddc3ec5e39d780bd930907/src/main/thrift/parquet.thrift#L912
181    pub fn with_column_stats_policy(mut self, policy: ParquetStatisticsPolicy) -> Self {
182        self.metadata_options.set_column_stats_policy(policy);
183        self
184    }
185
186    /// Sets the decoding policy for [`size_statistics`] in the Parquet `ColumnMetaData`.
187    ///
188    /// [`size_statistics`]:
189    /// https://github.com/apache/parquet-format/blob/786142e26740487930ddc3ec5e39d780bd930907/src/main/thrift/parquet.thrift#L936
190    pub fn with_size_stats_policy(mut self, policy: ParquetStatisticsPolicy) -> Self {
191        self.metadata_options.set_size_stats_policy(policy);
192        self
193    }
194
195    /// Seal the builder and return the read options
196    pub fn build(self) -> ReadOptions {
197        let props = self
198            .props
199            .unwrap_or_else(|| ReaderProperties::builder().build());
200        ReadOptions {
201            predicates: self.predicates,
202            enable_page_index: self.enable_page_index,
203            props,
204            metadata_options: self.metadata_options,
205        }
206    }
207}
208
209/// A collection of options for reading a Parquet file.
210///
211/// Predicates are currently only supported on row group metadata.
212/// All predicates will be chained using 'AND' to filter the row groups.
213pub struct ReadOptions {
214    predicates: Vec<ReadGroupPredicate>,
215    enable_page_index: bool,
216    props: ReaderProperties,
217    metadata_options: ParquetMetaDataOptions,
218}
219
220impl<R: 'static + ChunkReader> SerializedFileReader<R> {
221    /// Creates file reader from a Parquet file.
222    /// Returns an error if the Parquet file does not exist or is corrupt.
223    pub fn new(chunk_reader: R) -> Result<Self> {
224        let metadata = ParquetMetaDataReader::new().parse_and_finish(&chunk_reader)?;
225        let props = Arc::new(ReaderProperties::builder().build());
226        Ok(Self {
227            chunk_reader: Arc::new(chunk_reader),
228            metadata: Arc::new(metadata),
229            props,
230        })
231    }
232
233    /// Creates file reader from a Parquet file with read options.
234    /// Returns an error if the Parquet file does not exist or is corrupt.
235    pub fn new_with_options(chunk_reader: R, options: ReadOptions) -> Result<Self> {
236        let mut metadata = ParquetMetaDataReader::new()
237            .with_metadata_options(Some(options.metadata_options.clone()))
238            .parse_and_finish(&chunk_reader)?;
239        let mut predicates = options.predicates;
240
241        if !predicates.is_empty() {
242            let row_groups = metadata.row_groups();
243            let mut filtered = None;
244
245            // Only clone retained row groups if a predicate rejects a row group.
246            for (i, rg_meta) in row_groups.iter().enumerate() {
247                let keep = predicates.iter_mut().all(|predicate| predicate(rg_meta, i));
248                if !keep {
249                    filtered.get_or_insert_with(|| row_groups[..i].to_vec());
250                } else if let Some(kept) = &mut filtered {
251                    kept.push(rg_meta.clone());
252                }
253            }
254
255            if let Some(row_groups) = filtered {
256                metadata = metadata.into_builder().set_row_groups(row_groups).build();
257            }
258        }
259
260        // If page indexes are desired, build them with the filtered set of row groups
261        if options.enable_page_index {
262            let mut reader = ParquetMetaDataReader::new_with_metadata(metadata)
263                .with_page_index_policy(PageIndexPolicy::Required);
264            reader.read_page_indexes(&chunk_reader)?;
265            metadata = reader.finish()?;
266        }
267
268        Ok(Self {
269            chunk_reader: Arc::new(chunk_reader),
270            metadata: Arc::new(metadata),
271            props: Arc::new(options.props),
272        })
273    }
274}
275
276/// Get midpoint offset for a row group
277fn get_midpoint_offset(meta: &RowGroupMetaData) -> i64 {
278    let col = meta.column(0);
279    let mut offset = col.data_page_offset();
280    if let Some(dic_offset) = col.dictionary_page_offset()
281        && offset > dic_offset
282    {
283        offset = dic_offset
284    }
285    offset + meta.compressed_size() / 2
286}
287
288impl<R: 'static + ChunkReader> FileReader for SerializedFileReader<R> {
289    fn metadata(&self) -> &ParquetMetaData {
290        &self.metadata
291    }
292
293    fn num_row_groups(&self) -> usize {
294        self.metadata.num_row_groups()
295    }
296
297    fn get_row_group(&self, i: usize) -> Result<Box<dyn RowGroupReader + '_>> {
298        let row_group_metadata = self.metadata.row_group(i);
299        // Row groups should be processed sequentially.
300        let props = Arc::clone(&self.props);
301        let f = Arc::clone(&self.chunk_reader);
302        let page_index = self.metadata.page_index_for_row_group(i);
303        Ok(Box::new(SerializedRowGroupReader::new(
304            f,
305            row_group_metadata,
306            page_index,
307            props,
308        )?))
309    }
310
311    fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter<'_>> {
312        RowIter::from_file(projection, self)
313    }
314}
315
316/// A serialized implementation for Parquet [`RowGroupReader`].
317pub struct SerializedRowGroupReader<'a, R: ChunkReader> {
318    chunk_reader: Arc<R>,
319    metadata: &'a RowGroupMetaData,
320    page_index: RowGroupPageIndex,
321    props: ReaderPropertiesPtr,
322    bloom_filters: Vec<Option<Sbbf>>,
323}
324
325impl<'a, R: ChunkReader> SerializedRowGroupReader<'a, R> {
326    /// Creates new row group reader from a file, row group metadata and custom config.
327    pub fn new(
328        chunk_reader: Arc<R>,
329        metadata: &'a RowGroupMetaData,
330        page_index: RowGroupPageIndex,
331        props: ReaderPropertiesPtr,
332    ) -> Result<Self> {
333        let bloom_filters = if props.read_bloom_filter() {
334            metadata
335                .columns()
336                .iter()
337                .map(|col| Sbbf::read_from_column_chunk(col, &*chunk_reader))
338                .collect::<Result<Vec<_>>>()?
339        } else {
340            std::iter::repeat_n(None, metadata.columns().len()).collect()
341        };
342        Ok(Self {
343            chunk_reader,
344            metadata,
345            page_index,
346            props,
347            bloom_filters,
348        })
349    }
350}
351
352impl<R: 'static + ChunkReader> RowGroupReader for SerializedRowGroupReader<'_, R> {
353    fn metadata(&self) -> &RowGroupMetaData {
354        self.metadata
355    }
356
357    fn num_columns(&self) -> usize {
358        self.metadata.num_columns()
359    }
360
361    // TODO: fix PARQUET-816
362    fn get_column_page_reader(&self, i: usize) -> Result<Box<dyn PageReader>> {
363        let col = self.metadata.column(i);
364
365        // TODO(ets): push page index into page reader so we don't have to clone here
366        let page_locations = self.page_index.page_locations(i).cloned();
367
368        let props = Arc::clone(&self.props);
369        Ok(Box::new(SerializedPageReader::new_with_properties(
370            Arc::clone(&self.chunk_reader),
371            col,
372            usize::try_from(self.metadata.num_rows())?,
373            page_locations,
374            props,
375        )?))
376    }
377
378    /// get bloom filter for the `i`th column
379    fn get_column_bloom_filter(&self, i: usize) -> Option<&Sbbf> {
380        self.bloom_filters[i].as_ref()
381    }
382
383    fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter<'_>> {
384        RowIter::from_row_group(projection, self)
385    }
386}
387
388/// Decodes a [`Page`] from the provided `buffer`
389pub(crate) fn decode_page(
390    page_header: PageHeader,
391    buffer: Bytes,
392    physical_type: Type,
393    decompressor: Option<&mut Box<dyn Codec>>,
394) -> Result<Page> {
395    // Verify the 32-bit CRC checksum of the page
396    #[cfg(feature = "crc")]
397    if let Some(expected_crc) = page_header.crc {
398        let crc = crc32fast::hash(&buffer);
399        if crc != expected_crc as u32 {
400            return Err(general_err!("Page CRC checksum mismatch"));
401        }
402    }
403
404    // When processing data page v2, depending on enabled compression for the
405    // page, we should account for uncompressed data ('offset') of
406    // repetition and definition levels.
407    //
408    // We always use 0 offset for other pages other than v2, `true` flag means
409    // that compression will be applied if decompressor is defined
410    let (offset, can_decompress): (usize, bool) = match page_header.data_page_header_v2 {
411        Some(ref header_v2) => {
412            if header_v2.definition_levels_byte_length < 0
413                || header_v2.repetition_levels_byte_length < 0
414                || header_v2.definition_levels_byte_length + header_v2.repetition_levels_byte_length
415                    > page_header.uncompressed_page_size
416            {
417                return Err(general_err!(
418                    "DataPage v2 header contains implausible values \
419                        for definition_levels_byte_length ({}) \
420                        and repetition_levels_byte_length ({}) \
421                        given DataPage header provides uncompressed_page_size ({})",
422                    header_v2.definition_levels_byte_length,
423                    header_v2.repetition_levels_byte_length,
424                    page_header.uncompressed_page_size
425                ));
426            }
427            (
428                usize::try_from(
429                    header_v2.definition_levels_byte_length
430                        + header_v2.repetition_levels_byte_length,
431                )?,
432                // When is_compressed flag is missing the page is considered compressed
433                header_v2.is_compressed.unwrap_or(true),
434            )
435        }
436        None => (0, true),
437    };
438
439    let buffer = match decompressor {
440        Some(decompressor) if can_decompress => {
441            let uncompressed_page_size = usize::try_from(page_header.uncompressed_page_size)?;
442            if offset > buffer.len() || offset > uncompressed_page_size {
443                return Err(general_err!("Invalid page header"));
444            }
445            let decompressed_size = uncompressed_page_size - offset;
446            let mut decompressed = Vec::with_capacity(uncompressed_page_size);
447            decompressed.extend_from_slice(&buffer[..offset]);
448            // decompressed size of zero corresponds to a page with no non-null values
449            // see https://github.com/apache/parquet-format/blob/master/README.md#data-pages
450            if decompressed_size > 0 {
451                let compressed = &buffer[offset..];
452                decompressor.decompress(compressed, &mut decompressed, Some(decompressed_size))?;
453            }
454
455            if decompressed.len() != uncompressed_page_size {
456                return Err(general_err!(
457                    "Actual decompressed size doesn't match the expected one ({} vs {})",
458                    decompressed.len(),
459                    uncompressed_page_size
460                ));
461            }
462
463            Bytes::from(decompressed)
464        }
465        _ => buffer,
466    };
467
468    let result = match page_header.r#type {
469        PageType::DICTIONARY_PAGE => {
470            let dict_header = page_header.dictionary_page_header.as_ref().ok_or_else(|| {
471                ParquetError::General("Missing dictionary page header".to_string())
472            })?;
473            let is_sorted = dict_header.is_sorted.unwrap_or(false);
474            Page::DictionaryPage {
475                buf: buffer,
476                num_values: dict_header.num_values.try_into()?,
477                encoding: dict_header.encoding,
478                is_sorted,
479            }
480        }
481        PageType::DATA_PAGE => {
482            let header = page_header
483                .data_page_header
484                .ok_or_else(|| ParquetError::General("Missing V1 data page header".to_string()))?;
485            Page::DataPage {
486                buf: buffer,
487                num_values: header.num_values.try_into()?,
488                encoding: header.encoding,
489                def_level_encoding: header.definition_level_encoding,
490                rep_level_encoding: header.repetition_level_encoding,
491                statistics: statistics::from_thrift_page_stats(physical_type, header.statistics)?,
492            }
493        }
494        PageType::DATA_PAGE_V2 => {
495            let header = page_header
496                .data_page_header_v2
497                .ok_or_else(|| ParquetError::General("Missing V2 data page header".to_string()))?;
498            let is_compressed = header.is_compressed.unwrap_or(true);
499            Page::DataPageV2 {
500                buf: buffer,
501                num_values: header.num_values.try_into()?,
502                encoding: header.encoding,
503                num_nulls: header.num_nulls.try_into()?,
504                num_rows: header.num_rows.try_into()?,
505                def_levels_byte_len: header.definition_levels_byte_length.try_into()?,
506                rep_levels_byte_len: header.repetition_levels_byte_length.try_into()?,
507                is_compressed,
508                statistics: statistics::from_thrift_page_stats(physical_type, header.statistics)?,
509            }
510        }
511        PageType::INDEX_PAGE => {
512            // For unknown page type (e.g., INDEX_PAGE), skip and read next.
513            return Err(general_err!(
514                "Page type {:?} is not supported",
515                page_header.r#type
516            ));
517        }
518    };
519
520    Ok(result)
521}
522
523/// Location of a dictionary page skipped without decoding.
524///
525/// ```text
526///            skip, and the reader can defer         first data page that
527///            (record location, zero decompress)     needs the dictionary
528///   NotSeen ─────────────────────────────▶ Deferred ─────────────────────▶ Decoded
529///      │                                                                      ▲
530///      ├── read hits the dictionary page (eager path, unchanged) ─────────────┤
531///      │                                                                      │
532///      └── skip, but the reader cannot defer ─────────────────────────────────┘
533///          (`supports_deferred_dictionary()` is false: read it eagerly,
534///           because a skipped page this reader cannot hand back is lost)
535/// ```
536///
537/// A chunk skipped end to end stays `Deferred`: its dictionary is never
538/// decompressed, which is the point. Skipping *values* needs only the RLE
539/// index cursor, and skipping whole pages needs nothing at all, so the only
540/// consumer of dictionary *contents* is value decoding. Random re-read is
541/// what makes the deferral safe: [`ChunkReader`] hands out readers at any
542/// offset, and the async path's in-memory column chunk already holds the
543/// page's bytes.
544enum DeferredDictionaryPage {
545    /// `Values` state: the header was already parsed on the skip path, only
546    /// the page body was left untouched.
547    Header {
548        data_start: u64,
549        header: Box<PageHeader>,
550    },
551    /// `Pages` state: the offset index records the page location; header and
552    /// body are both still unread.
553    Location(PageLocation),
554}
555
556enum SerializedPageReaderState {
557    Values {
558        /// The current byte offset in the reader
559        /// Note that offset is u64 (i.e., not usize) to support 32-bit architectures such as WASM
560        offset: u64,
561
562        /// The length of the chunk in bytes
563        /// Note that remaining_bytes is u64 (i.e., not usize) to support 32-bit architectures such as WASM
564        remaining_bytes: u64,
565
566        // If the next page header has already been "peeked", we will cache it and it`s length here
567        next_page_header: Option<Box<PageHeader>>,
568
569        /// The index of the data page within this column chunk
570        page_index: usize,
571
572        /// Whether the next page is expected to be a dictionary page
573        require_dictionary: bool,
574    },
575    Pages {
576        /// Remaining page locations
577        page_locations: VecDeque<PageLocation>,
578        /// Remaining dictionary location if any
579        dictionary_page: Option<PageLocation>,
580        /// The total number of rows in this column chunk
581        total_rows: usize,
582        /// The index of the data page within this column chunk
583        page_index: usize,
584    },
585}
586
587#[derive(Default)]
588pub(crate) struct SerializedPageReaderContext {
589    /// Controls decoding of page-level statistics
590    pub(crate) read_stats: bool,
591    /// Crypto context carrying objects required for decryption
592    #[cfg(feature = "encryption")]
593    pub(crate) crypto_context: Option<Arc<CryptoContext>>,
594}
595
596/// A serialized implementation for Parquet [`PageReader`].
597pub struct SerializedPageReader<R: ChunkReader> {
598    /// The chunk reader
599    reader: Arc<R>,
600
601    /// A dictionary page skipped past without decoding. See
602    /// [`DeferredDictionaryPage`] for the state diagram.
603    deferred_dictionary: Option<DeferredDictionaryPage>,
604
605    /// The compression codec for this column chunk. Only set for non-PLAIN codec.
606    decompressor: Option<Box<dyn Codec>>,
607
608    /// Column chunk type.
609    physical_type: Type,
610
611    state: SerializedPageReaderState,
612
613    context: SerializedPageReaderContext,
614}
615
616impl<R: ChunkReader> SerializedPageReader<R> {
617    /// Creates a new serialized page reader from a chunk reader and metadata
618    pub fn new(
619        reader: Arc<R>,
620        column_chunk_metadata: &ColumnChunkMetaData,
621        total_rows: usize,
622        page_locations: Option<Vec<PageLocation>>,
623    ) -> Result<Self> {
624        let props = Arc::new(ReaderProperties::builder().build());
625        SerializedPageReader::new_with_properties(
626            reader,
627            column_chunk_metadata,
628            total_rows,
629            page_locations,
630            props,
631        )
632    }
633
634    /// Stub No-op implementation when encryption is disabled.
635    #[cfg(all(feature = "arrow", not(feature = "encryption")))]
636    pub(crate) fn add_crypto_context(
637        self,
638        _rg_idx: usize,
639        _column_idx: usize,
640        _parquet_meta_data: &ParquetMetaData,
641        _column_chunk_metadata: &ColumnChunkMetaData,
642    ) -> Result<SerializedPageReader<R>> {
643        Ok(self)
644    }
645
646    /// Adds any necessary crypto context to this page reader, if encryption is enabled.
647    #[cfg(feature = "encryption")]
648    pub(crate) fn add_crypto_context(
649        mut self,
650        rg_idx: usize,
651        column_idx: usize,
652        parquet_meta_data: &ParquetMetaData,
653        column_chunk_metadata: &ColumnChunkMetaData,
654    ) -> Result<SerializedPageReader<R>> {
655        let Some(file_decryptor) = parquet_meta_data.file_decryptor() else {
656            return Ok(self);
657        };
658        let Some(crypto_metadata) = column_chunk_metadata.crypto_metadata() else {
659            return Ok(self);
660        };
661        let crypto_context =
662            CryptoContext::for_column(file_decryptor, crypto_metadata, rg_idx, column_idx)?;
663        self.context.crypto_context = Some(Arc::new(crypto_context));
664        Ok(self)
665    }
666
667    /// Creates a new serialized page with custom options.
668    pub fn new_with_properties(
669        reader: Arc<R>,
670        meta: &ColumnChunkMetaData,
671        total_rows: usize,
672        page_locations: Option<Vec<PageLocation>>,
673        props: ReaderPropertiesPtr,
674    ) -> Result<Self> {
675        let decompressor = create_codec(meta.compression(), props.codec_options())?;
676        let (start, len) = meta.byte_range();
677
678        let state = match page_locations {
679            Some(locations) => {
680                // If the offset of the first page doesn't match the start of the column chunk
681                // then the preceding space must contain a dictionary page.
682                let dictionary_page = match locations.first() {
683                    Some(dict_offset) if dict_offset.offset as u64 != start => Some(PageLocation {
684                        offset: start as i64,
685                        compressed_page_size: (dict_offset.offset as u64 - start) as i32,
686                        first_row_index: 0,
687                    }),
688                    _ => None,
689                };
690
691                SerializedPageReaderState::Pages {
692                    page_locations: locations.into(),
693                    dictionary_page,
694                    total_rows,
695                    page_index: 0,
696                }
697            }
698            None => SerializedPageReaderState::Values {
699                offset: start,
700                remaining_bytes: len,
701                next_page_header: None,
702                page_index: 0,
703                require_dictionary: meta.dictionary_page_offset().is_some(),
704            },
705        };
706        let mut context = SerializedPageReaderContext::default();
707        if props.read_page_stats() {
708            context.read_stats = true;
709        }
710        Ok(Self {
711            reader,
712            deferred_dictionary: None,
713            decompressor,
714            state,
715            physical_type: meta.column_type(),
716            context,
717        })
718    }
719
720    /// Similar to `peek_next_page`, but returns the offset of the next page instead of the page metadata.
721    /// Unlike page metadata, an offset can uniquely identify a page.
722    ///
723    /// This is used when we need to read parquet with row-filter, and we don't want to decompress the page twice.
724    /// This function allows us to check if the next page is being cached or read previously.
725    #[cfg(test)]
726    fn peek_next_page_offset(&mut self) -> Result<Option<u64>> {
727        match &mut self.state {
728            SerializedPageReaderState::Values {
729                offset,
730                remaining_bytes,
731                next_page_header,
732                page_index,
733                require_dictionary,
734            } => {
735                loop {
736                    if *remaining_bytes == 0 {
737                        return Ok(None);
738                    }
739                    return if let Some(header) = next_page_header.as_ref() {
740                        if let Ok(_page_meta) = PageMetadata::try_from(&**header) {
741                            Ok(Some(*offset))
742                        } else {
743                            // For unknown page type (e.g., INDEX_PAGE), skip and read next.
744                            *next_page_header = None;
745                            continue;
746                        }
747                    } else {
748                        let mut read = self.reader.get_read(*offset)?;
749                        let (header_len, header) = Self::read_page_header_len(
750                            &self.context,
751                            &mut read,
752                            *page_index,
753                            *require_dictionary,
754                        )?;
755                        *offset += header_len as u64;
756                        *remaining_bytes -= header_len as u64;
757                        let page_meta = if let Ok(_page_meta) = PageMetadata::try_from(&header) {
758                            Ok(Some(*offset))
759                        } else {
760                            // For unknown page type (e.g., INDEX_PAGE), skip and read next.
761                            continue;
762                        };
763                        *next_page_header = Some(Box::new(header));
764                        page_meta
765                    };
766                }
767            }
768            SerializedPageReaderState::Pages {
769                page_locations,
770                dictionary_page,
771                ..
772            } => {
773                if let Some(page) = dictionary_page {
774                    Ok(Some(page.offset as u64))
775                } else if let Some(page) = page_locations.front() {
776                    Ok(Some(page.offset as u64))
777                } else {
778                    Ok(None)
779                }
780            }
781        }
782    }
783
784    fn read_page_header_len<T: Read>(
785        context: &SerializedPageReaderContext,
786        input: &mut T,
787        page_index: usize,
788        dictionary_page: bool,
789    ) -> Result<(usize, PageHeader)> {
790        /// A wrapper around a [`std::io::Read`] that keeps track of the bytes read
791        struct TrackedRead<R> {
792            inner: R,
793            bytes_read: usize,
794        }
795
796        impl<R: Read> Read for TrackedRead<R> {
797            fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
798                let v = self.inner.read(buf)?;
799                self.bytes_read += v;
800                Ok(v)
801            }
802        }
803
804        let mut tracked = TrackedRead {
805            inner: input,
806            bytes_read: 0,
807        };
808        let header = context.read_page_header(&mut tracked, page_index, dictionary_page)?;
809        Ok((tracked.bytes_read, header))
810    }
811}
812
813/// Reads (and decrypts, if `context` carries a crypto context) the page header stored
814/// at the front of `buffer`, returning the header and the number of bytes of `buffer`
815/// it occupies.
816///
817/// This is exposed for callers that need to decode a single page directly from an
818/// already-fetched byte range, outside of the normal [`SerializedPageReader`] iteration
819/// -- e.g. decoding a dictionary page standalone.
820pub(crate) fn read_page_header_len_from_bytes(
821    context: &SerializedPageReaderContext,
822    buffer: &[u8],
823    page_index: usize,
824    dictionary_page: bool,
825) -> Result<(usize, PageHeader)> {
826    let mut input = std::io::Cursor::new(buffer);
827    let header = context.read_page_header(&mut input, page_index, dictionary_page)?;
828    let header_len = input.position() as usize;
829    Ok((header_len, header))
830}
831
832#[cfg(not(feature = "encryption"))]
833impl SerializedPageReaderContext {
834    pub(crate) fn read_page_header<T: Read>(
835        &self,
836        input: &mut T,
837        _page_index: usize,
838        _dictionary_page: bool,
839    ) -> Result<PageHeader> {
840        let mut prot = ThriftReadInputProtocol::new(input);
841        if self.read_stats {
842            Ok(PageHeader::read_thrift(&mut prot)?)
843        } else {
844            Ok(PageHeader::read_thrift_without_stats(&mut prot)?)
845        }
846    }
847
848    pub(crate) fn decrypt_page_data<T>(
849        &self,
850        buffer: T,
851        _page_index: usize,
852        _dictionary_page: bool,
853    ) -> Result<T> {
854        Ok(buffer)
855    }
856}
857
858#[cfg(feature = "encryption")]
859impl SerializedPageReaderContext {
860    pub(crate) fn read_page_header<T: Read>(
861        &self,
862        input: &mut T,
863        page_index: usize,
864        dictionary_page: bool,
865    ) -> Result<PageHeader> {
866        match self.page_crypto_context(page_index, dictionary_page) {
867            None => {
868                let mut prot = ThriftReadInputProtocol::new(input);
869                if self.read_stats {
870                    Ok(PageHeader::read_thrift(&mut prot)?)
871                } else {
872                    use crate::file::metadata::thrift::PageHeader;
873
874                    Ok(PageHeader::read_thrift_without_stats(&mut prot)?)
875                }
876            }
877            Some(page_crypto_context) => {
878                let data_decryptor = page_crypto_context.data_decryptor();
879                let aad = page_crypto_context.create_page_header_aad()?;
880
881                let buf = read_and_decrypt(data_decryptor, input, aad.as_ref()).map_err(|_| {
882                    ParquetError::General(format!(
883                        "Error decrypting page header for column {}, decryption key may be wrong",
884                        page_crypto_context.column_ordinal
885                    ))
886                })?;
887
888                let mut prot = ThriftSliceInputProtocol::new(buf.as_slice());
889                if self.read_stats {
890                    Ok(PageHeader::read_thrift(&mut prot)?)
891                } else {
892                    Ok(PageHeader::read_thrift_without_stats(&mut prot)?)
893                }
894            }
895        }
896    }
897
898    pub(crate) fn decrypt_page_data<T>(
899        &self,
900        buffer: T,
901        page_index: usize,
902        dictionary_page: bool,
903    ) -> Result<T>
904    where
905        T: AsRef<[u8]>,
906        T: From<Vec<u8>>,
907    {
908        let page_crypto_context = self.page_crypto_context(page_index, dictionary_page);
909        if let Some(page_crypto_context) = page_crypto_context {
910            let decryptor = page_crypto_context.data_decryptor();
911            let aad = page_crypto_context.create_page_aad()?;
912            let decrypted = decryptor.decrypt(buffer.as_ref(), &aad)?;
913            Ok(T::from(decrypted))
914        } else {
915            Ok(buffer)
916        }
917    }
918
919    fn page_crypto_context(
920        &self,
921        page_index: usize,
922        dictionary_page: bool,
923    ) -> Option<Arc<CryptoContext>> {
924        self.crypto_context.as_ref().map(|c| {
925            Arc::new(if dictionary_page {
926                c.for_dictionary_page()
927            } else {
928                c.with_page_ordinal(page_index)
929            })
930        })
931    }
932}
933
934impl<R: ChunkReader> Iterator for SerializedPageReader<R> {
935    type Item = Result<Page>;
936
937    fn next(&mut self) -> Option<Self::Item> {
938        self.get_next_page().transpose()
939    }
940}
941
942fn verify_page_header_len(header_len: usize, remaining_bytes: u64) -> Result<()> {
943    if header_len as u64 > remaining_bytes {
944        return Err(eof_err!("Invalid page header"));
945    }
946    Ok(())
947}
948
949pub(crate) fn verify_page_size(
950    compressed_size: i32,
951    uncompressed_size: i32,
952    remaining_bytes: u64,
953) -> Result<()> {
954    // The page's compressed size should not exceed the remaining bytes that are
955    // available to read. The page's uncompressed size is the expected size
956    // after decompression, which can never be negative.
957    if compressed_size < 0 || compressed_size as u64 > remaining_bytes || uncompressed_size < 0 {
958        return Err(eof_err!("Invalid page header"));
959    }
960    Ok(())
961}
962
963impl<R: ChunkReader> PageReader for SerializedPageReader<R> {
964    fn get_next_page(&mut self) -> Result<Option<Page>> {
965        loop {
966            let page = match &mut self.state {
967                SerializedPageReaderState::Values {
968                    offset,
969                    remaining_bytes: remaining,
970                    next_page_header,
971                    page_index,
972                    require_dictionary,
973                } => {
974                    if *remaining == 0 {
975                        return Ok(None);
976                    }
977
978                    let mut read = self.reader.get_read(*offset)?;
979                    let header = if let Some(header) = next_page_header.take() {
980                        *header
981                    } else {
982                        let (header_len, header) = Self::read_page_header_len(
983                            &self.context,
984                            &mut read,
985                            *page_index,
986                            *require_dictionary,
987                        )?;
988                        verify_page_header_len(header_len, *remaining)?;
989                        *offset += header_len as u64;
990                        *remaining -= header_len as u64;
991                        header
992                    };
993                    verify_page_size(
994                        header.compressed_page_size,
995                        header.uncompressed_page_size,
996                        *remaining,
997                    )?;
998                    let data_len = header.compressed_page_size as usize;
999                    let data_start = *offset;
1000                    *offset += data_len as u64;
1001                    *remaining -= data_len as u64;
1002
1003                    if header.r#type == PageType::INDEX_PAGE {
1004                        continue;
1005                    }
1006
1007                    let buffer = self.reader.get_bytes(data_start, data_len)?;
1008
1009                    let buffer =
1010                        self.context
1011                            .decrypt_page_data(buffer, *page_index, *require_dictionary)?;
1012
1013                    let page = decode_page(
1014                        header,
1015                        buffer,
1016                        self.physical_type,
1017                        self.decompressor.as_mut(),
1018                    )?;
1019                    if page.is_data_page() {
1020                        *page_index += 1;
1021                    } else if page.is_dictionary_page() {
1022                        *require_dictionary = false;
1023                    }
1024                    page
1025                }
1026                SerializedPageReaderState::Pages {
1027                    page_locations,
1028                    dictionary_page,
1029                    page_index,
1030                    ..
1031                } => {
1032                    let (front, is_dictionary_page) = match dictionary_page.take() {
1033                        Some(front) => (front, true),
1034                        None => match page_locations.pop_front() {
1035                            Some(front) => (front, false),
1036                            None => return Ok(None),
1037                        },
1038                    };
1039
1040                    let page_len = usize::try_from(front.compressed_page_size)?;
1041                    let buffer = self.reader.get_bytes(front.offset as u64, page_len)?;
1042
1043                    let (offset, header) = read_page_header_len_from_bytes(
1044                        &self.context,
1045                        buffer.as_ref(),
1046                        *page_index,
1047                        is_dictionary_page,
1048                    )?;
1049                    let bytes = buffer.slice(offset..);
1050                    let bytes =
1051                        self.context
1052                            .decrypt_page_data(bytes, *page_index, is_dictionary_page)?;
1053
1054                    if !is_dictionary_page {
1055                        *page_index += 1;
1056                    }
1057                    decode_page(
1058                        header,
1059                        bytes,
1060                        self.physical_type,
1061                        self.decompressor.as_mut(),
1062                    )?
1063                }
1064            };
1065
1066            return Ok(Some(page));
1067        }
1068    }
1069
1070    fn peek_next_page(&mut self) -> Result<Option<PageMetadata>> {
1071        match &mut self.state {
1072            SerializedPageReaderState::Values {
1073                offset,
1074                remaining_bytes,
1075                next_page_header,
1076                page_index,
1077                require_dictionary,
1078            } => {
1079                loop {
1080                    if *remaining_bytes == 0 {
1081                        return Ok(None);
1082                    }
1083                    return if let Some(header) = next_page_header.as_ref() {
1084                        if let Ok(page_meta) = (&**header).try_into() {
1085                            Ok(Some(page_meta))
1086                        } else {
1087                            // For unknown page type (e.g., INDEX_PAGE), skip and read next.
1088                            *next_page_header = None;
1089                            continue;
1090                        }
1091                    } else {
1092                        let mut read = self.reader.get_read(*offset)?;
1093                        let (header_len, header) = Self::read_page_header_len(
1094                            &self.context,
1095                            &mut read,
1096                            *page_index,
1097                            *require_dictionary,
1098                        )?;
1099                        verify_page_header_len(header_len, *remaining_bytes)?;
1100                        *offset += header_len as u64;
1101                        *remaining_bytes -= header_len as u64;
1102                        let page_meta = if let Ok(page_meta) = (&header).try_into() {
1103                            Ok(Some(page_meta))
1104                        } else {
1105                            // For unknown page type (e.g., INDEX_PAGE), skip and read next.
1106                            continue;
1107                        };
1108                        *next_page_header = Some(Box::new(header));
1109                        page_meta
1110                    };
1111                }
1112            }
1113            SerializedPageReaderState::Pages {
1114                page_locations,
1115                dictionary_page,
1116                total_rows,
1117                page_index: _,
1118            } => {
1119                if dictionary_page.is_some() {
1120                    Ok(Some(PageMetadata {
1121                        num_rows: None,
1122                        num_levels: None,
1123                        is_dict: true,
1124                    }))
1125                } else if let Some(page) = page_locations.front() {
1126                    let next_rows = page_locations
1127                        .get(1)
1128                        .map(|x| x.first_row_index as usize)
1129                        .unwrap_or(*total_rows);
1130
1131                    Ok(Some(PageMetadata {
1132                        num_rows: Some(next_rows - page.first_row_index as usize),
1133                        num_levels: None,
1134                        is_dict: false,
1135                    }))
1136                } else {
1137                    Ok(None)
1138                }
1139            }
1140        }
1141    }
1142
1143    fn skip_next_page(&mut self) -> Result<()> {
1144        // Retain skipped dictionary pages so they can be decoded on demand.
1145        // See [`DeferredDictionaryPage`] for the state diagram.
1146        let deferred = match &mut self.state {
1147            SerializedPageReaderState::Values {
1148                offset,
1149                remaining_bytes,
1150                next_page_header,
1151                page_index,
1152                require_dictionary,
1153            } => {
1154                let mut deferred = None;
1155                if let Some(buffered_header) = next_page_header.take() {
1156                    verify_page_size(
1157                        buffered_header.compressed_page_size,
1158                        buffered_header.uncompressed_page_size,
1159                        *remaining_bytes,
1160                    )?;
1161                    // The next page header has already been peeked, so the
1162                    // offset already points at the page body.
1163                    let body_len = buffered_header.compressed_page_size as u64;
1164                    if buffered_header.r#type == PageType::DICTIONARY_PAGE {
1165                        deferred = Some(DeferredDictionaryPage::Header {
1166                            data_start: *offset,
1167                            header: buffered_header,
1168                        });
1169                    }
1170                    *offset += body_len;
1171                    *remaining_bytes -= body_len;
1172                } else {
1173                    let mut read = self.reader.get_read(*offset)?;
1174                    let (header_len, header) = Self::read_page_header_len(
1175                        &self.context,
1176                        &mut read,
1177                        *page_index,
1178                        *require_dictionary,
1179                    )?;
1180                    verify_page_header_len(header_len, *remaining_bytes)?;
1181                    verify_page_size(
1182                        header.compressed_page_size,
1183                        header.uncompressed_page_size,
1184                        *remaining_bytes,
1185                    )?;
1186                    let data_page_size = header.compressed_page_size as u64;
1187                    let is_dict = header.r#type == PageType::DICTIONARY_PAGE;
1188                    if is_dict {
1189                        deferred = Some(DeferredDictionaryPage::Header {
1190                            data_start: *offset + header_len as u64,
1191                            header: Box::new(header),
1192                        });
1193                    }
1194                    *offset += header_len as u64 + data_page_size;
1195                    *remaining_bytes -= header_len as u64 + data_page_size;
1196                }
1197                if deferred.is_some() {
1198                    *require_dictionary = false;
1199                } else {
1200                    *page_index += 1;
1201                }
1202                deferred
1203            }
1204            SerializedPageReaderState::Pages {
1205                page_locations,
1206                dictionary_page,
1207                page_index,
1208                ..
1209            } => {
1210                if dictionary_page.is_some() {
1211                    // Consume the dictionary page, keeping its location.
1212                    dictionary_page.take().map(DeferredDictionaryPage::Location)
1213                } else {
1214                    // If no dictionary page exists, simply pop the data page from page_locations
1215                    if page_locations.pop_front().is_some() {
1216                        *page_index += 1;
1217                    }
1218                    None
1219                }
1220            }
1221        };
1222        if deferred.is_some() {
1223            self.deferred_dictionary = deferred;
1224        }
1225        Ok(())
1226    }
1227
1228    fn supports_deferred_dictionary(&self) -> bool {
1229        match &self.state {
1230            // Walking headers identifies dictionaries independently of metadata.
1231            SerializedPageReaderState::Values { .. } => true,
1232            // With an offset index, a dictionary is recoverable only when its
1233            // location can be inferred before the first data-page location, in
1234            // which case `dictionary_page` will be `Some`.
1235            SerializedPageReaderState::Pages {
1236                dictionary_page, ..
1237            } => dictionary_page.is_some(),
1238        }
1239    }
1240
1241    fn take_deferred_dictionary(&mut self) -> Result<Option<Page>> {
1242        let Some(deferred) = self.deferred_dictionary.take() else {
1243            return Ok(None);
1244        };
1245        let page = match deferred {
1246            DeferredDictionaryPage::Header { data_start, header } => {
1247                let data_len = header.compressed_page_size as usize;
1248                let buffer = self.reader.get_bytes(data_start, data_len)?;
1249                let buffer = self.context.decrypt_page_data(buffer, 0, true)?;
1250                decode_page(
1251                    *header,
1252                    buffer,
1253                    self.physical_type,
1254                    self.decompressor.as_mut(),
1255                )?
1256            }
1257            DeferredDictionaryPage::Location(front) => {
1258                let page_len = usize::try_from(front.compressed_page_size)?;
1259                let buffer = self.reader.get_bytes(front.offset as u64, page_len)?;
1260                let (offset, header) =
1261                    read_page_header_len_from_bytes(&self.context, buffer.as_ref(), 0, true)?;
1262                let bytes = buffer.slice(offset..);
1263                let bytes = self.context.decrypt_page_data(bytes, 0, true)?;
1264                decode_page(
1265                    header,
1266                    bytes,
1267                    self.physical_type,
1268                    self.decompressor.as_mut(),
1269                )?
1270            }
1271        };
1272        Ok(Some(page))
1273    }
1274
1275    fn at_record_boundary(&mut self) -> Result<bool> {
1276        match &mut self.state {
1277            SerializedPageReaderState::Values { .. } => match self.peek_next_page()? {
1278                None => Ok(true),
1279                // V2 data pages must start at record boundaries per the parquet
1280                // spec, so the current page ends at one.
1281                Some(metadata) => Ok(metadata.num_rows.is_some()),
1282            },
1283            SerializedPageReaderState::Pages { .. } => Ok(true),
1284        }
1285    }
1286}
1287
1288#[cfg(test)]
1289mod tests {
1290    use std::cell::RefCell;
1291    use std::collections::HashSet;
1292    use std::rc::Rc;
1293
1294    use bytes::Buf;
1295
1296    use crate::file::page_index::column_index::{
1297        ByteArrayColumnIndex, ColumnIndexMetaData, PrimitiveColumnIndex,
1298    };
1299    use crate::file::properties::{EnabledStatistics, WriterProperties};
1300
1301    use crate::basic::{self, BoundaryOrder, ColumnOrder, Compression, Encoding, SortOrder};
1302    use crate::column::reader::ColumnReader;
1303    use crate::data_type::private::ParquetValueType;
1304    use crate::data_type::{AsBytes, FixedLenByteArrayType, Int32Type};
1305    use crate::file::metadata::thrift::DataPageHeaderV2;
1306    use crate::file::writer::SerializedFileWriter;
1307    use crate::parquet_thrift::{ReadThrift, ThriftSliceInputProtocol};
1308    use crate::record::RowAccessor;
1309    use crate::schema::parser::parse_message_type;
1310    use crate::util::test_common::file_util::{get_test_file, get_test_path};
1311
1312    use super::*;
1313
1314    #[test]
1315    fn test_decode_page_invalid_offset() {
1316        let page_header = PageHeader {
1317            r#type: PageType::DATA_PAGE_V2,
1318            uncompressed_page_size: 10,
1319            compressed_page_size: 10,
1320            data_page_header: None,
1321            index_page_header: None,
1322            dictionary_page_header: None,
1323            crc: None,
1324            data_page_header_v2: Some(DataPageHeaderV2 {
1325                num_nulls: 0,
1326                num_rows: 0,
1327                num_values: 0,
1328                encoding: Encoding::PLAIN,
1329                definition_levels_byte_length: 11,
1330                repetition_levels_byte_length: 0,
1331                is_compressed: None,
1332                statistics: None,
1333            }),
1334        };
1335
1336        let buffer = Bytes::new();
1337        let err = decode_page(page_header, buffer, Type::INT32, None).unwrap_err();
1338        assert!(
1339            err.to_string()
1340                .contains("DataPage v2 header contains implausible values")
1341        );
1342    }
1343
1344    #[test]
1345    fn test_decode_unsupported_page() {
1346        let mut page_header = PageHeader {
1347            r#type: PageType::INDEX_PAGE,
1348            uncompressed_page_size: 10,
1349            compressed_page_size: 10,
1350            data_page_header: None,
1351            index_page_header: None,
1352            dictionary_page_header: None,
1353            crc: None,
1354            data_page_header_v2: None,
1355        };
1356        let buffer = Bytes::new();
1357        let err = decode_page(page_header.clone(), buffer.clone(), Type::INT32, None).unwrap_err();
1358        assert_eq!(
1359            err.to_string(),
1360            "Parquet error: Page type INDEX_PAGE is not supported"
1361        );
1362
1363        page_header.data_page_header_v2 = Some(DataPageHeaderV2 {
1364            num_nulls: 0,
1365            num_rows: 0,
1366            num_values: 0,
1367            encoding: Encoding::PLAIN,
1368            definition_levels_byte_length: 11,
1369            repetition_levels_byte_length: 0,
1370            is_compressed: None,
1371            statistics: None,
1372        });
1373        let err = decode_page(page_header, buffer, Type::INT32, None).unwrap_err();
1374        assert!(
1375            err.to_string()
1376                .contains("DataPage v2 header contains implausible values")
1377        );
1378    }
1379
1380    #[test]
1381    fn test_cursor_and_file_has_the_same_behaviour() {
1382        let mut buf: Vec<u8> = Vec::new();
1383        get_test_file("alltypes_plain.parquet")
1384            .read_to_end(&mut buf)
1385            .unwrap();
1386        let cursor = Bytes::from(buf);
1387        let read_from_cursor = SerializedFileReader::new(cursor).unwrap();
1388
1389        let test_file = get_test_file("alltypes_plain.parquet");
1390        let read_from_file = SerializedFileReader::new(test_file).unwrap();
1391
1392        let file_iter = read_from_file.get_row_iter(None).unwrap();
1393        let cursor_iter = read_from_cursor.get_row_iter(None).unwrap();
1394
1395        for (a, b) in file_iter.zip(cursor_iter) {
1396            assert_eq!(a.unwrap(), b.unwrap())
1397        }
1398    }
1399
1400    #[test]
1401    fn test_file_reader_try_from() {
1402        // Valid file path
1403        let test_file = get_test_file("alltypes_plain.parquet");
1404        let test_path_buf = get_test_path("alltypes_plain.parquet");
1405        let test_path = test_path_buf.as_path();
1406        let test_path_str = test_path.to_str().unwrap();
1407
1408        let reader = SerializedFileReader::try_from(test_file);
1409        assert!(reader.is_ok());
1410
1411        let reader = SerializedFileReader::try_from(test_path);
1412        assert!(reader.is_ok());
1413
1414        let reader = SerializedFileReader::try_from(test_path_str);
1415        assert!(reader.is_ok());
1416
1417        let reader = SerializedFileReader::try_from(test_path_str.to_string());
1418        assert!(reader.is_ok());
1419
1420        // Invalid file path
1421        let test_path = Path::new("invalid.parquet");
1422        let test_path_str = test_path.to_str().unwrap();
1423
1424        let reader = SerializedFileReader::try_from(test_path);
1425        assert!(reader.is_err());
1426
1427        let reader = SerializedFileReader::try_from(test_path_str);
1428        assert!(reader.is_err());
1429
1430        let reader = SerializedFileReader::try_from(test_path_str.to_string());
1431        assert!(reader.is_err());
1432    }
1433
1434    #[test]
1435    fn test_file_reader_into_iter() {
1436        let path = get_test_path("alltypes_plain.parquet");
1437        let reader = SerializedFileReader::try_from(path.as_path()).unwrap();
1438        let iter = reader.into_iter();
1439        let values: Vec<_> = iter.flat_map(|x| x.unwrap().get_int(0)).collect();
1440
1441        assert_eq!(values, &[4, 5, 6, 7, 2, 3, 0, 1]);
1442    }
1443
1444    #[test]
1445    fn test_file_reader_into_iter_project() {
1446        let path = get_test_path("alltypes_plain.parquet");
1447        let reader = SerializedFileReader::try_from(path.as_path()).unwrap();
1448        let schema = "message schema { OPTIONAL INT32 id; }";
1449        let proj = parse_message_type(schema).ok();
1450        let iter = reader.into_iter().project(proj).unwrap();
1451        let values: Vec<_> = iter.flat_map(|x| x.unwrap().get_int(0)).collect();
1452
1453        assert_eq!(values, &[4, 5, 6, 7, 2, 3, 0, 1]);
1454    }
1455
1456    #[test]
1457    fn test_reuse_file_chunk() {
1458        // This test covers the case of maintaining the correct start position in a file
1459        // stream for each column reader after initializing and moving to the next one
1460        // (without necessarily reading the entire column).
1461        let test_file = get_test_file("alltypes_plain.parquet");
1462        let reader = SerializedFileReader::new(test_file).unwrap();
1463        let row_group = reader.get_row_group(0).unwrap();
1464
1465        let mut page_readers = Vec::new();
1466        for i in 0..row_group.num_columns() {
1467            page_readers.push(row_group.get_column_page_reader(i).unwrap());
1468        }
1469
1470        // Now buffer each col reader, we do not expect any failures like:
1471        // General("underlying Thrift error: end of file")
1472        for mut page_reader in page_readers {
1473            assert!(page_reader.get_next_page().is_ok());
1474        }
1475    }
1476
1477    #[test]
1478    fn test_file_reader() {
1479        let test_file = get_test_file("alltypes_plain.parquet");
1480        let reader_result = SerializedFileReader::new(test_file);
1481        assert!(reader_result.is_ok());
1482        let reader = reader_result.unwrap();
1483
1484        // Test contents in Parquet metadata
1485        let metadata = reader.metadata();
1486        assert_eq!(metadata.num_row_groups(), 1);
1487
1488        // Test contents in file metadata
1489        let file_metadata = metadata.file_metadata();
1490        assert!(file_metadata.created_by().is_some());
1491        assert_eq!(
1492            file_metadata.created_by().unwrap(),
1493            "impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)"
1494        );
1495        assert!(file_metadata.key_value_metadata().is_none());
1496        assert_eq!(file_metadata.num_rows(), 8);
1497        assert_eq!(file_metadata.version(), 1);
1498        assert_eq!(file_metadata.column_orders(), None);
1499
1500        // Test contents in row group metadata
1501        let row_group_metadata = metadata.row_group(0);
1502        assert_eq!(row_group_metadata.num_columns(), 11);
1503        assert_eq!(row_group_metadata.num_rows(), 8);
1504        assert_eq!(row_group_metadata.total_byte_size(), 671);
1505        // Check each column order
1506        for i in 0..row_group_metadata.num_columns() {
1507            assert_eq!(file_metadata.column_order(i), ColumnOrder::UNDEFINED);
1508        }
1509
1510        // Test row group reader
1511        let row_group_reader_result = reader.get_row_group(0);
1512        assert!(row_group_reader_result.is_ok());
1513        let row_group_reader: Box<dyn RowGroupReader> = row_group_reader_result.unwrap();
1514        assert_eq!(
1515            row_group_reader.num_columns(),
1516            row_group_metadata.num_columns()
1517        );
1518        assert_eq!(
1519            row_group_reader.metadata().total_byte_size(),
1520            row_group_metadata.total_byte_size()
1521        );
1522
1523        // Test page readers
1524        // TODO: test for every column
1525        let page_reader_0_result = row_group_reader.get_column_page_reader(0);
1526        assert!(page_reader_0_result.is_ok());
1527        let mut page_reader_0: Box<dyn PageReader> = page_reader_0_result.unwrap();
1528        let mut page_count = 0;
1529        while let Some(page) = page_reader_0.get_next_page().unwrap() {
1530            let is_expected_page = match page {
1531                Page::DictionaryPage {
1532                    buf,
1533                    num_values,
1534                    encoding,
1535                    is_sorted,
1536                } => {
1537                    assert_eq!(buf.len(), 32);
1538                    assert_eq!(num_values, 8);
1539                    assert_eq!(encoding, Encoding::PLAIN_DICTIONARY);
1540                    assert!(!is_sorted);
1541                    true
1542                }
1543                Page::DataPage {
1544                    buf,
1545                    num_values,
1546                    encoding,
1547                    def_level_encoding,
1548                    rep_level_encoding,
1549                    statistics,
1550                } => {
1551                    assert_eq!(buf.len(), 11);
1552                    assert_eq!(num_values, 8);
1553                    assert_eq!(encoding, Encoding::PLAIN_DICTIONARY);
1554                    assert_eq!(def_level_encoding, Encoding::RLE);
1555                    #[expect(deprecated)]
1556                    let expected_rep_level_encoding = Encoding::BIT_PACKED;
1557                    assert_eq!(rep_level_encoding, expected_rep_level_encoding);
1558                    assert!(statistics.is_none());
1559                    true
1560                }
1561                Page::DataPageV2 { .. } => false,
1562            };
1563            assert!(is_expected_page);
1564            page_count += 1;
1565        }
1566        assert_eq!(page_count, 2);
1567    }
1568
1569    #[test]
1570    fn test_file_reader_datapage_v2() {
1571        let test_file = get_test_file("datapage_v2.snappy.parquet");
1572        let reader_result = SerializedFileReader::new(test_file);
1573        assert!(reader_result.is_ok());
1574        let reader = reader_result.unwrap();
1575
1576        // Test contents in Parquet metadata
1577        let metadata = reader.metadata();
1578        assert_eq!(metadata.num_row_groups(), 1);
1579
1580        // Test contents in file metadata
1581        let file_metadata = metadata.file_metadata();
1582        assert!(file_metadata.created_by().is_some());
1583        assert_eq!(
1584            file_metadata.created_by().unwrap(),
1585            "parquet-mr version 1.8.1 (build 4aba4dae7bb0d4edbcf7923ae1339f28fd3f7fcf)"
1586        );
1587        assert!(file_metadata.key_value_metadata().is_some());
1588        assert_eq!(file_metadata.key_value_metadata().unwrap().len(), 1);
1589
1590        assert_eq!(file_metadata.num_rows(), 5);
1591        assert_eq!(file_metadata.version(), 1);
1592        assert_eq!(file_metadata.column_orders(), None);
1593
1594        let row_group_metadata = metadata.row_group(0);
1595
1596        // Check each column order
1597        for i in 0..row_group_metadata.num_columns() {
1598            assert_eq!(file_metadata.column_order(i), ColumnOrder::UNDEFINED);
1599        }
1600
1601        // Test row group reader
1602        let row_group_reader_result = reader.get_row_group(0);
1603        assert!(row_group_reader_result.is_ok());
1604        let row_group_reader: Box<dyn RowGroupReader> = row_group_reader_result.unwrap();
1605        assert_eq!(
1606            row_group_reader.num_columns(),
1607            row_group_metadata.num_columns()
1608        );
1609        assert_eq!(
1610            row_group_reader.metadata().total_byte_size(),
1611            row_group_metadata.total_byte_size()
1612        );
1613
1614        // Test page readers
1615        // TODO: test for every column
1616        let page_reader_0_result = row_group_reader.get_column_page_reader(0);
1617        assert!(page_reader_0_result.is_ok());
1618        let mut page_reader_0: Box<dyn PageReader> = page_reader_0_result.unwrap();
1619        let mut page_count = 0;
1620        while let Some(page) = page_reader_0.get_next_page().unwrap() {
1621            let is_expected_page = match page {
1622                Page::DictionaryPage {
1623                    buf,
1624                    num_values,
1625                    encoding,
1626                    is_sorted,
1627                } => {
1628                    assert_eq!(buf.len(), 7);
1629                    assert_eq!(num_values, 1);
1630                    assert_eq!(encoding, Encoding::PLAIN);
1631                    assert!(!is_sorted);
1632                    true
1633                }
1634                Page::DataPageV2 {
1635                    buf,
1636                    num_values,
1637                    encoding,
1638                    num_nulls,
1639                    num_rows,
1640                    def_levels_byte_len,
1641                    rep_levels_byte_len,
1642                    is_compressed,
1643                    statistics,
1644                } => {
1645                    assert_eq!(buf.len(), 4);
1646                    assert_eq!(num_values, 5);
1647                    assert_eq!(encoding, Encoding::RLE_DICTIONARY);
1648                    assert_eq!(num_nulls, 1);
1649                    assert_eq!(num_rows, 5);
1650                    assert_eq!(def_levels_byte_len, 2);
1651                    assert_eq!(rep_levels_byte_len, 0);
1652                    assert!(is_compressed);
1653                    assert!(statistics.is_none()); // page stats are no longer read
1654                    true
1655                }
1656                Page::DataPage { .. } => false,
1657            };
1658            assert!(is_expected_page);
1659            page_count += 1;
1660        }
1661        assert_eq!(page_count, 2);
1662    }
1663
1664    #[cfg_attr(miri, ignore)] // calls native Zstd code unsupported by Miri
1665    #[test]
1666    fn test_file_reader_empty_compressed_datapage_v2() {
1667        // this file has a compressed datapage that un-compresses to 0 bytes
1668        let test_file = get_test_file("page_v2_empty_compressed.parquet");
1669        let reader_result = SerializedFileReader::new(test_file);
1670        assert!(reader_result.is_ok());
1671        let reader = reader_result.unwrap();
1672
1673        // Test contents in Parquet metadata
1674        let metadata = reader.metadata();
1675        assert_eq!(metadata.num_row_groups(), 1);
1676
1677        // Test contents in file metadata
1678        let file_metadata = metadata.file_metadata();
1679        assert!(file_metadata.created_by().is_some());
1680        assert_eq!(
1681            file_metadata.created_by().unwrap(),
1682            "parquet-cpp-arrow version 14.0.2"
1683        );
1684        assert!(file_metadata.key_value_metadata().is_some());
1685        assert_eq!(file_metadata.key_value_metadata().unwrap().len(), 1);
1686
1687        assert_eq!(file_metadata.num_rows(), 10);
1688        assert_eq!(file_metadata.version(), 2);
1689        let expected_order = ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::SIGNED);
1690        assert_eq!(
1691            file_metadata.column_orders(),
1692            Some(vec![expected_order].as_ref())
1693        );
1694
1695        let row_group_metadata = metadata.row_group(0);
1696
1697        // Check each column order
1698        for i in 0..row_group_metadata.num_columns() {
1699            assert_eq!(file_metadata.column_order(i), expected_order);
1700        }
1701
1702        // Test row group reader
1703        let row_group_reader_result = reader.get_row_group(0);
1704        assert!(row_group_reader_result.is_ok());
1705        let row_group_reader: Box<dyn RowGroupReader> = row_group_reader_result.unwrap();
1706        assert_eq!(
1707            row_group_reader.num_columns(),
1708            row_group_metadata.num_columns()
1709        );
1710        assert_eq!(
1711            row_group_reader.metadata().total_byte_size(),
1712            row_group_metadata.total_byte_size()
1713        );
1714
1715        // Test page readers
1716        let page_reader_0_result = row_group_reader.get_column_page_reader(0);
1717        assert!(page_reader_0_result.is_ok());
1718        let mut page_reader_0: Box<dyn PageReader> = page_reader_0_result.unwrap();
1719        let mut page_count = 0;
1720        while let Some(page) = page_reader_0.get_next_page().unwrap() {
1721            let is_expected_page = match page {
1722                Page::DictionaryPage {
1723                    buf,
1724                    num_values,
1725                    encoding,
1726                    is_sorted,
1727                } => {
1728                    assert_eq!(buf.len(), 0);
1729                    assert_eq!(num_values, 0);
1730                    assert_eq!(encoding, Encoding::PLAIN);
1731                    assert!(!is_sorted);
1732                    true
1733                }
1734                Page::DataPageV2 {
1735                    buf,
1736                    num_values,
1737                    encoding,
1738                    num_nulls,
1739                    num_rows,
1740                    def_levels_byte_len,
1741                    rep_levels_byte_len,
1742                    is_compressed,
1743                    statistics,
1744                } => {
1745                    assert_eq!(buf.len(), 3);
1746                    assert_eq!(num_values, 10);
1747                    assert_eq!(encoding, Encoding::RLE_DICTIONARY);
1748                    assert_eq!(num_nulls, 10);
1749                    assert_eq!(num_rows, 10);
1750                    assert_eq!(def_levels_byte_len, 2);
1751                    assert_eq!(rep_levels_byte_len, 0);
1752                    assert!(is_compressed);
1753                    assert!(statistics.is_none()); // page stats are no longer read
1754                    true
1755                }
1756                Page::DataPage { .. } => false,
1757            };
1758            assert!(is_expected_page);
1759            page_count += 1;
1760        }
1761        assert_eq!(page_count, 2);
1762    }
1763
1764    #[test]
1765    fn test_file_reader_empty_datapage_v2() {
1766        // this file has 0 bytes compressed datapage that un-compresses to 0 bytes
1767        let test_file = get_test_file("datapage_v2_empty_datapage.snappy.parquet");
1768        let reader_result = SerializedFileReader::new(test_file);
1769        assert!(reader_result.is_ok());
1770        let reader = reader_result.unwrap();
1771
1772        // Test contents in Parquet metadata
1773        let metadata = reader.metadata();
1774        assert_eq!(metadata.num_row_groups(), 1);
1775
1776        // Test contents in file metadata
1777        let file_metadata = metadata.file_metadata();
1778        assert!(file_metadata.created_by().is_some());
1779        assert_eq!(
1780            file_metadata.created_by().unwrap(),
1781            "parquet-mr version 1.13.1 (build db4183109d5b734ec5930d870cdae161e408ddba)"
1782        );
1783        assert!(file_metadata.key_value_metadata().is_some());
1784        assert_eq!(file_metadata.key_value_metadata().unwrap().len(), 2);
1785
1786        assert_eq!(file_metadata.num_rows(), 1);
1787        assert_eq!(file_metadata.version(), 1);
1788        let expected_order = ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::SIGNED);
1789        assert_eq!(
1790            file_metadata.column_orders(),
1791            Some(vec![expected_order].as_ref())
1792        );
1793
1794        let row_group_metadata = metadata.row_group(0);
1795
1796        // Check each column order
1797        for i in 0..row_group_metadata.num_columns() {
1798            assert_eq!(file_metadata.column_order(i), expected_order);
1799        }
1800
1801        // Test row group reader
1802        let row_group_reader_result = reader.get_row_group(0);
1803        assert!(row_group_reader_result.is_ok());
1804        let row_group_reader: Box<dyn RowGroupReader> = row_group_reader_result.unwrap();
1805        assert_eq!(
1806            row_group_reader.num_columns(),
1807            row_group_metadata.num_columns()
1808        );
1809        assert_eq!(
1810            row_group_reader.metadata().total_byte_size(),
1811            row_group_metadata.total_byte_size()
1812        );
1813
1814        // Test page readers
1815        let page_reader_0_result = row_group_reader.get_column_page_reader(0);
1816        assert!(page_reader_0_result.is_ok());
1817        let mut page_reader_0: Box<dyn PageReader> = page_reader_0_result.unwrap();
1818        let mut page_count = 0;
1819        while let Some(page) = page_reader_0.get_next_page().unwrap() {
1820            let is_expected_page = match page {
1821                Page::DataPageV2 {
1822                    buf,
1823                    num_values,
1824                    encoding,
1825                    num_nulls,
1826                    num_rows,
1827                    def_levels_byte_len,
1828                    rep_levels_byte_len,
1829                    is_compressed,
1830                    statistics,
1831                } => {
1832                    assert_eq!(buf.len(), 2);
1833                    assert_eq!(num_values, 1);
1834                    assert_eq!(encoding, Encoding::PLAIN);
1835                    assert_eq!(num_nulls, 1);
1836                    assert_eq!(num_rows, 1);
1837                    assert_eq!(def_levels_byte_len, 2);
1838                    assert_eq!(rep_levels_byte_len, 0);
1839                    assert!(is_compressed);
1840                    assert!(statistics.is_none());
1841                    true
1842                }
1843                _ => false,
1844            };
1845            assert!(is_expected_page);
1846            page_count += 1;
1847        }
1848        assert_eq!(page_count, 1);
1849    }
1850
1851    fn get_serialized_page_reader<R: ChunkReader>(
1852        file_reader: &SerializedFileReader<R>,
1853        row_group_idx: usize,
1854        column: usize,
1855    ) -> Result<SerializedPageReader<R>> {
1856        let row_group = {
1857            let row_group_metadata = file_reader.metadata.row_group(row_group_idx);
1858            let props = Arc::clone(&file_reader.props);
1859            let f = Arc::clone(&file_reader.chunk_reader);
1860            let page_index = file_reader.metadata.page_index_for_row_group(row_group_idx);
1861            SerializedRowGroupReader::new(f, row_group_metadata, page_index, props)?
1862        };
1863
1864        let col = row_group.metadata.column(column);
1865        let page_locations = file_reader
1866            .metadata
1867            .page_index()
1868            .map(|pi| pi.page_locations(row_group_idx, column))
1869            .unwrap_or(None)
1870            .cloned();
1871
1872        let props = Arc::clone(&row_group.props);
1873        SerializedPageReader::new_with_properties(
1874            Arc::clone(&row_group.chunk_reader),
1875            col,
1876            usize::try_from(row_group.metadata.num_rows())?,
1877            page_locations,
1878            props,
1879        )
1880    }
1881
1882    #[test]
1883    fn test_peek_next_page_offset_matches_actual() -> Result<()> {
1884        let test_file = get_test_file("alltypes_plain.parquet");
1885        let reader = SerializedFileReader::new(test_file)?;
1886
1887        let mut offset_set = HashSet::new();
1888        let num_row_groups = reader.metadata.num_row_groups();
1889        for row_group in 0..num_row_groups {
1890            let num_columns = reader.metadata.row_group(row_group).num_columns();
1891            for column in 0..num_columns {
1892                let mut page_reader = get_serialized_page_reader(&reader, row_group, column)?;
1893
1894                while let Ok(Some(page_offset)) = page_reader.peek_next_page_offset() {
1895                    match &page_reader.state {
1896                        SerializedPageReaderState::Pages {
1897                            page_locations,
1898                            dictionary_page,
1899                            ..
1900                        } => {
1901                            if let Some(page) = dictionary_page {
1902                                assert_eq!(page.offset as u64, page_offset);
1903                            } else if let Some(page) = page_locations.front() {
1904                                assert_eq!(page.offset as u64, page_offset);
1905                            } else {
1906                                unreachable!()
1907                            }
1908                        }
1909                        SerializedPageReaderState::Values {
1910                            offset,
1911                            next_page_header,
1912                            ..
1913                        } => {
1914                            assert!(next_page_header.is_some());
1915                            assert_eq!(*offset, page_offset);
1916                        }
1917                    }
1918                    let page = page_reader.get_next_page()?;
1919                    assert!(page.is_some());
1920                    let newly_inserted = offset_set.insert(page_offset);
1921                    assert!(newly_inserted);
1922                }
1923            }
1924        }
1925
1926        Ok(())
1927    }
1928
1929    #[test]
1930    fn test_page_iterator() {
1931        let file = get_test_file("alltypes_plain.parquet");
1932        let file_reader = Arc::new(SerializedFileReader::new(file).unwrap());
1933
1934        let mut page_iterator = FilePageIterator::new(0, file_reader.clone()).unwrap();
1935
1936        // read first page
1937        let page = page_iterator.next();
1938        assert!(page.is_some());
1939        assert!(page.unwrap().is_ok());
1940
1941        // reach end of file
1942        let page = page_iterator.next();
1943        assert!(page.is_none());
1944
1945        let row_group_indices = Box::new(0..1);
1946        let mut page_iterator =
1947            FilePageIterator::with_row_groups(0, row_group_indices, file_reader).unwrap();
1948
1949        // read first page
1950        let page = page_iterator.next();
1951        assert!(page.is_some());
1952        assert!(page.unwrap().is_ok());
1953
1954        // reach end of file
1955        let page = page_iterator.next();
1956        assert!(page.is_none());
1957    }
1958
1959    #[test]
1960    fn test_file_reader_key_value_metadata() {
1961        let file = get_test_file("binary.parquet");
1962        let file_reader = Arc::new(SerializedFileReader::new(file).unwrap());
1963
1964        let metadata = file_reader
1965            .metadata
1966            .file_metadata()
1967            .key_value_metadata()
1968            .unwrap();
1969
1970        assert_eq!(metadata.len(), 3);
1971
1972        assert_eq!(metadata[0].key, "parquet.proto.descriptor");
1973
1974        assert_eq!(metadata[1].key, "writer.model.name");
1975        assert_eq!(metadata[1].value, Some("protobuf".to_owned()));
1976
1977        assert_eq!(metadata[2].key, "parquet.proto.class");
1978        assert_eq!(metadata[2].value, Some("foo.baz.Foobaz$Event".to_owned()));
1979    }
1980
1981    #[test]
1982    fn test_file_reader_optional_metadata() {
1983        // file with optional metadata: bloom filters, encoding stats, column index and offset index.
1984        let file = get_test_file("data_index_bloom_encoding_stats.parquet");
1985        let options = ReadOptionsBuilder::new()
1986            .with_encoding_stats_as_mask(false)
1987            .build();
1988        let file_reader = Arc::new(SerializedFileReader::new_with_options(file, options).unwrap());
1989
1990        let row_group_metadata = file_reader.metadata.row_group(0);
1991        let col0_metadata = row_group_metadata.column(0);
1992
1993        // test optional bloom filter offset
1994        assert_eq!(col0_metadata.bloom_filter_offset().unwrap(), 192);
1995
1996        // test page encoding stats
1997        let page_encoding_stats = &col0_metadata.page_encoding_stats().unwrap()[0];
1998
1999        assert_eq!(page_encoding_stats.page_type, basic::PageType::DATA_PAGE);
2000        assert_eq!(page_encoding_stats.encoding, Encoding::PLAIN);
2001        assert_eq!(page_encoding_stats.count, 1);
2002
2003        // test optional column index offset
2004        assert_eq!(col0_metadata.column_index_offset().unwrap(), 156);
2005        assert_eq!(col0_metadata.column_index_length().unwrap(), 25);
2006
2007        // test optional offset index offset
2008        assert_eq!(col0_metadata.offset_index_offset().unwrap(), 181);
2009        assert_eq!(col0_metadata.offset_index_length().unwrap(), 11);
2010    }
2011
2012    #[test]
2013    fn test_file_reader_page_stats_mask() {
2014        let file = get_test_file("alltypes_tiny_pages.parquet");
2015        let options = ReadOptionsBuilder::new()
2016            .with_encoding_stats_as_mask(true)
2017            .build();
2018        let file_reader = Arc::new(SerializedFileReader::new_with_options(file, options).unwrap());
2019
2020        let row_group_metadata = file_reader.metadata.row_group(0);
2021
2022        // test page encoding stats
2023        let page_encoding_stats = row_group_metadata
2024            .column(0)
2025            .page_encoding_stats_mask()
2026            .unwrap();
2027        assert!(page_encoding_stats.is_only(Encoding::PLAIN));
2028        let page_encoding_stats = row_group_metadata
2029            .column(2)
2030            .page_encoding_stats_mask()
2031            .unwrap();
2032        assert!(page_encoding_stats.is_only(Encoding::PLAIN_DICTIONARY));
2033    }
2034
2035    #[test]
2036    fn test_file_reader_page_stats_skipped() {
2037        let file = get_test_file("alltypes_tiny_pages.parquet");
2038
2039        // test skipping all
2040        let options = ReadOptionsBuilder::new()
2041            .with_encoding_stats_policy(ParquetStatisticsPolicy::SkipAll)
2042            .with_column_stats_policy(ParquetStatisticsPolicy::SkipAll)
2043            .build();
2044        let file_reader = Arc::new(
2045            SerializedFileReader::new_with_options(file.try_clone().unwrap(), options).unwrap(),
2046        );
2047
2048        let row_group_metadata = file_reader.metadata.row_group(0);
2049        for column in row_group_metadata.columns() {
2050            assert!(column.page_encoding_stats().is_none());
2051            assert!(column.page_encoding_stats_mask().is_none());
2052            assert!(column.statistics().is_none());
2053        }
2054
2055        // test skipping all but one column
2056        let options = ReadOptionsBuilder::new()
2057            .with_encoding_stats_as_mask(true)
2058            .with_encoding_stats_policy(ParquetStatisticsPolicy::skip_except(&[0]))
2059            .with_column_stats_policy(ParquetStatisticsPolicy::skip_except(&[0]))
2060            .build();
2061        let file_reader = Arc::new(
2062            SerializedFileReader::new_with_options(file.try_clone().unwrap(), options).unwrap(),
2063        );
2064
2065        let row_group_metadata = file_reader.metadata.row_group(0);
2066        for (idx, column) in row_group_metadata.columns().iter().enumerate() {
2067            assert!(column.page_encoding_stats().is_none());
2068            assert_eq!(column.page_encoding_stats_mask().is_some(), idx == 0);
2069            assert_eq!(column.statistics().is_some(), idx == 0);
2070        }
2071    }
2072
2073    #[test]
2074    fn test_file_reader_size_stats_skipped() {
2075        let file = get_test_file("repeated_primitive_no_list.parquet");
2076
2077        // test skipping all
2078        let options = ReadOptionsBuilder::new()
2079            .with_size_stats_policy(ParquetStatisticsPolicy::SkipAll)
2080            .build();
2081        let file_reader = Arc::new(
2082            SerializedFileReader::new_with_options(file.try_clone().unwrap(), options).unwrap(),
2083        );
2084
2085        let row_group_metadata = file_reader.metadata.row_group(0);
2086        for column in row_group_metadata.columns() {
2087            assert!(column.repetition_level_histogram().is_none());
2088            assert!(column.definition_level_histogram().is_none());
2089            assert!(column.unencoded_byte_array_data_bytes().is_none());
2090        }
2091
2092        // test skipping all but one column
2093        let options = ReadOptionsBuilder::new()
2094            .with_encoding_stats_as_mask(true)
2095            .with_size_stats_policy(ParquetStatisticsPolicy::skip_except(&[1]))
2096            .build();
2097        let file_reader = Arc::new(
2098            SerializedFileReader::new_with_options(file.try_clone().unwrap(), options).unwrap(),
2099        );
2100
2101        let row_group_metadata = file_reader.metadata.row_group(0);
2102        for (idx, column) in row_group_metadata.columns().iter().enumerate() {
2103            assert_eq!(column.repetition_level_histogram().is_some(), idx == 1);
2104            assert_eq!(column.definition_level_histogram().is_some(), idx == 1);
2105            assert_eq!(column.unencoded_byte_array_data_bytes().is_some(), idx == 1);
2106        }
2107    }
2108
2109    #[test]
2110    fn test_file_reader_with_no_filter() -> Result<()> {
2111        let test_file = get_test_file("alltypes_plain.parquet");
2112        let origin_reader = SerializedFileReader::new(test_file)?;
2113        // test initial number of row groups
2114        let metadata = origin_reader.metadata();
2115        assert_eq!(metadata.num_row_groups(), 1);
2116        Ok(())
2117    }
2118
2119    #[test]
2120    fn test_file_reader_filter_row_groups_with_predicate() -> Result<()> {
2121        let test_file = get_test_file("alltypes_plain.parquet");
2122        let read_options = ReadOptionsBuilder::new()
2123            .with_predicate(Box::new(|_, _| false))
2124            .build();
2125        let reader = SerializedFileReader::new_with_options(test_file, read_options)?;
2126        let metadata = reader.metadata();
2127        assert_eq!(metadata.num_row_groups(), 0);
2128        Ok(())
2129    }
2130
2131    #[test]
2132    fn test_file_reader_filter_row_groups_with_range() -> Result<()> {
2133        let test_file = get_test_file("alltypes_plain.parquet");
2134        let origin_reader = SerializedFileReader::new(test_file)?;
2135        // test initial number of row groups
2136        let metadata = origin_reader.metadata();
2137        assert_eq!(metadata.num_row_groups(), 1);
2138        let mid = get_midpoint_offset(metadata.row_group(0));
2139
2140        let test_file = get_test_file("alltypes_plain.parquet");
2141        let read_options = ReadOptionsBuilder::new().with_range(0, mid + 1).build();
2142        let reader = SerializedFileReader::new_with_options(test_file, read_options)?;
2143        let metadata = reader.metadata();
2144        assert_eq!(metadata.num_row_groups(), 1);
2145
2146        let test_file = get_test_file("alltypes_plain.parquet");
2147        let read_options = ReadOptionsBuilder::new().with_range(0, mid).build();
2148        let reader = SerializedFileReader::new_with_options(test_file, read_options)?;
2149        let metadata = reader.metadata();
2150        assert_eq!(metadata.num_row_groups(), 0);
2151        Ok(())
2152    }
2153
2154    #[test]
2155    #[cfg_attr(miri, ignore)] // Takes too long
2156    fn test_file_reader_filter_row_groups_and_range() -> Result<()> {
2157        let test_file = get_test_file("alltypes_tiny_pages.parquet");
2158        let origin_reader = SerializedFileReader::new(test_file)?;
2159        let metadata = origin_reader.metadata();
2160        let mid = get_midpoint_offset(metadata.row_group(0));
2161
2162        // true, true predicate
2163        let test_file = get_test_file("alltypes_tiny_pages.parquet");
2164        let read_options = ReadOptionsBuilder::new()
2165            .with_page_index()
2166            .with_predicate(Box::new(|_, _| true))
2167            .with_range(mid, mid + 1)
2168            .build();
2169        let reader = SerializedFileReader::new_with_options(test_file, read_options)?;
2170        let metadata = reader.metadata();
2171        assert_eq!(metadata.num_row_groups(), 1);
2172        assert!(metadata.page_index().is_some());
2173
2174        // true, false predicate
2175        let test_file = get_test_file("alltypes_tiny_pages.parquet");
2176        let read_options = ReadOptionsBuilder::new()
2177            .with_page_index()
2178            .with_predicate(Box::new(|_, _| true))
2179            .with_range(0, mid)
2180            .build();
2181        let reader = SerializedFileReader::new_with_options(test_file, read_options)?;
2182        let metadata = reader.metadata();
2183        assert_eq!(metadata.num_row_groups(), 0);
2184        assert!(metadata.page_index().is_none());
2185
2186        // false, true predicate
2187        let test_file = get_test_file("alltypes_tiny_pages.parquet");
2188        let read_options = ReadOptionsBuilder::new()
2189            .with_page_index()
2190            .with_predicate(Box::new(|_, _| false))
2191            .with_range(mid, mid + 1)
2192            .build();
2193        let reader = SerializedFileReader::new_with_options(test_file, read_options)?;
2194        let metadata = reader.metadata();
2195        assert_eq!(metadata.num_row_groups(), 0);
2196        assert!(metadata.page_index().is_none());
2197
2198        // false, false predicate
2199        let test_file = get_test_file("alltypes_tiny_pages.parquet");
2200        let read_options = ReadOptionsBuilder::new()
2201            .with_page_index()
2202            .with_predicate(Box::new(|_, _| false))
2203            .with_range(0, mid)
2204            .build();
2205        let reader = SerializedFileReader::new_with_options(test_file, read_options)?;
2206        let metadata = reader.metadata();
2207        assert_eq!(metadata.num_row_groups(), 0);
2208        assert!(metadata.page_index().is_none());
2209        Ok(())
2210    }
2211
2212    #[test]
2213    fn test_file_reader_invalid_metadata() {
2214        let data = [
2215            255, 172, 1, 0, 50, 82, 65, 73, 1, 0, 0, 0, 169, 168, 168, 162, 87, 255, 16, 0, 0, 0,
2216            80, 65, 82, 49,
2217        ];
2218        let ret = SerializedFileReader::new(Bytes::copy_from_slice(&data));
2219        assert_eq!(
2220            ret.err().unwrap().to_string(),
2221            "Parquet error: Expected list element type of Struct but got List"
2222        );
2223    }
2224
2225    #[test]
2226    fn test_file_reader_empty_row_groups_with_zero_list_header() {
2227        let schema = parse_message_type("message test_schema { OPTIONAL INT32 k; }").unwrap();
2228        let mut out = Vec::new();
2229        SerializedFileWriter::new(&mut out, Arc::new(schema), Default::default())
2230            .unwrap()
2231            .close()
2232            .unwrap();
2233
2234        // fastparquet writes the empty row_groups list header as 0x00 instead
2235        // of 0x0C (size 0, struct). Patch the footer to match: num_rows = 0
2236        // (field 3, i64) followed by the row_groups field header and list header.
2237        let pos = out
2238            .windows(4)
2239            .position(|w| w == [0x16, 0x00, 0x19, 0x0C])
2240            .expect("empty row_groups list header not found in footer");
2241        out[pos + 3] = 0x00;
2242
2243        let reader = SerializedFileReader::new(Bytes::from(out)).unwrap();
2244        assert_eq!(reader.metadata().num_row_groups(), 0);
2245        assert_eq!(reader.metadata().file_metadata().num_rows(), 0);
2246    }
2247
2248    #[test]
2249    // Use java parquet-tools get below pageIndex info
2250    // !```
2251    // parquet-tools column-index ./data_index_bloom_encoding_stats.parquet
2252    // row group 0:
2253    // column index for column String:
2254    // Boundary order: ASCENDING
2255    // page-0  :
2256    // null count                 min                                  max
2257    // 0                          Hello                                today
2258    //
2259    // offset index for column String:
2260    // page-0   :
2261    // offset   compressed size       first row index
2262    // 4               152                     0
2263    ///```
2264    //
2265    fn test_page_index_reader() {
2266        let test_file = get_test_file("data_index_bloom_encoding_stats.parquet");
2267        let builder = ReadOptionsBuilder::new();
2268        //enable read page index
2269        let options = builder.with_page_index().build();
2270        let reader_result = SerializedFileReader::new_with_options(test_file, options);
2271        let reader = reader_result.unwrap();
2272
2273        // Test contents in Parquet metadata
2274        let metadata = reader.metadata();
2275        assert_eq!(metadata.num_row_groups(), 1);
2276
2277        let page_index = metadata.page_index().expect("page index should be present");
2278
2279        // only one row group
2280        let Some(ColumnIndexMetaData::BYTE_ARRAY(index)) = page_index.column_index(0, 0) else {
2281            unreachable!()
2282        };
2283
2284        assert_eq!(index.boundary_order, BoundaryOrder::ASCENDING);
2285
2286        //only one page group
2287        assert_eq!(index.num_pages(), 1);
2288
2289        let min = index.min_value(0).unwrap();
2290        let max = index.max_value(0).unwrap();
2291        assert_eq!(b"Hello", min.as_bytes());
2292        assert_eq!(b"today", max.as_bytes());
2293
2294        // only one row group
2295        let offset_index = page_index
2296            .offset_index(0, 0)
2297            .expect("offset index should be present");
2298        let page_offset = offset_index
2299            .page_locations()
2300            .first()
2301            .expect("offset index too small");
2302
2303        assert_eq!(4, page_offset.offset);
2304        assert_eq!(152, page_offset.compressed_page_size);
2305        assert_eq!(0, page_offset.first_row_index);
2306    }
2307
2308    #[test]
2309    #[cfg_attr(miri, ignore)] // Takes too long
2310    fn test_page_index_reader_all_type() {
2311        let test_file = get_test_file("alltypes_tiny_pages_plain.parquet");
2312        let builder = ReadOptionsBuilder::new();
2313        //enable read page index
2314        let options = builder.with_page_index().build();
2315        let reader_result = SerializedFileReader::new_with_options(test_file, options);
2316        let reader = reader_result.unwrap();
2317
2318        // Test contents in Parquet metadata
2319        let metadata = reader.metadata();
2320        assert_eq!(metadata.num_row_groups(), 1);
2321
2322        let page_index = metadata.page_index().unwrap();
2323        let row_group_offset_indexes = metadata.page_index_for_row_group(0);
2324
2325        // only one row group
2326        let row_group_metadata = metadata.row_group(0);
2327
2328        //col0->id: INT32 UNCOMPRESSED DO:0 FPO:4 SZ:37325/37325/1.00 VC:7300 ENC:BIT_PACKED,RLE,PLAIN ST:[min: 0, max: 7299, num_nulls: 0]
2329        let ci = page_index.column_index(0, 0).unwrap();
2330        assert!(!ci.is_sorted());
2331        assert!(matches!(
2332            ci.get_boundary_order(),
2333            Some(BoundaryOrder::UNORDERED)
2334        ));
2335        if let ColumnIndexMetaData::INT32(index) = ci {
2336            check_native_page_index(
2337                index,
2338                325,
2339                get_row_group_min_max_bytes(row_group_metadata, 0),
2340                BoundaryOrder::UNORDERED,
2341            );
2342            assert_eq!(
2343                row_group_offset_indexes.page_locations(0).unwrap().len(),
2344                325
2345            );
2346        } else {
2347            unreachable!()
2348        }
2349        //col1->bool_col:BOOLEAN UNCOMPRESSED DO:0 FPO:37329 SZ:3022/3022/1.00 VC:7300 ENC:BIT_PACKED,RLE,PLAIN ST:[min: false, max: true, num_nulls: 0]
2350        let ci = page_index.column_index(0, 1).unwrap();
2351        assert!(ci.is_sorted());
2352        if let ColumnIndexMetaData::BOOLEAN(index) = ci {
2353            assert_eq!(index.num_pages(), 82);
2354            assert_eq!(
2355                row_group_offset_indexes.page_locations(1).unwrap().len(),
2356                82
2357            );
2358        } else {
2359            unreachable!()
2360        }
2361        //col2->tinyint_col: INT32 UNCOMPRESSED DO:0 FPO:40351 SZ:37325/37325/1.00 VC:7300 ENC:BIT_PACKED,RLE,PLAIN ST:[min: 0, max: 9, num_nulls: 0]
2362        let ci = page_index.column_index(0, 2).unwrap();
2363        assert!(ci.is_sorted());
2364        if let ColumnIndexMetaData::INT32(index) = ci {
2365            check_native_page_index(
2366                index,
2367                325,
2368                get_row_group_min_max_bytes(row_group_metadata, 2),
2369                BoundaryOrder::ASCENDING,
2370            );
2371            assert_eq!(
2372                row_group_offset_indexes.page_locations(2).unwrap().len(),
2373                325
2374            );
2375        } else {
2376            unreachable!()
2377        }
2378        //col4->smallint_col: INT32 UNCOMPRESSED DO:0 FPO:77676 SZ:37325/37325/1.00 VC:7300 ENC:BIT_PACKED,RLE,PLAIN ST:[min: 0, max: 9, num_nulls: 0]
2379        let ci = page_index.column_index(0, 3).unwrap();
2380        assert!(ci.is_sorted());
2381        if let ColumnIndexMetaData::INT32(index) = ci {
2382            check_native_page_index(
2383                index,
2384                325,
2385                get_row_group_min_max_bytes(row_group_metadata, 3),
2386                BoundaryOrder::ASCENDING,
2387            );
2388            assert_eq!(
2389                row_group_offset_indexes.page_locations(3).unwrap().len(),
2390                325
2391            );
2392        } else {
2393            unreachable!()
2394        }
2395        //col5->smallint_col: INT32 UNCOMPRESSED DO:0 FPO:77676 SZ:37325/37325/1.00 VC:7300 ENC:BIT_PACKED,RLE,PLAIN ST:[min: 0, max: 9, num_nulls: 0]
2396        let ci = page_index.column_index(0, 4).unwrap();
2397        assert!(ci.is_sorted());
2398        if let ColumnIndexMetaData::INT32(index) = ci {
2399            check_native_page_index(
2400                index,
2401                325,
2402                get_row_group_min_max_bytes(row_group_metadata, 4),
2403                BoundaryOrder::ASCENDING,
2404            );
2405            assert_eq!(
2406                row_group_offset_indexes.page_locations(4).unwrap().len(),
2407                325
2408            );
2409        } else {
2410            unreachable!()
2411        }
2412        //col6->bigint_col: INT64 UNCOMPRESSED DO:0 FPO:152326 SZ:71598/71598/1.00 VC:7300 ENC:BIT_PACKED,RLE,PLAIN ST:[min: 0, max: 90, num_nulls: 0]
2413        let ci = page_index.column_index(0, 5).unwrap();
2414        assert!(!ci.is_sorted());
2415        if let ColumnIndexMetaData::INT64(index) = ci {
2416            check_native_page_index(
2417                index,
2418                528,
2419                get_row_group_min_max_bytes(row_group_metadata, 5),
2420                BoundaryOrder::UNORDERED,
2421            );
2422            assert_eq!(
2423                row_group_offset_indexes.page_locations(5).unwrap().len(),
2424                528
2425            );
2426        } else {
2427            unreachable!()
2428        }
2429        //col7->float_col: FLOAT UNCOMPRESSED DO:0 FPO:223924 SZ:37325/37325/1.00 VC:7300 ENC:BIT_PACKED,RLE,PLAIN ST:[min: -0.0, max: 9.9, num_nulls: 0]
2430        let ci = page_index.column_index(0, 6).unwrap();
2431        assert!(ci.is_sorted());
2432        if let ColumnIndexMetaData::FLOAT(index) = ci {
2433            check_native_page_index(
2434                index,
2435                325,
2436                get_row_group_min_max_bytes(row_group_metadata, 6),
2437                BoundaryOrder::ASCENDING,
2438            );
2439            assert_eq!(
2440                row_group_offset_indexes.page_locations(6).unwrap().len(),
2441                325
2442            );
2443        } else {
2444            unreachable!()
2445        }
2446        //col8->double_col: DOUBLE UNCOMPRESSED DO:0 FPO:261249 SZ:71598/71598/1.00 VC:7300 ENC:BIT_PACKED,RLE,PLAIN ST:[min: -0.0, max: 90.89999999999999, num_nulls: 0]
2447        let ci = page_index.column_index(0, 7).unwrap();
2448        assert!(!ci.is_sorted());
2449        if let ColumnIndexMetaData::DOUBLE(index) = ci {
2450            check_native_page_index(
2451                index,
2452                528,
2453                get_row_group_min_max_bytes(row_group_metadata, 7),
2454                BoundaryOrder::UNORDERED,
2455            );
2456            assert_eq!(
2457                row_group_offset_indexes.page_locations(7).unwrap().len(),
2458                528
2459            );
2460        } else {
2461            unreachable!()
2462        }
2463        //col9->date_string_col: BINARY UNCOMPRESSED DO:0 FPO:332847 SZ:111948/111948/1.00 VC:7300 ENC:BIT_PACKED,RLE,PLAIN ST:[min: 01/01/09, max: 12/31/10, num_nulls: 0]
2464        let ci = page_index.column_index(0, 8).unwrap();
2465        assert!(!ci.is_sorted());
2466        if let ColumnIndexMetaData::BYTE_ARRAY(index) = ci {
2467            check_byte_array_page_index(
2468                index,
2469                974,
2470                get_row_group_min_max_bytes(row_group_metadata, 8),
2471                BoundaryOrder::UNORDERED,
2472            );
2473            assert_eq!(
2474                row_group_offset_indexes.page_locations(8).unwrap().len(),
2475                974
2476            );
2477        } else {
2478            unreachable!()
2479        }
2480        //col10->string_col: BINARY UNCOMPRESSED DO:0 FPO:444795 SZ:45298/45298/1.00 VC:7300 ENC:BIT_PACKED,RLE,PLAIN ST:[min: 0, max: 9, num_nulls: 0]
2481        let ci = page_index.column_index(0, 9).unwrap();
2482        assert!(ci.is_sorted());
2483        if let ColumnIndexMetaData::BYTE_ARRAY(index) = ci {
2484            check_byte_array_page_index(
2485                index,
2486                352,
2487                get_row_group_min_max_bytes(row_group_metadata, 9),
2488                BoundaryOrder::ASCENDING,
2489            );
2490            assert_eq!(
2491                row_group_offset_indexes.page_locations(9).unwrap().len(),
2492                352
2493            );
2494        } else {
2495            unreachable!()
2496        }
2497        //col11->timestamp_col: INT96 UNCOMPRESSED DO:0 FPO:490093 SZ:111948/111948/1.00 VC:7300 ENC:BIT_PACKED,RLE,PLAIN ST:[num_nulls: 0, min/max not defined]
2498        // this columns lacks an index
2499        assert!(page_index.column_index(0, 10).is_none());
2500        //col12->year: INT32 UNCOMPRESSED DO:0 FPO:602041 SZ:37325/37325/1.00 VC:7300 ENC:BIT_PACKED,RLE,PLAIN ST:[min: 2009, max: 2010, num_nulls: 0]
2501        let ci = page_index.column_index(0, 11).unwrap();
2502        assert!(ci.is_sorted());
2503        if let ColumnIndexMetaData::INT32(index) = ci {
2504            check_native_page_index(
2505                index,
2506                325,
2507                get_row_group_min_max_bytes(row_group_metadata, 11),
2508                BoundaryOrder::ASCENDING,
2509            );
2510            assert_eq!(
2511                row_group_offset_indexes.page_locations(11).unwrap().len(),
2512                325
2513            );
2514        } else {
2515            unreachable!()
2516        }
2517        //col13->month: INT32 UNCOMPRESSED DO:0 FPO:639366 SZ:37325/37325/1.00 VC:7300 ENC:BIT_PACKED,RLE,PLAIN ST:[min: 1, max: 12, num_nulls: 0]
2518        let ci = page_index.column_index(0, 12).unwrap();
2519        assert!(!ci.is_sorted());
2520        if let ColumnIndexMetaData::INT32(index) = ci {
2521            check_native_page_index(
2522                index,
2523                325,
2524                get_row_group_min_max_bytes(row_group_metadata, 12),
2525                BoundaryOrder::UNORDERED,
2526            );
2527            assert_eq!(
2528                row_group_offset_indexes.page_locations(12).unwrap().len(),
2529                325
2530            );
2531        } else {
2532            unreachable!()
2533        }
2534    }
2535
2536    fn check_native_page_index<T: ParquetValueType>(
2537        row_group_index: &PrimitiveColumnIndex<T>,
2538        page_size: usize,
2539        min_max: (&[u8], &[u8]),
2540        boundary_order: BoundaryOrder,
2541    ) {
2542        assert_eq!(row_group_index.num_pages() as usize, page_size);
2543        assert_eq!(row_group_index.boundary_order, boundary_order);
2544        assert!(row_group_index.min_values().iter().all(|x| {
2545            x >= &T::try_from_le_slice(min_max.0).unwrap()
2546                && x <= &T::try_from_le_slice(min_max.1).unwrap()
2547        }));
2548    }
2549
2550    fn check_byte_array_page_index(
2551        row_group_index: &ByteArrayColumnIndex,
2552        page_size: usize,
2553        min_max: (&[u8], &[u8]),
2554        boundary_order: BoundaryOrder,
2555    ) {
2556        assert_eq!(row_group_index.num_pages() as usize, page_size);
2557        assert_eq!(row_group_index.boundary_order, boundary_order);
2558        for i in 0..row_group_index.num_pages() as usize {
2559            let x = row_group_index.min_value(i).unwrap();
2560            assert!(x >= min_max.0 && x <= min_max.1);
2561        }
2562    }
2563
2564    fn get_row_group_min_max_bytes(r: &RowGroupMetaData, col_num: usize) -> (&[u8], &[u8]) {
2565        let statistics = r.column(col_num).statistics().unwrap();
2566        (
2567            statistics.min_bytes_opt().unwrap_or_default(),
2568            statistics.max_bytes_opt().unwrap_or_default(),
2569        )
2570    }
2571
2572    #[test]
2573    #[cfg_attr(miri, ignore)] // Takes too long
2574    fn test_skip_next_page_with_dictionary_page() {
2575        let test_file = get_test_file("alltypes_tiny_pages.parquet");
2576        let builder = ReadOptionsBuilder::new();
2577        // enable read page index
2578        let options = builder.with_page_index().build();
2579        let reader_result = SerializedFileReader::new_with_options(test_file, options);
2580        let reader = reader_result.unwrap();
2581
2582        let row_group_reader = reader.get_row_group(0).unwrap();
2583
2584        // use 'string_col', Boundary order: UNORDERED, total 352 data pages and 1 dictionary page.
2585        let mut column_page_reader = row_group_reader.get_column_page_reader(9).unwrap();
2586
2587        let mut vec = vec![];
2588
2589        // Step 1: Peek and ensure dictionary page is correctly identified
2590        let meta = column_page_reader.peek_next_page().unwrap().unwrap();
2591        assert!(meta.is_dict);
2592
2593        // Step 2: Call skip_next_page to skip the dictionary page
2594        column_page_reader.skip_next_page().unwrap();
2595
2596        // Step 3: Read the next data page after skipping the dictionary page
2597        let page = column_page_reader.get_next_page().unwrap().unwrap();
2598        assert!(matches!(page.page_type(), basic::PageType::DATA_PAGE));
2599
2600        // Step 4: Continue reading remaining data pages and verify correctness
2601        for _i in 0..351 {
2602            // 352 total pages, 1 dictionary page is skipped
2603            let meta = column_page_reader.peek_next_page().unwrap().unwrap();
2604            assert!(!meta.is_dict); // Verify no dictionary page here
2605            vec.push(meta);
2606
2607            let page = column_page_reader.get_next_page().unwrap().unwrap();
2608            assert!(matches!(page.page_type(), basic::PageType::DATA_PAGE));
2609        }
2610
2611        // Step 5: Check if all pages are read
2612        assert!(column_page_reader.peek_next_page().unwrap().is_none());
2613        assert!(column_page_reader.get_next_page().unwrap().is_none());
2614
2615        // Step 6: Verify the number of data pages read (should be 351 data pages)
2616        assert_eq!(vec.len(), 351);
2617    }
2618
2619    #[test]
2620    #[cfg_attr(miri, ignore)] // Takes too long
2621    fn test_skip_page_with_offset_index() {
2622        let test_file = get_test_file("alltypes_tiny_pages_plain.parquet");
2623        let builder = ReadOptionsBuilder::new();
2624        //enable read page index
2625        let options = builder.with_page_index().build();
2626        let reader_result = SerializedFileReader::new_with_options(test_file, options);
2627        let reader = reader_result.unwrap();
2628
2629        let row_group_reader = reader.get_row_group(0).unwrap();
2630
2631        //use 'int_col', Boundary order: ASCENDING, total 325 pages.
2632        let mut column_page_reader = row_group_reader.get_column_page_reader(4).unwrap();
2633
2634        let mut vec = vec![];
2635
2636        for i in 0..325 {
2637            if i % 2 == 0 {
2638                vec.push(column_page_reader.get_next_page().unwrap().unwrap());
2639            } else {
2640                column_page_reader.skip_next_page().unwrap();
2641            }
2642        }
2643        //check read all pages.
2644        assert!(column_page_reader.peek_next_page().unwrap().is_none());
2645        assert!(column_page_reader.get_next_page().unwrap().is_none());
2646
2647        assert_eq!(vec.len(), 163);
2648    }
2649
2650    #[test]
2651    fn test_skip_page_without_offset_index() {
2652        let test_file = get_test_file("alltypes_tiny_pages_plain.parquet");
2653
2654        // use default SerializedFileReader without read offsetIndex
2655        let reader_result = SerializedFileReader::new(test_file);
2656        let reader = reader_result.unwrap();
2657
2658        let row_group_reader = reader.get_row_group(0).unwrap();
2659
2660        //use 'int_col', Boundary order: ASCENDING, total 325 pages.
2661        let mut column_page_reader = row_group_reader.get_column_page_reader(4).unwrap();
2662
2663        let mut vec = vec![];
2664
2665        for i in 0..325 {
2666            if i % 2 == 0 {
2667                vec.push(column_page_reader.get_next_page().unwrap().unwrap());
2668            } else {
2669                column_page_reader.peek_next_page().unwrap().unwrap();
2670                column_page_reader.skip_next_page().unwrap();
2671            }
2672        }
2673        //check read all pages.
2674        assert!(column_page_reader.peek_next_page().unwrap().is_none());
2675        assert!(column_page_reader.get_next_page().unwrap().is_none());
2676
2677        assert_eq!(vec.len(), 163);
2678    }
2679
2680    #[test]
2681    #[cfg_attr(miri, ignore)] // Takes too long
2682    fn test_peek_page_with_dictionary_page() {
2683        let test_file = get_test_file("alltypes_tiny_pages.parquet");
2684        let builder = ReadOptionsBuilder::new();
2685        //enable read page index
2686        let options = builder.with_page_index().build();
2687        let reader_result = SerializedFileReader::new_with_options(test_file, options);
2688        let reader = reader_result.unwrap();
2689        let row_group_reader = reader.get_row_group(0).unwrap();
2690
2691        //use 'string_col', Boundary order: UNORDERED, total 352 data ages and 1 dictionary page.
2692        let mut column_page_reader = row_group_reader.get_column_page_reader(9).unwrap();
2693
2694        let mut vec = vec![];
2695
2696        let meta = column_page_reader.peek_next_page().unwrap().unwrap();
2697        assert!(meta.is_dict);
2698        let page = column_page_reader.get_next_page().unwrap().unwrap();
2699        assert!(matches!(page.page_type(), basic::PageType::DICTIONARY_PAGE));
2700
2701        for i in 0..352 {
2702            let meta = column_page_reader.peek_next_page().unwrap().unwrap();
2703            // have checked with `parquet-tools column-index   -c string_col  ./alltypes_tiny_pages.parquet`
2704            // page meta has two scenarios(21, 20) of num_rows expect last page has 11 rows.
2705            if i != 351 {
2706                assert!((meta.num_rows == Some(21)) || (meta.num_rows == Some(20)));
2707            } else {
2708                // last page first row index is 7290, total row count is 7300
2709                // because first row start with zero, last page row count should be 10.
2710                assert_eq!(meta.num_rows, Some(10));
2711            }
2712            assert!(!meta.is_dict);
2713            vec.push(meta);
2714            let page = column_page_reader.get_next_page().unwrap().unwrap();
2715            assert!(matches!(page.page_type(), basic::PageType::DATA_PAGE));
2716        }
2717
2718        //check read all pages.
2719        assert!(column_page_reader.peek_next_page().unwrap().is_none());
2720        assert!(column_page_reader.get_next_page().unwrap().is_none());
2721
2722        assert_eq!(vec.len(), 352);
2723    }
2724
2725    #[test]
2726    fn test_peek_page_with_dictionary_page_without_offset_index() {
2727        let test_file = get_test_file("alltypes_tiny_pages.parquet");
2728
2729        let reader_result = SerializedFileReader::new(test_file);
2730        let reader = reader_result.unwrap();
2731        let row_group_reader = reader.get_row_group(0).unwrap();
2732
2733        //use 'string_col', Boundary order: UNORDERED, total 352 data ages and 1 dictionary page.
2734        let mut column_page_reader = row_group_reader.get_column_page_reader(9).unwrap();
2735
2736        let mut vec = vec![];
2737
2738        let meta = column_page_reader.peek_next_page().unwrap().unwrap();
2739        assert!(meta.is_dict);
2740        let page = column_page_reader.get_next_page().unwrap().unwrap();
2741        assert!(matches!(page.page_type(), basic::PageType::DICTIONARY_PAGE));
2742
2743        for i in 0..352 {
2744            let meta = column_page_reader.peek_next_page().unwrap().unwrap();
2745            // have checked with `parquet-tools column-index   -c string_col  ./alltypes_tiny_pages.parquet`
2746            // page meta has two scenarios(21, 20) of num_rows expect last page has 11 rows.
2747            if i != 351 {
2748                assert!((meta.num_levels == Some(21)) || (meta.num_levels == Some(20)));
2749            } else {
2750                // last page first row index is 7290, total row count is 7300
2751                // because first row start with zero, last page row count should be 10.
2752                assert_eq!(meta.num_levels, Some(10));
2753            }
2754            assert!(!meta.is_dict);
2755            vec.push(meta);
2756            let page = column_page_reader.get_next_page().unwrap().unwrap();
2757            assert!(matches!(page.page_type(), basic::PageType::DATA_PAGE));
2758        }
2759
2760        //check read all pages.
2761        assert!(column_page_reader.peek_next_page().unwrap().is_none());
2762        assert!(column_page_reader.get_next_page().unwrap().is_none());
2763
2764        assert_eq!(vec.len(), 352);
2765    }
2766
2767    #[test]
2768    fn test_fixed_length_index() {
2769        let message_type = "
2770        message test_schema {
2771          OPTIONAL FIXED_LEN_BYTE_ARRAY (11) value (DECIMAL(25,2));
2772        }
2773        ";
2774
2775        let schema = parse_message_type(message_type).unwrap();
2776        let mut out = Vec::with_capacity(1024);
2777        let mut writer =
2778            SerializedFileWriter::new(&mut out, Arc::new(schema), Default::default()).unwrap();
2779
2780        let mut r = writer.next_row_group().unwrap();
2781        let mut c = r.next_column().unwrap().unwrap();
2782        c.typed::<FixedLenByteArrayType>()
2783            .write_batch(
2784                &[vec![0; 11].into(), vec![5; 11].into(), vec![3; 11].into()],
2785                Some(&[1, 1, 0, 1]),
2786                None,
2787            )
2788            .unwrap();
2789        c.close().unwrap();
2790        r.close().unwrap();
2791        writer.close().unwrap();
2792
2793        let b = Bytes::from(out);
2794        let options = ReadOptionsBuilder::new().with_page_index().build();
2795        let reader = SerializedFileReader::new_with_options(b, options).unwrap();
2796        let page_index = reader.metadata().page_index().unwrap();
2797
2798        match page_index.column_index(0, 0) {
2799            Some(ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(v)) => {
2800                assert_eq!(v.num_pages(), 1);
2801                assert_eq!(v.null_count(0).unwrap(), 1);
2802                assert_eq!(v.min_value(0).unwrap(), &[0; 11]);
2803                assert_eq!(v.max_value(0).unwrap(), &[5; 11]);
2804            }
2805            _ => unreachable!(),
2806        }
2807    }
2808
2809    #[test]
2810    fn test_multi_gz() {
2811        let file = get_test_file("concatenated_gzip_members.parquet");
2812        let reader = SerializedFileReader::new(file).unwrap();
2813        let row_group_reader = reader.get_row_group(0).unwrap();
2814        match row_group_reader.get_column_reader(0).unwrap() {
2815            ColumnReader::Int64ColumnReader(mut reader) => {
2816                let mut buffer = Vec::with_capacity(1024);
2817                let mut def_levels = Vec::with_capacity(1024);
2818                let (num_records, num_values, num_levels) = reader
2819                    .read_records(1024, Some(&mut def_levels), None, &mut buffer)
2820                    .unwrap();
2821
2822                assert_eq!(num_records, 513);
2823                assert_eq!(num_values, 513);
2824                assert_eq!(num_levels, 513);
2825
2826                let expected: Vec<i64> = (1..514).collect();
2827                assert_eq!(&buffer, &expected);
2828            }
2829            _ => unreachable!(),
2830        }
2831    }
2832
2833    #[test]
2834    #[cfg_attr(miri, ignore)] // Takes too long
2835    fn test_byte_stream_split_extended() {
2836        let path = format!(
2837            "{}/byte_stream_split_extended.gzip.parquet",
2838            arrow::util::test_util::parquet_test_data(),
2839        );
2840        let file = File::open(path).unwrap();
2841        let reader = Box::new(SerializedFileReader::new(file).expect("Failed to create reader"));
2842
2843        // Use full schema as projected schema
2844        let mut iter = reader
2845            .get_row_iter(None)
2846            .expect("Failed to create row iterator");
2847
2848        let mut start = 0;
2849        let end = reader.metadata().file_metadata().num_rows();
2850
2851        let check_row = |row: Result<Row, ParquetError>| {
2852            assert!(row.is_ok());
2853            let r = row.unwrap();
2854            assert_eq!(r.get_float16(0).unwrap(), r.get_float16(1).unwrap());
2855            assert_eq!(r.get_float(2).unwrap(), r.get_float(3).unwrap());
2856            assert_eq!(r.get_double(4).unwrap(), r.get_double(5).unwrap());
2857            assert_eq!(r.get_int(6).unwrap(), r.get_int(7).unwrap());
2858            assert_eq!(r.get_long(8).unwrap(), r.get_long(9).unwrap());
2859            assert_eq!(r.get_bytes(10).unwrap(), r.get_bytes(11).unwrap());
2860            assert_eq!(r.get_decimal(12).unwrap(), r.get_decimal(13).unwrap());
2861        };
2862
2863        while start < end {
2864            match iter.next() {
2865                Some(row) => check_row(row),
2866                None => break,
2867            }
2868            start += 1;
2869        }
2870    }
2871
2872    #[test]
2873    fn test_filtered_rowgroup_metadata() {
2874        let message_type = "
2875            message test_schema {
2876                REQUIRED INT32 a;
2877            }
2878        ";
2879        let schema = Arc::new(parse_message_type(message_type).unwrap());
2880        let props = Arc::new(
2881            WriterProperties::builder()
2882                .set_statistics_enabled(EnabledStatistics::Page)
2883                .build(),
2884        );
2885        let mut file: File = tempfile::tempfile().unwrap();
2886        let mut file_writer = SerializedFileWriter::new(&mut file, schema, props).unwrap();
2887        let data = [1, 2, 3, 4, 5];
2888
2889        // write 5 row groups
2890        for idx in 0..5 {
2891            let data_i: Vec<i32> = data.iter().map(|x| x * (idx + 1)).collect();
2892            let mut row_group_writer = file_writer.next_row_group().unwrap();
2893            if let Some(mut writer) = row_group_writer.next_column().unwrap() {
2894                writer
2895                    .typed::<Int32Type>()
2896                    .write_batch(data_i.as_slice(), None, None)
2897                    .unwrap();
2898                writer.close().unwrap();
2899            }
2900            row_group_writer.close().unwrap();
2901            file_writer.flushed_row_groups();
2902        }
2903        let file_metadata = file_writer.close().unwrap();
2904
2905        assert_eq!(file_metadata.file_metadata().num_rows(), 25);
2906        assert_eq!(file_metadata.num_row_groups(), 5);
2907
2908        // Stateful predicates must run once per group in order, with short-circuiting
2909        // and original indexes, whether or not any groups are rejected.
2910        for filter in [false, true] {
2911            let calls = Rc::new(RefCell::new(Vec::new()));
2912            let first_calls = Rc::clone(&calls);
2913            let second_calls = Rc::clone(&calls);
2914            let mut next_index = 0;
2915            let read_options = ReadOptionsBuilder::new()
2916                .with_page_index()
2917                .with_predicate(Box::new(move |rg_meta, i| {
2918                    assert_eq!(i, next_index);
2919                    assert_eq!(rg_meta.ordinal(), Some(i as i32));
2920                    next_index += 1;
2921                    first_calls.borrow_mut().push((i, 0));
2922                    !filter || i != 1
2923                }))
2924                .with_predicate(Box::new(move |_, i| {
2925                    second_calls.borrow_mut().push((i, 1));
2926                    !filter || i != 3
2927                }))
2928                .build();
2929            let reader =
2930                SerializedFileReader::new_with_options(file.try_clone().unwrap(), read_options)
2931                    .unwrap();
2932            let metadata = reader.metadata();
2933            let expected_ordinals = if filter {
2934                vec![0, 2, 4]
2935            } else {
2936                vec![0, 1, 2, 3, 4]
2937            };
2938            let ordinals: Vec<_> = metadata
2939                .row_groups()
2940                .iter()
2941                .map(|rg| rg.ordinal().unwrap())
2942                .collect();
2943            assert_eq!(ordinals, expected_ordinals);
2944            assert!(metadata.page_index().is_some_and(|idx| idx.is_complete()));
2945
2946            let expected_calls: Vec<_> = (0..5)
2947                .flat_map(|i| {
2948                    if filter && i == 1 {
2949                        vec![(i, 0)]
2950                    } else {
2951                        vec![(i, 0), (i, 1)]
2952                    }
2953                })
2954                .collect();
2955            assert_eq!(*calls.borrow(), expected_calls);
2956        }
2957
2958        // read only the 3rd row group
2959        let read_options = ReadOptionsBuilder::new()
2960            .with_page_index()
2961            .with_predicate(Box::new(|rgmeta, _| rgmeta.ordinal().unwrap_or(0) == 2))
2962            .build();
2963        let reader =
2964            SerializedFileReader::new_with_options(file.try_clone().unwrap(), read_options)
2965                .unwrap();
2966        let metadata = reader.metadata();
2967
2968        // check we got the expected row group
2969        assert_eq!(metadata.num_row_groups(), 1);
2970        assert_eq!(metadata.row_group(0).ordinal(), Some(2));
2971
2972        // check we only got the relevant page indexes
2973        assert!(metadata.page_index().is_some_and(|idx| idx.is_complete()));
2974        let page_index = metadata.page_index().unwrap();
2975
2976        let col_stats = metadata.row_group(0).column(0).statistics().unwrap();
2977        let pg_idx = page_index.column_index(0, 0);
2978        let off_idx_i = page_index.offset_index(0, 0);
2979
2980        // test that we got the index matching the row group
2981        match pg_idx {
2982            Some(ColumnIndexMetaData::INT32(int_idx)) => {
2983                let min = col_stats.min_bytes_opt().unwrap().get_i32_le();
2984                let max = col_stats.max_bytes_opt().unwrap().get_i32_le();
2985                assert_eq!(int_idx.min_value(0), Some(min).as_ref());
2986                assert_eq!(int_idx.max_value(0), Some(max).as_ref());
2987            }
2988            _ => panic!("wrong stats type"),
2989        }
2990
2991        // check offset index matches too
2992        assert_eq!(
2993            off_idx_i.as_ref().unwrap().page_locations[0].offset,
2994            metadata.row_group(0).column(0).data_page_offset()
2995        );
2996
2997        // read non-contiguous row groups
2998        let read_options = ReadOptionsBuilder::new()
2999            .with_page_index()
3000            .with_predicate(Box::new(|rgmeta, _| rgmeta.ordinal().unwrap_or(0) % 2 == 1))
3001            .build();
3002        let reader =
3003            SerializedFileReader::new_with_options(file.try_clone().unwrap(), read_options)
3004                .unwrap();
3005        let metadata = reader.metadata();
3006
3007        // check we got the expected row groups
3008        assert_eq!(metadata.num_row_groups(), 2);
3009        assert_eq!(metadata.row_group(0).ordinal(), Some(1));
3010        assert_eq!(metadata.row_group(1).ordinal(), Some(3));
3011
3012        // check we only got the relevant page indexes
3013        assert!(metadata.page_index().is_some_and(|idx| idx.is_complete()));
3014
3015        let page_index = metadata.page_index().unwrap();
3016
3017        for rg_idx in 0..metadata.num_row_groups() {
3018            let col_stats = metadata.row_group(rg_idx).column(0).statistics().unwrap();
3019            let pg_idx = page_index.column_index(rg_idx, 0);
3020            let off_idx_i = page_index.offset_index(rg_idx, 0);
3021
3022            // test that we got the index matching the row group
3023            match pg_idx {
3024                Some(ColumnIndexMetaData::INT32(int_idx)) => {
3025                    let min = col_stats.min_bytes_opt().unwrap().get_i32_le();
3026                    let max = col_stats.max_bytes_opt().unwrap().get_i32_le();
3027                    assert_eq!(int_idx.min_value(0), Some(min).as_ref());
3028                    assert_eq!(int_idx.max_value(0), Some(max).as_ref());
3029                }
3030                _ => panic!("wrong stats type"),
3031            }
3032
3033            // check offset index matches too
3034            assert_eq!(
3035                off_idx_i.as_ref().unwrap().page_locations[0].offset,
3036                metadata.row_group(rg_idx).column(0).data_page_offset()
3037            );
3038        }
3039    }
3040
3041    #[test]
3042    fn test_reuse_schema() {
3043        let file = get_test_file("alltypes_plain.parquet");
3044        let file_reader = SerializedFileReader::new(file.try_clone().unwrap()).unwrap();
3045        let schema = file_reader.metadata().file_metadata().schema_descr_ptr();
3046        let expected = file_reader.metadata;
3047
3048        let options = ReadOptionsBuilder::new()
3049            .with_parquet_schema(schema)
3050            .build();
3051        let file_reader = SerializedFileReader::new_with_options(file, options).unwrap();
3052
3053        assert_eq!(expected.as_ref(), file_reader.metadata.as_ref());
3054        // Should have used the same schema instance
3055        assert!(Arc::ptr_eq(
3056            &expected.file_metadata().schema_descr_ptr(),
3057            &file_reader.metadata.file_metadata().schema_descr_ptr()
3058        ));
3059    }
3060
3061    #[test]
3062    fn test_read_unknown_logical_type() {
3063        let file = get_test_file("unknown-logical-type.parquet");
3064        let reader = SerializedFileReader::new(file).expect("Error opening file");
3065
3066        let schema = reader.metadata().file_metadata().schema_descr();
3067        assert_eq!(
3068            schema.column(0).logical_type_ref(),
3069            Some(&basic::LogicalType::String)
3070        );
3071        assert_eq!(
3072            schema.column(1).logical_type_ref(),
3073            Some(&basic::LogicalType::_Unknown { field_id: 2555 })
3074        );
3075        assert_eq!(schema.column(1).physical_type(), Type::BYTE_ARRAY);
3076
3077        let mut iter = reader
3078            .get_row_iter(None)
3079            .expect("Failed to create row iterator");
3080
3081        let mut num_rows = 0;
3082        while iter.next().is_some() {
3083            num_rows += 1;
3084        }
3085        assert_eq!(num_rows, reader.metadata().file_metadata().num_rows());
3086    }
3087
3088    /// Builds an in-memory single-row-group file with one snappy-compressed,
3089    /// dictionary-encoded INT32 column holding `1..=5`, and returns its bytes
3090    /// together with the byte range of the dictionary page *body* (the
3091    /// compressed payload after the page header), so tests can corrupt
3092    /// exactly the part lazy skipping promises never to touch.
3093    fn dict_column_file() -> (Bytes, std::ops::Range<usize>) {
3094        let message_type = "
3095            message test_schema {
3096                REQUIRED INT32 a;
3097            }
3098        ";
3099        let schema = Arc::new(parse_message_type(message_type).unwrap());
3100        let props = Arc::new(
3101            WriterProperties::builder()
3102                .set_compression(Compression::SNAPPY)
3103                .build(),
3104        );
3105        let mut out = Vec::new();
3106        let mut writer = SerializedFileWriter::new(&mut out, schema, props).unwrap();
3107        let mut rg = writer.next_row_group().unwrap();
3108        let mut col = rg.next_column().unwrap().unwrap();
3109        col.typed::<Int32Type>()
3110            .write_batch(&[1, 2, 3, 4, 5], None, None)
3111            .unwrap();
3112        col.close().unwrap();
3113        rg.close().unwrap();
3114        writer.close().unwrap();
3115        let bytes = Bytes::from(out);
3116
3117        let reader = SerializedFileReader::new(bytes.clone()).unwrap();
3118        let col_meta = reader.metadata().row_group(0).column(0).clone();
3119        let dict_offset = col_meta.dictionary_page_offset().unwrap() as usize;
3120        let data_offset = col_meta.data_page_offset() as usize;
3121        // Parse the dictionary page header to find where its body starts.
3122        let mut prot = ThriftSliceInputProtocol::new(&bytes[dict_offset..data_offset]);
3123        let _header = PageHeader::read_thrift(&mut prot).unwrap();
3124        let header_len = (data_offset - dict_offset) - prot.as_slice().len();
3125        (bytes, dict_offset + header_len..data_offset)
3126    }
3127
3128    /// Zeroes the dictionary page body: an all-zero snappy stream declares an
3129    /// uncompressed length of zero, so decoding it fails deterministically on
3130    /// the decompressed-size check. Any code path that decompresses the
3131    /// dictionary therefore errors; a path that genuinely never touches it
3132    /// succeeds.
3133    fn corrupt_dictionary_body(bytes: &Bytes, body: &std::ops::Range<usize>) -> Bytes {
3134        let mut v = bytes.to_vec();
3135        v[body.clone()].fill(0);
3136        Bytes::from(v)
3137    }
3138
3139    fn int32_reader(bytes: Bytes) -> crate::column::reader::ColumnReaderImpl<Int32Type> {
3140        let reader = SerializedFileReader::new(bytes.clone()).unwrap();
3141        let meta = reader.metadata();
3142        let descr = meta.file_metadata().schema_descr().column(0);
3143        let col_meta = meta.row_group(0).column(0).clone();
3144        let total_rows = meta.row_group(0).num_rows() as usize;
3145        let pages =
3146            SerializedPageReader::new(Arc::new(bytes), &col_meta, total_rows, None).unwrap();
3147        crate::column::reader::ColumnReaderImpl::<Int32Type>::new(descr, Box::new(pages))
3148    }
3149
3150    /// Skipping the whole chunk never decompresses the dictionary: with the
3151    /// dictionary body corrupted so that any decompression errors, the skip
3152    /// still succeeds. Reverting the lazy branch in
3153    /// `GenericColumnReader::skip_records` turns this test red.
3154    #[test]
3155    fn skipping_a_whole_chunk_never_decodes_the_dictionary() {
3156        let (bytes, body) = dict_column_file();
3157        let corrupted = corrupt_dictionary_body(&bytes, &body);
3158        let mut reader = int32_reader(corrupted);
3159        let skipped = reader.skip_records(5).unwrap();
3160        assert_eq!(skipped, 5);
3161    }
3162
3163    /// The deferred dictionary is genuinely decoded the moment a data page
3164    /// needs it: the same corruption that whole-chunk skipping never notices
3165    /// fails a skip that has to open the data page.
3166    #[test]
3167    fn a_partial_skip_pays_the_deferred_dictionary() {
3168        let (bytes, body) = dict_column_file();
3169        let corrupted = corrupt_dictionary_body(&bytes, &body);
3170        let mut reader = int32_reader(corrupted);
3171        let err = reader.skip_records(2).unwrap_err().to_string();
3172        assert!(
3173            err.contains("corrupt") || err.contains("size"),
3174            "expected the deferred dictionary decode to fail on \
3175             decompression, got: {err}"
3176        );
3177    }
3178
3179    /// Skip-then-read returns the right values: the dictionary deferred by
3180    /// the skip is installed before the first value decodes.
3181    #[test]
3182    fn skip_then_read_decodes_correct_values() {
3183        let (bytes, _) = dict_column_file();
3184        let mut reader = int32_reader(bytes);
3185        assert_eq!(reader.skip_records(2).unwrap(), 2);
3186        let mut values = Vec::new();
3187        let (records, _, read) = reader.read_records(3, None, None, &mut values).unwrap();
3188        assert_eq!(records, 3);
3189        assert_eq!(read, 3);
3190        assert_eq!(values, vec![3, 4, 5]);
3191    }
3192
3193    /// The eager path is untouched: reading from the start still installs the
3194    /// dictionary through `get_next_page` and decodes every value.
3195    #[test]
3196    fn plain_read_is_unchanged() {
3197        let (bytes, _) = dict_column_file();
3198        let mut reader = int32_reader(bytes);
3199        let mut values = Vec::new();
3200        let (records, _, read) = reader.read_records(5, None, None, &mut values).unwrap();
3201        assert_eq!((records, read), (5, 5));
3202        assert_eq!(values, vec![1, 2, 3, 4, 5]);
3203    }
3204
3205    /// A [`PageReader`] that implements only what existed before deferral: it
3206    /// forwards the required trait methods and takes the defaults for
3207    /// `supports_deferred_dictionary` (`false`) and `take_deferred_dictionary`
3208    /// (`Ok(None)`). This stands in for every third-party implementation, none
3209    /// of which can retain a dictionary that `skip_next_page` skips past.
3210    struct NonDeferringPageReader(SerializedPageReader<Bytes>);
3211
3212    impl Iterator for NonDeferringPageReader {
3213        type Item = Result<Page>;
3214
3215        fn next(&mut self) -> Option<Self::Item> {
3216            self.get_next_page().transpose()
3217        }
3218    }
3219
3220    impl PageReader for NonDeferringPageReader {
3221        fn get_next_page(&mut self) -> Result<Option<Page>> {
3222            self.0.get_next_page()
3223        }
3224
3225        fn peek_next_page(&mut self) -> Result<Option<PageMetadata>> {
3226            self.0.peek_next_page()
3227        }
3228
3229        fn skip_next_page(&mut self) -> Result<()> {
3230            self.0.skip_next_page()
3231        }
3232    }
3233
3234    fn non_deferring_int32_reader(
3235        bytes: Bytes,
3236    ) -> crate::column::reader::ColumnReaderImpl<Int32Type> {
3237        let reader = SerializedFileReader::new(bytes.clone()).unwrap();
3238        let meta = reader.metadata();
3239        let descr = meta.file_metadata().schema_descr().column(0);
3240        let col_meta = meta.row_group(0).column(0).clone();
3241        let total_rows = meta.row_group(0).num_rows() as usize;
3242        let pages =
3243            SerializedPageReader::new(Arc::new(bytes), &col_meta, total_rows, None).unwrap();
3244        crate::column::reader::ColumnReaderImpl::<Int32Type>::new(
3245            descr,
3246            Box::new(NonDeferringPageReader(pages)),
3247        )
3248    }
3249
3250    /// A page reader that cannot give a skipped dictionary back must still get
3251    /// the eager path. Skipping past its dictionary would drop it, and the
3252    /// first dictionary-encoded data page decoded afterwards would have no
3253    /// dictionary to decode against — a panic in `set_data`, not an error.
3254    ///
3255    /// Removing the `supports_deferred_dictionary` gate in
3256    /// `GenericColumnReader::skip_records` turns this test red.
3257    #[test]
3258    fn a_reader_that_cannot_defer_still_gets_its_dictionary() {
3259        let (bytes, _) = dict_column_file();
3260        let mut reader = non_deferring_int32_reader(bytes);
3261
3262        assert_eq!(reader.skip_records(2).unwrap(), 2);
3263
3264        let mut values = Vec::new();
3265        let (records, _, read) = reader.read_records(3, None, None, &mut values).unwrap();
3266        assert_eq!(records, 3);
3267        assert_eq!(read, 3);
3268        assert_eq!(values, vec![3, 4, 5]);
3269    }
3270
3271    /// The same reader must also survive a whole-chunk skip. Nothing is
3272    /// decoded, so nothing needs the dictionary, but the eager read still has
3273    /// to leave the page cursor in a consistent place.
3274    #[test]
3275    fn a_reader_that_cannot_defer_survives_a_whole_chunk_skip() {
3276        let (bytes, _) = dict_column_file();
3277        let mut reader = non_deferring_int32_reader(bytes);
3278
3279        assert_eq!(reader.skip_records(5).unwrap(), 5);
3280
3281        let mut values = Vec::new();
3282        let (records, _, _) = reader.read_records(1, None, None, &mut values).unwrap();
3283        assert_eq!(records, 0, "chunk is exhausted");
3284        assert!(values.is_empty());
3285    }
3286
3287    /// The probe has to describe what the reader can actually do. Without page
3288    /// locations the reader walks page headers, so it recognises a dictionary
3289    /// by its header type and can always defer.
3290    #[test]
3291    fn values_state_supports_deferral() {
3292        let (bytes, _) = dict_column_file();
3293        let reader = SerializedFileReader::new(bytes.clone()).unwrap();
3294        let meta = reader.metadata();
3295        let col_meta = meta.row_group(0).column(0).clone();
3296        let total_rows = meta.row_group(0).num_rows() as usize;
3297        let pages =
3298            SerializedPageReader::new(Arc::new(bytes), &col_meta, total_rows, None).unwrap();
3299
3300        assert!(pages.supports_deferred_dictionary());
3301    }
3302
3303    /// With page locations, a dictionary is only representable when the
3304    /// metadata recorded an offset for it. `dict_column_file` writes one, so
3305    /// the synthesised `dictionary_page` is present and deferral is safe.
3306    #[test]
3307    fn offset_index_state_with_a_dictionary_offset_supports_deferral() {
3308        let (bytes, _) = dict_column_file();
3309        let reader = SerializedFileReader::new(bytes.clone()).unwrap();
3310        let meta = reader.metadata();
3311        let col_meta = meta.row_group(0).column(0).clone();
3312        let total_rows = meta.row_group(0).num_rows() as usize;
3313        let (dict_offset, data_offset) = (
3314            col_meta.dictionary_page_offset().unwrap(),
3315            col_meta.data_page_offset(),
3316        );
3317        assert_ne!(
3318            dict_offset, data_offset,
3319            "fixture must record a dictionary offset"
3320        );
3321
3322        let locations = vec![PageLocation {
3323            offset: data_offset,
3324            compressed_page_size: 0,
3325            first_row_index: 0,
3326        }];
3327        let pages =
3328            SerializedPageReader::new(Arc::new(bytes), &col_meta, total_rows, Some(locations))
3329                .unwrap();
3330
3331        assert!(pages.supports_deferred_dictionary());
3332    }
3333
3334    /// When the metadata records no dictionary offset, `byte_range()` starts
3335    /// at the first data page, so the synthesised `dictionary_page` is `None`
3336    /// and a dictionary inlined ahead of that page is indistinguishable from a
3337    /// data page location. Skipping it would drop it, so the probe must say
3338    /// so and let the column reader read it eagerly instead.
3339    #[test]
3340    fn offset_index_state_without_a_dictionary_offset_declines_deferral() {
3341        let (bytes, _) = dict_column_file();
3342        let reader = SerializedFileReader::new(bytes.clone()).unwrap();
3343        let meta = reader.metadata();
3344        let total_rows = meta.row_group(0).num_rows() as usize;
3345
3346        // Drop the dictionary offset, reproducing the older parquet-mr writers
3347        // that inline a dictionary without recording where it is.
3348        let col_meta = meta.row_group(0).column(0).clone();
3349        let stripped = col_meta
3350            .clone()
3351            .into_builder()
3352            .set_dictionary_page_offset(None)
3353            .build()
3354            .unwrap();
3355
3356        let locations = vec![PageLocation {
3357            offset: stripped.data_page_offset(),
3358            compressed_page_size: 0,
3359            first_row_index: 0,
3360        }];
3361        let pages =
3362            SerializedPageReader::new(Arc::new(bytes), &stripped, total_rows, Some(locations))
3363                .unwrap();
3364
3365        assert!(
3366            !pages.supports_deferred_dictionary(),
3367            "an offset-less dictionary cannot be recovered from the offset-index state"
3368        );
3369        // Note this combination is unlikely to occur in practice: a writer old
3370        // enough to omit `dictionary_page_offset` predates the offset index,
3371        // so it would not emit page locations either. The probe answers
3372        // defensively rather than relying on that.
3373    }
3374
3375    /// Alternating skip and read is the shape a `RowSelection` produces, and
3376    /// it exercises the deferral more than once per chunk: the first skip
3377    /// defers, the first read installs, and every later skip has to cope with
3378    /// a dictionary that is already installed.
3379    #[test]
3380    fn alternating_skip_and_read_decodes_correct_values() {
3381        let (bytes, _) = dict_column_file();
3382        let mut reader = int32_reader(bytes);
3383
3384        assert_eq!(reader.skip_records(1).unwrap(), 1);
3385        let mut first = Vec::new();
3386        let (records, _, _) = reader.read_records(2, None, None, &mut first).unwrap();
3387        assert_eq!(records, 2);
3388        assert_eq!(first, vec![2, 3]);
3389
3390        assert_eq!(reader.skip_records(1).unwrap(), 1);
3391        let mut second = Vec::new();
3392        let (records, _, _) = reader.read_records(1, None, None, &mut second).unwrap();
3393        assert_eq!(records, 1);
3394        assert_eq!(second, vec![5]);
3395    }
3396
3397    /// A wrapper that forwards `take_deferred_dictionary` but not
3398    /// `supports_deferred_dictionary`, which is the easy mistake to make when
3399    /// adding the new methods to an existing wrapper.
3400    struct HalfForwardingPageReader(SerializedPageReader<Bytes>);
3401
3402    impl Iterator for HalfForwardingPageReader {
3403        type Item = Result<Page>;
3404
3405        fn next(&mut self) -> Option<Self::Item> {
3406            self.get_next_page().transpose()
3407        }
3408    }
3409
3410    impl PageReader for HalfForwardingPageReader {
3411        fn get_next_page(&mut self) -> Result<Option<Page>> {
3412            self.0.get_next_page()
3413        }
3414
3415        fn peek_next_page(&mut self) -> Result<Option<PageMetadata>> {
3416            self.0.peek_next_page()
3417        }
3418
3419        fn skip_next_page(&mut self) -> Result<()> {
3420            self.0.skip_next_page()
3421        }
3422
3423        fn take_deferred_dictionary(&mut self) -> Result<Option<Page>> {
3424            self.0.take_deferred_dictionary()
3425        }
3426    }
3427
3428    /// Forwarding only half the pair must stay correct: the probe still
3429    /// reports `false`, so the skip path reads the dictionary eagerly and the
3430    /// forwarded `take_deferred_dictionary` simply never has anything to
3431    /// return. Slower than forwarding both, never wrong.
3432    #[test]
3433    fn forwarding_only_take_deferred_dictionary_is_still_correct() {
3434        let (bytes, _) = dict_column_file();
3435        let reader = SerializedFileReader::new(bytes.clone()).unwrap();
3436        let meta = reader.metadata();
3437        let descr = meta.file_metadata().schema_descr().column(0);
3438        let col_meta = meta.row_group(0).column(0).clone();
3439        let total_rows = meta.row_group(0).num_rows() as usize;
3440        let pages =
3441            SerializedPageReader::new(Arc::new(bytes), &col_meta, total_rows, None).unwrap();
3442        let mut reader = crate::column::reader::ColumnReaderImpl::<Int32Type>::new(
3443            descr,
3444            Box::new(HalfForwardingPageReader(pages)),
3445        );
3446
3447        assert_eq!(reader.skip_records(2).unwrap(), 2);
3448        let mut values = Vec::new();
3449        let (records, _, _) = reader.read_records(3, None, None, &mut values).unwrap();
3450        assert_eq!(records, 3);
3451        assert_eq!(values, vec![3, 4, 5]);
3452    }
3453}