Skip to main content

parquet/file/metadata/
push_decoder.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use crate::DecodeResult;
6#[cfg(feature = "encryption")]
7use crate::encryption::decrypt::FileDecryptionProperties;
8use crate::errors::{ParquetError, Result};
9use crate::file::FOOTER_SIZE;
10use crate::file::metadata::page_index::PageIndex;
11use crate::file::metadata::parser::{MetadataParser, parse_page_index};
12use crate::file::metadata::{
13    ColumnChunkMask, FooterTail, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions,
14};
15use crate::file::reader::ChunkReader;
16use bytes::Bytes;
17use std::ops::Range;
18use std::sync::Arc;
19
20/// A push decoder for [`ParquetMetaData`].
21///
22/// This structure implements a push API for decoding Parquet metadata, which
23/// decouples IO from the metadata decoding logic (sometimes referred to as
24/// [Sans-IO]).
25///
26/// See [`ParquetMetaDataReader`] for a pull-based API that incorporates IO and
27/// is simpler to use for basic use cases. This decoder is best for customizing
28/// your IO operations to minimize bytes read, prefetch data, or use async IO.
29///
30/// [Sans-IO]: https://sans-io.readthedocs.io
31/// [`ParquetMetaDataReader`]: crate::file::metadata::ParquetMetaDataReader
32///
33/// # Example
34///
35/// The most basic usage is to feed the decoder with the necessary byte ranges
36/// as requested as shown below. This minimizes the number of bytes read, but
37/// requires the most IO operations - one to read the footer and then one
38/// to read the metadata, and possibly more if page indexes are requested.
39///
40#[cfg_attr(
41    feature = "arrow",
42    doc = r#"
43```rust
44# use std::ops::Range;
45# use bytes::Bytes;
46# use arrow_array::record_batch;
47# use parquet::DecodeResult;
48# use parquet::arrow::ArrowWriter;
49# use parquet::errors::ParquetError;
50# use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataPushDecoder};
51#
52# fn decode_metadata() -> Result<ParquetMetaData, ParquetError> {
53# let file_bytes = {
54#   let mut buffer = vec![0];
55#   let batch = record_batch!(("a", Int32, [1, 2, 3])).unwrap();
56#   let mut writer = ArrowWriter::try_new(&mut buffer, batch.schema(), None).unwrap();
57#   writer.write(&batch).unwrap();
58#   writer.close().unwrap();
59#   Bytes::from(buffer)
60# };
61# // mimic IO by returning a function that returns the bytes for a given range
62# let get_range = |range: &Range<u64>| -> Bytes {
63#    let start = range.start as usize;
64#     let end = range.end as usize;
65#    file_bytes.slice(start..end)
66# };
67#
68# let file_len = file_bytes.len() as u64;
69// The `ParquetMetaDataPushDecoder` needs to know the file length.
70let mut decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
71// try to decode the metadata. If more data is needed, the decoder will tell you what ranges
72loop {
73    match decoder.try_decode() {
74       Ok(DecodeResult::Data(metadata)) => { return Ok(metadata); } // decode successful
75       Ok(DecodeResult::NeedsData(ranges)) => {
76          // The decoder needs more data
77          //
78          // In this example, we call a function that returns the bytes for each given range.
79          // In a real application, you would likely read the data from a file or network.
80          let data = ranges.iter().map(|range| get_range(range)).collect();
81          // Push the data into the decoder and try to decode again on the next iteration.
82          decoder.push_ranges(ranges, data).unwrap();
83       }
84       Ok(DecodeResult::Finished) => { unreachable!("returned metadata in previous match arm") }
85       Err(e) => return Err(e),
86    }
87}
88# }
89```
90"#
91)]
92///
93/// # Example with "prefetching"
94///
95/// By default, the [`ParquetMetaDataPushDecoder`] will request only the exact byte
96/// ranges it needs. This minimizes the number of bytes read, however it
97/// requires at least two IO operations to read the metadata - one to read the
98/// footer and then one to read the metadata.
99///
100/// If the file has a "Page Index" (see [Self::with_page_index_policy]), three
101/// IO operations are required to read the metadata, as the page index is
102/// not part of the normal metadata footer.
103///
104/// To reduce the number of IO operations in systems with high per operation
105/// overhead (e.g. cloud storage), you can "prefetch" the data and then push
106/// the data into the decoder before calling [`Self::try_decode`]. If you do
107/// not push enough bytes, the decoder will return the ranges that are still
108/// needed.
109///
110/// This approach can also be used when you have the entire file already in memory
111/// for other reasons.
112#[cfg_attr(
113    feature = "arrow",
114    doc = r#"
115```rust
116# use std::ops::Range;
117# use bytes::Bytes;
118# use arrow_array::record_batch;
119# use parquet::DecodeResult;
120# use parquet::arrow::ArrowWriter;
121# use parquet::errors::ParquetError;
122# use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataPushDecoder};
123#
124# fn decode_metadata() -> Result<ParquetMetaData, ParquetError> {
125# let file_bytes = {
126#   let mut buffer = vec![0];
127#   let batch = record_batch!(("a", Int32, [1, 2, 3])).unwrap();
128#   let mut writer = ArrowWriter::try_new(&mut buffer, batch.schema(), None).unwrap();
129#   writer.write(&batch).unwrap();
130#   writer.close().unwrap();
131#   Bytes::from(buffer)
132# };
133#
134let file_len = file_bytes.len() as u64;
135// For this example, we "prefetch" all the bytes which we have in memory,
136// but in a real application, you would likely read a chunk from the end
137// for example 1MB.
138let prefetched_bytes = file_bytes.clone();
139let mut decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
140// push the prefetched bytes into the decoder
141decoder.push_ranges(vec![0..file_len], vec![prefetched_bytes]).unwrap();
142// The decoder will now be able to decode the metadata. Note in a real application,
143// unless you can guarantee that the pushed data is enough to decode the metadata,
144// you still need to call `try_decode` in a loop until it returns `DecodeResult::Data`
145// as shown in  the previous example
146    match decoder.try_decode() {
147        Ok(DecodeResult::Data(metadata)) => { return Ok(metadata); } // decode successful
148        other => { panic!("expected DecodeResult::Data, got: {other:?}") }
149    }
150# }
151```
152"#
153)]
154///
155/// # Example using [`AsyncRead`]
156///
157/// [`ParquetMetaDataPushDecoder`] is designed to work with any data source that can
158/// provide byte ranges, including async IO sources. However, it does not
159/// implement async IO itself. To use async IO, you simply write an async
160/// wrapper around it that reads the required byte ranges and pushes them into the
161/// decoder.
162#[cfg_attr(
163    feature = "arrow",
164    doc = r#"
165```rust
166# use std::ops::Range;
167# use bytes::Bytes;
168use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt};
169# use arrow_array::record_batch;
170# use parquet::DecodeResult;
171# use parquet::arrow::ArrowWriter;
172# use parquet::errors::ParquetError;
173# use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataPushDecoder};
174#
175// This function decodes Parquet Metadata from anything that implements
176// [`AsyncRead`] and [`AsyncSeek`] such as a tokio::fs::File
177async fn decode_metadata(
178  file_len: u64,
179  mut async_source: impl AsyncRead + AsyncSeek + Unpin
180) -> Result<ParquetMetaData, ParquetError> {
181  // We need a ParquetMetaDataPushDecoder to decode the metadata.
182  let mut decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
183  loop {
184    match decoder.try_decode() {
185       Ok(DecodeResult::Data(metadata)) => { return Ok(metadata); } // decode successful
186       Ok(DecodeResult::NeedsData(ranges)) => {
187          // The decoder needs more data
188          //
189          // In this example we use the AsyncRead and AsyncSeek traits to read the
190          // required ranges from the async source.
191          let mut data = Vec::with_capacity(ranges.len());
192          for range in &ranges {
193            let mut buffer = vec![0; (range.end - range.start) as usize];
194            async_source.seek(std::io::SeekFrom::Start(range.start)).await?;
195            async_source.read_exact(&mut buffer).await?;
196            data.push(Bytes::from(buffer));
197          }
198          // Push the data into the decoder and try to decode again on the next iteration.
199          decoder.push_ranges(ranges, data).unwrap();
200       }
201       Ok(DecodeResult::Finished) => { unreachable!("returned metadata in previous match arm") }
202       Err(e) => return Err(e),
203    }
204  }
205}
206```
207"#
208)]
209/// [`AsyncRead`]: tokio::io::AsyncRead
210#[derive(Debug)]
211pub struct ParquetMetaDataPushDecoder {
212    /// Decoding state
213    state: DecodeState,
214    /// policy for loading ColumnIndex (part of the PageIndex)
215    column_index_policy: PageIndexPolicy,
216    /// policy for loading OffsetIndex (part of the PageIndex)
217    offset_index_policy: PageIndexPolicy,
218    /// which rows and columns of the column index should be read
219    column_index_mask: ColumnChunkMask,
220    /// which rows and columns of the offset index should be read
221    offset_index_mask: ColumnChunkMask,
222    /// Page-index range already requested for the current policies and masks.
223    pending_page_index_range: Option<Range<u64>>,
224    /// Underlying buffers
225    buffers: crate::util::push_buffers::PushBuffers,
226    /// Encryption API
227    metadata_parser: MetadataParser,
228}
229
230impl ParquetMetaDataPushDecoder {
231    /// Create a new `ParquetMetaDataPushDecoder` with the given file length.
232    ///
233    /// By default, this will read page indexes and column indexes. See
234    /// [`ParquetMetaDataPushDecoder::with_page_index_policy`] for more detail.
235    ///
236    /// See examples on [`ParquetMetaDataPushDecoder`].
237    pub fn try_new(file_len: u64) -> Result<Self> {
238        if file_len < 8 {
239            return Err(ParquetError::General(format!(
240                "Parquet files are at least 8 bytes long, but file length is {file_len}"
241            )));
242        }
243
244        Ok(Self {
245            state: DecodeState::ReadingFooter,
246            column_index_policy: PageIndexPolicy::Optional,
247            offset_index_policy: PageIndexPolicy::Optional,
248            column_index_mask: ColumnChunkMask::all(),
249            offset_index_mask: ColumnChunkMask::all(),
250            pending_page_index_range: None,
251            buffers: crate::util::push_buffers::PushBuffers::new(file_len),
252            metadata_parser: MetadataParser::new(),
253        })
254    }
255
256    /// Begin decoding from the given footer tail.
257    pub(crate) fn try_new_with_footer_tail(file_len: u64, footer_tail: FooterTail) -> Result<Self> {
258        let mut new_self = Self::try_new(file_len)?;
259        new_self.state = DecodeState::ReadingMetadata(footer_tail);
260        Ok(new_self)
261    }
262
263    /// Create a decoder with the given `ParquetMetaData` already known.
264    ///
265    /// This can be used to parse and populate the page index structures
266    /// after the metadata has already been decoded.
267    ///
268    /// The page indexes can then be decoded in one of two ways:
269    ///
270    /// * [`Self::try_decode`] attaches the decoded page indexes to the returned
271    ///   metadata, replacing (or clearing) any existing page indexes
272    ///   See [`ParquetMetaDataReader::read_page_indexes`] for details.
273    /// * [`Self::try_decode_page_index`] returns the decoded page indexes
274    ///   separately and leaves any page indexes already attached to `metadata`
275    ///   unchanged.
276    ///
277    /// [`ParquetMetaDataReader::read_page_indexes`]: [`crate::file::metadata::ParquetMetaDataReader::read_page_indexes`]
278    pub fn try_new_with_metadata(file_len: u64, metadata: ParquetMetaData) -> Result<Self> {
279        let mut new_self = Self::try_new(file_len)?;
280        new_self.state = DecodeState::ReadingPageIndex(Box::new(metadata));
281        Ok(new_self)
282    }
283
284    /// Enable or disable reading the page index structures described in
285    /// "[Parquet page index] Layout to Support Page Skipping".
286    ///
287    /// Defaults to [`PageIndexPolicy::Optional`]
288    ///
289    /// This requires
290    /// 1. The Parquet file to have been written with page indexes
291    /// 2. Additional data to be pushed into the decoder (as the page indexes are not part of the thrift footer)
292    ///
293    /// [Parquet page index]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
294    pub fn with_page_index_policy(mut self, page_index_policy: PageIndexPolicy) -> Self {
295        self.column_index_policy = page_index_policy;
296        self.offset_index_policy = page_index_policy;
297        self.pending_page_index_range = None;
298        self
299    }
300
301    /// Set the policy for reading the ColumnIndex (part of the PageIndex)
302    pub fn with_column_index_policy(mut self, column_index_policy: PageIndexPolicy) -> Self {
303        self.column_index_policy = column_index_policy;
304        self.pending_page_index_range = None;
305        self
306    }
307
308    /// Set the policy for reading the OffsetIndex (part of the PageIndex)
309    pub fn with_offset_index_policy(mut self, offset_index_policy: PageIndexPolicy) -> Self {
310        self.offset_index_policy = offset_index_policy;
311        self.pending_page_index_range = None;
312        self
313    }
314
315    // TODO: deprecate the following as part of https://github.com/apache/arrow-rs/issues/11395
316
317    /// Select the row groups and columns for which both page index structures are read.
318    pub fn with_page_index_mask(mut self, mask: ColumnChunkMask) -> Self {
319        self.column_index_mask = mask.clone();
320        self.offset_index_mask = mask;
321        self.pending_page_index_range = None;
322        self
323    }
324
325    /// Select the row groups and columns for which column indexes are read.
326    pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self {
327        self.column_index_mask = mask;
328        self.pending_page_index_range = None;
329        self
330    }
331
332    /// Select the row groups and columns for which offset indexes are read.
333    pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self {
334        self.offset_index_mask = mask;
335        self.pending_page_index_range = None;
336        self
337    }
338
339    /// Set the options to use when decoding the Parquet metadata.
340    pub fn with_metadata_options(mut self, options: Option<Arc<ParquetMetaDataOptions>>) -> Self {
341        self.metadata_parser = self.metadata_parser.with_metadata_options(options);
342        self
343    }
344
345    #[cfg(feature = "encryption")]
346    /// Provide decryption properties for decoding encrypted Parquet files
347    pub fn with_file_decryption_properties(
348        mut self,
349        file_decryption_properties: Option<std::sync::Arc<FileDecryptionProperties>>,
350    ) -> Self {
351        self.metadata_parser = self
352            .metadata_parser
353            .with_file_decryption_properties(file_decryption_properties);
354        self
355    }
356
357    /// Push the data into the decoder's buffer.
358    ///
359    /// The decoder does not immediately attempt to decode the metadata
360    /// after pushing data. Instead, it accumulates the pushed data until you
361    /// call [`Self::try_decode`].
362    ///
363    /// # Determining required data:
364    ///
365    /// To determine what ranges are required to decode the metadata, you can
366    /// either:
367    ///
368    /// 1. Call [`Self::try_decode`] first to get the exact ranges required (see
369    ///    example on [`Self`])
370    ///
371    /// 2. Speculatively push any data that you have available, which may
372    ///    include more than the footer data or requested bytes.
373    ///
374    /// Speculatively pushing data can be used when  "prefetching" data. See
375    /// example on [`Self`]
376    pub fn push_ranges(&mut self, ranges: Vec<Range<u64>>, buffers: Vec<Bytes>) -> Result<()> {
377        if matches!(&self.state, DecodeState::Finished) {
378            return Err(general_err!(
379                "ParquetMetaDataPushDecoder: cannot push data after decoding is finished"
380            ));
381        }
382        self.buffers.push_ranges(ranges, buffers)?;
383        Ok(())
384    }
385
386    /// Pushes a single range of data into the decoder's buffer.
387    pub fn push_range(&mut self, range: Range<u64>, buffer: Bytes) -> Result<()> {
388        if matches!(&self.state, DecodeState::Finished) {
389            return Err(general_err!(
390                "ParquetMetaDataPushDecoder: cannot push data after decoding is finished"
391            ));
392        }
393        self.buffers.push_range(range, buffer)?;
394        Ok(())
395    }
396
397    /// Clear any staged byte ranges currently buffered for future decode work.
398    pub fn clear_all_ranges(&mut self) {
399        self.buffers.clear_all_ranges();
400    }
401
402    /// Try to decode the metadata from the pushed data, returning the
403    /// decoded metadata or an error if not enough data is available.
404    pub fn try_decode(&mut self) -> Result<DecodeResult<ParquetMetaData>> {
405        let file_len = self.buffers.file_len();
406        let footer_len = FOOTER_SIZE as u64;
407        loop {
408            match std::mem::replace(&mut self.state, DecodeState::Intermediate) {
409                DecodeState::ReadingFooter => {
410                    // need to have the last 8 bytes of the file to decode the metadata
411                    let footer_start = file_len.saturating_sub(footer_len);
412                    let footer_range = footer_start..file_len;
413
414                    if !self.buffers.has_range(&footer_range) {
415                        self.state = DecodeState::ReadingFooter;
416                        return Ok(needs_range(footer_range));
417                    }
418                    let footer_bytes = self.get_bytes(&footer_range)?;
419                    let footer_tail = FooterTail::try_from(footer_bytes.as_ref())?;
420
421                    self.state = DecodeState::ReadingMetadata(footer_tail);
422                }
423
424                DecodeState::ReadingMetadata(footer_tail) => {
425                    let metadata_len: u64 = footer_tail.metadata_length() as u64;
426                    // A corrupt footer can claim more metadata than the file holds
427                    let needed = footer_len + metadata_len;
428                    let Some(metadata_start) = file_len.checked_sub(needed) else {
429                        return Err(eof_err!(
430                            "Parquet file too small. Size is {file_len} but need {needed}"
431                        ));
432                    };
433                    let metadata_end = metadata_start + metadata_len;
434                    let metadata_range = metadata_start..metadata_end;
435
436                    if !self.buffers.has_range(&metadata_range) {
437                        self.state = DecodeState::ReadingMetadata(footer_tail);
438                        return Ok(needs_range(metadata_range));
439                    }
440
441                    let metadata = self.metadata_parser.decode_metadata(
442                        &self.get_bytes(&metadata_range)?,
443                        footer_tail.is_encrypted_footer(),
444                    )?;
445                    // Note: ReadingPageIndex first checks if page indexes are needed
446                    // and is a no-op if not
447                    self.state = DecodeState::ReadingPageIndex(Box::new(metadata));
448                }
449
450                DecodeState::ReadingPageIndex(metadata) => {
451                    match self.try_decode_page_index_for_metadata(metadata)? {
452                        DecodeResult::NeedsData(ranges) => {
453                            return Ok(DecodeResult::NeedsData(ranges));
454                        }
455                        DecodeResult::Data((mut metadata, page_index)) => {
456                            // Install the newly decoded indexes, or clear any existing indexes if
457                            // none were requested or present in the file.
458                            if let Some(page_index) = page_index {
459                                metadata.set_page_index(Some(Arc::new(page_index)));
460                            } else {
461                                metadata.set_page_index(None);
462                            }
463                            return Ok(DecodeResult::Data(*metadata));
464                        }
465                        DecodeResult::Finished => return Ok(DecodeResult::Finished),
466                    }
467                }
468
469                DecodeState::Finished => return Ok(DecodeResult::Finished),
470                DecodeState::Intermediate => {
471                    return Err(general_err!(
472                        "ParquetMetaDataPushDecoder: internal error, invalid state"
473                    ));
474                }
475            }
476        }
477    }
478
479    /// Try to decode the page indexes from the pushed data, returning the stored
480    /// [`ParquetMetaData`] together with the decoded [`PageIndex`].
481    ///
482    /// This method can only be called once the footer metadata is known: either
483    /// after [`Self::try_new_with_metadata`], or after [`Self::try_decode`] has
484    /// decoded the footer metadata and requested page index data. Calling it
485    /// earlier returns an error and leaves the decoder unchanged.
486    ///
487    /// This method is useful when you want to:
488    /// 1. First decode the footer metadata without reading the page indexes
489    ///    (using [`PageIndexPolicy::Skip`])
490    /// 2. Later selectively decode a subset of the page indexes using
491    ///    [`Self::with_column_index_mask`] or [`Self::with_offset_index_mask`]
492    ///
493    /// When the data can be accessed through a [`ChunkReader`], prefer
494    /// [`ParquetMetaDataReader::read_page_index_and_finish`] (or
495    /// [`ParquetMetaDataReader::load_page_index_and_finish`] for asynchronous
496    /// I/O), which handles the decoder state machine and I/O requests. See the
497    /// [`read_page_index_and_finish` example][`ParquetMetaDataReader::read_page_index_and_finish`]
498    /// for the complete two-stage workflow.
499    ///
500    /// # Returns
501    ///
502    /// * `Ok(DecodeResult::Data((metadata, Some(page_index))))` - Successfully decoded the
503    ///   requested page indexes based on the configured policies and masks. The newly decoded page
504    ///   index is returned separately; any page index already attached to `metadata` is unchanged.
505    /// * `Ok(DecodeResult::Data((metadata, None)))` - No page indexes were requested or found
506    ///   according to the configured policies and masks. The stored metadata is still returned.
507    /// * `Ok(DecodeResult::NeedsData(ranges))` - More data is needed to decode
508    ///   the page indexes. Push the requested ranges and call this method again.
509    /// * `Ok(DecodeResult::Finished)` - The decoder has finished and no more
510    ///   data can be decoded
511    /// * `Err(_)` - An error occurred during decoding
512    ///
513    /// [`ParquetMetaDataReader::read_page_index_and_finish`]: crate::file::metadata::ParquetMetaDataReader::read_page_index_and_finish
514    /// [`ParquetMetaDataReader::load_page_index_and_finish`]: crate::file::metadata::ParquetMetaDataReader::load_page_index_and_finish
515    pub fn try_decode_page_index(
516        &mut self,
517    ) -> Result<DecodeResult<(ParquetMetaData, Option<PageIndex>)>> {
518        match std::mem::replace(&mut self.state, DecodeState::Intermediate) {
519            DecodeState::ReadingPageIndex(metadata) => {
520                Ok(match self.try_decode_page_index_for_metadata(metadata)? {
521                    DecodeResult::NeedsData(ranges) => DecodeResult::NeedsData(ranges),
522                    DecodeResult::Data((metadata, page_index)) => {
523                        DecodeResult::Data((*metadata, page_index))
524                    }
525                    DecodeResult::Finished => DecodeResult::Finished,
526                })
527            }
528            DecodeState::Finished => {
529                self.state = DecodeState::Finished;
530                Ok(DecodeResult::Finished)
531            }
532            other => {
533                // Restore the original state so the decoder remains usable
534                self.state = other;
535                Err(general_err!(
536                    "ParquetMetaDataPushDecoder: try_decode_page_index requires a decoder created with try_new_with_metadata"
537                ))
538            }
539        }
540    }
541
542    fn try_decode_page_index_for_metadata(
543        &mut self,
544        metadata: Box<ParquetMetaData>,
545    ) -> Result<DecodeResult<(Box<ParquetMetaData>, Option<PageIndex>)>> {
546        // Reuse the requested range on resume. Only keep it while waiting for bytes.
547        let range = self.pending_page_index_range.take().or_else(|| {
548            range_for_page_index(
549                &metadata,
550                self.column_index_policy,
551                self.offset_index_policy,
552                &self.column_index_mask,
553                &self.offset_index_mask,
554            )
555        });
556
557        let Some(page_index_range) = range else {
558            self.state = DecodeState::Finished;
559            return Ok(DecodeResult::Data((metadata, None)));
560        };
561
562        if !self.buffers.has_range(&page_index_range) {
563            self.pending_page_index_range = Some(page_index_range.clone());
564            self.state = DecodeState::ReadingPageIndex(metadata);
565            return Ok(DecodeResult::NeedsData(vec![page_index_range]));
566        }
567
568        let buffer = match self.get_bytes(&page_index_range) {
569            Ok(buffer) => buffer,
570            Err(error) => {
571                self.state = DecodeState::ReadingPageIndex(metadata);
572                return Err(error);
573            }
574        };
575        let page_index = match parse_page_index(
576            &metadata,
577            self.column_index_policy,
578            self.offset_index_policy,
579            &self.column_index_mask,
580            &self.offset_index_mask,
581            &buffer,
582            page_index_range.start,
583        ) {
584            Ok(page_index) => page_index,
585            Err(error) => {
586                self.state = DecodeState::ReadingPageIndex(metadata);
587                return Err(error);
588            }
589        };
590
591        self.state = DecodeState::Finished;
592        Ok(DecodeResult::Data((metadata, page_index)))
593    }
594
595    /// Returns the bytes for the given range from the internal buffer
596    fn get_bytes(&self, range: &Range<u64>) -> Result<Bytes> {
597        let start = range.start;
598        let raw_len = range.end - range.start;
599        let len: usize = raw_len.try_into().map_err(|_| {
600            ParquetError::General(format!(
601                "ParquetMetaDataPushDecoder: Range length too large to fit in usize: {raw_len}",
602            ))
603        })?;
604        self.buffers.get_bytes(start, len)
605    }
606}
607
608/// returns a DecodeResults that describes needing the given range
609fn needs_range<T: std::fmt::Debug>(range: Range<u64>) -> DecodeResult<T> {
610    DecodeResult::NeedsData(vec![range])
611}
612
613/// Decoding state machine
614#[derive(Debug)]
615enum DecodeState {
616    /// Reading the last 8 bytes of the file
617    ReadingFooter,
618    /// Reading the metadata thrift structure
619    ReadingMetadata(FooterTail),
620    // Actively reading the page index
621    ReadingPageIndex(Box<ParquetMetaData>),
622    // Decoding is complete
623    Finished,
624    /// State left during the `try_decode` method so something valid is present.
625    /// This state should never be observed.
626    Intermediate,
627}
628
629/// Returns the byte range needed to read the offset/page indexes, based on the
630/// specified policies and masks
631///
632/// Returns None if no page indexes are needed
633pub(crate) fn range_for_page_index(
634    metadata: &ParquetMetaData,
635    column_index_policy: PageIndexPolicy,
636    offset_index_policy: PageIndexPolicy,
637    column_index_mask: &ColumnChunkMask,
638    offset_index_mask: &ColumnChunkMask,
639) -> Option<Range<u64>> {
640    let mut range: Option<Range<u64>> = None;
641    for (policy, mask, column_index) in [
642        (column_index_policy, column_index_mask, true),
643        (offset_index_policy, offset_index_mask, false),
644    ] {
645        if policy == PageIndexPolicy::Skip {
646            continue;
647        }
648        for row_group in mask.row_group_indices(metadata.num_row_groups()) {
649            for column in mask.column_indices(metadata.row_group(row_group).num_columns()) {
650                let column = metadata.row_group(row_group).column(column);
651                let index = if column_index {
652                    column.column_index_range()
653                } else {
654                    column.offset_index_range()
655                };
656                if let Some(index) = index {
657                    range = Some(match range {
658                        Some(range) => range.start.min(index.start)..range.end.max(index.end),
659                        None => index,
660                    });
661                }
662            }
663        }
664    }
665    range
666}
667
668// These tests use the arrow writer to create a parquet file in memory
669// so they need the arrow feature and the test feature
670#[cfg(all(test, feature = "arrow"))]
671mod tests {
672    use super::*;
673    use crate::arrow::ArrowWriter;
674    use crate::file::metadata::page_index::PageIndexProvider;
675    use crate::file::properties::WriterProperties;
676    use arrow_array::{ArrayRef, Int64Array, RecordBatch, StringViewArray};
677    use bytes::Bytes;
678    use std::fmt::Debug;
679    use std::ops::Range;
680    use std::sync::{Arc, LazyLock};
681
682    /// It is possible to decode the metadata from the entire file at once before being asked
683    #[test]
684    fn test_metadata_decoder_all_data() {
685        let file_len = test_file_len();
686        let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
687        // Push the entire file data into the metadata decoder
688        push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![test_file_range()]);
689
690        // should be able to decode the metadata without needing more data
691        let metadata = expect_data(metadata_decoder.try_decode());
692
693        assert_eq!(metadata.num_row_groups(), 2);
694        assert_eq!(metadata.row_group(0).num_rows(), 200);
695        assert_eq!(metadata.row_group(1).num_rows(), 200);
696        assert!(metadata.page_index().is_some_and(|idx| idx.is_complete()));
697    }
698
699    /// A footer whose metadata length exceeds the file length is an error, not a panic
700    #[test]
701    fn test_metadata_decoder_metadata_length_exceeds_file() {
702        // 4 bytes of data, then a footer claiming 1000 bytes of metadata
703        let mut file = vec![0u8; 4];
704        file.extend_from_slice(&1000u32.to_le_bytes());
705        file.extend_from_slice(b"PAR1");
706        let file_len = file.len() as u64;
707
708        let mut decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
709        decoder.push_range(0..file_len, Bytes::from(file)).unwrap();
710        let err = decoder.try_decode().unwrap_err();
711        assert_eq!(
712            err.to_string(),
713            "EOF: Parquet file too small. Size is 12 but need 1008"
714        );
715    }
716
717    #[test]
718    fn test_range_for_page_index_respects_column_mask() {
719        let mut decoder = ParquetMetaDataPushDecoder::try_new(test_file_len()).unwrap();
720        push_ranges_to_metadata_decoder(&mut decoder, vec![test_file_range()]);
721        let metadata = expect_data(decoder.try_decode());
722
723        let all = range_for_page_index(
724            &metadata,
725            PageIndexPolicy::Required,
726            PageIndexPolicy::Skip,
727            &ColumnChunkMask::all(),
728            &ColumnChunkMask::all(),
729        )
730        .unwrap();
731        let masked = range_for_page_index(
732            &metadata,
733            PageIndexPolicy::Required,
734            PageIndexPolicy::Skip,
735            &ColumnChunkMask::columns([0]),
736            &ColumnChunkMask::all(),
737        )
738        .unwrap();
739
740        assert!(all.start <= masked.start && masked.end <= all.end);
741        assert!(masked.end - masked.start < all.end - all.start);
742    }
743
744    #[test]
745    fn test_decode_page_index_after_metadata() {
746        let metadata = test_metadata_without_page_index();
747        assert!(metadata.page_index().is_none());
748
749        let mut decoder =
750            ParquetMetaDataPushDecoder::try_new_with_metadata(test_file_len(), metadata)
751                .unwrap()
752                .with_column_index_policy(PageIndexPolicy::Required)
753                .with_offset_index_policy(PageIndexPolicy::Required)
754                .with_column_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2]))
755                .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([0], [1]));
756
757        let ranges = expect_needs_data(decoder.try_decode_page_index());
758        assert_eq!(ranges.len(), 1);
759        push_ranges_to_metadata_decoder(&mut decoder, ranges);
760
761        let (metadata, page_index) = expect_data(decoder.try_decode_page_index());
762        assert!(metadata.page_index().is_none());
763        let page_index = page_index.unwrap();
764        expect_finished(decoder.try_decode_page_index());
765
766        for row_group_idx in 0..2 {
767            for column_idx in 0..3 {
768                assert_eq!(
769                    page_index.column_index(row_group_idx, column_idx).is_some(),
770                    row_group_idx == 1 && matches!(column_idx, 0 | 2)
771                );
772                assert_eq!(
773                    page_index.offset_index(row_group_idx, column_idx).is_some(),
774                    row_group_idx == 0 && column_idx == 1
775                );
776            }
777        }
778    }
779
780    #[test]
781    fn test_change_page_index_selection_while_waiting() {
782        type Decoder = ParquetMetaDataPushDecoder;
783        let metadata = test_metadata_without_page_index();
784        let check = |initial: fn(Decoder) -> Decoder, change: fn(Decoder) -> Decoder| {
785            let new_decoder = || {
786                initial(Decoder::try_new_with_metadata(test_file_len(), metadata.clone()).unwrap())
787            };
788            let mut decoder = new_decoder();
789            let original = expect_needs_data(decoder.try_decode());
790            decoder = change(decoder);
791
792            let mut fresh = change(new_decoder());
793            let expected = expect_needs_data(fresh.try_decode());
794            assert_ne!(original, expected);
795            assert_eq!(expect_needs_data(decoder.try_decode()), expected);
796            push_ranges_to_metadata_decoder(&mut decoder, expected.clone());
797            push_ranges_to_metadata_decoder(&mut fresh, expected);
798            assert_eq!(
799                expect_data(decoder.try_decode()),
800                expect_data(fresh.try_decode())
801            );
802        };
803
804        let all = |decoder| decoder;
805        let one_chunk = |decoder: Decoder| {
806            decoder.with_page_index_mask(ColumnChunkMask::row_groups_and_columns([0], [1]))
807        };
808        check(all, |d| d.with_column_index_policy(PageIndexPolicy::Skip));
809        check(all, |d| d.with_offset_index_policy(PageIndexPolicy::Skip));
810        check(all, one_chunk);
811        check(one_chunk, |d| {
812            d.with_column_index_mask(ColumnChunkMask::all())
813        });
814        check(one_chunk, |d| {
815            d.with_offset_index_mask(ColumnChunkMask::all())
816        });
817    }
818
819    #[test]
820    fn test_missing_required_offset_index_after_pending_request() {
821        let mut metadata = test_metadata_without_page_index().into_builder();
822        let mut groups = metadata.take_row_groups();
823        let column = &mut groups[0].columns_mut()[0];
824        *column = column
825            .clone()
826            .into_builder()
827            .set_offset_index_offset(None)
828            .set_offset_index_length(None)
829            .build()
830            .unwrap();
831        let metadata = metadata.set_row_groups(groups).build();
832        let mut decoder =
833            ParquetMetaDataPushDecoder::try_new_with_metadata(test_file_len(), metadata).unwrap();
834
835        // The other chunks still require index bytes under either policy.
836        expect_needs_data(decoder.try_decode());
837        decoder = decoder.with_offset_index_policy(PageIndexPolicy::Required);
838        let ranges = expect_needs_data(decoder.try_decode());
839        push_ranges_to_metadata_decoder(&mut decoder, ranges);
840        let error = decoder.try_decode().unwrap_err();
841        assert!(
842            matches!(error, ParquetError::General(message) if message == "missing offset index")
843        );
844        assert!(decoder.pending_page_index_range.is_none());
845    }
846
847    #[test]
848    fn test_page_index_decoder_no_indexes_requested() {
849        let metadata = test_metadata_without_page_index();
850        let mut decoder =
851            ParquetMetaDataPushDecoder::try_new_with_metadata(test_file_len(), metadata).unwrap();
852        expect_needs_data(decoder.try_decode_page_index());
853        decoder = decoder.with_page_index_policy(PageIndexPolicy::Skip);
854        assert!(expect_data(decoder.try_decode_page_index()).1.is_none());
855        expect_finished(decoder.try_decode_page_index());
856
857        let metadata = test_metadata_without_page_index();
858        let mut decoder =
859            ParquetMetaDataPushDecoder::try_new_with_metadata(test_file_len(), metadata)
860                .unwrap()
861                .with_page_index_mask(ColumnChunkMask::none());
862
863        assert!(expect_data(decoder.try_decode_page_index()).1.is_none());
864        expect_finished(decoder.try_decode_page_index());
865    }
866
867    #[test]
868    fn test_page_index_decoder_requires_metadata() {
869        let mut decoder = ParquetMetaDataPushDecoder::try_new(test_file_len()).unwrap();
870        let err = decoder.try_decode_page_index().unwrap_err();
871        assert!(
872            err.to_string()
873                .contains("requires a decoder created with try_new_with_metadata"),
874            "unexpected error: {err}"
875        );
876    }
877
878    #[test]
879    fn test_page_index_decoder_restores_state_after_decode_error() {
880        let metadata = test_metadata_without_page_index();
881        let mut decoder =
882            ParquetMetaDataPushDecoder::try_new_with_metadata(test_file_len(), metadata)
883                .unwrap()
884                .with_page_index_policy(PageIndexPolicy::Required);
885
886        let ranges = expect_needs_data(decoder.try_decode_page_index());
887        let invalid_data = ranges
888            .iter()
889            .map(|range| Bytes::from(vec![0; (range.end - range.start) as usize]))
890            .collect();
891        decoder.push_ranges(ranges.clone(), invalid_data).unwrap();
892        decoder.try_decode_page_index().unwrap_err();
893        assert!(decoder.pending_page_index_range.is_none());
894
895        decoder.clear_all_ranges();
896        push_ranges_to_metadata_decoder(&mut decoder, ranges);
897        let (metadata, page_index) = expect_data(decoder.try_decode_page_index());
898        assert!(metadata.page_index().is_none());
899        assert!(page_index.is_some_and(|index| index.is_complete()));
900    }
901
902    /// A failed call to `try_decode_page_index` must not leave the decoder in
903    /// an unusable state: decoding the metadata via `try_decode` should still
904    /// work afterwards, at every stage of the footer decoding state machine.
905    #[test]
906    fn test_decode_after_failed_page_index_decode() {
907        let file_len = test_file_len();
908        let mut decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
909
910        // Before any data is pushed, the decoder is waiting for the footer
911        decoder.try_decode_page_index().unwrap_err();
912        let footer_ranges = expect_needs_data(decoder.try_decode());
913        assert_eq!(footer_ranges.len(), 1);
914        push_ranges_to_metadata_decoder(&mut decoder, footer_ranges);
915
916        // The footer is available but not yet decoded
917        decoder.try_decode_page_index().unwrap_err();
918        let metadata_ranges = expect_needs_data(decoder.try_decode());
919        assert_eq!(metadata_ranges.len(), 1);
920        push_ranges_to_metadata_decoder(&mut decoder, metadata_ranges);
921
922        // The metadata is available but not yet decoded
923        decoder.try_decode_page_index().unwrap_err();
924        let index_ranges = expect_needs_data(decoder.try_decode());
925        assert_eq!(index_ranges.len(), 1);
926
927        // The metadata is decoded and the decoder is waiting for page index
928        // data, so try_decode_page_index is now valid and asks for the same range
929        let index_ranges_again = expect_needs_data(decoder.try_decode_page_index());
930        assert_eq!(index_ranges, index_ranges_again);
931        assert_eq!(
932            decoder.pending_page_index_range.as_ref(),
933            index_ranges.first()
934        );
935        push_ranges_to_metadata_decoder(&mut decoder, index_ranges.clone());
936        decoder.clear_all_ranges();
937        assert_eq!(expect_needs_data(decoder.try_decode()), index_ranges);
938        push_ranges_to_metadata_decoder(&mut decoder, index_ranges);
939
940        // The regular decode path still works and attaches the page index
941        let metadata = expect_data(decoder.try_decode());
942        assert!(decoder.pending_page_index_range.is_none());
943        assert_eq!(metadata.num_row_groups(), 2);
944        assert!(metadata.page_index().is_some_and(|idx| idx.is_complete()));
945
946        // Once finished, both entry points keep reporting Finished
947        expect_finished(decoder.try_decode_page_index());
948        expect_finished(decoder.try_decode_page_index());
949        expect_finished(decoder.try_decode());
950    }
951
952    /// It is possible to feed some, but not all, of the footer into the metadata decoder
953    /// before asked. This avoids multiple IO requests
954    #[test]
955    fn test_metadata_decoder_prefetch_success() {
956        let file_len = test_file_len();
957        let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
958        // simulate pre-fetching the last 2k bytes of the file without asking the decoder
959        let prefetch_range = (file_len - 2 * 1024)..file_len;
960        push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![prefetch_range]);
961
962        // expect the decoder has enough data to decode the metadata
963        let metadata = expect_data(metadata_decoder.try_decode());
964        expect_finished(metadata_decoder.try_decode());
965        assert_eq!(metadata.num_row_groups(), 2);
966        assert_eq!(metadata.row_group(0).num_rows(), 200);
967        assert_eq!(metadata.row_group(1).num_rows(), 200);
968        assert!(metadata.page_index().is_some_and(|idx| idx.is_complete()));
969    }
970
971    /// It is possible to pre-fetch some, but not all, of the necessary data
972    /// data
973    #[test]
974    fn test_metadata_decoder_prefetch_retry() {
975        let file_len = test_file_len();
976        let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
977        // simulate pre-fetching the last 1500 bytes of the file.
978        // this is enough to read the footer thrift metadata, but not the offset indexes
979        let prefetch_range = (file_len - 1500)..file_len;
980        push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![prefetch_range]);
981
982        // expect another request is needed to read the offset indexes (note
983        // try_decode only returns NeedsData once, whereas without any prefetching it would
984        // return NeedsData three times)
985        let ranges = expect_needs_data(metadata_decoder.try_decode());
986        push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges);
987
988        // expect the decoder has enough data to decode the metadata
989        let metadata = expect_data(metadata_decoder.try_decode());
990        expect_finished(metadata_decoder.try_decode());
991
992        assert_eq!(metadata.num_row_groups(), 2);
993        assert_eq!(metadata.row_group(0).num_rows(), 200);
994        assert_eq!(metadata.row_group(1).num_rows(), 200);
995        assert!(metadata.page_index().is_some_and(|idx| idx.is_complete()));
996    }
997
998    #[test]
999    fn test_metadata_decoder_clear_all_ranges() {
1000        let file_len = test_file_len();
1001        let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
1002
1003        metadata_decoder
1004            .push_range(test_file_range(), TEST_FILE_DATA.clone())
1005            .unwrap();
1006        assert_eq!(metadata_decoder.buffers.buffered_bytes(), test_file_len());
1007
1008        metadata_decoder.clear_all_ranges();
1009        assert_eq!(metadata_decoder.buffers.buffered_bytes(), 0);
1010
1011        let ranges = expect_needs_data(metadata_decoder.try_decode());
1012        assert_eq!(ranges, vec![test_file_len() - 8..test_file_len()]);
1013    }
1014
1015    /// Decode the metadata incrementally, simulating a scenario where exactly the data needed
1016    /// is read in each step
1017    #[test]
1018    fn test_metadata_decoder_incremental() {
1019        let file_len = TEST_FILE_DATA.len() as u64;
1020        let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
1021        let ranges = expect_needs_data(metadata_decoder.try_decode());
1022        assert_eq!(ranges.len(), 1);
1023        assert_eq!(ranges[0], test_file_len() - 8..test_file_len());
1024        push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges);
1025
1026        // expect the first request to read the footer
1027        let ranges = expect_needs_data(metadata_decoder.try_decode());
1028        push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges);
1029
1030        // expect the second request to read the offset indexes
1031        let ranges = expect_needs_data(metadata_decoder.try_decode());
1032        // since we're reading all indexes, there should be a single range needed
1033        assert_eq!(ranges.len(), 1);
1034        push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges);
1035
1036        // expect the third request to read the actual data
1037        let metadata = expect_data(metadata_decoder.try_decode());
1038        expect_finished(metadata_decoder.try_decode());
1039
1040        assert_eq!(metadata.num_row_groups(), 2);
1041        assert_eq!(metadata.row_group(0).num_rows(), 200);
1042        assert_eq!(metadata.row_group(1).num_rows(), 200);
1043        assert!(metadata.page_index().is_some_and(|idx| idx.is_complete()));
1044    }
1045
1046    /// Decode the metadata incrementally, but without reading the page indexes
1047    /// (so only two requests)
1048    #[test]
1049    fn test_metadata_decoder_incremental_no_page_index() {
1050        let file_len = TEST_FILE_DATA.len() as u64;
1051        let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len)
1052            .unwrap()
1053            .with_page_index_policy(PageIndexPolicy::Skip);
1054        let ranges = expect_needs_data(metadata_decoder.try_decode());
1055        assert_eq!(ranges.len(), 1);
1056        assert_eq!(ranges[0], test_file_len() - 8..test_file_len());
1057        push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges);
1058
1059        // expect the first request to read the footer
1060        let ranges = expect_needs_data(metadata_decoder.try_decode());
1061        push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges);
1062
1063        // expect NO second request to read the offset indexes, should just cough up the metadata
1064        let metadata = expect_data(metadata_decoder.try_decode());
1065        expect_finished(metadata_decoder.try_decode());
1066
1067        assert_eq!(metadata.num_row_groups(), 2);
1068        assert_eq!(metadata.row_group(0).num_rows(), 200);
1069        assert_eq!(metadata.row_group(1).num_rows(), 200);
1070        assert!(metadata.page_index().is_none()); // of course, we did not read the page index
1071    }
1072
1073    static TEST_BATCH: LazyLock<RecordBatch> = LazyLock::new(|| {
1074        // Input batch has 400 rows, with 3 columns: "a", "b", "c"
1075        // Note c is a different types (so the data page sizes will be different)
1076        let a: ArrayRef = Arc::new(Int64Array::from_iter_values(0..400));
1077        let b: ArrayRef = Arc::new(Int64Array::from_iter_values(400..800));
1078        let c: ArrayRef = Arc::new(StringViewArray::from_iter_values((0..400).map(|i| {
1079            if i % 2 == 0 {
1080                format!("string_{i}")
1081            } else {
1082                format!("A string larger than 12 bytes and thus not inlined {i}")
1083            }
1084        })));
1085
1086        RecordBatch::try_from_iter(vec![("a", a), ("b", b), ("c", c)]).unwrap()
1087    });
1088
1089    /// Create a parquet file in memory for testing. See [`test_file_range`] for details.
1090    static TEST_FILE_DATA: LazyLock<Bytes> = LazyLock::new(|| {
1091        let input_batch = &TEST_BATCH;
1092        let mut output = Vec::new();
1093
1094        let writer_options = WriterProperties::builder()
1095            .set_max_row_group_row_count(Some(200))
1096            .set_data_page_row_count_limit(100)
1097            .build();
1098        let mut writer =
1099            ArrowWriter::try_new(&mut output, input_batch.schema(), Some(writer_options)).unwrap();
1100
1101        // since the limits are only enforced on batch boundaries, write the input
1102        // batch in chunks of 50
1103        let mut row_remain = input_batch.num_rows();
1104        while row_remain > 0 {
1105            let chunk_size = row_remain.min(50);
1106            let chunk = input_batch.slice(input_batch.num_rows() - row_remain, chunk_size);
1107            writer.write(&chunk).unwrap();
1108            row_remain -= chunk_size;
1109        }
1110        writer.close().unwrap();
1111        Bytes::from(output)
1112    });
1113
1114    /// Return the length of the test file in bytes
1115    fn test_file_len() -> u64 {
1116        TEST_FILE_DATA.len() as u64
1117    }
1118
1119    /// Return the range of the entire test file
1120    fn test_file_range() -> Range<u64> {
1121        0..test_file_len()
1122    }
1123
1124    /// Return a slice of the test file data from the given range
1125    pub fn test_file_slice(range: Range<u64>) -> Bytes {
1126        let start: usize = range.start.try_into().unwrap();
1127        let end: usize = range.end.try_into().unwrap();
1128        TEST_FILE_DATA.slice(start..end)
1129    }
1130
1131    /// Push the given ranges to the metadata decoder, simulating reading from a file
1132    fn push_ranges_to_metadata_decoder(
1133        metadata_decoder: &mut ParquetMetaDataPushDecoder,
1134        ranges: Vec<Range<u64>>,
1135    ) {
1136        let data = ranges
1137            .iter()
1138            .map(|range| test_file_slice(range.clone()))
1139            .collect::<Vec<_>>();
1140        metadata_decoder.push_ranges(ranges, data).unwrap();
1141    }
1142
1143    fn test_metadata_without_page_index() -> ParquetMetaData {
1144        let mut decoder = ParquetMetaDataPushDecoder::try_new(test_file_len())
1145            .unwrap()
1146            .with_page_index_policy(PageIndexPolicy::Skip);
1147        push_ranges_to_metadata_decoder(&mut decoder, vec![test_file_range()]);
1148        let metadata = expect_data(decoder.try_decode());
1149        assert!(metadata.page_index().is_none());
1150        metadata
1151    }
1152
1153    /// Expect that the [`DecodeResult`] is a [`DecodeResult::Data`] and return the corresponding element
1154    fn expect_data<T: Debug>(result: Result<DecodeResult<T>>) -> T {
1155        match result.expect("Expected Ok(DecodeResult::Data(T))") {
1156            DecodeResult::Data(data) => data,
1157            result => panic!("Expected DecodeResult::Data, got {result:?}"),
1158        }
1159    }
1160
1161    /// Expect that the [`DecodeResult`] is a [`DecodeResult::NeedsData`] and return the corresponding ranges
1162    fn expect_needs_data<T: Debug>(result: Result<DecodeResult<T>>) -> Vec<Range<u64>> {
1163        match result.expect("Expected Ok(DecodeResult::NeedsData(ranges))") {
1164            DecodeResult::NeedsData(ranges) => ranges,
1165            result => panic!("Expected DecodeResult::NeedsData, got {result:?}"),
1166        }
1167    }
1168
1169    fn expect_finished<T: Debug>(result: Result<DecodeResult<T>>) {
1170        match result.expect("Expected Ok(DecodeResult::Finished)") {
1171            DecodeResult::Finished => {}
1172            result => panic!("Expected DecodeResult::Finished, got {result:?}"),
1173        }
1174    }
1175}