Skip to main content

parquet/arrow/push_decoder/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! [`ParquetPushDecoder`]: decodes Parquet data with data provided by the
6//! caller (rather than from an underlying reader).
7
8mod reader_builder;
9mod remaining;
10mod scan_plan;
11
12use crate::DecodeResult;
13pub use crate::arrow::arrow_reader::RowGroupSelection;
14use crate::arrow::arrow_reader::{
15    ArrowReaderBuilder, ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReader,
16};
17use crate::errors::ParquetError;
18use crate::file::metadata::ParquetMetaData;
19pub use crate::util::push_buffers::PushBuffers;
20use arrow_array::RecordBatch;
21use bytes::Bytes;
22use reader_builder::{RowGroupReaderBuilder, RowGroupReaderBuilderParts};
23use remaining::{RemainingRowGroups, RemainingRowGroupsParts};
24use scan_plan::RowBudget;
25pub use scan_plan::{PageKind, PlannedRange, ScanPlan};
26use std::ops::Range;
27use std::sync::Arc;
28
29/// A builder for [`ParquetPushDecoder`].
30///
31/// To create a new decoder, use [`ParquetPushDecoderBuilder::try_new_decoder`].
32///
33/// You can decode the metadata from a Parquet file using either
34/// [`ParquetMetadataReader`] or [`ParquetMetaDataPushDecoder`].
35///
36/// [`ParquetMetadataReader`]: crate::file::metadata::ParquetMetaDataReader
37/// [`ParquetMetaDataPushDecoder`]: crate::file::metadata::ParquetMetaDataPushDecoder
38///
39/// Note the "input" type is `u64` which represents the length of the Parquet file
40/// being decoded. This is needed to initialize the internal buffers that track
41/// what data has been provided to the decoder.
42///
43/// # Example
44/// ```
45/// # use std::ops::Range;
46/// # use std::sync::Arc;
47/// # use bytes::Bytes;
48/// # use arrow_array::record_batch;
49/// # use parquet::DecodeResult;
50/// # use parquet::arrow::push_decoder::ParquetPushDecoderBuilder;
51/// # use parquet::arrow::ArrowWriter;
52/// # use parquet::file::metadata::ParquetMetaDataPushDecoder;
53/// # let file_bytes = {
54/// #   let mut buffer = vec![];
55/// #   let batch = record_batch!(("a", Int32, [1, 2, 3])).unwrap();
56/// #   let mut writer = ArrowWriter::try_new(&mut buffer, batch.schema(), None).unwrap();
57/// #   writer.write(&batch).unwrap();
58/// #   writer.close().unwrap();
59/// #   Bytes::from(buffer)
60/// # };
61/// # // mimic IO by returning a function that returns the bytes for a given range
62/// # let get_range = |range: &Range<u64>| -> Bytes {
63/// #    let start = range.start as usize;
64/// #     let end = range.end as usize;
65/// #    file_bytes.slice(start..end)
66/// # };
67/// # let file_length = file_bytes.len() as u64;
68/// # let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_length).unwrap();
69/// # metadata_decoder.push_ranges(vec![0..file_length], vec![file_bytes.clone()]).unwrap();
70/// # let DecodeResult::Data(parquet_metadata) = metadata_decoder.try_decode().unwrap() else { panic!("failed to decode metadata") };
71/// # let parquet_metadata = Arc::new(parquet_metadata);
72/// // The file length and metadata are required to create the decoder
73/// let mut decoder =
74///     ParquetPushDecoderBuilder::try_new_decoder(parquet_metadata)
75///       .unwrap()
76///       // Optionally configure the decoder, e.g. batch size
77///       .with_batch_size(1024)
78///       // Build the decoder
79///       .build()
80///       .unwrap();
81///
82///     // In a loop, ask the decoder what it needs next, and provide it with the required data
83///     loop {
84///         match decoder.try_decode().unwrap() {
85///             DecodeResult::NeedsData(ranges) => {
86///                 // The decoder needs more data. Fetch the data for the given ranges
87///                 let data = ranges.iter().map(|r| get_range(r)).collect::<Vec<_>>();
88///                 // Push the data to the decoder
89///                 decoder.push_ranges(ranges, data).unwrap();
90///                 // After pushing the data, we can try to decode again on the next iteration
91///             }
92///             DecodeResult::Data(batch) => {
93///                 // Successfully decoded a batch of data
94///                 assert!(batch.num_rows() > 0);
95///             }
96///             DecodeResult::Finished => {
97///                 // The decoder has finished decoding exit the loop
98///                 break;
99///             }
100///         }
101///     }
102/// ```
103///
104/// # Adaptive scans
105///
106/// The scan strategy is not fixed once [`build`](Self::build) is called: it
107/// can be changed *while decoding*, at row-group boundaries.
108///
109/// The important API for this is [`ParquetPushDecoder::try_next_reader`].
110/// Unlike [`try_decode`](ParquetPushDecoder::try_decode), which barrels
111/// straight through row-group boundaries, `try_next_reader` returns once per
112/// row group — leaving a clean window *between* row groups. At any such
113/// boundary, [`ParquetPushDecoder::into_builder`] hands back a
114/// `ParquetPushDecoderBuilder` for the row groups not yet decoded. Change any
115/// option on it (projection, row filter, row selection policy, …) and
116/// [`build`](Self::build) a fresh decoder that resumes from the next row
117/// group. This is how a query engine promotes or demotes filters — for
118/// example turning a row filter on or off — based on the selectivity observed
119/// in the row groups decoded so far.
120///
121/// ```
122/// # use std::ops::Range;
123/// # use std::sync::Arc;
124/// # use bytes::Bytes;
125/// # use arrow_array::record_batch;
126/// # use parquet::DecodeResult;
127/// # use parquet::arrow::ProjectionMask;
128/// # use parquet::arrow::push_decoder::ParquetPushDecoderBuilder;
129/// # use parquet::arrow::ArrowWriter;
130/// # use parquet::file::metadata::ParquetMetaDataPushDecoder;
131/// # use parquet::file::properties::WriterProperties;
132/// # let file_bytes = {
133/// #   let batch = record_batch!(
134/// #       ("a", Int32, [1, 2, 3, 4, 5, 6]),
135/// #       ("b", Int32, [6, 5, 4, 3, 2, 1])
136/// #   ).unwrap();
137/// #   // Small row groups so the test file has two of them.
138/// #   let props = WriterProperties::builder().set_max_row_group_row_count(Some(3)).build();
139/// #   let mut buffer = vec![];
140/// #   let mut writer = ArrowWriter::try_new(&mut buffer, batch.schema(), Some(props)).unwrap();
141/// #   writer.write(&batch).unwrap();
142/// #   writer.close().unwrap();
143/// #   Bytes::from(buffer)
144/// # };
145/// # let get_range = |r: &Range<u64>| file_bytes.slice(r.start as usize..r.end as usize);
146/// # let file_length = file_bytes.len() as u64;
147/// # let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_length).unwrap();
148/// # metadata_decoder.push_ranges(vec![0..file_length], vec![file_bytes.clone()]).unwrap();
149/// # let DecodeResult::Data(parquet_metadata) = metadata_decoder.try_decode().unwrap() else { panic!() };
150/// # let parquet_metadata = Arc::new(parquet_metadata);
151/// let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(parquet_metadata)
152///     .unwrap()
153///     .build()
154///     .unwrap();
155///
156/// // Drive the decoder one row group at a time with `try_next_reader`.
157/// loop {
158///     match decoder.try_next_reader().unwrap() {
159///         DecodeResult::NeedsData(ranges) => {
160///             // Fetch and hand over the bytes the decoder asked for.
161///             let data = ranges.iter().map(|r| get_range(r)).collect();
162///             decoder.push_ranges(ranges, data).unwrap();
163///         }
164///         DecodeResult::Data(reader) => {
165///             // Decode this row group's batches.
166///             for batch in reader {
167///                 assert!(batch.unwrap().num_rows() > 0);
168///             }
169///             // We are now at a row-group boundary. Based on whatever stats
170///             // were gathered, optionally change strategy for the row groups
171///             // still to come: drop or promote a row filter, narrow or widen
172///             // the projection, etc.
173///             if decoder.is_at_row_group_boundary() && decoder.row_groups_remaining() > 0 {
174///                 let builder = decoder.into_builder().unwrap();
175///                 // e.g. column "b" turned out not to be needed.
176///                 let projection = ProjectionMask::columns(builder.parquet_schema(), ["a"]);
177///                 decoder = builder.with_projection(projection).build().unwrap();
178///             }
179///         }
180///         DecodeResult::Finished => break,
181///     }
182/// }
183/// ```
184pub type ParquetPushDecoderBuilder = ArrowReaderBuilder<PushDecoderInput>;
185
186/// The `input` of a [`ParquetPushDecoderBuilder`].
187///
188/// The shared [`ArrowReaderBuilder`] is generic over an `input`. The sync and
189/// async builders read from a file or async reader; the push decoder has no
190/// reader, so its input is the [`PushBuffers`] that caller-pushed bytes
191/// accumulate in (empty for a fresh builder).
192#[derive(Debug, Default)]
193pub struct PushDecoderInput {
194    /// Bytes pushed into the decoder, awaiting decode.
195    buffers: PushBuffers,
196}
197
198/// Methods for building a ParquetDecoder. See the base [`ArrowReaderBuilder`] for
199/// more options that can be configured.
200impl ParquetPushDecoderBuilder {
201    /// Create a new `ParquetDecoderBuilder` for configuring a Parquet decoder for the given file.
202    ///
203    /// See [`ParquetMetadataDecoder`] for a builder that can read the metadata from a Parquet file.
204    ///
205    /// [`ParquetMetadataDecoder`]: crate::file::metadata::ParquetMetaDataPushDecoder
206    ///
207    /// See example on [`ParquetPushDecoderBuilder`]
208    pub fn try_new_decoder(parquet_metadata: Arc<ParquetMetaData>) -> Result<Self, ParquetError> {
209        Self::try_new_decoder_with_options(parquet_metadata, ArrowReaderOptions::default())
210    }
211
212    /// Create a new `ParquetDecoderBuilder` for configuring a Parquet decoder for the given file
213    /// with the given reader options.
214    ///
215    /// This is similar to [`Self::try_new_decoder`] but allows configuring
216    /// options such as Arrow schema
217    pub fn try_new_decoder_with_options(
218        parquet_metadata: Arc<ParquetMetaData>,
219        arrow_reader_options: ArrowReaderOptions,
220    ) -> Result<Self, ParquetError> {
221        let arrow_reader_metadata =
222            ArrowReaderMetadata::try_new(parquet_metadata, arrow_reader_options)?;
223        Ok(Self::new_with_metadata(arrow_reader_metadata))
224    }
225
226    /// Create a new `ParquetDecoderBuilder` given [`ArrowReaderMetadata`].
227    ///
228    /// See [`ArrowReaderMetadata::try_new`] for how to create the metadata from
229    /// the Parquet metadata and reader options.
230    pub fn new_with_metadata(arrow_reader_metadata: ArrowReaderMetadata) -> Self {
231        Self::new_builder(PushDecoderInput::default(), arrow_reader_metadata)
232    }
233
234    /// Provide a preexisting [`PushBuffers`] for the built decoder to read
235    /// from, so bytes already fetched are not requested again.
236    ///
237    /// [`Self::build`] releases the bytes that the decoder does not read:
238    /// the bytes outside the read column chunks of the row groups that the
239    /// decoder reads.
240    pub fn with_buffers(self, buffers: PushBuffers) -> Self {
241        Self {
242            input: PushDecoderInput { buffers },
243            ..self
244        }
245    }
246
247    /// Select row groups and rows using row-group-local coordinates.
248    ///
249    /// Entries are decoded in the supplied order, and omitted row groups are
250    /// skipped. A row group listed more than once is decoded once per entry.
251    /// A `None` selection reads the entire row group. A selection shorter
252    /// than its row group skips the trailing rows, while a selection longer
253    /// than its row group returns an error from [`Self::build`].
254    ///
255    /// [`ArrowReaderBuilder::with_offset`] and
256    /// [`ArrowReaderBuilder::with_limit`] apply after the row-group-local
257    /// selections and any [`ArrowReaderBuilder::with_row_filter`], across the
258    /// plan as a whole and in the supplied order: the offset skips the first N
259    /// remaining rows and the limit caps the total rows emitted, regardless of
260    /// which row group they come from.
261    ///
262    /// This configuration is mutually exclusive with
263    /// [`ArrowReaderBuilder::with_row_groups`] and
264    /// [`ArrowReaderBuilder::with_row_selection`]. Combining them returns an
265    /// error from [`Self::build`]. Calling this method more than once replaces
266    /// the previous row-group-local configuration.
267    ///
268    /// This method is defined on the push decoder rather than
269    /// [`ArrowReaderBuilder`] because the synchronous reader does not support
270    /// row-group-local selections. The async stream builder exposes the same
271    /// method because it uses the push decoder internally.
272    ///
273    /// For example, if row groups 0 and 2 each contain 200 rows, the legacy
274    /// combination of `with_row_groups(vec![0, 2])` and a global selection for
275    /// rows 10..15 of row group 0 and all of row group 2 can be expressed as:
276    ///
277    /// ```no_run
278    /// # use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
279    /// # use parquet::arrow::push_decoder::{ParquetPushDecoderBuilder, RowGroupSelection};
280    /// # fn configure(builder: ParquetPushDecoderBuilder) -> ParquetPushDecoderBuilder {
281    /// builder.with_row_group_selections(vec![
282    ///     RowGroupSelection::new(0, Some(RowSelection::from(vec![
283    ///         RowSelector::skip(10),
284    ///         RowSelector::select(5),
285    ///     ]))),
286    ///     RowGroupSelection::new(2, None),
287    /// ])
288    /// # }
289    /// ```
290    pub fn with_row_group_selections(
291        mut self,
292        row_group_selections: Vec<RowGroupSelection>,
293    ) -> Self {
294        self.row_group_plan
295            .set_row_group_selections(row_group_selections);
296        self
297    }
298
299    /// Create a [`ParquetPushDecoder`] with the configured options
300    pub fn build(self) -> Result<ParquetPushDecoder, ParquetError> {
301        let Self {
302            input: PushDecoderInput { buffers },
303            metadata: parquet_metadata,
304            schema,
305            fields,
306            batch_size,
307            row_group_plan,
308            projection,
309            filter,
310            limit,
311            offset,
312            metrics,
313            row_selection_policy,
314            max_predicate_cache_size,
315        } = self;
316
317        // Evaluate eligible same-projection predicates as one predicate.
318        let filter = filter.map(|filter| {
319            filter.fuse_same_projection(parquet_metadata.file_metadata().schema_descr())
320        });
321
322        let has_predicates = filter
323            .as_ref()
324            .is_some_and(|filter| !filter.predicates.is_empty());
325
326        // Prepare to build RowGroup readers. `buffers` carries any bytes the
327        // caller already pushed (preserved across `into_builder`); a fresh
328        // builder supplies an empty `PushBuffers`.
329        let row_group_reader_builder = RowGroupReaderBuilder::new(
330            batch_size,
331            projection,
332            Arc::clone(&parquet_metadata),
333            fields,
334            filter,
335            metrics,
336            max_predicate_cache_size,
337            buffers,
338            row_selection_policy,
339        );
340
341        // Initialize the decoder with the configured options
342        let mut remaining_row_groups = RemainingRowGroups::new(
343            schema,
344            parquet_metadata,
345            row_group_plan,
346            RowBudget::new(offset, limit),
347            has_predicates,
348            row_group_reader_builder,
349        )?;
350        // The buffers can hold bytes that this decoder does not read. For
351        // example, the buffers of a rebuilt decoder can hold the bytes of row
352        // groups that the new configuration skips.
353        remaining_row_groups.release_unread_bytes();
354
355        Ok(ParquetPushDecoder {
356            state: ParquetDecoderState::ReadingRowGroup {
357                remaining_row_groups: Box::new(remaining_row_groups),
358            },
359        })
360    }
361}
362
363/// Reassemble a [`ParquetPushDecoderBuilder`] from a decoder's not-yet-decoded
364/// state — the inverse of [`ParquetPushDecoderBuilder::build`]. The rebuilt
365/// builder carries the remaining row-group plan, offset/limit budget, and
366/// buffered bytes.
367fn builder_from_remaining(parts: RemainingRowGroupsParts) -> ParquetPushDecoderBuilder {
368    let RemainingRowGroupsParts {
369        metadata,
370        schema,
371        row_group_plan,
372        offset,
373        limit,
374        reader_builder,
375    } = parts;
376    let RowGroupReaderBuilderParts {
377        batch_size,
378        projection,
379        fields,
380        filter,
381        max_predicate_cache_size,
382        metrics,
383        row_selection_policy,
384        buffers,
385    } = reader_builder;
386
387    ArrowReaderBuilder {
388        input: PushDecoderInput::default(),
389        metadata,
390        schema,
391        fields,
392        batch_size,
393        row_group_plan,
394        projection,
395        filter,
396        row_selection_policy,
397        limit,
398        offset,
399        metrics,
400        max_predicate_cache_size,
401    }
402    // Carry the decoder's already-fetched bytes across the rebuild so the new
403    // decoder does not re-request them.
404    .with_buffers(buffers)
405}
406
407/// A push based Parquet Decoder
408///
409/// See [`ParquetPushDecoderBuilder`] for an example of how to build and use the decoder.
410///
411/// [`ParquetPushDecoder`] is a low level API for decoding Parquet data without an
412/// underlying reader for performing IO, and thus offers fine grained control
413/// over how data is fetched and decoded.
414///
415/// When more data is needed to make progress, instead of reading data directly
416/// from a reader, the decoder returns [`DecodeResult`] indicating what ranges
417/// are needed. Once the caller provides the requested ranges via
418/// [`Self::push_ranges`], they try to decode again by calling
419/// [`Self::try_decode`].
420///
421/// The decoder's internal state tracks what has been already decoded and what
422/// is needed next.
423#[derive(Debug)]
424pub struct ParquetPushDecoder {
425    /// The inner state.
426    ///
427    /// This state is consumed on every transition and a new state is produced
428    /// so the Rust compiler can ensure that the state is always valid and
429    /// transitions are not missed.
430    state: ParquetDecoderState,
431}
432
433impl ParquetPushDecoder {
434    /// Attempt to decode the next batch of data, or return what data is needed
435    ///
436    /// The the decoder communicates the next state with a [`DecodeResult`]
437    ///
438    /// See full example in [`ParquetPushDecoderBuilder`]
439    ///
440    /// ```no_run
441    /// # use parquet::arrow::push_decoder::ParquetPushDecoder;
442    /// use parquet::DecodeResult;
443    /// # fn get_decoder() -> ParquetPushDecoder { unimplemented!() }
444    /// # fn push_data(decoder: &mut ParquetPushDecoder, ranges: Vec<std::ops::Range<u64>>) { unimplemented!() }
445    /// let mut decoder = get_decoder();
446    /// loop {
447    ///    match decoder.try_decode().unwrap() {
448    ///       DecodeResult::NeedsData(ranges) => {
449    ///         // The decoder needs more data. Fetch the data for the given ranges
450    ///         // call decoder.push_ranges(ranges, data) and call again
451    ///         push_data(&mut decoder, ranges);
452    ///       }
453    ///       DecodeResult::Data(batch) => {
454    ///         // Successfully decoded the next batch of data
455    ///         println!("Got batch with {} rows", batch.num_rows());
456    ///       }
457    ///       DecodeResult::Finished => {
458    ///         // The decoder has finished decoding all data
459    ///         break;
460    ///       }
461    ///    }
462    /// }
463    ///```
464    pub fn try_decode(&mut self) -> Result<DecodeResult<RecordBatch>, ParquetError> {
465        let current_state = std::mem::replace(&mut self.state, ParquetDecoderState::Finished);
466        let (new_state, decode_result) = current_state.try_next_batch()?;
467        self.state = new_state;
468        Ok(decode_result)
469    }
470
471    /// Return a [`ParquetRecordBatchReader`] that reads the next set of rows, or
472    /// return what data is needed to produce it.
473    ///
474    /// This API can be used to get a reader for decoding the next set of
475    /// RecordBatches while proceeding to begin fetching data for the set (e.g
476    /// row group)
477    ///
478    /// Example
479    /// ```no_run
480    /// # use parquet::arrow::push_decoder::ParquetPushDecoder;
481    /// use parquet::DecodeResult;
482    /// # fn get_decoder() -> ParquetPushDecoder { unimplemented!() }
483    /// # fn push_data(decoder: &mut ParquetPushDecoder, ranges: Vec<std::ops::Range<u64>>) { unimplemented!() }
484    /// let mut decoder = get_decoder();
485    /// loop {
486    ///    match decoder.try_next_reader().unwrap() {
487    ///       DecodeResult::NeedsData(ranges) => {
488    ///         // The decoder needs more data. Fetch the data for the given ranges
489    ///         // call decoder.push_ranges(ranges, data) and call again
490    ///         push_data(&mut decoder, ranges);
491    ///       }
492    ///       DecodeResult::Data(reader) => {
493    ///          // spawn a thread to read the batches in parallel
494    ///          // with fetching the next row group / data
495    ///          std::thread::spawn(move || {
496    ///            for batch in reader {
497    ///              let batch = batch.unwrap();
498    ///              println!("Got batch with {} rows", batch.num_rows());
499    ///            }
500    ///         });
501    ///       }
502    ///       DecodeResult::Finished => {
503    ///         // The decoder has finished decoding all data
504    ///         break;
505    ///       }
506    ///    }
507    /// }
508    ///```
509    pub fn try_next_reader(
510        &mut self,
511    ) -> Result<DecodeResult<ParquetRecordBatchReader>, ParquetError> {
512        let current_state = std::mem::replace(&mut self.state, ParquetDecoderState::Finished);
513        let (new_state, decode_result) = current_state.try_next_reader()?;
514        self.state = new_state;
515        Ok(decode_result)
516    }
517
518    /// Push data into the decoder for processing
519    ///
520    /// This is a convenience wrapper around [`Self::push_ranges`] for pushing a
521    /// single range of data.
522    ///
523    /// Note this can be the entire file or just a part of it. If it is part of the file,
524    /// the ranges should correspond to the data ranges requested by the decoder.
525    ///
526    /// # Memory Usage
527    ///
528    /// The decoder releases the pushed bytes that it will not read again (for
529    /// example, the bytes of a row group when it is done with that row group).
530    /// However, it does not release *all* pushed bytes: it keeps the bytes of
531    /// row groups that it does not read (for example, row groups without
532    /// selected rows, or after the limit). Call [`Self::clear_all_ranges`] to
533    /// release them.
534    ///
535    /// See example in [`ParquetPushDecoderBuilder`]
536    pub fn push_range(&mut self, range: Range<u64>, data: Bytes) -> Result<(), ParquetError> {
537        self.push_ranges(vec![range], vec![data])
538    }
539
540    /// Push data into the decoder for processing
541    ///
542    /// This should correspond to the data ranges requested by the decoder.
543    ///
544    /// If you fetch data before the decoder requests it, for example with
545    /// [`Self::scan_plan`], keep it in your own cache and push exactly the
546    /// ranges in each [`DecodeResult::NeedsData`]:
547    ///
548    /// * The decoder does not use a requested range that is split over two
549    ///   pushed buffers.
550    /// * The decoder releases a pushed buffer only if its range is equal to a
551    ///   requested range. Other pushed data stays buffered.
552    pub fn push_ranges(
553        &mut self,
554        ranges: Vec<Range<u64>>,
555        data: Vec<Bytes>,
556    ) -> Result<(), ParquetError> {
557        let current_state = std::mem::replace(&mut self.state, ParquetDecoderState::Finished);
558        self.state = current_state.push_data(ranges, data)?;
559        Ok(())
560    }
561
562    /// Returns the total number of buffered bytes in the decoder
563    ///
564    /// This is the sum of the size of all [`Bytes`] that has been pushed to the
565    /// decoder but not yet consumed.
566    ///
567    /// Note that this does not include any overhead of the internal data
568    /// structures and that since [`Bytes`] are ref counted memory, this may not
569    /// reflect additional memory usage.
570    ///
571    /// This can be used to monitor memory usage of the decoder.
572    pub fn buffered_bytes(&self) -> u64 {
573        self.state.buffered_bytes()
574    }
575
576    /// Clear any staged byte ranges currently buffered for future decode work.
577    ///
578    /// This clears byte ranges still owned by the decoder's internal
579    /// `PushBuffers`. It does not affect any data that has already been handed
580    /// off to an active [`ParquetRecordBatchReader`].
581    pub fn clear_all_ranges(&mut self) {
582        self.state.clear_all_ranges();
583    }
584
585    /// True iff the decoder is at a row-group boundary, where
586    /// [`Self::into_builder`] can reconfigure the scan.
587    ///
588    /// A boundary is "between row groups": the previous row group's
589    /// [`ParquetRecordBatchReader`] has been fully extracted (via
590    /// [`Self::try_next_reader`]) or fully drained (via [`Self::try_decode`]),
591    /// and the next row group has not yet been planned. While
592    /// [`Self::try_decode`] is iterating an active row group's reader this
593    /// returns `false`; with [`Self::try_next_reader`] there is a clean
594    /// window between two consecutive returns where this is `true`.
595    pub fn is_at_row_group_boundary(&self) -> bool {
596        self.state.is_at_row_group_boundary()
597    }
598
599    /// Number of row groups left to decode after the one currently in flight.
600    /// Useful as a "should I bother reconfiguring the scan?" signal.
601    pub fn row_groups_remaining(&self) -> usize {
602        self.state.row_groups_remaining()
603    }
604
605    /// Returns the row-group index that the next call to
606    /// [`Self::try_next_reader`] will yield a reader for, after applying
607    /// any internal skipping (row selection emptiness, exhausted budget,
608    /// finished state).
609    ///
610    /// Safe to call at any time. When called mid-row-group (i.e. while a
611    /// previously-emitted [`ParquetRecordBatchReader`] is still being
612    /// drained), the returned index refers to the row group that
613    /// [`Self::try_next_reader`] will produce *after* the current one.
614    ///
615    /// Returns `Ok(None)` when:
616    /// - the decoder has no more row groups to read, or
617    /// - every remaining row group would be skipped.
618    ///
619    /// This method does not mutate decoder state. It is useful for
620    /// callers that maintain per-row-group state in lock-step with the
621    /// decoder (e.g. dynamic row-group pruners) to determine which row
622    /// group the next reader corresponds to, since
623    /// [`Self::try_next_reader`] may silently advance past row groups
624    /// based on filtering and other criteria.
625    pub fn peek_next_row_group(&self) -> Result<Option<usize>, ParquetError> {
626        self.state.peek_next_row_group()
627    }
628
629    /// Returns the byte ranges that this decoder may still read, in the order
630    /// that decoding needs them. See [`ScanPlan`].
631    pub fn scan_plan(&self) -> ScanPlan {
632        self.state.scan_plan()
633    }
634
635    /// Decompose this decoder back into a [`ParquetPushDecoderBuilder`] for the
636    /// row groups that have *not* yet been decoded.
637    ///
638    /// This is the API for *adaptive* scans. Drive the decoder with
639    /// [`Self::try_next_reader`]; at any row-group boundary, call
640    /// `into_builder` to recover a builder, adjust it with the usual
641    /// [`ParquetPushDecoderBuilder`] setters, and
642    /// [`build`](ParquetPushDecoderBuilder::build) a fresh decoder that resumes
643    /// from the next row group:
644    ///
645    /// ```no_run
646    /// # use parquet::arrow::push_decoder::ParquetPushDecoder;
647    /// # use parquet::arrow::arrow_reader::RowFilter;
648    /// # fn get_decoder() -> ParquetPushDecoder { unimplemented!() }
649    /// # fn new_filter() -> RowFilter { unimplemented!() }
650    /// let mut decoder = get_decoder();
651    /// // ... drive `decoder.try_next_reader()` for a few row groups ...
652    /// if decoder.is_at_row_group_boundary() && decoder.row_groups_remaining() > 0 {
653    ///     decoder = decoder
654    ///         .into_builder()
655    ///         .unwrap()
656    ///         // any builder option can be changed here, e.g. promote a
657    ///         // filter into a row filter based on observed selectivity
658    ///         .with_row_filter(new_filter())
659    ///         .build()
660    ///         .unwrap();
661    /// }
662    /// ```
663    ///
664    /// The returned builder preserves the not-yet-decoded row-group plan,
665    /// including any row-group-local selections, together with the remaining
666    /// offset/limit budget. Rows from already-decoded row groups are not
667    /// produced again. Every other option — projection, row filter, row
668    /// selection policy, batch size, metrics, predicate-cache size — is left
669    /// exactly as the decoder had it and can be overridden before
670    /// [`build`](ParquetPushDecoderBuilder::build).
671    ///
672    /// Because the preserved plan pins the remaining row groups, the mutual
673    /// exclusion documented on
674    /// [`with_row_group_selections`](ParquetPushDecoderBuilder::with_row_group_selections)
675    /// applies to the rebuilt builder as well: calling it on a builder rebuilt
676    /// from a decoder configured with `with_row_groups` / `with_row_selection`
677    /// (or vice versa) returns an error from `build`.
678    ///
679    /// # Errors
680    ///
681    /// Returns `Err(ParquetError::General)` when the decoder is not at a
682    /// row-group boundary (check [`Self::is_at_row_group_boundary`] first) or
683    /// has already finished. The decoder is consumed either way.
684    ///
685    /// # Buffered bytes
686    ///
687    /// The decoder's buffered bytes are carried across the rebuild: bytes
688    /// already fetched for row groups the new configuration still reads are
689    /// not re-requested. [`build`](ParquetPushDecoderBuilder::build) releases
690    /// the bytes that the new configuration does not read. See
691    /// [`push_range`](Self::push_range) for more details.
692    pub fn into_builder(self) -> Result<ParquetPushDecoderBuilder, ParquetError> {
693        self.state.into_builder()
694    }
695}
696
697/// Internal state machine for the [`ParquetPushDecoder`]
698#[derive(Debug)]
699enum ParquetDecoderState {
700    /// Waiting for data needed to decode the next RowGroup
701    ReadingRowGroup {
702        remaining_row_groups: Box<RemainingRowGroups>,
703    },
704    /// The decoder is actively decoding a RowGroup
705    DecodingRowGroup {
706        /// Current active reader
707        record_batch_reader: Box<ParquetRecordBatchReader>,
708        remaining_row_groups: Box<RemainingRowGroups>,
709    },
710    /// The decoder has finished processing all data
711    Finished,
712}
713
714impl ParquetDecoderState {
715    /// If actively reading a RowGroup, return the currently active
716    /// ParquetRecordBatchReader and advance to the next group.
717    fn try_next_reader(
718        self,
719    ) -> Result<(Self, DecodeResult<ParquetRecordBatchReader>), ParquetError> {
720        let mut current_state = self;
721        loop {
722            let (next_state, decode_result) = current_state.transition()?;
723            // if more data is needed to transition, can't proceed further without it
724            match decode_result {
725                DecodeResult::NeedsData(ranges) => {
726                    return Ok((next_state, DecodeResult::NeedsData(ranges)));
727                }
728                // act next based on state
729                DecodeResult::Data(()) | DecodeResult::Finished => {}
730            }
731            match next_state {
732                // not ready to read yet, continue transitioning
733                Self::ReadingRowGroup { .. } => current_state = next_state,
734                // have a reader ready, so return it and set ourself to ReadingRowGroup
735                Self::DecodingRowGroup {
736                    record_batch_reader,
737                    remaining_row_groups,
738                } => {
739                    let result = DecodeResult::Data(*record_batch_reader);
740                    let next_state = Self::ReadingRowGroup {
741                        remaining_row_groups,
742                    };
743                    return Ok((next_state, result));
744                }
745                Self::Finished => {
746                    return Ok((Self::Finished, DecodeResult::Finished));
747                }
748            }
749        }
750    }
751
752    /// Current state --> next state + output
753    ///
754    /// This function is called to get the next RecordBatch
755    ///
756    /// This structure is used to reduce the indentation level of the main loop
757    /// in try_build
758    fn try_next_batch(self) -> Result<(Self, DecodeResult<RecordBatch>), ParquetError> {
759        let mut current_state = self;
760        loop {
761            let (new_state, decode_result) = current_state.transition()?;
762            // if more data is needed to transition, can't proceed further without it
763            match decode_result {
764                DecodeResult::NeedsData(ranges) => {
765                    return Ok((new_state, DecodeResult::NeedsData(ranges)));
766                }
767                // act next based on state
768                DecodeResult::Data(()) | DecodeResult::Finished => {}
769            }
770            match new_state {
771                // not ready to read yet, continue transitioning
772                Self::ReadingRowGroup { .. } => current_state = new_state,
773                // have a reader ready, so decode the next batch
774                Self::DecodingRowGroup {
775                    mut record_batch_reader,
776                    remaining_row_groups,
777                } => {
778                    match record_batch_reader.next() {
779                        // Successfully decoded a batch, return it
780                        Some(Ok(batch)) => {
781                            let result = DecodeResult::Data(batch);
782                            let next_state = Self::DecodingRowGroup {
783                                record_batch_reader,
784                                remaining_row_groups,
785                            };
786                            return Ok((next_state, result));
787                        }
788                        // No more batches in this row group, move to the next row group
789                        None => {
790                            current_state = Self::ReadingRowGroup {
791                                remaining_row_groups,
792                            }
793                        }
794                        // some error occurred while decoding, so return that
795                        Some(Err(e)) => {
796                            // TODO: preserve ArrowError in ParquetError (rather than convert to a string)
797                            return Err(ParquetError::ArrowError(e.to_string()));
798                        }
799                    }
800                }
801                Self::Finished => {
802                    return Ok((Self::Finished, DecodeResult::Finished));
803                }
804            }
805        }
806    }
807
808    /// Transition to the next state with a reader (data can be produced), if not end of stream
809    ///
810    /// This function is called in a loop until the decoder is ready to return
811    /// data (has the required pages buffered) or is finished.
812    fn transition(self) -> Result<(Self, DecodeResult<()>), ParquetError> {
813        // result returned when there is data ready
814        let data_ready = DecodeResult::Data(());
815        match self {
816            Self::ReadingRowGroup {
817                mut remaining_row_groups,
818            } => {
819                match remaining_row_groups.try_next_reader()? {
820                    // If we have a next reader, we can transition to decoding it
821                    DecodeResult::Data(record_batch_reader) => {
822                        // Transition to decoding the row group
823                        Ok((
824                            Self::DecodingRowGroup {
825                                record_batch_reader: Box::new(record_batch_reader),
826                                remaining_row_groups,
827                            },
828                            data_ready,
829                        ))
830                    }
831                    DecodeResult::NeedsData(ranges) => {
832                        // If we need more data, we return the ranges needed and stay in Reading
833                        // RowGroup state
834                        Ok((
835                            Self::ReadingRowGroup {
836                                remaining_row_groups,
837                            },
838                            DecodeResult::NeedsData(ranges),
839                        ))
840                    }
841                    // If there are no more readers, we are finished
842                    DecodeResult::Finished => {
843                        // No more row groups to read, we are finished
844                        Ok((Self::Finished, DecodeResult::Finished))
845                    }
846                }
847            }
848            // if we are already in DecodingRowGroup, just return data ready
849            Self::DecodingRowGroup { .. } => Ok((self, data_ready)),
850            // if finished, just return finished
851            Self::Finished => Ok((self, DecodeResult::Finished)),
852        }
853    }
854
855    /// Push data, and transition state if needed
856    ///
857    /// This should correspond to the data ranges requested by the decoder
858    pub fn push_data(
859        self,
860        ranges: Vec<Range<u64>>,
861        data: Vec<Bytes>,
862    ) -> Result<Self, ParquetError> {
863        match self {
864            ParquetDecoderState::ReadingRowGroup {
865                mut remaining_row_groups,
866            } => {
867                // Push data to the RowGroupReaderBuilder
868                remaining_row_groups.push_data(ranges, data)?;
869                Ok(ParquetDecoderState::ReadingRowGroup {
870                    remaining_row_groups,
871                })
872            }
873            // it is ok to get data before we asked for it
874            ParquetDecoderState::DecodingRowGroup {
875                record_batch_reader,
876                mut remaining_row_groups,
877            } => {
878                remaining_row_groups.push_data(ranges, data)?;
879                Ok(ParquetDecoderState::DecodingRowGroup {
880                    record_batch_reader,
881                    remaining_row_groups,
882                })
883            }
884            ParquetDecoderState::Finished => Err(ParquetError::General(
885                "Cannot push data to a finished decoder".to_string(),
886            )),
887        }
888    }
889
890    /// How many bytes are currently buffered in the decoder?
891    fn buffered_bytes(&self) -> u64 {
892        match self {
893            ParquetDecoderState::ReadingRowGroup {
894                remaining_row_groups,
895            } => remaining_row_groups.buffered_bytes(),
896            ParquetDecoderState::DecodingRowGroup {
897                record_batch_reader: _,
898                remaining_row_groups,
899            } => remaining_row_groups.buffered_bytes(),
900            ParquetDecoderState::Finished => 0,
901        }
902    }
903
904    /// Clear any staged ranges currently buffered in the decoder.
905    fn clear_all_ranges(&mut self) {
906        match self {
907            ParquetDecoderState::ReadingRowGroup {
908                remaining_row_groups,
909            } => remaining_row_groups.clear_all_ranges(),
910            ParquetDecoderState::DecodingRowGroup {
911                record_batch_reader: _,
912                remaining_row_groups,
913            } => remaining_row_groups.clear_all_ranges(),
914            ParquetDecoderState::Finished => {}
915        }
916    }
917
918    fn is_at_row_group_boundary(&self) -> bool {
919        match self {
920            ParquetDecoderState::ReadingRowGroup {
921                remaining_row_groups,
922            } => remaining_row_groups.is_at_row_group_boundary(),
923            // Mid-row-group: the active reader holds an `ArrayReader` and
924            // `ReadPlan` keyed to the *current* projection/filter; rebuilding
925            // would require throwing that work away.
926            ParquetDecoderState::DecodingRowGroup { .. } => false,
927            ParquetDecoderState::Finished => false,
928        }
929    }
930
931    /// See [`ParquetPushDecoder::scan_plan`].
932    fn scan_plan(&self) -> ScanPlan {
933        match self {
934            ParquetDecoderState::ReadingRowGroup {
935                remaining_row_groups,
936            }
937            | ParquetDecoderState::DecodingRowGroup {
938                remaining_row_groups,
939                ..
940            } => remaining_row_groups.scan_plan(),
941            ParquetDecoderState::Finished => ScanPlan::empty(),
942        }
943    }
944
945    fn row_groups_remaining(&self) -> usize {
946        match self {
947            ParquetDecoderState::ReadingRowGroup {
948                remaining_row_groups,
949            } => remaining_row_groups.row_groups_remaining(),
950            ParquetDecoderState::DecodingRowGroup {
951                remaining_row_groups,
952                ..
953            } => remaining_row_groups.row_groups_remaining(),
954            ParquetDecoderState::Finished => 0,
955        }
956    }
957
958    /// See [`ParquetPushDecoder::peek_next_row_group`] for the public
959    /// API contract. This inner method delegates to the underlying
960    /// [`RemainingRowGroups`] for both `ReadingRowGroup` and
961    /// `DecodingRowGroup`: mid-row-group the answer is the row group
962    /// `try_next_reader` will produce *after* the active one finishes,
963    /// which is exactly what `RemainingRowGroups::peek_next_row_group`
964    /// computes from the queued frontier.
965    fn peek_next_row_group(&self) -> Result<Option<usize>, ParquetError> {
966        match self {
967            ParquetDecoderState::ReadingRowGroup {
968                remaining_row_groups,
969            } => remaining_row_groups.peek_next_row_group(),
970            ParquetDecoderState::DecodingRowGroup {
971                remaining_row_groups,
972                ..
973            } => remaining_row_groups.peek_next_row_group(),
974            ParquetDecoderState::Finished => Ok(None),
975        }
976    }
977
978    fn into_builder(self) -> Result<ParquetPushDecoderBuilder, ParquetError> {
979        let remaining_row_groups = match self {
980            ParquetDecoderState::ReadingRowGroup {
981                remaining_row_groups,
982            } => remaining_row_groups,
983            ParquetDecoderState::DecodingRowGroup { .. } => {
984                return Err(ParquetError::General(
985                    "into_builder called while a row group is being decoded; \
986                     check is_at_row_group_boundary() first"
987                        .to_string(),
988                ));
989            }
990            ParquetDecoderState::Finished => {
991                return Err(ParquetError::General(
992                    "into_builder called on a finished decoder".to_string(),
993                ));
994            }
995        };
996        if !remaining_row_groups.is_at_row_group_boundary() {
997            return Err(ParquetError::General(
998                "into_builder called mid-row-group; check is_at_row_group_boundary() first"
999                    .to_string(),
1000            ));
1001        }
1002        Ok(builder_from_remaining(remaining_row_groups.into_parts()))
1003    }
1004}
1005
1006#[cfg(test)]
1007mod test {
1008    use super::*;
1009    use crate::DecodeResult;
1010    use crate::arrow::arrow_reader::{ArrowPredicateFn, RowFilter, RowSelection, RowSelector};
1011    use crate::arrow::push_decoder::reader_builder::column_chunk_range;
1012    use crate::arrow::push_decoder::{
1013        ParquetPushDecoder, ParquetPushDecoderBuilder, RowGroupSelection,
1014    };
1015    use crate::arrow::{ArrowWriter, ProjectionMask};
1016    use crate::errors::ParquetError;
1017    use crate::file::metadata::page_index::PageIndexBuilder;
1018    use crate::file::metadata::{PageIndexPolicy, ParquetMetaDataPushDecoder};
1019    use crate::file::properties::WriterProperties;
1020    use arrow::compute::kernels::cmp::{gt, lt};
1021    use arrow_array::cast::AsArray;
1022    use arrow_array::types::Int64Type;
1023    use arrow_array::{ArrayRef, Int64Array, RecordBatch, StringViewArray};
1024    use arrow_buffer::BooleanBuffer;
1025    use arrow_select::concat::concat_batches;
1026    use bytes::Bytes;
1027    use std::fmt::Debug;
1028    use std::ops::Range;
1029    use std::sync::{Arc, LazyLock};
1030
1031    /// Test decoder struct size (as they are copied around on each transition, they
1032    /// should not grow too large)
1033    #[test]
1034    fn test_decoder_size() {
1035        assert_eq!(std::mem::size_of::<ParquetDecoderState>(), 24);
1036    }
1037
1038    /// Decode the entire file at once, simulating a scenario where all data is
1039    /// available in memory
1040    #[test]
1041    fn test_decoder_all_data() {
1042        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
1043            .unwrap()
1044            .build()
1045            .unwrap();
1046
1047        decoder
1048            .push_range(test_file_range(), TEST_FILE_DATA.clone())
1049            .unwrap();
1050
1051        let results = vec![
1052            // first row group should be decoded without needing more data
1053            expect_data(decoder.try_decode()),
1054            // second row group should be decoded without needing more data
1055            expect_data(decoder.try_decode()),
1056        ];
1057        expect_finished(decoder.try_decode());
1058
1059        let all_output = concat_batches(&TEST_BATCH.schema(), &results).unwrap();
1060        // Check that the output matches the input batch
1061        assert_eq!(all_output, *TEST_BATCH);
1062    }
1063
1064    /// Decode the entire file incrementally, simulating a scenario where data is
1065    /// fetched as needed
1066    #[test]
1067    fn test_decoder_incremental() {
1068        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
1069            .unwrap()
1070            .build()
1071            .unwrap();
1072
1073        let mut results = vec![];
1074
1075        // First row group, expect a single request
1076        let ranges = expect_needs_data(decoder.try_decode());
1077        let num_bytes_requested: u64 = ranges.iter().map(|r| r.end - r.start).sum();
1078        push_ranges_to_decoder(&mut decoder, ranges);
1079        // The decoder should currently only store the data it needs to decode the first row group
1080        assert_eq!(decoder.buffered_bytes(), num_bytes_requested);
1081        results.push(expect_data(decoder.try_decode()));
1082        // the decoder should have consumed the data for the first row group and freed it
1083        assert_eq!(decoder.buffered_bytes(), 0);
1084
1085        // Second row group,
1086        let ranges = expect_needs_data(decoder.try_decode());
1087        let num_bytes_requested: u64 = ranges.iter().map(|r| r.end - r.start).sum();
1088        push_ranges_to_decoder(&mut decoder, ranges);
1089        // The decoder should currently only store the data it needs to decode the second row group
1090        assert_eq!(decoder.buffered_bytes(), num_bytes_requested);
1091        results.push(expect_data(decoder.try_decode()));
1092        // the decoder should have consumed the data for the second row group and freed it
1093        assert_eq!(decoder.buffered_bytes(), 0);
1094        expect_finished(decoder.try_decode());
1095
1096        // Check that the output matches the input batch
1097        let all_output = concat_batches(&TEST_BATCH.schema(), &results).unwrap();
1098        assert_eq!(all_output, *TEST_BATCH);
1099    }
1100
1101    /// Releasing staged ranges should free speculative buffers without affecting
1102    /// the active row group reader.
1103    #[test]
1104    fn test_decoder_clear_all_ranges() {
1105        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
1106            .unwrap()
1107            .with_batch_size(100)
1108            .build()
1109            .unwrap();
1110
1111        decoder
1112            .push_range(test_file_range(), TEST_FILE_DATA.clone())
1113            .unwrap();
1114        assert_eq!(decoder.buffered_bytes(), test_file_len());
1115
1116        // The current row group reader is built from the prefetched bytes.
1117        // The decoder releases the bytes of the first row group, and the rest
1118        // of the speculative full-file range remains staged in the decoder.
1119        let batch1 = expect_data(decoder.try_decode());
1120        assert_eq!(batch1, TEST_BATCH.slice(0, 100));
1121        assert_eq!(
1122            decoder.buffered_bytes(),
1123            test_file_len() - test_file_row_group_bytes(0)
1124        );
1125
1126        // All of the buffer is released
1127        decoder.clear_all_ranges();
1128        assert_eq!(decoder.buffered_bytes(), 0);
1129
1130        // The active reader still owns the current row group's bytes, so it can
1131        // continue decoding without consulting PushBuffers.
1132        let batch2 = expect_data(decoder.try_decode());
1133        assert_eq!(batch2, TEST_BATCH.slice(100, 100));
1134        assert_eq!(decoder.buffered_bytes(), 0);
1135
1136        // Moving to the next row group now requires the decoder to ask for data
1137        // again because the staged speculative ranges were released.
1138        let ranges = expect_needs_data(decoder.try_decode());
1139        let num_bytes_requested: u64 = ranges.iter().map(|r| r.end - r.start).sum();
1140        push_ranges_to_decoder(&mut decoder, ranges);
1141        assert_eq!(decoder.buffered_bytes(), num_bytes_requested);
1142
1143        let batch3 = expect_data(decoder.try_decode());
1144        assert_eq!(batch3, TEST_BATCH.slice(200, 100));
1145        assert_eq!(decoder.buffered_bytes(), 0);
1146
1147        let batch4 = expect_data(decoder.try_decode());
1148        assert_eq!(batch4, TEST_BATCH.slice(300, 100));
1149        assert_eq!(decoder.buffered_bytes(), 0);
1150
1151        expect_finished(decoder.try_decode());
1152    }
1153
1154    /// Decode the entire file incrementally, simulating partial reads
1155    #[test]
1156    fn test_decoder_partial() {
1157        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
1158            .unwrap()
1159            .build()
1160            .unwrap();
1161
1162        // First row group, expect a single request for all data needed to read "a" and "b"
1163        let ranges = expect_needs_data(decoder.try_decode());
1164        push_ranges_to_decoder(&mut decoder, ranges);
1165
1166        let batch1 = expect_data(decoder.try_decode());
1167        let expected1 = TEST_BATCH.slice(0, 200);
1168        assert_eq!(batch1, expected1);
1169
1170        // Second row group, this time provide the data in two steps
1171        let ranges = expect_needs_data(decoder.try_decode());
1172        let (ranges1, ranges2) = ranges.split_at(ranges.len() / 2);
1173        assert!(!ranges1.is_empty());
1174        assert!(!ranges2.is_empty());
1175        // push first half to simulate partial read
1176        push_ranges_to_decoder(&mut decoder, ranges1.to_vec());
1177
1178        // still expect more data
1179        let ranges = expect_needs_data(decoder.try_decode());
1180        assert_eq!(ranges, ranges2); // should be the remaining ranges
1181        // push empty ranges should be a no-op
1182        push_ranges_to_decoder(&mut decoder, vec![]);
1183        let ranges = expect_needs_data(decoder.try_decode());
1184        assert_eq!(ranges, ranges2); // should be the remaining ranges
1185        push_ranges_to_decoder(&mut decoder, ranges);
1186
1187        let batch2 = expect_data(decoder.try_decode());
1188        let expected2 = TEST_BATCH.slice(200, 200);
1189        assert_eq!(batch2, expected2);
1190
1191        expect_finished(decoder.try_decode());
1192    }
1193
1194    /// Push only the pages needed for each batch of a row group rather than
1195    /// the entire row group, and check whether the decoder can produce that
1196    /// batch before the rest of the row group's pages have been pushed.
1197    #[test]
1198    fn test_decoder_first_pages_only() {
1199        let metadata = test_file_parquet_metadata_with_offset_index();
1200        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(Arc::clone(&metadata))
1201            .unwrap()
1202            // Each data page has 100 rows, so a batch of 100 rows needs only
1203            // the first data page of each column
1204            .with_batch_size(100)
1205            .build()
1206            .unwrap();
1207
1208        // Row group 0: the decoder asks for the entire column chunk of each
1209        // of the three columns "a", "b", and "c"
1210        let ranges = expect_needs_data(decoder.try_decode());
1211        assert_eq!(ranges, vec![4..1860, 1860..3716, 3716..11062]);
1212
1213        // Compute the ranges that cover only the first data page of each
1214        // column (and the dictionary page which precedes it)
1215        let page_index = metadata.page_index_for_row_group(0);
1216        let row_group = metadata.row_group(0);
1217        let mut first_page_ranges = vec![];
1218        let mut second_page_ranges = vec![];
1219        for (idx, column) in row_group.columns().iter().enumerate() {
1220            let (start, len) = column.byte_range();
1221            let locations = page_index.page_locations(idx).unwrap();
1222            assert_eq!(locations.len(), 2, "expected 2 data pages per column chunk");
1223            let second_page_start = locations[1].offset as u64;
1224            first_page_ranges.push(start..second_page_start);
1225            second_page_ranges.push(second_page_start..start + len);
1226        }
1227        // Note the first range for each column includes the dictionary page
1228        assert_eq!(first_page_ranges, vec![4..1734, 1860..3590, 3716..10936]);
1229        assert_eq!(
1230            second_page_ranges,
1231            vec![1734..1860, 3590..3716, 10936..11062]
1232        );
1233
1234        // Push only the first page of each column. This is all the data
1235        // needed to decode the first batch of 100 rows.
1236        push_ranges_to_decoder(&mut decoder, first_page_ranges);
1237
1238        // Note will likely change as part of
1239        // <https://github.com/apache/arrow-rs/issues/6946>
1240
1241        // decoder still reports it needs the (complete) ranges
1242        // it originally asked for, and does not produce a batch.
1243        let ranges = expect_needs_data(decoder.try_decode());
1244        assert_eq!(ranges, vec![4..1860, 1860..3716, 3716..11062]);
1245
1246        // Pushing the second pages as separate ranges does not help either:
1247        // the decoder does not coalesce adjacent pushed ranges, so a requested
1248        // range is only satisfied by a single pushed buffer that covers it.
1249        push_ranges_to_decoder(&mut decoder, second_page_ranges);
1250        let ranges = expect_needs_data(decoder.try_decode());
1251        assert_eq!(ranges, vec![4..1860, 1860..3716, 3716..11062]);
1252
1253        // Only once the exact ranges originally requested are pushed does the
1254        // decoder produce batches.
1255        push_ranges_to_decoder(&mut decoder, ranges);
1256        let batch = expect_data(decoder.try_decode());
1257        assert_eq!(batch, TEST_BATCH.slice(0, 100));
1258        let batch = expect_data(decoder.try_decode());
1259        assert_eq!(batch, TEST_BATCH.slice(100, 100));
1260    }
1261
1262    /// Push one buffer for each `NeedsData`, from the start of the first
1263    /// requested range to the end of the last one, and check that the
1264    /// decoder releases the bytes of each row group when it is done with it.
1265    ///
1266    /// <https://github.com/apache/arrow-rs/issues/11233>
1267    fn decode_with_coalesced_pushes(mut decoder: ParquetPushDecoder) -> Vec<RecordBatch> {
1268        let mut batches = vec![];
1269        loop {
1270            match decoder.try_decode().unwrap() {
1271                DecodeResult::NeedsData(ranges) => {
1272                    let start = ranges.iter().map(|r| r.start).min().unwrap();
1273                    let end = ranges.iter().map(|r| r.end).max().unwrap();
1274                    let coalesced = start..end;
1275                    push_ranges_to_decoder(&mut decoder, vec![coalesced]);
1276                }
1277                DecodeResult::Data(batch) => {
1278                    // The test file has one batch per row group.
1279                    assert_eq!(decoder.buffered_bytes(), 0);
1280                    batches.push(batch);
1281                }
1282                DecodeResult::Finished => return batches,
1283            }
1284        }
1285    }
1286
1287    /// The requested ranges are adjacent, so the pushed buffer holds only
1288    /// requested bytes.
1289    #[test]
1290    fn test_decoder_releases_coalesced_push() {
1291        let decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
1292            .unwrap()
1293            .build()
1294            .unwrap();
1295        let batches = decode_with_coalesced_pushes(decoder);
1296        let all_output = concat_batches(&TEST_BATCH.schema(), &batches).unwrap();
1297        assert_eq!(all_output, *TEST_BATCH);
1298    }
1299
1300    /// Column "b" is not read, so the pushed buffer also holds the bytes of
1301    /// "b", which the decoder did not request.
1302    #[test]
1303    fn test_decoder_releases_unrequested_bytes_of_row_group() {
1304        let builder =
1305            ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1306        let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1307        let decoder = builder
1308            .with_projection(ProjectionMask::columns(&schema_descr, ["a", "c"]))
1309            .build()
1310            .unwrap();
1311        let batches = decode_with_coalesced_pushes(decoder);
1312        let all_output = concat_batches(&batches[0].schema(), &batches).unwrap();
1313        assert_eq!(all_output, TEST_BATCH.project(&[0, 2]).unwrap());
1314    }
1315
1316    /// A plan that reads a row group two times keeps its bytes until the
1317    /// second read.
1318    #[test]
1319    fn test_decoder_keeps_bytes_of_a_row_group_that_is_read_again() {
1320        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
1321            .unwrap()
1322            .with_row_groups(vec![0, 1, 0])
1323            .build()
1324            .unwrap();
1325        prefetch_test_file(&mut decoder);
1326        // Row group 0 is read again, so the decoder keeps its bytes.
1327        assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(0, 200));
1328        assert_eq!(decoder.buffered_bytes(), test_file_len());
1329
1330        assert_eq!(
1331            expect_data(decoder.try_decode()),
1332            TEST_BATCH.slice(200, 200)
1333        );
1334        assert_eq!(
1335            decoder.buffered_bytes(),
1336            test_file_len() - test_file_row_group_bytes(1)
1337        );
1338
1339        assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(0, 200));
1340        assert_eq!(
1341            decoder.buffered_bytes(),
1342            test_file_len() - test_file_row_group_bytes(0) - test_file_row_group_bytes(1)
1343        );
1344        expect_finished(decoder.try_decode());
1345    }
1346
1347    /// With a predicate, the decoder requests the bytes of the predicate
1348    /// column first, then the bytes of the other output columns.
1349    #[test]
1350    fn test_decoder_releases_coalesced_push_with_predicate() {
1351        let builder =
1352            ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1353        let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1354        let row_filter_a = ArrowPredicateFn::new(
1355            ProjectionMask::columns(&schema_descr, ["a"]),
1356            |batch: RecordBatch| {
1357                let scalar_250 = Int64Array::new_scalar(250);
1358                let column = batch.column(0).as_primitive::<Int64Type>();
1359                gt(column, &scalar_250)
1360            },
1361        );
1362        let decoder = builder
1363            .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
1364            .build()
1365            .unwrap();
1366        let batches = decode_with_coalesced_pushes(decoder);
1367        let all_output = concat_batches(&TEST_BATCH.schema(), &batches).unwrap();
1368        assert_eq!(all_output, TEST_BATCH.slice(251, 149));
1369    }
1370
1371    /// Decode multiple columns "a" and "b", expect that the decoder requests
1372    /// only a single request per row group
1373    #[test]
1374    fn test_decoder_selection_does_one_request() {
1375        let builder =
1376            ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1377
1378        let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1379
1380        let mut decoder = builder
1381            .with_projection(
1382                ProjectionMask::columns(&schema_descr, ["a", "b"]), // read "a", "b"
1383            )
1384            .build()
1385            .unwrap();
1386
1387        // First row group, expect a single request for all data needed to read "a" and "b"
1388        let ranges = expect_needs_data(decoder.try_decode());
1389        push_ranges_to_decoder(&mut decoder, ranges);
1390
1391        let batch1 = expect_data(decoder.try_decode());
1392        let expected1 = TEST_BATCH.slice(0, 200).project(&[0, 1]).unwrap();
1393        assert_eq!(batch1, expected1);
1394
1395        // Second row group, similarly expect a single request for all data needed to read "a" and "b"
1396        let ranges = expect_needs_data(decoder.try_decode());
1397        push_ranges_to_decoder(&mut decoder, ranges);
1398
1399        let batch2 = expect_data(decoder.try_decode());
1400        let expected2 = TEST_BATCH.slice(200, 200).project(&[0, 1]).unwrap();
1401        assert_eq!(batch2, expected2);
1402
1403        expect_finished(decoder.try_decode());
1404    }
1405
1406    /// Decode with a filter that requires multiple requests, but only provide part
1407    /// of the data needed for the filter at a time simulating partial reads.
1408    #[test]
1409    fn test_decoder_single_filter_partial() {
1410        let builder =
1411            ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1412
1413        // Values in column "a" range 0..399
1414        // First filter: "a" > 250  (nothing in Row Group 0, both data pages in Row Group 1)
1415        let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1416
1417        // a > 250
1418        let row_filter_a = ArrowPredicateFn::new(
1419            // claim to use both a and b so we get two ranges requests for the filter pages
1420            ProjectionMask::columns(&schema_descr, ["a", "b"]),
1421            |batch: RecordBatch| {
1422                let scalar_250 = Int64Array::new_scalar(250);
1423                let column = batch.column(0).as_primitive::<Int64Type>();
1424                gt(column, &scalar_250)
1425            },
1426        );
1427
1428        let mut decoder = builder
1429            .with_projection(
1430                // read only column "a" to test that filter pages are reused
1431                ProjectionMask::columns(&schema_descr, ["a"]), // read "a"
1432            )
1433            .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
1434            .build()
1435            .unwrap();
1436
1437        // First row group, evaluating filters
1438        let ranges = expect_needs_data(decoder.try_decode());
1439        // only provide half the ranges
1440        let (ranges1, ranges2) = ranges.split_at(ranges.len() / 2);
1441        assert!(!ranges1.is_empty());
1442        assert!(!ranges2.is_empty());
1443        push_ranges_to_decoder(&mut decoder, ranges1.to_vec());
1444        // still expect more data
1445        let ranges = expect_needs_data(decoder.try_decode());
1446        assert_eq!(ranges, ranges2); // should be the remaining ranges
1447        let ranges = expect_needs_data(decoder.try_decode());
1448        assert_eq!(ranges, ranges2); // should be the remaining ranges
1449        push_ranges_to_decoder(&mut decoder, ranges2.to_vec());
1450
1451        // Since no rows in the first row group pass the filters, there is no
1452        // additional requests to read data pages for "b" here
1453
1454        // Second row group
1455        let ranges = expect_needs_data(decoder.try_decode());
1456        push_ranges_to_decoder(&mut decoder, ranges);
1457
1458        let batch = expect_data(decoder.try_decode());
1459        let expected = TEST_BATCH.slice(251, 149).project(&[0]).unwrap();
1460        assert_eq!(batch, expected);
1461
1462        expect_finished(decoder.try_decode());
1463    }
1464
1465    /// Decode with a filter where we also skip one of the RowGroups via a RowSelection
1466    #[test]
1467    fn test_decoder_single_filter_and_row_selection() {
1468        let builder =
1469            ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1470
1471        // Values in column "a" range 0..399
1472        // First filter: "a" > 250  (nothing in Row Group 0, last data page in Row Group 1)
1473        let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1474
1475        // a > 250
1476        let row_filter_a = ArrowPredicateFn::new(
1477            ProjectionMask::columns(&schema_descr, ["a"]),
1478            |batch: RecordBatch| {
1479                let scalar_250 = Int64Array::new_scalar(250);
1480                let column = batch.column(0).as_primitive::<Int64Type>();
1481                gt(column, &scalar_250)
1482            },
1483        );
1484
1485        let mut decoder = builder
1486            .with_projection(
1487                // read only column "a" to test that filter pages are reused
1488                ProjectionMask::columns(&schema_descr, ["b"]), // read "b"
1489            )
1490            .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
1491            .with_row_selection(RowSelection::from(vec![
1492                RowSelector::skip(200),   // skip first row group
1493                RowSelector::select(100), // first 100 rows of second row group
1494                RowSelector::skip(100),
1495            ]))
1496            .build()
1497            .unwrap();
1498
1499        // expect the first row group to be filtered out (no filter is evaluated due to row selection)
1500
1501        // First row group, first filter (a > 250)
1502        let ranges = expect_needs_data(decoder.try_decode());
1503        push_ranges_to_decoder(&mut decoder, ranges);
1504
1505        // Second row group
1506        let ranges = expect_needs_data(decoder.try_decode());
1507        push_ranges_to_decoder(&mut decoder, ranges);
1508
1509        let batch = expect_data(decoder.try_decode());
1510        let expected = TEST_BATCH.slice(251, 49).project(&[1]).unwrap();
1511        assert_eq!(batch, expected);
1512
1513        expect_finished(decoder.try_decode());
1514    }
1515
1516    /// Decode with multiple filters that require multiple requests
1517    #[test]
1518    fn test_decoder_multi_filters() {
1519        // Create a decoder for decoding parquet data (note it does not have any IO / readers)
1520        let builder =
1521            ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1522
1523        // Values in column "a" range 0..399
1524        // Values in column "b" range 400..799
1525        // First filter: "a" > 175  (last data page in Row Group 0)
1526        // Second filter: "b" < 625 (last data page in Row Group 0 and first DataPage in RowGroup 1)
1527        let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1528
1529        // a > 175
1530        let row_filter_a = ArrowPredicateFn::new(
1531            ProjectionMask::columns(&schema_descr, ["a"]),
1532            |batch: RecordBatch| {
1533                let scalar_175 = Int64Array::new_scalar(175);
1534                let column = batch.column(0).as_primitive::<Int64Type>();
1535                gt(column, &scalar_175)
1536            },
1537        );
1538
1539        // b < 625
1540        let row_filter_b = ArrowPredicateFn::new(
1541            ProjectionMask::columns(&schema_descr, ["b"]),
1542            |batch: RecordBatch| {
1543                let scalar_625 = Int64Array::new_scalar(625);
1544                let column = batch.column(0).as_primitive::<Int64Type>();
1545                lt(column, &scalar_625)
1546            },
1547        );
1548
1549        let mut decoder = builder
1550            .with_projection(
1551                ProjectionMask::columns(&schema_descr, ["c"]), // read "c"
1552            )
1553            .with_row_filter(RowFilter::new(vec![
1554                Box::new(row_filter_a),
1555                Box::new(row_filter_b),
1556            ]))
1557            .build()
1558            .unwrap();
1559
1560        // First row group, first filter (a > 175)
1561        let ranges = expect_needs_data(decoder.try_decode());
1562        push_ranges_to_decoder(&mut decoder, ranges);
1563
1564        // first row group, second filter (b < 625)
1565        let ranges = expect_needs_data(decoder.try_decode());
1566        push_ranges_to_decoder(&mut decoder, ranges);
1567
1568        // first row group, data pages for "c"
1569        let ranges = expect_needs_data(decoder.try_decode());
1570        push_ranges_to_decoder(&mut decoder, ranges);
1571
1572        // expect the first batch to be decoded: rows 176..199, column "c"
1573        let batch1 = expect_data(decoder.try_decode());
1574        let expected1 = TEST_BATCH.slice(176, 24).project(&[2]).unwrap();
1575        assert_eq!(batch1, expected1);
1576
1577        // Second row group, first filter (a > 175)
1578        let ranges = expect_needs_data(decoder.try_decode());
1579        push_ranges_to_decoder(&mut decoder, ranges);
1580
1581        // Second row group, second filter (b < 625)
1582        let ranges = expect_needs_data(decoder.try_decode());
1583        push_ranges_to_decoder(&mut decoder, ranges);
1584
1585        // Second row group, data pages for "c"
1586        let ranges = expect_needs_data(decoder.try_decode());
1587        push_ranges_to_decoder(&mut decoder, ranges);
1588
1589        // expect the second batch to be decoded: rows 200..224, column "c"
1590        let batch2 = expect_data(decoder.try_decode());
1591        let expected2 = TEST_BATCH.slice(200, 25).project(&[2]).unwrap();
1592        assert_eq!(batch2, expected2);
1593
1594        expect_finished(decoder.try_decode());
1595    }
1596
1597    /// Consecutive filters on the same projection share one decode stream.
1598    #[test]
1599    fn test_decoder_same_projection_filters() {
1600        let builder =
1601            ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1602        let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1603        let projection_a = ProjectionMask::columns(&schema_descr, ["a"]);
1604
1605        let row_filter_gt = ArrowPredicateFn::new(projection_a.clone(), |batch: RecordBatch| {
1606            let column = batch.column(0).as_primitive::<Int64Type>();
1607            gt(column, &Int64Array::new_scalar(175))
1608        });
1609        let row_filter_lt = ArrowPredicateFn::new(projection_a, |batch: RecordBatch| {
1610            let column = batch.column(0).as_primitive::<Int64Type>();
1611            assert!(column.iter().flatten().all(|value| value > 175));
1612            lt(column, &Int64Array::new_scalar(190))
1613        });
1614
1615        let mut decoder = builder
1616            .with_projection(ProjectionMask::columns(&schema_descr, ["c"]))
1617            .with_row_filter(RowFilter::new(vec![
1618                Box::new(row_filter_gt),
1619                Box::new(row_filter_lt),
1620            ]))
1621            .with_batch_size(50)
1622            .build()
1623            .unwrap();
1624
1625        // One filter-column request, followed by the selected output page.
1626        let ranges = expect_needs_data(decoder.try_decode());
1627        push_ranges_to_decoder(&mut decoder, ranges);
1628        let ranges = expect_needs_data(decoder.try_decode());
1629        push_ranges_to_decoder(&mut decoder, ranges);
1630
1631        let batch = expect_data(decoder.try_decode());
1632        let expected = TEST_BATCH.slice(176, 14).project(&[2]).unwrap();
1633        assert_eq!(batch, expected);
1634
1635        // Row group 1 is rejected by the fused predicates, so no output-column
1636        // request is made.
1637        let ranges = expect_needs_data(decoder.try_decode());
1638        push_ranges_to_decoder(&mut decoder, ranges);
1639        expect_finished(decoder.try_decode());
1640    }
1641
1642    /// Fused same-projection filters honour the output limit: evaluation
1643    /// stops once enough rows survive the whole group.
1644    #[test]
1645    fn test_decoder_same_projection_filters_with_limit() {
1646        use std::sync::atomic::{AtomicUsize, Ordering};
1647
1648        let builder =
1649            ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1650        let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1651        let projection_a = ProjectionMask::columns(&schema_descr, ["a"]);
1652
1653        let rows_evaluated = Arc::new(AtomicUsize::new(0));
1654        let rows_evaluated_for_predicate = Arc::clone(&rows_evaluated);
1655        let row_filter_gt =
1656            ArrowPredicateFn::new(projection_a.clone(), move |batch: RecordBatch| {
1657                rows_evaluated_for_predicate.fetch_add(batch.num_rows(), Ordering::Relaxed);
1658                let column = batch.column(0).as_primitive::<Int64Type>();
1659                gt(column, &Int64Array::new_scalar(175))
1660            });
1661        let row_filter_lt = ArrowPredicateFn::new(projection_a, |batch: RecordBatch| {
1662            let column = batch.column(0).as_primitive::<Int64Type>();
1663            lt(column, &Int64Array::new_scalar(190))
1664        });
1665
1666        let mut decoder = builder
1667            .with_projection(ProjectionMask::columns(&schema_descr, ["c"]))
1668            .with_row_filter(RowFilter::new(vec![
1669                Box::new(row_filter_gt),
1670                Box::new(row_filter_lt),
1671            ]))
1672            .with_batch_size(10)
1673            .with_limit(5)
1674            .build()
1675            .unwrap();
1676
1677        // One filter-column request for the fused group, then the output page.
1678        let ranges = expect_needs_data(decoder.try_decode());
1679        push_ranges_to_decoder(&mut decoder, ranges);
1680        let ranges = expect_needs_data(decoder.try_decode());
1681        push_ranges_to_decoder(&mut decoder, ranges);
1682
1683        let batch = expect_data(decoder.try_decode());
1684        let expected = TEST_BATCH.slice(176, 5).project(&[2]).unwrap();
1685        assert_eq!(batch, expected);
1686
1687        // The limit was satisfied by row group 0.
1688        expect_finished(decoder.try_decode());
1689
1690        // Row 181 is the 6th match, so at most the batch holding it (rows
1691        // 180..189) is evaluated.
1692        let evaluated = rows_evaluated.load(Ordering::Relaxed);
1693        assert!(evaluated <= 190, "evaluated {evaluated} rows");
1694    }
1695
1696    /// Decode with a filter that uses a column that is also projected, and expect
1697    /// that the filter pages are reused (don't refetch them)
1698    #[test]
1699    fn test_decoder_reuses_filter_pages() {
1700        // Create a decoder for decoding parquet data (note it does not have any IO / readers)
1701        let builder =
1702            ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1703
1704        // Values in column "a" range 0..399
1705        // First filter: "a" > 250  (nothing in Row Group 0, last data page in Row Group 1)
1706        let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1707
1708        // a > 250
1709        let row_filter_a = ArrowPredicateFn::new(
1710            ProjectionMask::columns(&schema_descr, ["a"]),
1711            |batch: RecordBatch| {
1712                let scalar_250 = Int64Array::new_scalar(250);
1713                let column = batch.column(0).as_primitive::<Int64Type>();
1714                gt(column, &scalar_250)
1715            },
1716        );
1717
1718        let mut decoder = builder
1719            .with_projection(
1720                // read only column "a" to test that filter pages are reused
1721                ProjectionMask::columns(&schema_descr, ["a"]), // read "a"
1722            )
1723            .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
1724            .build()
1725            .unwrap();
1726
1727        // First row group, first filter (a > 175)
1728        let ranges = expect_needs_data(decoder.try_decode());
1729        push_ranges_to_decoder(&mut decoder, ranges);
1730
1731        // expect the first row group to be filtered out (no rows match)
1732
1733        // Second row group, first filter (a > 250)
1734        let ranges = expect_needs_data(decoder.try_decode());
1735        push_ranges_to_decoder(&mut decoder, ranges);
1736
1737        // expect that the second row group is decoded: rows 251..399, column "a"
1738        // Note that the filter pages for "a" should be reused and no additional data
1739        // should be requested
1740        let batch = expect_data(decoder.try_decode());
1741        let expected = TEST_BATCH.slice(251, 149).project(&[0]).unwrap();
1742        assert_eq!(batch, expected);
1743
1744        expect_finished(decoder.try_decode());
1745    }
1746
1747    #[test]
1748    fn test_decoder_empty_filters() {
1749        let builder =
1750            ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1751        let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1752
1753        // only read column "c", but with empty filters
1754        let mut decoder = builder
1755            .with_projection(
1756                ProjectionMask::columns(&schema_descr, ["c"]), // read "c"
1757            )
1758            .with_row_filter(RowFilter::new(vec![
1759                // empty filters should be ignored
1760            ]))
1761            .build()
1762            .unwrap();
1763
1764        // First row group
1765        let ranges = expect_needs_data(decoder.try_decode());
1766        push_ranges_to_decoder(&mut decoder, ranges);
1767
1768        // expect the first batch to be decoded: rows 0..199, column "c"
1769        let batch1 = expect_data(decoder.try_decode());
1770        let expected1 = TEST_BATCH.slice(0, 200).project(&[2]).unwrap();
1771        assert_eq!(batch1, expected1);
1772
1773        // Second row group,
1774        let ranges = expect_needs_data(decoder.try_decode());
1775        push_ranges_to_decoder(&mut decoder, ranges);
1776
1777        // expect the second batch to be decoded: rows 200..399, column "c"
1778        let batch2 = expect_data(decoder.try_decode());
1779        let expected2 = TEST_BATCH.slice(200, 200).project(&[2]).unwrap();
1780
1781        assert_eq!(batch2, expected2);
1782
1783        expect_finished(decoder.try_decode());
1784    }
1785
1786    /// When filter pushdown is combined with a `LIMIT`, the predicate must
1787    /// not be evaluated for rows beyond the `limit`-th match.
1788    ///
1789    /// Filter `a > 175` produces 24 matches in row group 0 (rows 176..199).
1790    /// With `limit = 10`, only the first 10 matches (rows 176..185) should be
1791    /// emitted, AND the predicate counter should observe that evaluation was
1792    /// short-circuited.
1793    #[test]
1794    fn test_decoder_filter_with_limit_short_circuits_within_row_group() {
1795        use std::sync::atomic::{AtomicUsize, Ordering};
1796
1797        let builder =
1798            ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1799        let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1800
1801        let rows_filtered = Arc::new(AtomicUsize::new(0));
1802        let rows_filtered_for_predicate = Arc::clone(&rows_filtered);
1803
1804        let row_filter_a = ArrowPredicateFn::new(
1805            ProjectionMask::columns(&schema_descr, ["a"]),
1806            move |batch: RecordBatch| {
1807                rows_filtered_for_predicate.fetch_add(batch.num_rows(), Ordering::Relaxed);
1808                let scalar_175 = Int64Array::new_scalar(175);
1809                let column = batch.column(0).as_primitive::<Int64Type>();
1810                gt(column, &scalar_175)
1811            },
1812        );
1813
1814        // Use a small batch size so the row group is evaluated across
1815        // multiple predicate batches; that is the regime where Layer 2's
1816        // short-circuit saves predicate evaluation work. Matching rows are
1817        // 176..199 (24 rows); with batch_size = 10 those span batches 17, 18,
1818        // and 19 (rows 170..199). A limit of 10 should stop filter evaluation
1819        // in the middle of batch 18.
1820        let mut decoder = builder
1821            .with_projection(ProjectionMask::columns(&schema_descr, ["a"]))
1822            .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
1823            .with_batch_size(10)
1824            .with_limit(10)
1825            .build()
1826            .unwrap();
1827
1828        // First row group: filter columns fetch (predicate is evaluated here)
1829        let ranges = expect_needs_data(decoder.try_decode());
1830        push_ranges_to_decoder(&mut decoder, ranges);
1831
1832        // The first 10 matching rows come out: 176..185, column "a"
1833        let batch = expect_data(decoder.try_decode());
1834        let expected = TEST_BATCH.slice(176, 10).project(&[0]).unwrap();
1835        assert_eq!(batch, expected);
1836
1837        // no data for row group 1 should be requested — the limit
1838        // was satisfied by row group 0 and the `Start` state for row group 1
1839        // short-circuits to `Finished`.
1840        expect_finished(decoder.try_decode());
1841
1842        // Row 186 is the 11th match; the scan should stop no later than the
1843        // batch containing it (batch 18 of 10 rows = rows 180..189), so at
1844        // most 190 rows are evaluated.
1845        let evaluated = rows_filtered.load(Ordering::Relaxed);
1846        assert!(
1847            evaluated <= 190,
1848            "predicate evaluated {evaluated} rows; expected ≤ 190 (stop within batch containing 11th match)"
1849        );
1850    }
1851
1852    /// Once the limit has been satisfied by a prior row group, subsequent
1853    /// row groups should be skipped entirely — no data request for their
1854    /// filter columns.
1855    #[test]
1856    fn test_decoder_filter_with_limit_skips_later_row_groups() {
1857        let builder =
1858            ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1859        let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1860
1861        // `a > 175` matches rows 176..199 in row group 0 (24 matches) and
1862        // 200..399 in row group 1 (200 matches). With limit = 5, all matches
1863        // should come from row group 0.
1864        let row_filter_a = ArrowPredicateFn::new(
1865            ProjectionMask::columns(&schema_descr, ["a"]),
1866            |batch: RecordBatch| {
1867                let scalar_175 = Int64Array::new_scalar(175);
1868                let column = batch.column(0).as_primitive::<Int64Type>();
1869                gt(column, &scalar_175)
1870            },
1871        );
1872
1873        let mut decoder = builder
1874            .with_projection(ProjectionMask::columns(&schema_descr, ["a"]))
1875            .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
1876            .with_limit(5)
1877            .build()
1878            .unwrap();
1879
1880        // Row group 0: fetch filter pages
1881        let ranges = expect_needs_data(decoder.try_decode());
1882        push_ranges_to_decoder(&mut decoder, ranges);
1883
1884        // First 5 matches: 176..180
1885        let batch = expect_data(decoder.try_decode());
1886        let expected = TEST_BATCH.slice(176, 5).project(&[0]).unwrap();
1887        assert_eq!(batch, expected);
1888
1889        // Row group 1 must NOT request data — the limit is already satisfied
1890        // so `Start` in row group 1 short-circuits to `Finished`.
1891        expect_finished(decoder.try_decode());
1892    }
1893
1894    /// The predicate short-circuit must account for `self.offset` as well as
1895    /// `self.limit`. The post-predicate `with_offset` step skips that many
1896    /// already-selected rows before `with_limit` counts output rows — so the
1897    /// predicate must retain at least `offset + limit` matches. Without the
1898    /// fix, Layer 2 caps at just `limit` and the later `with_offset` consumes
1899    /// all of them, producing 0 rows instead of `limit`.
1900    ///
1901    /// `a > 175` matches rows 176..199 in row group 0 (24 matches). With
1902    /// `offset = 10, limit = 5`, the expected output is rows 186..190 (the
1903    /// 11th through 15th matches).
1904    #[test]
1905    fn test_decoder_filter_with_offset_and_limit() {
1906        let builder =
1907            ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1908        let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1909
1910        let row_filter_a = ArrowPredicateFn::new(
1911            ProjectionMask::columns(&schema_descr, ["a"]),
1912            |batch: RecordBatch| {
1913                let scalar_175 = Int64Array::new_scalar(175);
1914                let column = batch.column(0).as_primitive::<Int64Type>();
1915                gt(column, &scalar_175)
1916            },
1917        );
1918
1919        let mut decoder = builder
1920            .with_projection(ProjectionMask::columns(&schema_descr, ["a"]))
1921            .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
1922            .with_offset(10)
1923            .with_limit(5)
1924            .build()
1925            .unwrap();
1926
1927        let ranges = expect_needs_data(decoder.try_decode());
1928        push_ranges_to_decoder(&mut decoder, ranges);
1929
1930        let batch = expect_data(decoder.try_decode());
1931        let expected = TEST_BATCH.slice(186, 5).project(&[0]).unwrap();
1932        assert_eq!(batch, expected);
1933
1934        expect_finished(decoder.try_decode());
1935    }
1936
1937    /// The limit short-circuit must also be correct when the limited predicate
1938    /// is the last predicate in a multi-predicate chain.
1939    ///
1940    /// `a > 175` first narrows row group 0 to rows 176..199. The final
1941    /// predicate `b < 625` is then evaluated only over those 24 rows, all of
1942    /// which match. With `limit = 10`, the final output should still be rows
1943    /// 176..185, and the second predicate should stop before consuming all 24
1944    /// selected rows.
1945    #[test]
1946    fn test_decoder_multi_filters_with_limit() {
1947        use std::sync::atomic::{AtomicUsize, Ordering};
1948
1949        let builder =
1950            ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1951        let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1952
1953        let first_predicate_rows = Arc::new(AtomicUsize::new(0));
1954        let second_predicate_rows = Arc::new(AtomicUsize::new(0));
1955
1956        let first_predicate_rows_for_filter = Arc::clone(&first_predicate_rows);
1957        let row_filter_a = ArrowPredicateFn::new(
1958            ProjectionMask::columns(&schema_descr, ["a"]),
1959            move |batch: RecordBatch| {
1960                first_predicate_rows_for_filter.fetch_add(batch.num_rows(), Ordering::Relaxed);
1961                let scalar_175 = Int64Array::new_scalar(175);
1962                let column = batch.column(0).as_primitive::<Int64Type>();
1963                gt(column, &scalar_175)
1964            },
1965        );
1966
1967        let second_predicate_rows_for_filter = Arc::clone(&second_predicate_rows);
1968        let row_filter_b = ArrowPredicateFn::new(
1969            ProjectionMask::columns(&schema_descr, ["b"]),
1970            move |batch: RecordBatch| {
1971                second_predicate_rows_for_filter.fetch_add(batch.num_rows(), Ordering::Relaxed);
1972                let scalar_625 = Int64Array::new_scalar(625);
1973                let column = batch.column(0).as_primitive::<Int64Type>();
1974                lt(column, &scalar_625)
1975            },
1976        );
1977
1978        let mut decoder = builder
1979            .with_projection(ProjectionMask::columns(&schema_descr, ["c"]))
1980            .with_row_filter(RowFilter::new(vec![
1981                Box::new(row_filter_a),
1982                Box::new(row_filter_b),
1983            ]))
1984            .with_batch_size(10)
1985            .with_limit(10)
1986            .build()
1987            .unwrap();
1988
1989        // Row group 0, first predicate
1990        let ranges = expect_needs_data(decoder.try_decode());
1991        push_ranges_to_decoder(&mut decoder, ranges);
1992
1993        // Row group 0, second predicate
1994        let ranges = expect_needs_data(decoder.try_decode());
1995        push_ranges_to_decoder(&mut decoder, ranges);
1996
1997        // Final projected data
1998        let ranges = expect_needs_data(decoder.try_decode());
1999        push_ranges_to_decoder(&mut decoder, ranges);
2000
2001        let batch = expect_data(decoder.try_decode());
2002        let expected = TEST_BATCH.slice(176, 10).project(&[2]).unwrap();
2003        assert_eq!(batch, expected);
2004
2005        // The overall limit was satisfied by row group 0.
2006        expect_finished(decoder.try_decode());
2007
2008        assert_eq!(first_predicate_rows.load(Ordering::Relaxed), 200);
2009        assert!(
2010            second_predicate_rows.load(Ordering::Relaxed) < 24,
2011            "final predicate should short-circuit before consuming all 24 rows selected by the first predicate"
2012        );
2013    }
2014
2015    /// When a row selection already exists, limiting the predicate must still
2016    /// preserve alignment with that prior selection.
2017    ///
2018    /// The explicit selection narrows row group 0 to rows 150..199. Applying
2019    /// `a > 175` over that selection yields rows 176..199. With `limit = 10`,
2020    /// the decoder should emit rows 176..185 and stop without evaluating the
2021    /// remaining selected rows.
2022    #[test]
2023    fn test_decoder_filter_with_row_selection_and_limit() {
2024        use std::sync::atomic::{AtomicUsize, Ordering};
2025
2026        let builder =
2027            ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
2028        let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
2029
2030        let rows_filtered = Arc::new(AtomicUsize::new(0));
2031        let rows_filtered_for_predicate = Arc::clone(&rows_filtered);
2032
2033        let row_filter_a = ArrowPredicateFn::new(
2034            ProjectionMask::columns(&schema_descr, ["a"]),
2035            move |batch: RecordBatch| {
2036                rows_filtered_for_predicate.fetch_add(batch.num_rows(), Ordering::Relaxed);
2037                let scalar_175 = Int64Array::new_scalar(175);
2038                let column = batch.column(0).as_primitive::<Int64Type>();
2039                gt(column, &scalar_175)
2040            },
2041        );
2042
2043        let mut decoder = builder
2044            .with_projection(ProjectionMask::columns(&schema_descr, ["a"]))
2045            .with_row_selection(RowSelection::from(vec![
2046                RowSelector::skip(150),
2047                RowSelector::select(50),
2048            ]))
2049            .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
2050            .with_batch_size(10)
2051            .with_limit(10)
2052            .build()
2053            .unwrap();
2054
2055        let ranges = expect_needs_data(decoder.try_decode());
2056        push_ranges_to_decoder(&mut decoder, ranges);
2057
2058        let batch = expect_data(decoder.try_decode());
2059        let expected = TEST_BATCH.slice(176, 10).project(&[0]).unwrap();
2060        assert_eq!(batch, expected);
2061
2062        expect_finished(decoder.try_decode());
2063
2064        assert!(
2065            rows_filtered.load(Ordering::Relaxed) < 50,
2066            "predicate should short-circuit before consuming all 50 rows from the explicit row selection"
2067        );
2068    }
2069
2070    #[test]
2071    fn test_decoder_offset_limit() {
2072        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2073            .unwrap()
2074            // skip entire first row group (200 rows) and first 25 rows of second row group
2075            .with_offset(225)
2076            // and limit to 20 rows
2077            .with_limit(20)
2078            .build()
2079            .unwrap();
2080
2081        // First row group should be skipped,
2082
2083        // Second row group
2084        let ranges = expect_needs_data(decoder.try_decode());
2085        push_ranges_to_decoder(&mut decoder, ranges);
2086
2087        // expect the first and only batch to be decoded
2088        let batch1 = expect_data(decoder.try_decode());
2089        let expected1 = TEST_BATCH.slice(225, 20);
2090        assert_eq!(batch1, expected1);
2091
2092        expect_finished(decoder.try_decode());
2093    }
2094
2095    #[test]
2096    fn test_decoder_try_next_reader_offset_limit() {
2097        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2098            .unwrap()
2099            .with_offset(225)
2100            .with_limit(20)
2101            .build()
2102            .unwrap();
2103
2104        let ranges = expect_needs_data(decoder.try_next_reader());
2105        push_ranges_to_decoder(&mut decoder, ranges);
2106
2107        let reader = expect_data(decoder.try_next_reader());
2108        let batches = reader
2109            .map(|batch| batch.expect("expected decoded batch"))
2110            .collect::<Vec<_>>();
2111        let output = concat_batches(&TEST_BATCH.schema(), &batches).unwrap();
2112        assert_eq!(output, TEST_BATCH.slice(225, 20));
2113
2114        expect_finished(decoder.try_next_reader());
2115    }
2116
2117    #[test]
2118    fn test_decoder_row_group_selection() {
2119        // take only the second row group
2120        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2121            .unwrap()
2122            .with_row_groups(vec![1])
2123            .build()
2124            .unwrap();
2125
2126        // First row group should be skipped,
2127
2128        // Second row group
2129        let ranges = expect_needs_data(decoder.try_decode());
2130        push_ranges_to_decoder(&mut decoder, ranges);
2131
2132        // expect the first and only batch to be decoded
2133        let batch1 = expect_data(decoder.try_decode());
2134        let expected1 = TEST_BATCH.slice(200, 200);
2135        assert_eq!(batch1, expected1);
2136
2137        expect_finished(decoder.try_decode());
2138    }
2139
2140    #[test]
2141    fn test_decoder_row_selection() {
2142        // take only the second row group
2143        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2144            .unwrap()
2145            .with_row_selection(RowSelection::from(vec![
2146                RowSelector::skip(225),  // skip first row group and 25 rows of second])
2147                RowSelector::select(20), // take 20 rows
2148            ]))
2149            .build()
2150            .unwrap();
2151
2152        // First row group should be skipped,
2153
2154        // Second row group
2155        let ranges = expect_needs_data(decoder.try_decode());
2156        push_ranges_to_decoder(&mut decoder, ranges);
2157
2158        // expect the first and only batch to be decoded
2159        let batch1 = expect_data(decoder.try_decode());
2160        let expected1 = TEST_BATCH.slice(225, 20);
2161        assert_eq!(batch1, expected1);
2162
2163        expect_finished(decoder.try_decode());
2164    }
2165
2166    #[test]
2167    fn test_decoder_row_group_local_selections() {
2168        let bitmap_selection = RowSelection::from_boolean_buffer(BooleanBuffer::from(
2169            (0..45)
2170                .map(|row_idx| (25..45).contains(&row_idx))
2171                .collect::<Vec<_>>(),
2172        ));
2173        let rle_selection =
2174            RowSelection::from(vec![RowSelector::skip(190), RowSelector::select(10)]);
2175
2176        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2177            .unwrap()
2178            .with_row_group_selections(vec![
2179                RowGroupSelection::new(1, Some(bitmap_selection)),
2180                RowGroupSelection::new(0, Some(rle_selection)),
2181            ])
2182            .build()
2183            .unwrap();
2184        prefetch_test_file(&mut decoder);
2185
2186        // Row-group-local selections use local coordinates and preserve the
2187        // supplied row-group order. The bitmap is intentionally shorter than
2188        // RG1, so rows after its 45th local row are skipped.
2189        assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(225, 20));
2190        assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(190, 10));
2191        expect_finished(decoder.try_decode());
2192    }
2193
2194    #[test]
2195    fn test_row_group_local_selections_respect_offset_and_limit() {
2196        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2197            .unwrap()
2198            .with_row_group_selections(vec![
2199                RowGroupSelection::new(1, None),
2200                RowGroupSelection::new(0, None),
2201            ])
2202            .with_offset(195)
2203            .with_limit(10)
2204            .build()
2205            .unwrap();
2206        prefetch_test_file(&mut decoder);
2207
2208        assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(395, 5));
2209        assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(0, 5));
2210        expect_finished(decoder.try_decode());
2211    }
2212
2213    #[test]
2214    fn test_row_group_local_selections_allow_duplicate_row_groups() {
2215        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2216            .unwrap()
2217            .with_row_group_selections(vec![
2218                RowGroupSelection::new(0, None),
2219                RowGroupSelection::new(0, None),
2220            ])
2221            .build()
2222            .unwrap();
2223        prefetch_test_file(&mut decoder);
2224
2225        let expected = TEST_BATCH.slice(0, 200);
2226        assert_eq!(expect_data(decoder.try_decode()), expected);
2227        assert_eq!(expect_data(decoder.try_decode()), expected);
2228        expect_finished(decoder.try_decode());
2229    }
2230
2231    #[test]
2232    fn test_short_row_group_local_selection_skips_trailing_rows() {
2233        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2234            .unwrap()
2235            .with_row_group_selections(vec![RowGroupSelection::new(
2236                1,
2237                Some(RowSelection::from(vec![
2238                    RowSelector::skip(5),
2239                    RowSelector::select(3),
2240                ])),
2241            )])
2242            .build()
2243            .unwrap();
2244        prefetch_test_file(&mut decoder);
2245
2246        assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(205, 3));
2247        expect_finished(decoder.try_decode());
2248    }
2249
2250    #[test]
2251    fn test_empty_row_group_local_selections_read_nothing() {
2252        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2253            .unwrap()
2254            .with_row_group_selections(vec![])
2255            .build()
2256            .unwrap();
2257
2258        expect_finished(decoder.try_decode());
2259    }
2260
2261    /// A `RowFilter` narrows each row group's local selection rather than
2262    /// replacing it: the predicate is evaluated against the locally selected
2263    /// rows, including when the selection is shorter than its row group and
2264    /// the row groups are supplied out of order.
2265    ///
2266    /// RG1 contributes local rows 10..20 ("a" 210..220), all of which pass
2267    /// `a > 195`; RG0 contributes local rows 190..200 ("a" 190..200), of which
2268    /// only 196..200 pass.
2269    fn row_group_local_selections_with_filter() -> ParquetPushDecoderBuilder {
2270        let builder =
2271            ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
2272        let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
2273
2274        // Values in column "a" range 0..399
2275        let row_filter_a = ArrowPredicateFn::new(
2276            ProjectionMask::columns(&schema_descr, ["a"]),
2277            |batch: RecordBatch| {
2278                let scalar_195 = Int64Array::new_scalar(195);
2279                let column = batch.column(0).as_primitive::<Int64Type>();
2280                gt(column, &scalar_195)
2281            },
2282        );
2283
2284        builder
2285            .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
2286            .with_row_group_selections(vec![
2287                RowGroupSelection::new(
2288                    1,
2289                    Some(RowSelection::from(vec![
2290                        RowSelector::skip(10),
2291                        RowSelector::select(10),
2292                    ])),
2293                ),
2294                RowGroupSelection::new(
2295                    0,
2296                    Some(RowSelection::from(vec![
2297                        RowSelector::skip(190),
2298                        RowSelector::select(10),
2299                    ])),
2300                ),
2301            ])
2302    }
2303
2304    #[test]
2305    fn test_row_group_local_selections_with_row_filter() {
2306        let mut decoder = row_group_local_selections_with_filter().build().unwrap();
2307        prefetch_test_file(&mut decoder);
2308
2309        assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(210, 10));
2310        assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(196, 4));
2311        expect_finished(decoder.try_decode());
2312    }
2313
2314    /// `into_builder` mid-scan must carry both the row filter and the
2315    /// still-local selection of the not-yet-decoded row group, so the rebuilt
2316    /// decoder produces exactly what an uninterrupted scan would have.
2317    #[test]
2318    fn test_into_builder_preserves_local_selections_with_row_filter() {
2319        let mut decoder = row_group_local_selections_with_filter().build().unwrap();
2320        prefetch_test_file(&mut decoder);
2321
2322        let reader1 = expect_data(decoder.try_next_reader());
2323        let batches1: Vec<_> = reader1.collect::<Result<_, _>>().unwrap();
2324        let batch1 = concat_batches(&TEST_BATCH.schema(), &batches1).unwrap();
2325        assert_eq!(batch1, TEST_BATCH.slice(210, 10));
2326
2327        // Only RG0 and its local `skip(190) + select(10)` remain.
2328        assert!(decoder.is_at_row_group_boundary());
2329        assert_eq!(decoder.row_groups_remaining(), 1);
2330        let mut decoder = decoder.into_builder().unwrap().build().unwrap();
2331
2332        let reader0 = expect_data(decoder.try_next_reader());
2333        let batches0: Vec<_> = reader0.collect::<Result<_, _>>().unwrap();
2334        let batch0 = concat_batches(&TEST_BATCH.schema(), &batches0).unwrap();
2335        assert_eq!(batch0, TEST_BATCH.slice(196, 4));
2336        expect_finished(decoder.try_next_reader());
2337    }
2338
2339    #[test]
2340    fn test_row_group_local_selection_replaces_previous_configuration() {
2341        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2342            .unwrap()
2343            .with_row_group_selections(vec![RowGroupSelection::new(0, None)])
2344            .with_row_group_selections(vec![RowGroupSelection::new(1, None)])
2345            .build()
2346            .unwrap();
2347        prefetch_test_file(&mut decoder);
2348
2349        // The second call replaces the first. `None` reads RG1 in full, and
2350        // the omitted RG0 is skipped.
2351        assert_eq!(
2352            expect_data(decoder.try_decode()),
2353            TEST_BATCH.slice(200, 200)
2354        );
2355        expect_finished(decoder.try_decode());
2356    }
2357
2358    #[test]
2359    fn test_legacy_row_groups_and_row_selection_remain_composable() {
2360        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2361            .unwrap()
2362            .with_row_groups(vec![1])
2363            .with_row_selection(RowSelection::from(vec![
2364                RowSelector::skip(25),
2365                RowSelector::select(20),
2366            ]))
2367            .build()
2368            .unwrap();
2369        prefetch_test_file(&mut decoder);
2370
2371        assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(225, 20));
2372        expect_finished(decoder.try_decode());
2373    }
2374
2375    #[test]
2376    fn test_row_group_local_selection_is_mutually_exclusive_with_legacy_configuration() {
2377        let metadata = test_file_parquet_metadata();
2378        let new_builder =
2379            || ParquetPushDecoderBuilder::try_new_decoder(Arc::clone(&metadata)).unwrap();
2380        let local_selection = || vec![RowGroupSelection::new(0, None)];
2381        let global_selection = || RowSelection::from(vec![RowSelector::select(1)]);
2382        let assert_conflict = |builder: ParquetPushDecoderBuilder| {
2383            let error = builder.build().unwrap_err();
2384            assert_eq!(
2385                error.to_string(),
2386                "Parquet error: with_row_group_selections cannot be combined with with_row_groups or with_row_selection"
2387            );
2388        };
2389
2390        assert_conflict(
2391            new_builder()
2392                .with_row_groups(vec![0])
2393                .with_row_group_selections(local_selection()),
2394        );
2395        assert_conflict(
2396            new_builder()
2397                .with_row_group_selections(local_selection())
2398                .with_row_groups(vec![0]),
2399        );
2400        assert_conflict(
2401            new_builder()
2402                .with_row_selection(global_selection())
2403                .with_row_group_selections(local_selection()),
2404        );
2405        assert_conflict(
2406            new_builder()
2407                .with_row_group_selections(local_selection())
2408                .with_row_selection(global_selection()),
2409        );
2410    }
2411
2412    #[test]
2413    fn test_row_group_local_selection_validates_row_group_and_length_at_build() {
2414        let metadata = test_file_parquet_metadata();
2415
2416        let error = ParquetPushDecoderBuilder::try_new_decoder(Arc::clone(&metadata))
2417            .unwrap()
2418            .with_row_group_selections(vec![RowGroupSelection::new(
2419                0,
2420                Some(RowSelection::from(vec![RowSelector::select(201)])),
2421            )])
2422            .build()
2423            .unwrap_err();
2424        assert!(
2425            error.to_string().contains(
2426                "Row selection for row group 0 contains 201 rows, but the row group has 200"
2427            ),
2428            "unexpected error: {error}"
2429        );
2430
2431        let error = ParquetPushDecoderBuilder::try_new_decoder(metadata)
2432            .unwrap()
2433            .with_row_group_selections(vec![RowGroupSelection::new(2, None)])
2434            .build()
2435            .unwrap_err();
2436        assert!(
2437            error
2438                .to_string()
2439                .contains("Row group index 2 out of bounds for file with 2 row groups"),
2440            "unexpected error: {error}"
2441        );
2442    }
2443
2444    /// `peek_next_row_group` reports the index of the row group the
2445    /// next `try_next_reader` call will hand back, matching the
2446    /// frontier's internal skip logic.
2447    #[test]
2448    fn test_peek_next_row_group_basic() {
2449        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2450            .unwrap()
2451            .build()
2452            .unwrap();
2453
2454        // Two row groups (0, 1). At boundary before any read, peek should
2455        // see RG 0.
2456        assert_eq!(decoder.peek_next_row_group().unwrap(), Some(0));
2457        assert!(decoder.is_at_row_group_boundary());
2458
2459        let ranges = expect_needs_data(decoder.try_next_reader());
2460        push_ranges_to_decoder(&mut decoder, ranges);
2461        let reader = expect_data(decoder.try_next_reader());
2462        // Once the reader for RG 0 has been handed off, the decoder is
2463        // back at a boundary waiting for RG 1 — peek must reflect that
2464        // (the active reader lives outside the decoder).
2465        assert!(decoder.is_at_row_group_boundary());
2466        assert_eq!(decoder.peek_next_row_group().unwrap(), Some(1));
2467
2468        // Drain RG 0's reader and consume RG 1.
2469        for batch in reader {
2470            let _ = batch.unwrap();
2471        }
2472        let ranges = expect_needs_data(decoder.try_next_reader());
2473        push_ranges_to_decoder(&mut decoder, ranges);
2474        let reader = expect_data(decoder.try_next_reader());
2475        for batch in reader {
2476            let _ = batch.unwrap();
2477        }
2478
2479        // No row groups left.
2480        assert_eq!(decoder.peek_next_row_group().unwrap(), None);
2481    }
2482
2483    /// `peek_next_row_group` honors `with_row_groups` — restricting the
2484    /// scan to a single row group means peek reports only that one and
2485    /// then `None`.
2486    #[test]
2487    fn test_peek_next_row_group_respects_with_row_groups() {
2488        let decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2489            .unwrap()
2490            .with_row_groups(vec![1])
2491            .build()
2492            .unwrap();
2493
2494        assert_eq!(decoder.peek_next_row_group().unwrap(), Some(1));
2495    }
2496
2497    /// When a row-selection segment leaves the next row group with zero
2498    /// selected rows, `peek_next_row_group` mirrors
2499    /// `next_readable_row_group`'s skip: it returns the *following*
2500    /// row group instead of the empty one.
2501    #[test]
2502    fn test_peek_next_row_group_skips_empty_selection() {
2503        // Each row group has 200 rows. Skip all 200 of RG 0 plus 50 of
2504        // RG 1; the next reader will be for RG 1, not RG 0.
2505        let decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2506            .unwrap()
2507            .with_row_selection(RowSelection::from(vec![
2508                RowSelector::skip(250),
2509                RowSelector::select(100),
2510            ]))
2511            .build()
2512            .unwrap();
2513
2514        assert_eq!(decoder.peek_next_row_group().unwrap(), Some(1));
2515    }
2516
2517    /// `peek_next_row_group` returns `None` on a finished decoder.
2518    #[test]
2519    fn test_peek_next_row_group_finished() {
2520        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2521            .unwrap()
2522            .with_row_groups(vec![])
2523            .build()
2524            .unwrap();
2525
2526        // No row groups requested ⇒ already finished, no peek.
2527        expect_finished(decoder.try_next_reader());
2528        assert_eq!(decoder.peek_next_row_group().unwrap(), None);
2529    }
2530
2531    /// `peek_next_row_group` mirrors `next_readable_row_group`'s
2532    /// budget-skipping logic: with an `OFFSET` that consumes the
2533    /// entire first row group, peek must return the *following*
2534    /// row group, not RG 0 (no predicates, so budget skips apply).
2535    #[test]
2536    fn test_peek_next_row_group_skips_for_budget() {
2537        // Each row group has 200 rows. OFFSET 200 drains RG 0 entirely
2538        // and lands the decoder's first emitted reader at RG 1.
2539        let decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2540            .unwrap()
2541            .with_offset(200)
2542            .build()
2543            .unwrap();
2544
2545        assert_eq!(decoder.peek_next_row_group().unwrap(), Some(1));
2546    }
2547
2548    /// OFFSET larger than every remaining row group's row count
2549    /// exhausts the budget, so peek should report `None` — matching
2550    /// `next_readable_row_group`'s behavior of producing `Finished`.
2551    #[test]
2552    fn test_peek_next_row_group_budget_drains_all() {
2553        // 2 row groups × 200 rows = 400 total. OFFSET 500 cannot land
2554        // anywhere — every RG is skipped under the budget.
2555        let decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2556            .unwrap()
2557            .with_offset(500)
2558            .build()
2559            .unwrap();
2560
2561        assert_eq!(decoder.peek_next_row_group().unwrap(), None);
2562    }
2563
2564    /// `peek_next_row_group` is safe to call while a row group's reader
2565    /// is still being drained via `try_decode`. In `DecodingRowGroup`
2566    /// state it must report the row group `try_next_reader` will yield
2567    /// a reader for *after* the active one — never `None` just because
2568    /// the decoder is mid-row-group.
2569    #[test]
2570    fn test_peek_next_row_group_during_decoding_row_group() {
2571        // Two row groups (200 rows each), small batch size so the
2572        // decoder stays in `DecodingRowGroup` across multiple
2573        // `try_decode` calls within RG 0.
2574        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2575            .unwrap()
2576            .with_batch_size(100)
2577            .build()
2578            .unwrap();
2579        decoder
2580            .push_range(test_file_range(), TEST_FILE_DATA.clone())
2581            .unwrap();
2582
2583        // First batch of RG 0 — after this, the decoder is in
2584        // `DecodingRowGroup` state with the reader retained internally
2585        // and one more 100-row batch still owed for RG 0. Peek must
2586        // therefore look past the active RG to RG 1.
2587        let _batch0 = expect_data(decoder.try_decode());
2588        assert!(!decoder.is_at_row_group_boundary());
2589        assert_eq!(decoder.peek_next_row_group().unwrap(), Some(1));
2590
2591        // Drain the rest of RG 0; peek still reports RG 1.
2592        let _batch1 = expect_data(decoder.try_decode());
2593        assert_eq!(decoder.peek_next_row_group().unwrap(), Some(1));
2594
2595        // Move into RG 1 — peek now sees no further row groups.
2596        let _batch2 = expect_data(decoder.try_decode());
2597        assert!(!decoder.is_at_row_group_boundary());
2598        assert_eq!(decoder.peek_next_row_group().unwrap(), None);
2599
2600        let _batch3 = expect_data(decoder.try_decode());
2601        expect_finished(decoder.try_decode());
2602    }
2603
2604    /// Peeking is a read-only operation: calling it repeatedly between
2605    /// `try_next_reader` calls must never change which row group the
2606    /// reader path actually produces. Drives the decoder all the way
2607    /// through and asserts each peek/read pair agrees.
2608    #[test]
2609    fn test_peek_next_row_group_does_not_mutate_state() {
2610        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2611            .unwrap()
2612            .build()
2613            .unwrap();
2614
2615        // Two row groups expected — drive both, asserting peek/read agree.
2616        for expected_rg in [0usize, 1usize] {
2617            assert!(decoder.is_at_row_group_boundary());
2618
2619            // Multiple peeks before reading must all agree, and must not
2620            // disturb the upcoming read.
2621            let first_peek = decoder.peek_next_row_group().unwrap();
2622            let second_peek = decoder.peek_next_row_group().unwrap();
2623            let third_peek = decoder.peek_next_row_group().unwrap();
2624            assert_eq!(first_peek, Some(expected_rg));
2625            assert_eq!(first_peek, second_peek);
2626            assert_eq!(second_peek, third_peek);
2627
2628            // Now read for real and confirm the decoder hands back exactly
2629            // what peek promised.
2630            let ranges = expect_needs_data(decoder.try_next_reader());
2631            push_ranges_to_decoder(&mut decoder, ranges);
2632            let reader = expect_data(decoder.try_next_reader());
2633            for batch in reader {
2634                let _ = batch.unwrap();
2635            }
2636        }
2637
2638        // Decoder is drained. Peek must agree with `try_next_reader`'s
2639        // terminal state.
2640        assert_eq!(decoder.peek_next_row_group().unwrap(), None);
2641        expect_finished(decoder.try_next_reader());
2642    }
2643
2644    /// `into_builder` between row groups recovers a builder for the
2645    /// not-yet-decoded row groups; rebuilding it with a new row filter
2646    /// applies that filter to the subsequent row groups while leaving the
2647    /// already-decoded row group's results untouched.
2648    ///
2649    /// See the "Adaptive scans" section of [`ParquetPushDecoderBuilder`] for
2650    /// the high-level overview.
2651    #[test]
2652    fn test_into_builder_installs_filter_between_row_groups() {
2653        let schema_descr = test_file_parquet_metadata()
2654            .file_metadata()
2655            .schema_descr_ptr();
2656        let mut decoder = prefetched_decoder(1024);
2657
2658        // Reader for row group 0 — no filter.
2659        let reader0 = expect_data(decoder.try_next_reader());
2660        let batches0: Vec<_> = reader0.collect::<Result<_, _>>().unwrap();
2661        let batch0 = concat_batches(&TEST_BATCH.schema(), &batches0).unwrap();
2662        assert_eq!(batch0, TEST_BATCH.slice(0, 200));
2663
2664        // We're between row groups now. Rebuild with a filter on column "a".
2665        assert!(decoder.is_at_row_group_boundary());
2666        assert_eq!(decoder.row_groups_remaining(), 1);
2667        let filter =
2668            ArrowPredicateFn::new(ProjectionMask::columns(&schema_descr, ["a"]), |batch| {
2669                gt(batch.column(0), &Int64Array::new_scalar(250))
2670            });
2671        let mut decoder = decoder
2672            .into_builder()
2673            .unwrap()
2674            .with_row_filter(RowFilter::new(vec![Box::new(filter)]))
2675            .build()
2676            .unwrap();
2677
2678        // Reader for row group 1 — filter applied. The rebuilt decoder kept
2679        // the buffered bytes (see `test_into_builder_preserves_buffered_bytes`)
2680        // so no data needs to be re-supplied. Column "a" in RG1 has values
2681        // 200..399; `a > 250` keeps 251..399 = 149 rows.
2682        let reader1 = expect_data(decoder.try_next_reader());
2683        let batches1: Vec<_> = reader1.collect::<Result<_, _>>().unwrap();
2684        let batch1 = concat_batches(&TEST_BATCH.schema(), &batches1).unwrap();
2685        assert_eq!(batch1, TEST_BATCH.slice(251, 149));
2686        expect_finished(decoder.try_next_reader());
2687    }
2688
2689    /// `into_builder` is rejected while a row group's reader is being
2690    /// drained (`DecodingRowGroup`); the error points at
2691    /// `is_at_row_group_boundary`.
2692    #[test]
2693    fn test_into_builder_rejected_mid_row_group() {
2694        let mut decoder = prefetched_decoder(50);
2695
2696        // Decode one batch to land mid-row-group, inside `DecodingRowGroup`
2697        // with an active reader — not a boundary.
2698        expect_data(decoder.try_decode());
2699        assert!(!decoder.is_at_row_group_boundary());
2700
2701        let err = decoder.into_builder().unwrap_err();
2702        let err_msg = format!("{err}");
2703        assert!(
2704            err_msg.contains("is_at_row_group_boundary"),
2705            "unexpected error: {err_msg}"
2706        );
2707    }
2708
2709    /// `into_builder` is rejected once the decoder has finished.
2710    #[test]
2711    fn test_into_builder_rejected_on_finished_decoder() {
2712        let mut decoder = prefetched_decoder(1024);
2713        expect_data(decoder.try_decode());
2714        expect_data(decoder.try_decode());
2715        expect_finished(decoder.try_decode());
2716        assert!(!decoder.is_at_row_group_boundary());
2717
2718        let err = decoder.into_builder().unwrap_err();
2719        assert!(
2720            format!("{err}").contains("finished"),
2721            "unexpected error: {err}"
2722        );
2723    }
2724
2725    /// `try_next_reader` hands the active reader off to the caller and
2726    /// transitions the decoder back to `ReadingRowGroup` — so the caller
2727    /// can call `into_builder` even while still holding the returned
2728    /// reader. (The handed-off reader has no link back to the decoder's
2729    /// projection/filter; it has its own `ArrayReader` and `ReadPlan`.)
2730    #[test]
2731    fn test_into_builder_allowed_while_iterating_handed_off_reader() {
2732        let mut decoder = prefetched_decoder(1024);
2733
2734        let reader0 = expect_data(decoder.try_next_reader());
2735        // Decoder no longer owns the reader, so it considers itself
2736        // "between row groups".
2737        assert!(decoder.is_at_row_group_boundary());
2738        // Recovering the builder consumes the decoder but leaves `reader0`
2739        // valid: iterating it is independent of the decoder's state.
2740        let _builder = decoder.into_builder().unwrap();
2741        let batches: Vec<_> = reader0.collect::<Result<_, _>>().unwrap();
2742        let batch0 = concat_batches(&TEST_BATCH.schema(), &batches).unwrap();
2743        assert_eq!(batch0, TEST_BATCH.slice(0, 200));
2744    }
2745
2746    /// `into_builder` recovers a builder for the *remaining* row groups and
2747    /// carries the not-yet-consumed offset/limit budget, so a rebuilt
2748    /// decoder resumes where the original left off rather than restarting.
2749    #[test]
2750    fn test_into_builder_resumes_remaining_budget() {
2751        // limit = 250 spans both 200-row row groups: all 200 rows of RG0
2752        // plus the first 50 rows of RG1.
2753        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2754            .unwrap()
2755            .with_batch_size(1024)
2756            .with_limit(250)
2757            .build()
2758            .unwrap();
2759        prefetch_test_file(&mut decoder);
2760
2761        // RG0 contributes all 200 of its rows.
2762        let reader0 = expect_data(decoder.try_next_reader());
2763        let batches0: Vec<_> = reader0.collect::<Result<_, _>>().unwrap();
2764        let batch0 = concat_batches(&TEST_BATCH.schema(), &batches0).unwrap();
2765        assert_eq!(batch0, TEST_BATCH.slice(0, 200));
2766
2767        // Rebuild without changing anything: the remaining 50-row limit and
2768        // the not-yet-decoded RG1 must carry through (as do the buffers, so
2769        // no data needs re-supplying).
2770        assert!(decoder.is_at_row_group_boundary());
2771        let mut decoder = decoder.into_builder().unwrap().build().unwrap();
2772
2773        let reader1 = expect_data(decoder.try_next_reader());
2774        let batches1: Vec<_> = reader1.collect::<Result<_, _>>().unwrap();
2775        let batch1 = concat_batches(&TEST_BATCH.schema(), &batches1).unwrap();
2776        // Only the first 50 rows of RG1 (200..249) — the rest of the limit.
2777        assert_eq!(batch1, TEST_BATCH.slice(200, 50));
2778        expect_finished(decoder.try_next_reader());
2779    }
2780
2781    /// `into_builder` carries the decoder's buffered bytes across the
2782    /// rebuild: the rebuilt decoder keeps them and does not re-request data
2783    /// it already holds.
2784    #[test]
2785    fn test_into_builder_preserves_buffered_bytes() {
2786        let mut decoder = prefetched_decoder(1024);
2787        assert_eq!(decoder.buffered_bytes(), test_file_len());
2788
2789        // Drain RG0.
2790        let reader0 = expect_data(decoder.try_next_reader());
2791        let _: Vec<_> = reader0.collect::<Result<_, _>>().unwrap();
2792        // RG1's bytes are still staged inside the decoder.
2793        let buffered = decoder.buffered_bytes();
2794        assert!(buffered > 0);
2795
2796        // Rebuilding via into_builder keeps the staged bytes that RG1 reads,
2797        // and releases the other bytes of the prefetched file.
2798        let mut decoder = decoder.into_builder().unwrap().build().unwrap();
2799        assert!(decoder.buffered_bytes() < buffered);
2800        assert_eq!(decoder.buffered_bytes(), test_file_row_group_bytes(1));
2801
2802        // RG1's bytes are already buffered, so it decodes without a
2803        // `NeedsData` round-trip.
2804        let reader1 = expect_data(decoder.try_next_reader());
2805        let batches1: Vec<_> = reader1.collect::<Result<_, _>>().unwrap();
2806        let batch1 = concat_batches(&TEST_BATCH.schema(), &batches1).unwrap();
2807        assert_eq!(batch1, TEST_BATCH.slice(200, 200));
2808        expect_finished(decoder.try_next_reader());
2809    }
2810
2811    /// A rebuilt decoder releases the bytes of a row group that it skips,
2812    /// and keeps the bytes of the columns that it reads.
2813    #[test]
2814    fn test_into_builder_releases_bytes_of_skipped_row_groups() {
2815        let builder =
2816            ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
2817        let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
2818        let mut decoder = builder
2819            .with_row_groups(vec![1])
2820            .with_projection(ProjectionMask::columns(&schema_descr, ["a", "b"]))
2821            .build()
2822            .unwrap();
2823        prefetch_test_file(&mut decoder);
2824        assert!(decoder.is_at_row_group_boundary());
2825
2826        // Skip RG1 and read RG0, with column "c" too.
2827        let mut decoder = decoder
2828            .into_builder()
2829            .unwrap()
2830            .with_row_groups(vec![0])
2831            .with_projection(ProjectionMask::all())
2832            .build()
2833            .unwrap();
2834        assert_eq!(decoder.buffered_bytes(), test_file_row_group_bytes(0));
2835
2836        let reader = expect_data(decoder.try_next_reader());
2837        let batches: Vec<_> = reader.collect::<Result<_, _>>().unwrap();
2838        let batch = concat_batches(&TEST_BATCH.schema(), &batches).unwrap();
2839        assert_eq!(batch, TEST_BATCH.slice(0, 200));
2840        expect_finished(decoder.try_next_reader());
2841    }
2842
2843    /// A rebuilt decoder keeps the bytes of a column that only a predicate
2844    /// reads.
2845    #[test]
2846    fn test_into_builder_keeps_bytes_of_predicate_columns() {
2847        let builder =
2848            ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
2849        let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
2850        let row_filter_a = ArrowPredicateFn::new(
2851            ProjectionMask::columns(&schema_descr, ["a"]),
2852            |batch: RecordBatch| {
2853                let scalar_250 = Int64Array::new_scalar(250);
2854                let column = batch.column(0).as_primitive::<Int64Type>();
2855                gt(column, &scalar_250)
2856            },
2857        );
2858        let mut decoder = builder
2859            .with_row_groups(vec![1])
2860            .with_projection(ProjectionMask::columns(&schema_descr, ["b"]))
2861            .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
2862            .build()
2863            .unwrap();
2864        prefetch_test_file(&mut decoder);
2865
2866        // The rebuilt decoder keeps the bytes of "a" (the predicate) and "b"
2867        // (the output) of RG1, and releases the bytes of "c".
2868        let mut decoder = decoder.into_builder().unwrap().build().unwrap();
2869        let column_bytes = |column_idx: usize| {
2870            let range =
2871                column_chunk_range(test_file_parquet_metadata().row_group(1).column(column_idx));
2872            range.end - range.start
2873        };
2874        assert_eq!(decoder.buffered_bytes(), column_bytes(0) + column_bytes(1));
2875
2876        let reader = expect_data(decoder.try_next_reader());
2877        let batches: Vec<_> = reader.collect::<Result<_, _>>().unwrap();
2878        let batch = concat_batches(&batches[0].schema(), &batches).unwrap();
2879        assert_eq!(batch, TEST_BATCH.slice(251, 149).project(&[1]).unwrap());
2880        expect_finished(decoder.try_next_reader());
2881    }
2882
2883    #[test]
2884    fn test_into_builder_preserves_remaining_row_group_local_selections() {
2885        let bitmap_selection = RowSelection::from_boolean_buffer(BooleanBuffer::from(
2886            (0..45)
2887                .map(|row_idx| (25..45).contains(&row_idx))
2888                .collect::<Vec<_>>(),
2889        ));
2890        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2891            .unwrap()
2892            .with_row_group_selections(vec![
2893                RowGroupSelection::new(
2894                    0,
2895                    Some(RowSelection::from(vec![
2896                        RowSelector::skip(190),
2897                        RowSelector::select(10),
2898                    ])),
2899                ),
2900                RowGroupSelection::new(1, Some(bitmap_selection)),
2901            ])
2902            .build()
2903            .unwrap();
2904        prefetch_test_file(&mut decoder);
2905
2906        let reader0 = expect_data(decoder.try_next_reader());
2907        let batches0: Vec<_> = reader0.collect::<Result<_, _>>().unwrap();
2908        let batch0 = concat_batches(&TEST_BATCH.schema(), &batches0).unwrap();
2909        assert_eq!(batch0, TEST_BATCH.slice(190, 10));
2910
2911        // Rebuilding must carry only RG1 and its still-local bitmap selection.
2912        assert!(decoder.is_at_row_group_boundary());
2913        assert_eq!(decoder.row_groups_remaining(), 1);
2914        let mut decoder = decoder.into_builder().unwrap().build().unwrap();
2915
2916        let reader1 = expect_data(decoder.try_next_reader());
2917        let batches1: Vec<_> = reader1.collect::<Result<_, _>>().unwrap();
2918        let batch1 = concat_batches(&TEST_BATCH.schema(), &batches1).unwrap();
2919        assert_eq!(batch1, TEST_BATCH.slice(225, 20));
2920        expect_finished(decoder.try_next_reader());
2921    }
2922
2923    /// Drive the decoder incrementally. Start with a narrow projection,
2924    /// drain RG0, then `into_builder` and widen the projection to all three
2925    /// columns. The rebuilt decoder's `NeedsData` for RG1 must request
2926    /// bytes for *all three* columns, not just the originally-projected
2927    /// "a". The expected ranges are hardcoded because `TEST_BATCH` and the
2928    /// writer settings are static; this pins the layout cleanly without a
2929    /// parallel reference decoder.
2930    #[test]
2931    fn test_into_builder_expand_projection_requests_new_bytes() {
2932        let metadata = test_file_parquet_metadata();
2933        let schema_descr = metadata.file_metadata().schema_descr_ptr();
2934
2935        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(metadata)
2936            .unwrap()
2937            .with_batch_size(1024)
2938            .with_projection(ProjectionMask::columns(&schema_descr, ["a"]))
2939            .build()
2940            .unwrap();
2941
2942        // RG0: incrementally satisfy the narrow request — a single
2943        // contiguous range for the "a"-only projection.
2944        let ranges_rg0 = expect_needs_data(decoder.try_next_reader());
2945        assert_eq!(ranges_rg0, vec![4..1860]);
2946        push_ranges_to_decoder(&mut decoder, ranges_rg0);
2947
2948        let reader0 = expect_data(decoder.try_next_reader());
2949        let batches0: Vec<_> = reader0.collect::<Result<_, _>>().unwrap();
2950        let batch0 = concat_batches(&batches0[0].schema(), &batches0).unwrap();
2951        assert_eq!(batch0, TEST_BATCH.slice(0, 200).project(&[0]).unwrap());
2952
2953        // Widen the projection at the boundary.
2954        assert!(decoder.is_at_row_group_boundary());
2955        let mut decoder = decoder
2956            .into_builder()
2957            .unwrap()
2958            .with_projection(ProjectionMask::columns(&schema_descr, ["a", "b", "c"]))
2959            .build()
2960            .unwrap();
2961
2962        // RG1 now requests "a", "b", and "c" column chunks: ~1.8KiB each
2963        // for "a" and "b", ~7.5KiB for the StringView column "c".
2964        let ranges_rg1 = expect_needs_data(decoder.try_next_reader());
2965        assert_eq!(ranges_rg1, vec![11062..12918, 12918..14774, 14774..22230]);
2966        push_ranges_to_decoder(&mut decoder, ranges_rg1);
2967
2968        let reader1 = expect_data(decoder.try_next_reader());
2969        let batches1: Vec<_> = reader1.collect::<Result<_, _>>().unwrap();
2970        let batch1 = concat_batches(&TEST_BATCH.schema(), &batches1).unwrap();
2971        assert_eq!(batch1, TEST_BATCH.slice(200, 200));
2972        expect_finished(decoder.try_next_reader());
2973    }
2974
2975    /// Mirror of [`test_into_builder_expand_projection_requests_new_bytes`]:
2976    /// start with the full projection, drain RG0, then `into_builder` and
2977    /// narrow the projection to just column "a". RG1's `NeedsData` must
2978    /// request only the single "a" column-chunk range, not the three a
2979    /// wide projection would.
2980    #[test]
2981    fn test_into_builder_narrow_projection_requests_fewer_bytes() {
2982        let metadata = test_file_parquet_metadata();
2983        let schema_descr = metadata.file_metadata().schema_descr_ptr();
2984
2985        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(metadata)
2986            .unwrap()
2987            .with_batch_size(1024)
2988            .build()
2989            .unwrap();
2990
2991        // RG0 with the default (full) projection — three column ranges.
2992        let ranges_rg0 = expect_needs_data(decoder.try_next_reader());
2993        assert_eq!(ranges_rg0, vec![4..1860, 1860..3716, 3716..11062]);
2994        push_ranges_to_decoder(&mut decoder, ranges_rg0);
2995
2996        let reader0 = expect_data(decoder.try_next_reader());
2997        let batches0: Vec<_> = reader0.collect::<Result<_, _>>().unwrap();
2998        let batch0 = concat_batches(&TEST_BATCH.schema(), &batches0).unwrap();
2999        assert_eq!(batch0, TEST_BATCH.slice(0, 200));
3000
3001        // Narrow the projection at the boundary.
3002        assert!(decoder.is_at_row_group_boundary());
3003        let mut decoder = decoder
3004            .into_builder()
3005            .unwrap()
3006            .with_projection(ProjectionMask::columns(&schema_descr, ["a"]))
3007            .build()
3008            .unwrap();
3009
3010        // RG1 now requests column "a" only — a single 1856-byte range.
3011        let ranges_rg1 = expect_needs_data(decoder.try_next_reader());
3012        assert_eq!(ranges_rg1, vec![11062..12918]);
3013        push_ranges_to_decoder(&mut decoder, ranges_rg1);
3014
3015        let reader1 = expect_data(decoder.try_next_reader());
3016        let batches1: Vec<_> = reader1.collect::<Result<_, _>>().unwrap();
3017        let batch1 = concat_batches(&batches1[0].schema(), &batches1).unwrap();
3018        assert_eq!(batch1, TEST_BATCH.slice(200, 200).project(&[0]).unwrap());
3019        expect_finished(decoder.try_next_reader());
3020    }
3021
3022    /// Returns a batch with 400 rows, with 3 columns: "a", "b", "c"
3023    ///
3024    /// Note c is a different types (so the data page sizes will be different)
3025    static TEST_BATCH: LazyLock<RecordBatch> = LazyLock::new(|| {
3026        let a: ArrayRef = Arc::new(Int64Array::from_iter_values(0..400));
3027        let b: ArrayRef = Arc::new(Int64Array::from_iter_values(400..800));
3028        let c: ArrayRef = Arc::new(StringViewArray::from_iter_values((0..400).map(|i| {
3029            if i % 2 == 0 {
3030                format!("string_{i}")
3031            } else {
3032                format!("A string larger than 12 bytes and thus not inlined {i}")
3033            }
3034        })));
3035
3036        RecordBatch::try_from_iter(vec![("a", a), ("b", b), ("c", c)]).unwrap()
3037    });
3038
3039    /// Create a parquet file in memory for testing.
3040    ///
3041    /// See [`TEST_BATCH`] for the data in the file.
3042    ///
3043    /// Each column is written in 4 data pages, each with 100 rows, across 2
3044    /// row groups. Each column in each row group has two data pages.
3045    ///
3046    /// The data is split across row groups like this
3047    ///
3048    /// Column |   Values                | Data Page | Row Group
3049    /// -------|------------------------|-----------|-----------
3050    /// a      | 0..99                  | 1         | 0
3051    /// a      | 100..199               | 2         | 0
3052    /// a      | 200..299               | 1         | 1
3053    /// a      | 300..399               | 2         | 1
3054    ///
3055    /// b      | 400..499               | 1         | 0
3056    /// b      | 500..599               | 2         | 0
3057    /// b      | 600..699               | 1         | 1
3058    /// b      | 700..799               | 2         | 1
3059    ///
3060    /// c      | "string_0".."string_99"        | 1         | 0
3061    /// c      | "string_100".."string_199"     | 2         | 0
3062    /// c      | "string_200".."string_299"     | 1         | 1
3063    /// c      | "string_300".."string_399"     | 2         | 1
3064    static TEST_FILE_DATA: LazyLock<Bytes> = LazyLock::new(|| {
3065        let input_batch = &TEST_BATCH;
3066        let mut output = Vec::new();
3067
3068        let writer_options = WriterProperties::builder()
3069            .set_max_row_group_row_count(Some(200))
3070            .set_data_page_row_count_limit(100)
3071            .build();
3072        let mut writer =
3073            ArrowWriter::try_new(&mut output, input_batch.schema(), Some(writer_options)).unwrap();
3074
3075        // since the limits are only enforced on batch boundaries, write the input
3076        // batch in chunks of 50
3077        let mut row_remain = input_batch.num_rows();
3078        while row_remain > 0 {
3079            let chunk_size = row_remain.min(50);
3080            let chunk = input_batch.slice(input_batch.num_rows() - row_remain, chunk_size);
3081            writer.write(&chunk).unwrap();
3082            row_remain -= chunk_size;
3083        }
3084        writer.close().unwrap();
3085        Bytes::from(output)
3086    });
3087
3088    /// Return the length of [`TEST_FILE_DATA`], in bytes
3089    fn test_file_len() -> u64 {
3090        TEST_FILE_DATA.len() as u64
3091    }
3092
3093    /// Return a range that covers the entire [`TEST_FILE_DATA`]
3094    fn test_file_range() -> Range<u64> {
3095        0..test_file_len()
3096    }
3097
3098    /// Return a slice of the test file data from the given range
3099    pub fn test_file_slice(range: Range<u64>) -> Bytes {
3100        let start: usize = range.start.try_into().unwrap();
3101        let end: usize = range.end.try_into().unwrap();
3102        TEST_FILE_DATA.slice(start..end)
3103    }
3104
3105    /// return the metadata for the test file
3106    pub fn test_file_parquet_metadata() -> Arc<ParquetMetaData> {
3107        let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(test_file_len()).unwrap();
3108        push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![test_file_range()]);
3109        let metadata = metadata_decoder.try_decode().unwrap();
3110        let DecodeResult::Data(metadata) = metadata else {
3111            panic!("Expected metadata to be decoded successfully");
3112        };
3113        Arc::new(metadata)
3114    }
3115
3116    /// An offset index whose last page is longer than the column chunk gives
3117    /// an error, not a panic.
3118    #[test]
3119    fn test_decoder_offset_index_page_past_column_chunk() {
3120        let metadata = Arc::unwrap_or_clone(test_file_parquet_metadata_with_offset_index());
3121        let num_columns = metadata.row_group(0).num_columns();
3122        let mut page_index = PageIndexBuilder::new(metadata.num_row_groups(), num_columns);
3123        for row_group in 0..metadata.num_row_groups() {
3124            let row_group_index = metadata.page_index_for_row_group(row_group);
3125            for column in 0..num_columns {
3126                let mut offset_index = row_group_index.offset_index(column).unwrap().clone();
3127                if (row_group, column) == (0, 0) {
3128                    let last = offset_index.page_locations.last_mut().unwrap();
3129                    last.compressed_page_size += 1;
3130                }
3131                page_index
3132                    .try_put_offset_index(offset_index, row_group, column)
3133                    .unwrap();
3134            }
3135        }
3136        let metadata = metadata
3137            .into_builder()
3138            .set_page_index(Some(Arc::new(page_index.build())))
3139            .build();
3140
3141        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(Arc::new(metadata))
3142            .unwrap()
3143            .with_row_groups(vec![0])
3144            .build()
3145            .unwrap();
3146        let ranges = expect_needs_data(decoder.try_decode());
3147        push_ranges_to_decoder(&mut decoder, ranges);
3148        let err = loop {
3149            match decoder.try_decode() {
3150                Ok(DecodeResult::Data(_)) => {}
3151                Ok(other) => panic!("expected an error, got {other:?}"),
3152                Err(e) => break e.to_string(),
3153            }
3154        };
3155        assert!(err.contains("has 126 bytes, expected 127"), "{err}");
3156    }
3157
3158    /// return the metadata for the test file, including the offset index
3159    fn test_file_parquet_metadata_with_offset_index() -> Arc<ParquetMetaData> {
3160        let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(test_file_len())
3161            .unwrap()
3162            .with_offset_index_policy(PageIndexPolicy::Required);
3163        push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![test_file_range()]);
3164        let metadata = metadata_decoder.try_decode().unwrap();
3165        let DecodeResult::Data(metadata) = metadata else {
3166            panic!("Expected metadata to be decoded successfully");
3167        };
3168        Arc::new(metadata)
3169    }
3170
3171    /// Push the given ranges to the metadata decoder, simulating reading from a file
3172    fn push_ranges_to_metadata_decoder(
3173        metadata_decoder: &mut ParquetMetaDataPushDecoder,
3174        ranges: Vec<Range<u64>>,
3175    ) {
3176        let data = ranges
3177            .iter()
3178            .map(|range| test_file_slice(range.clone()))
3179            .collect::<Vec<_>>();
3180        metadata_decoder.push_ranges(ranges, data).unwrap();
3181    }
3182
3183    /// Push the entire test file into `decoder`.
3184    fn prefetch_test_file(decoder: &mut ParquetPushDecoder) {
3185        decoder
3186            .push_range(test_file_range(), TEST_FILE_DATA.clone())
3187            .unwrap();
3188    }
3189
3190    /// Build a decoder over the test file with the given batch size and
3191    /// prefetch the whole file into it.
3192    fn prefetched_decoder(batch_size: usize) -> ParquetPushDecoder {
3193        let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
3194            .unwrap()
3195            .with_batch_size(batch_size)
3196            .build()
3197            .unwrap();
3198        prefetch_test_file(&mut decoder);
3199        decoder
3200    }
3201
3202    /// The number of bytes in all column chunks of a row group of the test
3203    /// file.
3204    fn test_file_row_group_bytes(row_group_idx: usize) -> u64 {
3205        test_file_parquet_metadata()
3206            .row_group(row_group_idx)
3207            .columns()
3208            .iter()
3209            .map(|column| {
3210                let range = column_chunk_range(column);
3211                range.end - range.start
3212            })
3213            .sum()
3214    }
3215
3216    fn push_ranges_to_decoder(decoder: &mut ParquetPushDecoder, ranges: Vec<Range<u64>>) {
3217        let data = ranges
3218            .iter()
3219            .map(|range| test_file_slice(range.clone()))
3220            .collect::<Vec<_>>();
3221        decoder.push_ranges(ranges, data).unwrap();
3222    }
3223
3224    /// Expect that the [`DecodeResult`] is a [`DecodeResult::Data`] and return the corresponding element
3225    fn expect_data<T: Debug>(result: Result<DecodeResult<T>, ParquetError>) -> T {
3226        match result.expect("Expected Ok(DecodeResult::Data(T))") {
3227            DecodeResult::Data(data) => data,
3228            result => panic!("Expected DecodeResult::Data, got {result:?}"),
3229        }
3230    }
3231
3232    /// Expect that the [`DecodeResult`] is a [`DecodeResult::NeedsData`] and return the corresponding ranges
3233    fn expect_needs_data<T: Debug>(
3234        result: Result<DecodeResult<T>, ParquetError>,
3235    ) -> Vec<Range<u64>> {
3236        match result.expect("Expected Ok(DecodeResult::NeedsData(ranges))") {
3237            DecodeResult::NeedsData(ranges) => ranges,
3238            result => panic!("Expected DecodeResult::NeedsData, got {result:?}"),
3239        }
3240    }
3241
3242    fn expect_finished<T: Debug>(result: Result<DecodeResult<T>, ParquetError>) {
3243        match result.expect("Expected Ok(DecodeResult::Finished)") {
3244            DecodeResult::Finished => {}
3245            result => panic!("Expected DecodeResult::Finished, got {result:?}"),
3246        }
3247    }
3248}