Skip to main content

parquet/arrow/arrow_reader/
read_plan.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! [`ReadPlan`] and [`ReadPlanBuilder`] for determining which rows to read
6//! from a Parquet file
7
8use crate::arrow::array_reader::ArrayReader;
9use crate::arrow::arrow_reader::selection::{
10    LoadedRowRanges, RowSelectionPolicy, RowSelectionStrategy,
11};
12use crate::arrow::arrow_reader::{
13    ArrowPredicate, ParquetRecordBatchReader, RowSelection, RowSelectionCursor, RowSelector,
14};
15use crate::errors::{ParquetError, Result};
16use arrow_array::{Array, BooleanArray};
17use arrow_buffer::BooleanBuffer;
18use arrow_select::filter::prep_null_mask_filter;
19use std::sync::Arc;
20
21/// Options for [`ReadPlanBuilder::with_predicate_options`].
22pub struct PredicateOptions<'a> {
23    array_reader: Box<dyn ArrayReader>,
24    predicate: &'a mut dyn ArrowPredicate,
25    limit: Option<usize>,
26    total_rows: usize,
27}
28
29impl<'a> PredicateOptions<'a> {
30    /// Create options for evaluating `predicate` against rows produced by
31    /// `array_reader`.
32    ///
33    /// By default there is no match-count limit; the predicate is evaluated
34    /// over every row the reader yields. Use [`Self::with_limit`] to enable
35    /// early termination.
36    pub fn new(array_reader: Box<dyn ArrayReader>, predicate: &'a mut dyn ArrowPredicate) -> Self {
37        Self {
38            array_reader,
39            predicate,
40            limit: None,
41            total_rows: 0,
42        }
43    }
44
45    /// Stop scanning `array_reader` once `limit` matches have accumulated.
46    ///
47    /// Performance optimization for `LIMIT` / TopK: when the cumulative
48    /// `true_count` reaches `limit`, the current filter batch is truncated
49    /// at the `limit`-th match and remaining batches are never decoded.
50    ///
51    /// `limit` counts predicate matches, not output rows — callers applying
52    /// an offset must pass `offset + limit`.
53    ///
54    /// `total_rows` is the row count `array_reader` would yield if iterated
55    /// to completion. It is used to pad un-evaluated trailing rows as "not
56    /// selected" so the returned [`RowSelection`] covers the full row group.
57    ///
58    /// Only valid for the *last* predicate in a filter chain: intermediate
59    /// predicates' match counts do not map 1:1 to output rows.
60    pub fn with_limit(mut self, limit: usize, total_rows: usize) -> Self {
61        self.limit = Some(limit);
62        self.total_rows = total_rows;
63        self
64    }
65}
66
67/// A builder for [`ReadPlan`]
68#[derive(Clone, Debug)]
69pub struct ReadPlanBuilder {
70    batch_size: usize,
71    /// Which rows to select. Includes the result of all filters applied so far
72    selection: Option<RowSelection>,
73    /// Policy to use when materializing the row selection
74    row_selection_policy: RowSelectionPolicy,
75    /// Row ranges with page data loaded for the current projection.
76    loaded_row_ranges: Option<Arc<LoadedRowRanges>>,
77}
78
79impl ReadPlanBuilder {
80    /// Create a `ReadPlanBuilder` with the given batch size
81    pub fn new(batch_size: usize) -> Self {
82        Self {
83            batch_size,
84            selection: None,
85            row_selection_policy: RowSelectionPolicy::default(),
86            loaded_row_ranges: None,
87        }
88    }
89
90    /// Set the current selection to the given value
91    pub fn with_selection(mut self, selection: Option<RowSelection>) -> Self {
92        self.selection = selection;
93        self
94    }
95
96    /// Configure the policy to use when materialising the [`RowSelection`]
97    ///
98    /// Defaults to [`RowSelectionPolicy::Auto`]
99    pub fn with_row_selection_policy(mut self, policy: RowSelectionPolicy) -> Self {
100        self.row_selection_policy = policy;
101        self
102    }
103
104    pub(crate) fn with_loaded_row_ranges(mut self, ranges: Option<LoadedRowRanges>) -> Self {
105        self.loaded_row_ranges = ranges.map(Arc::new);
106        self
107    }
108
109    /// Returns the current row selection policy
110    pub fn row_selection_policy(&self) -> &RowSelectionPolicy {
111        &self.row_selection_policy
112    }
113
114    /// Returns the current selection, if any
115    pub fn selection(&self) -> Option<&RowSelection> {
116        self.selection.as_ref()
117    }
118
119    /// Specifies the number of rows in the row group, before filtering is applied.
120    ///
121    /// Returns a [`LimitedReadPlanBuilder`] that can apply
122    /// offset and limit.
123    ///
124    /// Call [`LimitedReadPlanBuilder::build_limited`] to apply the limits to this
125    /// selection.
126    pub(crate) fn limited(self, row_count: usize) -> LimitedReadPlanBuilder {
127        LimitedReadPlanBuilder::new(self, row_count)
128    }
129
130    /// Returns true if the current plan selects any rows
131    pub fn selects_any(&self) -> bool {
132        self.selection
133            .as_ref()
134            .map(|s| s.selects_any())
135            .unwrap_or(true)
136    }
137
138    /// Returns the number of rows selected, or `None` if all rows are selected.
139    pub fn num_rows_selected(&self) -> Option<usize> {
140        self.selection.as_ref().map(|s| s.row_count())
141    }
142
143    /// Returns the [`RowSelectionStrategy`] for this plan.
144    ///
145    /// Guarantees to return either `Selectors` or `Mask`, never `Auto`.
146    pub(crate) fn resolve_selection_strategy(&self) -> RowSelectionStrategy {
147        self.row_selection_policy.resolve(self.selection.as_ref())
148    }
149
150    /// Evaluates an [`ArrowPredicate`], updating this plan's `selection`
151    ///
152    /// If the current `selection` is `Some`, the resulting [`RowSelection`]
153    /// will be the conjunction of the existing selection and the rows selected
154    /// by `predicate`.
155    ///
156    /// Note: pre-existing selections may come from evaluating a previous predicate
157    /// or if the [`ParquetRecordBatchReader`] specified an explicit
158    /// [`RowSelection`] in addition to one or more predicates.
159    pub fn with_predicate(
160        self,
161        array_reader: Box<dyn ArrayReader>,
162        predicate: &mut dyn ArrowPredicate,
163    ) -> Result<Self> {
164        self.with_predicate_options(PredicateOptions::new(array_reader, predicate))
165    }
166
167    /// Evaluates an [`ArrowPredicate`] with the given [`PredicateOptions`],
168    /// updating this plan's `selection`.
169    ///
170    /// Like [`Self::with_predicate`], but allows additional options such as a
171    /// match-count limit for early termination (see
172    /// [`PredicateOptions::with_limit`]).
173    pub fn with_predicate_options(mut self, options: PredicateOptions<'_>) -> Result<Self> {
174        let PredicateOptions {
175            array_reader,
176            predicate,
177            limit,
178            total_rows,
179        } = options;
180
181        // Target length for the concatenated filter output:
182        // - Prior selection ⇒ the reader yields that many rows; `and_then`
183        //   below requires the filter output to match.
184        // - No prior selection ⇒ the reader yields `total_rows`. We only
185        //   need to pad when `limit` may short-circuit the loop; otherwise
186        //   iteration naturally exhausts.
187        let expected_rows = match self.selection.as_ref() {
188            Some(s) => Some(s.row_count()),
189            None => limit.map(|_| total_rows),
190        };
191
192        let reader = ParquetRecordBatchReader::new(array_reader, self.clone().build());
193        let mut filters = vec![];
194        let mut processed_rows: usize = 0;
195        let mut matched_rows: usize = 0;
196        for maybe_batch in reader {
197            let maybe_batch = maybe_batch?;
198            let input_rows = maybe_batch.num_rows();
199            let filter = predicate.evaluate(maybe_batch)?;
200            // Since user supplied predicate, check error here to catch bugs quickly
201            if filter.len() != input_rows {
202                return Err(arrow_err!(
203                    "ArrowPredicate predicate returned {} rows, expected {input_rows}",
204                    filter.len()
205                ));
206            }
207            let filter = match filter.null_count() {
208                0 => filter,
209                // RowSelection::from_filters expects non-null filters. Convert
210                // NULL predicate results to false so they are not selected.
211                _ => prep_null_mask_filter(&filter),
212            };
213
214            processed_rows += input_rows;
215
216            match limit {
217                Some(limit) if limit - matched_rows <= filter.len() => {
218                    let truncated = filter.take_n_true(limit - matched_rows);
219                    matched_rows += truncated.true_count();
220                    filters.push(truncated);
221                    if matched_rows >= limit {
222                        break;
223                    }
224                }
225                _ => {
226                    matched_rows += filter.true_count();
227                    filters.push(filter);
228                }
229            }
230        }
231
232        // Pad the tail so the filters cover `expected_rows` total. This keeps
233        // the invariant that the resulting `RowSelection` spans every row the
234        // reader would have produced — rows past the early break are marked
235        // "not selected". When no limit is set the loop always exhausts and
236        // no padding is needed.
237        if let Some(expected) = expected_rows
238            && processed_rows < expected
239        {
240            let pad_len = expected - processed_rows;
241            filters.push(BooleanArray::new(BooleanBuffer::new_unset(pad_len), None));
242        }
243
244        // If the predicate selected all rows, applying it is a no-op. With no
245        // prior selection this keeps selection as None, enabling coalesced page
246        // fetches; with a prior selection it avoids rebuilding the same
247        // selection.
248        let all_selected = filters.iter().all(|f| f.true_count() == f.len());
249        if all_selected {
250            return Ok(self);
251        }
252        let raw = match (self.selection.as_ref(), self.row_selection_policy) {
253            (Some(selection), _) if selection.as_mask().is_some() => {
254                RowSelection::from_filters_mask(&filters)
255            }
256            (None, RowSelectionPolicy::Auto { threshold }) => {
257                RowSelection::from_filters_auto(&filters, threshold)
258            }
259            _ => RowSelection::from_filters(&filters),
260        };
261        self.selection = match self.selection.take() {
262            Some(selection) => Some(selection.and_then(&raw)),
263            None => Some(raw),
264        };
265        Ok(self)
266    }
267
268    /// Create a final `ReadPlan` the read plan for the scan
269    pub fn build(mut self) -> ReadPlan {
270        // If selection is empty, truncate
271        if !self.selects_any() {
272            self.selection = Some(RowSelection::from(vec![]));
273        }
274
275        // Preferred strategy must not be Auto
276        let selection_strategy = self.resolve_selection_strategy();
277
278        let Self {
279            batch_size,
280            selection,
281            row_selection_policy: _,
282            loaded_row_ranges,
283        } = self;
284
285        let row_selection_cursor = selection
286            .map(|s| selection_strategy.build_cursor(s.trim(), loaded_row_ranges))
287            .unwrap_or_else(RowSelectionCursor::new_all);
288
289        ReadPlan {
290            batch_size,
291            row_selection_cursor,
292        }
293    }
294}
295
296/// Builder for [`ReadPlan`] that applies a limit and offset to the read plan
297///
298/// See [`ReadPlanBuilder::limited`] to create this builder.
299pub(crate) struct LimitedReadPlanBuilder {
300    /// The underlying builder
301    inner: ReadPlanBuilder,
302    /// Total number of rows in the row group before the selection, limit or
303    /// offset are applied
304    row_count: usize,
305    /// The offset to apply, if any
306    offset: Option<usize>,
307    /// The limit to apply, if any
308    limit: Option<usize>,
309}
310
311impl LimitedReadPlanBuilder {
312    /// Create a new `LimitedReadPlanBuilder` from the existing builder and number of rows
313    fn new(inner: ReadPlanBuilder, row_count: usize) -> Self {
314        Self {
315            inner,
316            row_count,
317            offset: None,
318            limit: None,
319        }
320    }
321
322    /// Set the offset to apply to the read plan
323    pub(crate) fn with_offset(mut self, offset: Option<usize>) -> Self {
324        self.offset = offset;
325        self
326    }
327
328    /// Set the limit to apply to the read plan
329    pub(crate) fn with_limit(mut self, limit: Option<usize>) -> Self {
330        self.limit = limit;
331        self
332    }
333
334    /// Apply offset and limit, updating the selection on the underlying builder
335    /// and returning it.
336    pub(crate) fn build_limited(self) -> ReadPlanBuilder {
337        let Self {
338            mut inner,
339            row_count,
340            offset,
341            limit,
342        } = self;
343
344        // If the selection is empty, truncate
345        if !inner.selects_any() {
346            inner.selection = Some(RowSelection::from(vec![]));
347        }
348
349        // If an offset is defined, apply it to the `selection`
350        if let Some(offset) = offset {
351            inner.selection = Some(match row_count.checked_sub(offset) {
352                None => RowSelection::from(vec![]),
353                Some(remaining) => inner
354                    .selection
355                    .map(|selection| selection.offset(offset))
356                    .unwrap_or_else(|| {
357                        RowSelection::from(vec![
358                            RowSelector::skip(offset),
359                            RowSelector::select(remaining),
360                        ])
361                    }),
362            });
363        }
364
365        // If a limit is defined, apply it to the final `selection`
366        if let Some(limit) = limit {
367            inner.selection = Some(
368                inner
369                    .selection
370                    .map(|selection| selection.limit(limit))
371                    .unwrap_or_else(|| {
372                        RowSelection::from(vec![RowSelector::select(limit.min(row_count))])
373                    }),
374            );
375        }
376
377        inner
378    }
379}
380
381/// A plan reading specific rows from a Parquet Row Group.
382///
383/// See [`ReadPlanBuilder`] to create `ReadPlan`s
384#[derive(Debug)]
385pub struct ReadPlan {
386    /// The number of rows to read in each batch
387    batch_size: usize,
388    /// Row ranges to be selected from the data source
389    row_selection_cursor: RowSelectionCursor,
390}
391
392impl ReadPlan {
393    /// Returns a mutable reference to the row selection cursor
394    pub fn row_selection_cursor_mut(&mut self) -> &mut RowSelectionCursor {
395        &mut self.row_selection_cursor
396    }
397
398    /// Return the number of rows to read in each output batch
399    #[inline(always)]
400    pub fn batch_size(&self) -> usize {
401        self.batch_size
402    }
403}
404
405#[cfg(test)]
406mod tests {
407    use super::*;
408
409    const DEFAULT_AUTO_THRESHOLD: usize = 32;
410
411    fn builder_with_selection(selection: RowSelection) -> ReadPlanBuilder {
412        ReadPlanBuilder::new(1024).with_selection(Some(selection))
413    }
414
415    fn predicate_plan(
416        pattern: Vec<bool>,
417        batch_size: usize,
418        limit: Option<usize>,
419    ) -> ReadPlanBuilder {
420        use crate::arrow::ProjectionMask;
421        use crate::arrow::array_reader::StructArrayReader;
422        use crate::arrow::array_reader::test_util::make_int32_page_reader;
423        use crate::arrow::arrow_reader::ArrowPredicateFn;
424        use arrow_schema::{DataType as ArrowType, Field, Fields};
425
426        let total_rows = pattern.len();
427        let data: Vec<i32> = (0..total_rows as i32).collect();
428        let levels = vec![0; total_rows];
429        let leaf = make_int32_page_reader(&data, &levels, &levels, 0, 0, None);
430        let struct_type = ArrowType::Struct(Fields::from(vec![Field::new(
431            "c0",
432            ArrowType::Int32,
433            false,
434        )]));
435        let struct_reader = StructArrayReader::new(struct_type, vec![leaf], 0, 0, false, None);
436
437        let mut offset = 0usize;
438        let mut predicate = ArrowPredicateFn::new(ProjectionMask::all(), move |batch| {
439            let end = offset + batch.num_rows();
440            let filter = BooleanArray::from(pattern[offset..end].to_vec());
441            offset = end;
442            Ok(filter)
443        });
444        let options = PredicateOptions::new(Box::new(struct_reader), &mut predicate);
445        let options = match limit {
446            Some(limit) => options.with_limit(limit, total_rows),
447            None => options,
448        };
449
450        ReadPlanBuilder::new(batch_size)
451            .with_predicate_options(options)
452            .unwrap()
453    }
454
455    fn first_n_matches(pattern: &[bool], limit: usize) -> Vec<bool> {
456        let mut remaining = limit;
457        pattern
458            .iter()
459            .map(|selected| {
460                if *selected && remaining != 0 {
461                    remaining -= 1;
462                    true
463                } else {
464                    false
465                }
466            })
467            .collect()
468    }
469
470    fn assert_limit_case(name: &str, pattern: Vec<bool>, batch_size: usize, limit: usize) {
471        let expected_bits = first_n_matches(&pattern, limit);
472        let expected = RowSelection::from_filters(&[BooleanArray::from(expected_bits)]);
473        let builder = predicate_plan(pattern, batch_size, Some(limit));
474        let actual = builder
475            .selection()
476            .unwrap_or_else(|| panic!("{name}: limited mixed predicate must produce a selection"));
477
478        assert_eq!(actual, &expected, "{name}: logical selection");
479
480        let current_strategy = expected.auto_selection_strategy(DEFAULT_AUTO_THRESHOLD);
481        assert_eq!(
482            builder.resolve_selection_strategy(),
483            current_strategy,
484            "{name}: Auto strategy"
485        );
486        assert_eq!(
487            actual.as_mask().is_some(),
488            current_strategy == RowSelectionStrategy::Mask,
489            "{name}: backing selected by capped Auto"
490        );
491    }
492
493    #[test]
494    fn preferred_selection_strategy_prefers_mask_by_default() {
495        let selection = RowSelection::from(vec![RowSelector::select(8)]);
496        let builder = builder_with_selection(selection);
497        assert_eq!(
498            builder.resolve_selection_strategy(),
499            RowSelectionStrategy::Mask
500        );
501    }
502
503    #[test]
504    fn preferred_selection_strategy_prefers_selectors_when_threshold_small() {
505        let selection = RowSelection::from(vec![RowSelector::select(8)]);
506        let builder = builder_with_selection(selection)
507            .with_row_selection_policy(RowSelectionPolicy::Auto { threshold: 1 });
508        assert_eq!(
509            builder.resolve_selection_strategy(),
510            RowSelectionStrategy::Selectors
511        );
512    }
513
514    #[test]
515    fn preferred_selection_strategy_handles_dense_mask_backing() {
516        let bits: Vec<_> = (0..16).map(|i| i % 2 == 0).collect();
517        let selection = RowSelection::from_boolean_buffer(BooleanBuffer::from(bits));
518        let builder = builder_with_selection(selection)
519            .with_row_selection_policy(RowSelectionPolicy::Auto { threshold: 4 });
520        assert_eq!(
521            builder.resolve_selection_strategy(),
522            RowSelectionStrategy::Mask
523        );
524    }
525
526    #[test]
527    fn preferred_selection_strategy_handles_sparse_mask_backing() {
528        let bits: Vec<_> = (0..128).map(|i| i < 64).collect();
529        let selection = RowSelection::from_boolean_buffer(BooleanBuffer::from(bits));
530        let builder = builder_with_selection(selection)
531            .with_row_selection_policy(RowSelectionPolicy::Auto { threshold: 32 });
532        assert_eq!(
533            builder.resolve_selection_strategy(),
534            RowSelectionStrategy::Selectors
535        );
536    }
537
538    #[test]
539    fn preferred_selection_strategy_preserves_mask_threshold_boundaries() {
540        let mask = BooleanBuffer::from(vec![true; 8]);
541
542        let empty = builder_with_selection(RowSelection::from_boolean_buffer(
543            BooleanBuffer::new_unset(0),
544        ));
545        assert_eq!(
546            empty.resolve_selection_strategy(),
547            RowSelectionStrategy::Mask
548        );
549
550        let disabled = builder_with_selection(RowSelection::from_boolean_buffer(mask.clone()))
551            .with_row_selection_policy(RowSelectionPolicy::Auto { threshold: 0 });
552        assert_eq!(
553            disabled.resolve_selection_strategy(),
554            RowSelectionStrategy::Selectors
555        );
556
557        // Equality does not satisfy the policy's strict inequality: 8 < 1 * 8.
558        let equal = builder_with_selection(RowSelection::from_boolean_buffer(mask.clone()))
559            .with_row_selection_policy(RowSelectionPolicy::Auto { threshold: 8 });
560        assert_eq!(
561            equal.resolve_selection_strategy(),
562            RowSelectionStrategy::Selectors
563        );
564
565        let above = builder_with_selection(RowSelection::from_boolean_buffer(mask))
566            .with_row_selection_policy(RowSelectionPolicy::Auto { threshold: 9 });
567        assert_eq!(
568            above.resolve_selection_strategy(),
569            RowSelectionStrategy::Mask
570        );
571    }
572
573    #[test]
574    #[cfg_attr(miri, ignore)] // Takes too long
575    fn preferred_selection_strategy_mask_matches_selector_backing() {
576        use rand::{RngExt, rng};
577
578        let mut rand = rng();
579        for _ in 0..200 {
580            let len = rand.random_range(0..256);
581            let bits: Vec<_> = (0..len).map(|_| rand.random_bool(0.5)).collect();
582            let mask_backed = RowSelection::from_boolean_buffer(BooleanBuffer::from(bits.clone()));
583            let selector_backed = RowSelection::from_filters(&[BooleanArray::from(bits)]);
584
585            for threshold in [0, 1, 2, 8, 32, 64, usize::MAX] {
586                assert_eq!(
587                    mask_backed.auto_selection_strategy(threshold),
588                    selector_backed.auto_selection_strategy(threshold),
589                    "strategy differs for len {len} and threshold {threshold}"
590                );
591            }
592        }
593    }
594
595    #[test]
596    fn mask_plan_trims_trailing_skips_before_chunking() {
597        let mut plan = ReadPlanBuilder::new(8)
598            .with_selection(Some(RowSelection::from(vec![
599                RowSelector::select(1),
600                RowSelector::skip(7),
601            ])))
602            .with_row_selection_policy(RowSelectionPolicy::Mask)
603            .build();
604        let RowSelectionCursor::Mask(cursor) = plan.row_selection_cursor_mut() else {
605            panic!("expected a Mask cursor");
606        };
607
608        let chunk = cursor.next_chunk(8).unwrap();
609        assert_eq!(chunk.chunk_rows, 1);
610        assert_eq!(chunk.selected_rows, 1);
611        assert!(cursor.is_empty());
612    }
613
614    #[test]
615    fn selectors_policy_lowers_mask_backed_selection() {
616        let selection = RowSelection::from_boolean_buffer(BooleanBuffer::from(vec![
617            true, false, false, true, true,
618        ]));
619        let mut plan = ReadPlanBuilder::new(4)
620            .with_selection(Some(selection))
621            .with_row_selection_policy(RowSelectionPolicy::Selectors)
622            .build();
623        let RowSelectionCursor::Selectors(cursor) = plan.row_selection_cursor_mut() else {
624            panic!("expected a Selectors cursor");
625        };
626
627        assert_eq!(cursor.next_selector(), RowSelector::select(1));
628        assert_eq!(cursor.next_selector(), RowSelector::skip(2));
629        assert_eq!(cursor.next_selector(), RowSelector::select(2));
630        assert!(cursor.is_empty());
631    }
632
633    #[test]
634    fn mask_backed_plan_respects_loaded_row_ranges() {
635        // Start from a non-byte-aligned BooleanBuffer::slice so the mask-backed
636        // cursor path (new_mask_from_buffer) is exercised with a bit offset.
637        // Rows 0 and 11 of 12 are selected; pages for rows [4, 10) are unloaded.
638        let mut bits = vec![false; 5];
639        bits.push(true); // row 0
640        bits.extend(std::iter::repeat_n(false, 10)); // rows 1..=10
641        bits.push(true); // row 11
642        bits.extend([true, false, true]); // trailing padding outside the slice
643        let mask = BooleanBuffer::from(bits).slice(5, 12);
644        let selection = RowSelection::from_boolean_buffer(mask);
645        assert!(selection.as_mask().is_some());
646
647        let loaded = LoadedRowRanges::from_selection(RowSelection::from(vec![
648            RowSelector::select(4),
649            RowSelector::skip(6),
650            RowSelector::select(2),
651        ]));
652
653        let mut plan = ReadPlanBuilder::new(12)
654            .with_selection(Some(selection))
655            .with_row_selection_policy(RowSelectionPolicy::Mask)
656            .with_loaded_row_ranges(Some(loaded))
657            .build();
658        let RowSelectionCursor::Mask(cursor) = plan.row_selection_cursor_mut() else {
659            panic!("expected a Mask cursor");
660        };
661
662        // The first chunk stops at its final selected row instead of carrying
663        // trailing skipped rows to the loaded range boundary.
664        let first = cursor.next_chunk(12).unwrap();
665        assert_eq!(first.initial_skip, 0);
666        assert_eq!(first.chunk_rows, 1);
667        assert_eq!(first.selected_rows, 1);
668
669        // The second chunk skips directly to the next selected row.
670        let second = cursor.next_chunk(12).unwrap();
671        assert_eq!(second.initial_skip, 10);
672        assert_eq!(second.chunk_rows, 1);
673        assert_eq!(second.selected_rows, 1);
674        assert!(cursor.is_empty());
675    }
676
677    #[test]
678    fn with_predicate_options_capped_auto_preserves_limit_and_padding_boundaries() {
679        let fragmented_early_limit = (0..37)
680            .map(|row| matches!(row, 0 | 3 | 7 | 9 | 12 | 18 | 24 | 36))
681            .collect();
682        assert_limit_case("fragmented early limit", fragmented_early_limit, 16, 3);
683
684        assert_limit_case(
685            "selector-friendly padded tail",
686            vec![true; 4_097],
687            1_024,
688            1_024,
689        );
690    }
691
692    #[test]
693    fn with_predicate_options_limit_pads_tail_when_no_prior_selection() {
694        use crate::arrow::ProjectionMask;
695        use crate::arrow::array_reader::StructArrayReader;
696        use crate::arrow::array_reader::test_util::make_int32_page_reader;
697        use crate::arrow::arrow_reader::ArrowPredicateFn;
698        use arrow_schema::{DataType as ArrowType, Field, Fields};
699
700        // 100 rows, all match the predicate. Limit stops the loop after 10
701        // matches — but the resulting RowSelection must still describe the
702        // full 100-row row group (90 trailing rows as "not selected"), not
703        // only the 10 rows we happened to evaluate before breaking.
704        const TOTAL_ROWS: usize = 100;
705        const LIMIT: usize = 10;
706
707        let data: Vec<i32> = (0..TOTAL_ROWS as i32).collect();
708        let levels = vec![0; TOTAL_ROWS];
709        let leaf = make_int32_page_reader(&data, &levels, &levels, 0, 0, None);
710        let struct_type = ArrowType::Struct(Fields::from(vec![Field::new(
711            "c0",
712            ArrowType::Int32,
713            false,
714        )]));
715        let struct_reader = StructArrayReader::new(struct_type, vec![leaf], 0, 0, false, None);
716
717        let mut predicate = ArrowPredicateFn::new(ProjectionMask::all(), |batch| {
718            Ok(BooleanArray::from(vec![true; batch.num_rows()]))
719        });
720
721        let builder = ReadPlanBuilder::new(16)
722            .with_predicate_options(
723                PredicateOptions::new(Box::new(struct_reader), &mut predicate)
724                    .with_limit(LIMIT, TOTAL_ROWS),
725            )
726            .unwrap();
727
728        let selection = builder
729            .selection()
730            .expect("limit-driven early break must produce a selection");
731
732        // `row_count` counts selected rows — must equal the limit.
733        assert_eq!(selection.row_count(), LIMIT);
734
735        // Total rows covered (selects + skips) must equal the full row group
736        // so downstream offset/limit math stays in absolute-row space.
737        assert_eq!(
738            selection.total_row_count(),
739            TOTAL_ROWS,
740            "selection must span the full row group, not only the prefix evaluated before the limit"
741        );
742    }
743
744    #[test]
745    fn with_predicate_options_preserves_mask_selection() {
746        use crate::arrow::ProjectionMask;
747        use crate::arrow::array_reader::StructArrayReader;
748        use crate::arrow::array_reader::test_util::make_int32_page_reader;
749        use crate::arrow::arrow_reader::ArrowPredicateFn;
750        use arrow_schema::{DataType as ArrowType, Field, Fields};
751
752        let data: Vec<i32> = (0..6).collect();
753        let levels = vec![0; data.len()];
754        let leaf = make_int32_page_reader(&data, &levels, &levels, 0, 0, None);
755        let struct_type = ArrowType::Struct(Fields::from(vec![Field::new(
756            "c0",
757            ArrowType::Int32,
758            false,
759        )]));
760        let struct_reader = StructArrayReader::new(struct_type, vec![leaf], 0, 0, false, None);
761
762        let prior = RowSelection::from_boolean_buffer(BooleanBuffer::from(vec![
763            true, false, true, true, false, true,
764        ]));
765        let mut filters = vec![BooleanArray::from(vec![true, false, true, false])];
766        let mut predicate = ArrowPredicateFn::new(ProjectionMask::all(), move |batch| {
767            assert_eq!(batch.num_rows(), 4);
768            Ok(filters.remove(0))
769        });
770
771        let builder = ReadPlanBuilder::new(16)
772            .with_selection(Some(prior))
773            .with_predicate_options(PredicateOptions::new(
774                Box::new(struct_reader),
775                &mut predicate,
776            ))
777            .unwrap();
778
779        let selection = builder.selection().unwrap();
780        assert!(selection.as_mask().is_some());
781
782        let expected = RowSelection::from_boolean_buffer(BooleanBuffer::from(vec![
783            true, false, false, true, false, false,
784        ]));
785        assert_eq!(selection, &expected);
786    }
787
788    #[test]
789    fn with_predicate_options_limit_handles_null_filters() {
790        use crate::arrow::ProjectionMask;
791        use crate::arrow::array_reader::StructArrayReader;
792        use crate::arrow::array_reader::test_util::make_int32_page_reader;
793        use crate::arrow::arrow_reader::ArrowPredicateFn;
794        use arrow_schema::{DataType as ArrowType, Field, Fields};
795
796        const TOTAL_ROWS: usize = 100;
797        const LIMIT: usize = 10;
798
799        let data: Vec<i32> = (0..TOTAL_ROWS as i32).collect();
800        let levels = vec![0; TOTAL_ROWS];
801        let leaf = make_int32_page_reader(&data, &levels, &levels, 0, 0, None);
802        let struct_type = ArrowType::Struct(Fields::from(vec![Field::new(
803            "c0",
804            ArrowType::Int32,
805            false,
806        )]));
807        let struct_reader = StructArrayReader::new(struct_type, vec![leaf], 0, 0, false, None);
808
809        let mut predicate = ArrowPredicateFn::new(ProjectionMask::all(), |batch| {
810            Ok((0..batch.num_rows())
811                .map(|i| match i % 4 {
812                    0 | 2 => Some(true),
813                    1 => None,
814                    _ => Some(false),
815                })
816                .collect::<BooleanArray>())
817        });
818
819        let builder = ReadPlanBuilder::new(16)
820            .with_predicate_options(
821                PredicateOptions::new(Box::new(struct_reader), &mut predicate)
822                    .with_limit(LIMIT, TOTAL_ROWS),
823            )
824            .unwrap();
825
826        let selection = builder
827            .selection()
828            .expect("limit-driven early break must produce a selection");
829
830        assert_eq!(selection.row_count(), LIMIT);
831
832        assert_eq!(selection.total_row_count(), TOTAL_ROWS);
833    }
834}