1mod data;
6mod filter;
7mod stages;
8
9use crate::arrow::ProjectionMask;
10use crate::arrow::array_reader::{ArrayReaderBuilder, CacheOptions, RowGroupCache};
11use crate::arrow::arrow_reader::metrics::ArrowReaderMetrics;
12use crate::arrow::arrow_reader::selection::{LoadedRowRanges, RowSelectionStrategy};
13use crate::arrow::arrow_reader::{
14 ParquetRecordBatchReader, PredicateOptions, ReadPlanBuilder, RowFilter, RowSelection,
15 RowSelectionPolicy,
16};
17use crate::arrow::in_memory_row_group::ColumnChunkData;
18use crate::arrow::push_decoder::reader_builder::data::DataRequestBuilder;
19use crate::arrow::push_decoder::reader_builder::filter::CacheInfo;
20use crate::arrow::push_decoder::scan_plan::{
21 BudgetedReadPlan, RowBudget, RowGroupFrontier, ScanPlanBuilder,
22};
23use crate::arrow::schema::ParquetField;
24use crate::errors::ParquetError;
25use crate::file::metadata::page_index::RowGroupPageIndex;
26use crate::file::metadata::{ColumnChunkMetaData, ParquetMetaData};
27use crate::util::push_buffers::PushBuffers;
28use bytes::Bytes;
29use data::DataRequest;
30use filter::AdvanceResult;
31use filter::FilterInfo;
32pub(crate) use stages::{Stage, StageSchedule};
33use std::ops::Range;
34use std::sync::{Arc, RwLock};
35
36#[derive(Debug)]
38struct RowGroupInfo {
39 row_group_idx: usize,
40 row_count: usize,
41 plan_builder: ReadPlanBuilder,
42 budget: RowBudget,
43}
44
45#[derive(Debug)]
47enum RowGroupDecoderState {
48 Start {
49 row_group_info: RowGroupInfo,
50 },
51 Filters {
53 row_group_info: RowGroupInfo,
54 column_chunks: Option<Vec<Option<Arc<ColumnChunkData>>>>,
56 filter_info: FilterInfo,
57 },
58 WaitingOnFilterData {
60 row_group_info: RowGroupInfo,
61 filter_info: FilterInfo,
62 data_request: DataRequest,
63 },
64 StartData {
66 row_group_info: RowGroupInfo,
67 column_chunks: Option<Vec<Option<Arc<ColumnChunkData>>>>,
69 cache_info: Option<CacheInfo>,
71 },
72 WaitingOnData {
74 row_group_info: RowGroupInfo,
75 data_request: DataRequest,
76 cache_info: Option<CacheInfo>,
78 },
79 Finished,
81}
82
83impl RowGroupDecoderState {
84 fn row_group_idx(&self) -> Option<usize> {
86 match self {
87 Self::Start { row_group_info }
88 | Self::Filters { row_group_info, .. }
89 | Self::WaitingOnFilterData { row_group_info, .. }
90 | Self::StartData { row_group_info, .. }
91 | Self::WaitingOnData { row_group_info, .. } => Some(row_group_info.row_group_idx),
92 Self::Finished => None,
93 }
94 }
95}
96
97pub(crate) fn column_chunk_range(column: &ColumnChunkMetaData) -> Range<u64> {
99 let (start, length) = column.byte_range();
100 start..start + length
101}
102
103#[derive(Debug)]
104pub(crate) enum RowGroupBuildResult {
105 Finished {
107 remaining_budget: RowBudget,
109 },
110 NeedsData(Vec<Range<u64>>),
112 Data {
114 batch_reader: ParquetRecordBatchReader,
115 remaining_budget: RowBudget,
117 },
118}
119
120#[derive(Debug)]
122struct NextState {
123 next_state: RowGroupDecoderState,
124 result: Option<RowGroupBuildResult>,
129}
130
131impl NextState {
132 fn again(next_state: RowGroupDecoderState) -> Self {
136 Self {
137 next_state,
138 result: None,
139 }
140 }
141
142 fn result(next_state: RowGroupDecoderState, result: RowGroupBuildResult) -> Self {
144 Self {
145 next_state,
146 result: Some(result),
147 }
148 }
149}
150
151#[derive(Debug)]
157pub(crate) struct RowGroupReaderBuilder {
158 batch_size: usize,
160
161 projection: ProjectionMask,
163
164 metadata: Arc<ParquetMetaData>,
166
167 fields: Option<Arc<ParquetField>>,
169
170 filter: Option<RowFilter>,
172
173 max_predicate_cache_size: usize,
177
178 metrics: ArrowReaderMetrics,
180
181 row_selection_policy: RowSelectionPolicy,
183
184 state: Option<RowGroupDecoderState>,
189
190 buffers: PushBuffers,
192
193 stages: Arc<StageSchedule>,
196}
197
198#[derive(Debug)]
204pub(crate) struct RowGroupReaderBuilderParts {
205 pub batch_size: usize,
206 pub projection: ProjectionMask,
207 pub fields: Option<Arc<ParquetField>>,
208 pub filter: Option<RowFilter>,
209 pub max_predicate_cache_size: usize,
210 pub metrics: ArrowReaderMetrics,
211 pub row_selection_policy: RowSelectionPolicy,
212 pub buffers: PushBuffers,
215}
216
217impl RowGroupReaderBuilder {
218 #[expect(clippy::too_many_arguments)]
220 pub(crate) fn new(
221 batch_size: usize,
222 projection: ProjectionMask,
223 metadata: Arc<ParquetMetaData>,
224 fields: Option<Arc<ParquetField>>,
225 filter: Option<RowFilter>,
226 metrics: ArrowReaderMetrics,
227 max_predicate_cache_size: usize,
228 buffers: PushBuffers,
229 row_selection_policy: RowSelectionPolicy,
230 ) -> Self {
231 let (predicate_projections, cache_projection) = match &filter {
232 Some(filter) => (
233 filter
234 .predicates
235 .iter()
236 .map(|predicate| predicate.projection().clone())
237 .collect(),
238 Self::compute_cache_projection_inner(
239 filter,
240 &projection,
241 &metadata,
242 max_predicate_cache_size,
243 ),
244 ),
245 None => (vec![], None),
246 };
247 let stages =
248 StageSchedule::new(projection.clone(), predicate_projections, cache_projection);
249 Self {
250 batch_size,
251 projection,
252 metadata,
253 fields,
254 filter,
255 metrics,
256 max_predicate_cache_size,
257 row_selection_policy,
258 state: Some(RowGroupDecoderState::Finished),
259 buffers,
260 stages: Arc::new(stages),
261 }
262 }
263
264 pub(crate) fn into_parts(self) -> RowGroupReaderBuilderParts {
268 let Self {
271 batch_size,
272 projection,
273 metadata: _,
274 fields,
275 filter,
276 max_predicate_cache_size,
277 metrics,
278 row_selection_policy,
279 state: _,
280 buffers,
281 stages: _,
283 } = self;
284 RowGroupReaderBuilderParts {
285 batch_size,
286 projection,
287 fields,
288 filter,
289 max_predicate_cache_size,
290 metrics,
291 row_selection_policy,
292 buffers,
293 }
294 }
295
296 pub fn push_data(
298 &mut self,
299 ranges: Vec<Range<u64>>,
300 buffers: Vec<Bytes>,
301 ) -> Result<(), ParquetError> {
302 self.buffers.push_ranges(ranges, buffers)
303 }
304
305 pub(crate) fn is_finished(&self) -> bool {
310 matches!(self.state, Some(RowGroupDecoderState::Finished))
311 }
312
313 pub fn buffered_bytes(&self) -> u64 {
315 self.buffers.buffered_bytes()
316 }
317
318 pub fn clear_all_ranges(&mut self) {
320 self.buffers.clear_all_ranges();
321 }
322
323 fn take_state(&mut self) -> Result<RowGroupDecoderState, ParquetError> {
331 self.state.take().ok_or_else(|| {
332 ParquetError::General(String::from(
333 "Internal Error: RowGroupReader in invalid state",
334 ))
335 })
336 }
337
338 pub(crate) fn has_active_row_group(&self) -> bool {
340 !matches!(self.state, Some(RowGroupDecoderState::Finished))
341 }
342
343 pub(crate) fn next_row_group(
345 &mut self,
346 row_group_idx: usize,
347 row_count: usize,
348 selection: Option<RowSelection>,
349 budget: RowBudget,
350 ) -> Result<(), ParquetError> {
351 let state = self.take_state()?;
352 if !matches!(state, RowGroupDecoderState::Finished) {
353 return Err(ParquetError::General(format!(
354 "Internal Error: next_row_group called while still reading a row group. Expected Finished state, got {state:?}"
355 )));
356 }
357 let plan_builder = ReadPlanBuilder::new(self.batch_size)
358 .with_selection(selection)
359 .with_row_selection_policy(self.row_selection_policy);
360
361 let row_group_info = RowGroupInfo {
362 row_group_idx,
363 row_count,
364 plan_builder,
365 budget,
366 };
367
368 self.state = Some(RowGroupDecoderState::Start { row_group_info });
369 Ok(())
370 }
371
372 pub(crate) fn try_build(&mut self) -> Result<RowGroupBuildResult, ParquetError> {
379 loop {
380 let current_state = self.take_state()?;
381 match self.try_transition(current_state)? {
383 NextState {
385 next_state,
386 result: Some(result),
387 } => {
388 self.state = Some(next_state);
390 return Ok(result);
391 }
392 NextState {
394 next_state,
395 result: None,
396 } => {
397 self.state = Some(next_state);
399 }
400 }
401 }
402 }
403
404 fn try_transition(
414 &mut self,
415 current_state: RowGroupDecoderState,
416 ) -> Result<NextState, ParquetError> {
417 let result = match current_state {
418 RowGroupDecoderState::Start { row_group_info } => {
419 debug_assert!(
420 !row_group_info.budget.is_exhausted(),
421 "RowGroupFrontier should not hand off row groups after the output limit is exhausted"
422 );
423
424 let column_chunks = None; let Some(filter) = self.filter.take() else {
427 return Ok(NextState::again(RowGroupDecoderState::StartData {
429 row_group_info,
430 column_chunks,
431 cache_info: None,
432 }));
433 };
434 if filter.predicates.is_empty() {
436 return Ok(NextState::again(RowGroupDecoderState::StartData {
437 row_group_info,
438 column_chunks,
439 cache_info: None,
440 }));
441 }
442
443 let cache_projection =
445 self.compute_cache_projection(row_group_info.row_group_idx, &filter);
446
447 let cache_info = CacheInfo::new(
448 cache_projection,
449 Arc::new(RwLock::new(RowGroupCache::new(
450 self.batch_size,
451 self.max_predicate_cache_size,
452 ))),
453 );
454
455 let filter_info = FilterInfo::new(filter, cache_info);
456 NextState::again(RowGroupDecoderState::Filters {
457 row_group_info,
458 filter_info,
459 column_chunks,
460 })
461 }
462 RowGroupDecoderState::Filters {
464 row_group_info,
465 column_chunks,
466 filter_info,
467 } => {
468 let RowGroupInfo {
469 row_group_idx,
470 row_count,
471 plan_builder,
472 budget,
473 } = row_group_info;
474
475 if !plan_builder.selects_any() {
477 self.filter = Some(filter_info.into_filter());
479 return Ok(NextState::result(
480 RowGroupDecoderState::Finished,
481 RowGroupBuildResult::Finished {
482 remaining_budget: budget,
483 },
484 ));
485 }
486
487 let fetch = self.stages.fetch(Stage::Predicate(filter_info.index()));
489
490 let data_request = DataRequestBuilder::new(
493 row_group_idx,
494 row_count,
495 self.batch_size,
496 &self.metadata,
497 fetch.projection, )
499 .with_selection(plan_builder.selection())
500 .with_cache_projection(fetch.cache_projection)
504 .with_column_chunks(column_chunks)
505 .build();
506
507 let row_group_info = RowGroupInfo {
508 row_group_idx,
509 row_count,
510 plan_builder,
511 budget,
512 };
513
514 NextState::again(RowGroupDecoderState::WaitingOnFilterData {
515 row_group_info,
516 filter_info,
517 data_request,
518 })
519 }
520 RowGroupDecoderState::WaitingOnFilterData {
521 row_group_info,
522 data_request,
523 mut filter_info,
524 } => {
525 let needed_ranges = data_request.needed_ranges(&self.buffers);
527 if !needed_ranges.is_empty() {
528 return Ok(NextState::result(
530 RowGroupDecoderState::WaitingOnFilterData {
531 row_group_info,
532 filter_info,
533 data_request,
534 },
535 RowGroupBuildResult::NeedsData(needed_ranges),
536 ));
537 }
538
539 let RowGroupInfo {
541 row_group_idx,
542 row_count,
543 mut plan_builder,
544 budget,
545 } = row_group_info;
546
547 let predicate = filter_info.current();
548
549 let row_group = data_request.try_into_in_memory_row_group(
550 row_group_idx,
551 row_count,
552 &self.metadata,
553 predicate.projection(),
554 &mut self.buffers,
555 )?;
556
557 let cache_options = filter_info.cache_builder().producer();
558
559 let array_reader = ArrayReaderBuilder::new(&row_group, &self.metrics)
560 .with_batch_size(self.batch_size)
561 .with_cache_options(Some(&cache_options))
562 .with_parquet_metadata(&self.metadata)
563 .build_array_reader(self.fields.as_deref(), predicate.projection())?;
564
565 plan_builder = plan_builder.with_row_selection_policy(self.row_selection_policy);
568
569 plan_builder = prepare_selection_for_page_skipping(
571 plan_builder,
572 predicate.projection(),
573 self.row_group_offset_index(row_group_idx),
574 self.metadata.file_metadata().schema_descr().num_columns(),
575 row_count,
576 );
577
578 let predicate_limit = filter_info
582 .is_last()
583 .then(|| budget.selected_row_limit())
584 .flatten();
585
586 let mut predicate_options =
590 PredicateOptions::new(array_reader, filter_info.current_mut());
591 if let Some(limit) = predicate_limit {
592 predicate_options = predicate_options.with_limit(limit, row_count);
593 }
594 plan_builder = plan_builder.with_predicate_options(predicate_options)?;
595
596 let row_group_info = RowGroupInfo {
597 row_group_idx,
598 row_count,
599 plan_builder,
600 budget,
601 };
602
603 let column_chunks = Some(row_group.column_chunks);
605
606 match filter_info.advance() {
608 AdvanceResult::Continue(filter_info) => {
609 NextState::again(RowGroupDecoderState::Filters {
610 row_group_info,
611 column_chunks,
612 filter_info,
613 })
614 }
615 AdvanceResult::Done(filter, cache_info) => {
617 assert!(self.filter.is_none());
619 self.filter = Some(filter);
620 NextState::again(RowGroupDecoderState::StartData {
621 row_group_info,
622 column_chunks,
623 cache_info: Some(cache_info),
624 })
625 }
626 }
627 }
628 RowGroupDecoderState::StartData {
629 row_group_info,
630 column_chunks,
631 cache_info,
632 } => {
633 let RowGroupInfo {
634 row_group_idx,
635 row_count,
636 plan_builder,
637 budget,
638 } = row_group_info;
639
640 let BudgetedReadPlan {
641 mut plan_builder,
642 rows_before_budget,
643 rows_after_budget,
644 remaining_budget,
645 } = budget.apply_to_plan(plan_builder, row_count);
646
647 if rows_before_budget == 0 {
648 return Ok(NextState::result(
650 RowGroupDecoderState::Finished,
651 RowGroupBuildResult::Finished { remaining_budget },
652 ));
653 }
654
655 if rows_after_budget == 0 {
656 return Ok(NextState::result(
658 RowGroupDecoderState::Finished,
659 RowGroupBuildResult::Finished { remaining_budget },
660 ));
661 }
662
663 let fetch = self.stages.fetch(Stage::Projection);
664 let data_request = DataRequestBuilder::new(
665 row_group_idx,
666 row_count,
667 self.batch_size,
668 &self.metadata,
669 fetch.projection,
670 )
671 .with_selection(plan_builder.selection())
672 .with_column_chunks(column_chunks)
673 .with_cache_projection(fetch.cache_projection)
674 .build();
675
676 plan_builder = plan_builder.with_row_selection_policy(self.row_selection_policy);
677
678 plan_builder = prepare_selection_for_page_skipping(
679 plan_builder,
680 &self.projection,
681 self.row_group_offset_index(row_group_idx),
682 self.metadata.file_metadata().schema_descr().num_columns(),
683 row_count,
684 );
685
686 let row_group_info = RowGroupInfo {
687 row_group_idx,
688 row_count,
689 plan_builder,
690 budget: remaining_budget,
691 };
692
693 NextState::again(RowGroupDecoderState::WaitingOnData {
694 row_group_info,
695 data_request,
696 cache_info,
697 })
698 }
699 RowGroupDecoderState::WaitingOnData {
701 row_group_info,
702 data_request,
703 cache_info,
704 } => {
705 let needed_ranges = data_request.needed_ranges(&self.buffers);
706 if !needed_ranges.is_empty() {
707 return Ok(NextState::result(
709 RowGroupDecoderState::WaitingOnData {
710 row_group_info,
711 data_request,
712 cache_info,
713 },
714 RowGroupBuildResult::NeedsData(needed_ranges),
715 ));
716 }
717
718 let RowGroupInfo {
720 row_group_idx,
721 row_count,
722 plan_builder,
723 budget,
724 } = row_group_info;
725
726 let row_group = data_request.try_into_in_memory_row_group(
727 row_group_idx,
728 row_count,
729 &self.metadata,
730 &self.projection,
731 &mut self.buffers,
732 )?;
733
734 let plan = plan_builder.build();
735
736 let array_reader_builder = ArrayReaderBuilder::new(&row_group, &self.metrics)
738 .with_batch_size(self.batch_size)
739 .with_parquet_metadata(&self.metadata);
740 let array_reader = if let Some(cache_info) = cache_info.as_ref() {
741 let cache_options: CacheOptions = cache_info.builder().consumer();
742 array_reader_builder
743 .with_cache_options(Some(&cache_options))
744 .build_array_reader(self.fields.as_deref(), &self.projection)
745 } else {
746 array_reader_builder
747 .build_array_reader(self.fields.as_deref(), &self.projection)
748 }?;
749
750 let reader = ParquetRecordBatchReader::new(array_reader, plan);
751 NextState::result(
752 RowGroupDecoderState::Finished,
753 RowGroupBuildResult::Data {
754 batch_reader: reader,
755 remaining_budget: budget,
756 },
757 )
758 }
759 RowGroupDecoderState::Finished => {
760 return Err(ParquetError::General(String::from(
761 "Internal Error: try_build called without an active row group",
762 )));
763 }
764 };
765 Ok(result)
766 }
767
768 pub(crate) fn active_row_group_idx(&self) -> Option<usize> {
770 self.state
771 .as_ref()
772 .and_then(RowGroupDecoderState::row_group_idx)
773 }
774
775 pub(crate) fn release_row_group(&mut self, row_group_idx: usize) {
780 let ranges: Vec<Range<u64>> = self
781 .metadata
782 .row_group(row_group_idx)
783 .columns()
784 .iter()
785 .map(column_chunk_range)
786 .collect();
787 self.buffers.release_ranges(&ranges);
788 }
789
790 pub(crate) fn release_unread_bytes(&mut self, row_groups: impl IntoIterator<Item = usize>) {
794 if self.buffers.buffered_bytes() == 0 {
795 return;
796 }
797 let mut read_columns = self.projection.clone();
798 if let Some(filter) = &self.filter {
799 for predicate in &filter.predicates {
800 read_columns.union(predicate.projection());
801 }
802 }
803 let mut keep = vec![];
804 for row_group_idx in row_groups {
805 let Some(row_group) = self.metadata.row_groups().get(row_group_idx) else {
806 continue;
807 };
808 keep.extend(
809 row_group
810 .columns()
811 .iter()
812 .enumerate()
813 .filter(|(column_idx, _)| read_columns.leaf_included(*column_idx))
814 .map(|(_, column)| column_chunk_range(column)),
815 );
816 }
817 self.buffers.retain_ranges(&keep);
818 }
819
820 pub(crate) fn scan_plan_builder(&self, frontier: RowGroupFrontier) -> ScanPlanBuilder {
823 ScanPlanBuilder::new(frontier, self.batch_size, Arc::clone(&self.stages))
824 }
825
826 fn compute_cache_projection(&self, row_group_idx: usize, filter: &RowFilter) -> ProjectionMask {
831 let meta = self.metadata.row_group(row_group_idx);
832 let cache_projection = Self::compute_cache_projection_inner(
833 filter,
834 &self.projection,
835 &self.metadata,
836 self.max_predicate_cache_size,
837 );
838 match cache_projection {
839 Some(projection) => projection,
840 None => ProjectionMask::none(meta.columns().len()),
841 }
842 }
843
844 fn compute_cache_projection_inner(
847 filter: &RowFilter,
848 projection: &ProjectionMask,
849 metadata: &ParquetMetaData,
850 max_predicate_cache_size: usize,
851 ) -> Option<ProjectionMask> {
852 if max_predicate_cache_size == 0 {
854 return None;
855 }
856 let mut cache_projection = filter.predicates.first()?.projection().clone();
857 for predicate in &filter.predicates {
858 cache_projection.union(predicate.projection());
859 }
860 cache_projection.intersect(projection);
861 cache_projection.without_nested_types(metadata.file_metadata().schema_descr())
863 }
864
865 fn row_group_offset_index(&self, row_group_idx: usize) -> Option<RowGroupPageIndex> {
867 if self
868 .metadata
869 .page_index()
870 .is_some_and(|pi| pi.has_offset_indexes())
871 {
872 Some(self.metadata.page_index_for_row_group(row_group_idx))
873 } else {
874 None
875 }
876 }
877}
878
879fn prepare_selection_for_page_skipping(
900 plan_builder: ReadPlanBuilder,
901 projection_mask: &ProjectionMask,
902 page_index: Option<RowGroupPageIndex>,
903 num_columns: usize,
904 total_rows: usize,
905) -> ReadPlanBuilder {
906 if plan_builder.selection().is_none() {
910 return plan_builder;
911 }
912
913 match plan_builder.resolve_selection_strategy() {
914 RowSelectionStrategy::Mask => {
915 let loaded = loaded_row_ranges_for_projection(
916 plan_builder.selection(),
917 projection_mask,
918 page_index,
919 num_columns,
920 total_rows,
921 );
922 plan_builder
923 .with_row_selection_policy(RowSelectionPolicy::Mask)
924 .with_loaded_row_ranges(loaded)
925 }
926 RowSelectionStrategy::Selectors => {
927 plan_builder.with_row_selection_policy(RowSelectionPolicy::Selectors)
928 }
929 }
930}
931
932fn loaded_row_ranges_for_projection(
934 selection: Option<&RowSelection>,
935 projection_mask: &ProjectionMask,
936 page_index: Option<RowGroupPageIndex>,
937 num_columns: usize,
938 total_rows: usize,
939) -> Option<LoadedRowRanges> {
940 let selection = selection?;
941 let page_index = page_index?;
942
943 (0..num_columns)
944 .into_iter()
945 .filter_map(|leaf_idx| {
946 let column_metadata = page_index.offset_index(leaf_idx)?;
947 let pages = column_metadata.page_locations();
948 (projection_mask.leaf_included(leaf_idx) && !pages.is_empty()).then(|| {
949 RowSelection::from_consecutive_ranges(
950 selection
951 .row_ranges_for_selected_pages(pages, total_rows)
952 .into_iter(),
953 total_rows,
954 )
955 })
956 })
957 .reduce(|loaded, column| loaded.intersection(&column))
958 .filter(|loaded| loaded.skipped_row_count() != 0)
959 .map(LoadedRowRanges::from_selection)
960}
961
962#[cfg(test)]
963mod tests {
964 use super::*;
965 use crate::arrow::array_reader::StructArrayReader;
966 use crate::arrow::array_reader::test_util::make_int32_page_reader;
967 use crate::arrow::arrow_reader::ArrowPredicateFn;
968 use crate::arrow::arrow_reader::{RowSelection, RowSelector};
969 use crate::file::metadata::page_index::{PageIndexBuilder, PageIndexProvider};
970 use crate::file::page_index::offset_index::{OffsetIndexMetaData, PageLocation};
971 use arrow_array::BooleanArray;
972 use arrow_schema::{DataType as ArrowType, Field, Fields};
973
974 #[test]
975 fn test_structure_size() {
977 assert_eq!(std::mem::size_of::<RowGroupDecoderState>(), 240);
978 }
979
980 #[test]
981 fn test_loaded_row_ranges_intersect_column_page_boundaries() {
982 let mut page_index = PageIndexBuilder::new(1, 2);
983 let column = |first_rows: &[i64]| OffsetIndexMetaData {
984 page_locations: first_rows
985 .iter()
986 .enumerate()
987 .map(|(idx, first_row_index)| PageLocation {
988 offset: (idx * 10) as i64,
989 compressed_page_size: 10,
990 first_row_index: *first_row_index,
991 })
992 .collect(),
993 unencoded_byte_array_data_bytes: None,
994 };
995 page_index
996 .try_put_offset_index(column(&[0, 4, 8]), 0, 0)
997 .unwrap();
998 page_index
999 .try_put_offset_index(column(&[0, 6, 10]), 0, 1)
1000 .unwrap();
1001 let page_index: Option<Arc<dyn PageIndexProvider>> = Some(Arc::new(page_index.build()));
1002 let page_index = RowGroupPageIndex::new(0, page_index);
1003 let selection = RowSelection::from(vec![
1004 RowSelector::skip(1),
1005 RowSelector::select(1),
1006 RowSelector::skip(9),
1007 RowSelector::select(1),
1008 ]);
1009
1010 let loaded = loaded_row_ranges_for_projection(
1011 Some(&selection),
1012 &ProjectionMask::all(),
1013 Some(page_index),
1014 2,
1015 12,
1016 )
1017 .unwrap();
1018
1019 assert_eq!(loaded.ranges(), &[0..4, 10..12]);
1020 }
1021
1022 #[test]
1023 fn test_page_skipping_preparation_preserves_first_predicate_auto_mask() {
1024 let policy = RowSelectionPolicy::Auto { threshold: 4 };
1025 let plan_builder = ReadPlanBuilder::new(4).with_row_selection_policy(policy);
1026
1027 let prepared =
1028 prepare_selection_for_page_skipping(plan_builder, &ProjectionMask::all(), None, 1, 12);
1029 assert_eq!(prepared.row_selection_policy(), &policy);
1030 assert!(prepared.selection().is_none());
1031
1032 let data: Vec<i32> = (0..12).collect();
1033 let levels = vec![0; data.len()];
1034 let leaf = make_int32_page_reader(&data, &levels, &levels, 0, 0, None);
1035 let struct_type = ArrowType::Struct(Fields::from(vec![Field::new(
1036 "c0",
1037 ArrowType::Int32,
1038 false,
1039 )]));
1040 let struct_reader = StructArrayReader::new(struct_type, vec![leaf], 0, 0, false, None);
1041 let mut offset = 0usize;
1042 let mut predicate = ArrowPredicateFn::new(ProjectionMask::all(), move |batch| {
1043 let end = offset + batch.num_rows();
1044 let filter =
1045 BooleanArray::from((offset..end).map(|row| row % 2 == 0).collect::<Vec<_>>());
1046 offset = end;
1047 Ok(filter)
1048 });
1049
1050 let prepared = prepared
1051 .with_predicate_options(PredicateOptions::new(
1052 Box::new(struct_reader),
1053 &mut predicate,
1054 ))
1055 .unwrap();
1056 let selection = prepared.selection().expect("first predicate selection");
1057 let reference = RowSelection::from_filters(&[BooleanArray::from(
1058 (0..12).map(|row| row % 2 == 0).collect::<Vec<_>>(),
1059 )]);
1060
1061 assert_eq!(selection, &reference);
1062 assert!(selection.as_mask().is_some());
1063 }
1064
1065 #[test]
1066 fn test_auto_keeps_mask_when_page_pruning_skips_pages() {
1067 let mut page_index = PageIndexBuilder::new(1, 1);
1068 page_index
1069 .try_put_offset_index(
1070 OffsetIndexMetaData {
1071 page_locations: [0, 2, 4, 6, 8, 10]
1072 .into_iter()
1073 .enumerate()
1074 .map(|(idx, first_row_index)| PageLocation {
1075 offset: (idx * 10) as i64,
1076 compressed_page_size: 10,
1077 first_row_index,
1078 })
1079 .collect(),
1080 unencoded_byte_array_data_bytes: None,
1081 },
1082 0,
1083 0,
1084 )
1085 .unwrap();
1086 let page_index: Option<Arc<dyn PageIndexProvider>> = Some(Arc::new(page_index.build()));
1087 let page_index = RowGroupPageIndex::new(0, page_index);
1088 let selection = RowSelection::from(vec![
1089 RowSelector::select(1),
1090 RowSelector::skip(10),
1091 RowSelector::select(1),
1092 ]);
1093 let plan_builder = ReadPlanBuilder::new(12)
1094 .with_selection(Some(selection))
1095 .with_row_selection_policy(RowSelectionPolicy::Auto { threshold: 32 });
1096
1097 let prepared = prepare_selection_for_page_skipping(
1098 plan_builder,
1099 &ProjectionMask::all(),
1100 Some(page_index),
1101 1,
1102 12,
1103 );
1104
1105 assert_eq!(prepared.row_selection_policy(), &RowSelectionPolicy::Mask);
1106 }
1107}