1mod reader_builder;
9mod remaining;
10mod scan_plan;
11
12use crate::DecodeResult;
13pub use crate::arrow::arrow_reader::RowGroupSelection;
14use crate::arrow::arrow_reader::{
15 ArrowReaderBuilder, ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReader,
16};
17use crate::errors::ParquetError;
18use crate::file::metadata::ParquetMetaData;
19pub use crate::util::push_buffers::PushBuffers;
20use arrow_array::RecordBatch;
21use bytes::Bytes;
22use reader_builder::{RowGroupReaderBuilder, RowGroupReaderBuilderParts};
23use remaining::{RemainingRowGroups, RemainingRowGroupsParts};
24use scan_plan::RowBudget;
25pub use scan_plan::{PageKind, PlannedRange, ScanPlan};
26use std::ops::Range;
27use std::sync::Arc;
28
29pub type ParquetPushDecoderBuilder = ArrowReaderBuilder<PushDecoderInput>;
185
186#[derive(Debug, Default)]
193pub struct PushDecoderInput {
194 buffers: PushBuffers,
196}
197
198impl ParquetPushDecoderBuilder {
201 pub fn try_new_decoder(parquet_metadata: Arc<ParquetMetaData>) -> Result<Self, ParquetError> {
209 Self::try_new_decoder_with_options(parquet_metadata, ArrowReaderOptions::default())
210 }
211
212 pub fn try_new_decoder_with_options(
218 parquet_metadata: Arc<ParquetMetaData>,
219 arrow_reader_options: ArrowReaderOptions,
220 ) -> Result<Self, ParquetError> {
221 let arrow_reader_metadata =
222 ArrowReaderMetadata::try_new(parquet_metadata, arrow_reader_options)?;
223 Ok(Self::new_with_metadata(arrow_reader_metadata))
224 }
225
226 pub fn new_with_metadata(arrow_reader_metadata: ArrowReaderMetadata) -> Self {
231 Self::new_builder(PushDecoderInput::default(), arrow_reader_metadata)
232 }
233
234 pub fn with_buffers(self, buffers: PushBuffers) -> Self {
241 Self {
242 input: PushDecoderInput { buffers },
243 ..self
244 }
245 }
246
247 pub fn with_row_group_selections(
291 mut self,
292 row_group_selections: Vec<RowGroupSelection>,
293 ) -> Self {
294 self.row_group_plan
295 .set_row_group_selections(row_group_selections);
296 self
297 }
298
299 pub fn build(self) -> Result<ParquetPushDecoder, ParquetError> {
301 let Self {
302 input: PushDecoderInput { buffers },
303 metadata: parquet_metadata,
304 schema,
305 fields,
306 batch_size,
307 row_group_plan,
308 projection,
309 filter,
310 limit,
311 offset,
312 metrics,
313 row_selection_policy,
314 max_predicate_cache_size,
315 } = self;
316
317 let filter = filter.map(|filter| {
319 filter.fuse_same_projection(parquet_metadata.file_metadata().schema_descr())
320 });
321
322 let has_predicates = filter
323 .as_ref()
324 .is_some_and(|filter| !filter.predicates.is_empty());
325
326 let row_group_reader_builder = RowGroupReaderBuilder::new(
330 batch_size,
331 projection,
332 Arc::clone(&parquet_metadata),
333 fields,
334 filter,
335 metrics,
336 max_predicate_cache_size,
337 buffers,
338 row_selection_policy,
339 );
340
341 let mut remaining_row_groups = RemainingRowGroups::new(
343 schema,
344 parquet_metadata,
345 row_group_plan,
346 RowBudget::new(offset, limit),
347 has_predicates,
348 row_group_reader_builder,
349 )?;
350 remaining_row_groups.release_unread_bytes();
354
355 Ok(ParquetPushDecoder {
356 state: ParquetDecoderState::ReadingRowGroup {
357 remaining_row_groups: Box::new(remaining_row_groups),
358 },
359 })
360 }
361}
362
363fn builder_from_remaining(parts: RemainingRowGroupsParts) -> ParquetPushDecoderBuilder {
368 let RemainingRowGroupsParts {
369 metadata,
370 schema,
371 row_group_plan,
372 offset,
373 limit,
374 reader_builder,
375 } = parts;
376 let RowGroupReaderBuilderParts {
377 batch_size,
378 projection,
379 fields,
380 filter,
381 max_predicate_cache_size,
382 metrics,
383 row_selection_policy,
384 buffers,
385 } = reader_builder;
386
387 ArrowReaderBuilder {
388 input: PushDecoderInput::default(),
389 metadata,
390 schema,
391 fields,
392 batch_size,
393 row_group_plan,
394 projection,
395 filter,
396 row_selection_policy,
397 limit,
398 offset,
399 metrics,
400 max_predicate_cache_size,
401 }
402 .with_buffers(buffers)
405}
406
407#[derive(Debug)]
424pub struct ParquetPushDecoder {
425 state: ParquetDecoderState,
431}
432
433impl ParquetPushDecoder {
434 pub fn try_decode(&mut self) -> Result<DecodeResult<RecordBatch>, ParquetError> {
465 let current_state = std::mem::replace(&mut self.state, ParquetDecoderState::Finished);
466 let (new_state, decode_result) = current_state.try_next_batch()?;
467 self.state = new_state;
468 Ok(decode_result)
469 }
470
471 pub fn try_next_reader(
510 &mut self,
511 ) -> Result<DecodeResult<ParquetRecordBatchReader>, ParquetError> {
512 let current_state = std::mem::replace(&mut self.state, ParquetDecoderState::Finished);
513 let (new_state, decode_result) = current_state.try_next_reader()?;
514 self.state = new_state;
515 Ok(decode_result)
516 }
517
518 pub fn push_range(&mut self, range: Range<u64>, data: Bytes) -> Result<(), ParquetError> {
537 self.push_ranges(vec![range], vec![data])
538 }
539
540 pub fn push_ranges(
553 &mut self,
554 ranges: Vec<Range<u64>>,
555 data: Vec<Bytes>,
556 ) -> Result<(), ParquetError> {
557 let current_state = std::mem::replace(&mut self.state, ParquetDecoderState::Finished);
558 self.state = current_state.push_data(ranges, data)?;
559 Ok(())
560 }
561
562 pub fn buffered_bytes(&self) -> u64 {
573 self.state.buffered_bytes()
574 }
575
576 pub fn clear_all_ranges(&mut self) {
582 self.state.clear_all_ranges();
583 }
584
585 pub fn is_at_row_group_boundary(&self) -> bool {
596 self.state.is_at_row_group_boundary()
597 }
598
599 pub fn row_groups_remaining(&self) -> usize {
602 self.state.row_groups_remaining()
603 }
604
605 pub fn peek_next_row_group(&self) -> Result<Option<usize>, ParquetError> {
626 self.state.peek_next_row_group()
627 }
628
629 pub fn scan_plan(&self) -> ScanPlan {
632 self.state.scan_plan()
633 }
634
635 pub fn into_builder(self) -> Result<ParquetPushDecoderBuilder, ParquetError> {
693 self.state.into_builder()
694 }
695}
696
697#[derive(Debug)]
699enum ParquetDecoderState {
700 ReadingRowGroup {
702 remaining_row_groups: Box<RemainingRowGroups>,
703 },
704 DecodingRowGroup {
706 record_batch_reader: Box<ParquetRecordBatchReader>,
708 remaining_row_groups: Box<RemainingRowGroups>,
709 },
710 Finished,
712}
713
714impl ParquetDecoderState {
715 fn try_next_reader(
718 self,
719 ) -> Result<(Self, DecodeResult<ParquetRecordBatchReader>), ParquetError> {
720 let mut current_state = self;
721 loop {
722 let (next_state, decode_result) = current_state.transition()?;
723 match decode_result {
725 DecodeResult::NeedsData(ranges) => {
726 return Ok((next_state, DecodeResult::NeedsData(ranges)));
727 }
728 DecodeResult::Data(()) | DecodeResult::Finished => {}
730 }
731 match next_state {
732 Self::ReadingRowGroup { .. } => current_state = next_state,
734 Self::DecodingRowGroup {
736 record_batch_reader,
737 remaining_row_groups,
738 } => {
739 let result = DecodeResult::Data(*record_batch_reader);
740 let next_state = Self::ReadingRowGroup {
741 remaining_row_groups,
742 };
743 return Ok((next_state, result));
744 }
745 Self::Finished => {
746 return Ok((Self::Finished, DecodeResult::Finished));
747 }
748 }
749 }
750 }
751
752 fn try_next_batch(self) -> Result<(Self, DecodeResult<RecordBatch>), ParquetError> {
759 let mut current_state = self;
760 loop {
761 let (new_state, decode_result) = current_state.transition()?;
762 match decode_result {
764 DecodeResult::NeedsData(ranges) => {
765 return Ok((new_state, DecodeResult::NeedsData(ranges)));
766 }
767 DecodeResult::Data(()) | DecodeResult::Finished => {}
769 }
770 match new_state {
771 Self::ReadingRowGroup { .. } => current_state = new_state,
773 Self::DecodingRowGroup {
775 mut record_batch_reader,
776 remaining_row_groups,
777 } => {
778 match record_batch_reader.next() {
779 Some(Ok(batch)) => {
781 let result = DecodeResult::Data(batch);
782 let next_state = Self::DecodingRowGroup {
783 record_batch_reader,
784 remaining_row_groups,
785 };
786 return Ok((next_state, result));
787 }
788 None => {
790 current_state = Self::ReadingRowGroup {
791 remaining_row_groups,
792 }
793 }
794 Some(Err(e)) => {
796 return Err(ParquetError::ArrowError(e.to_string()));
798 }
799 }
800 }
801 Self::Finished => {
802 return Ok((Self::Finished, DecodeResult::Finished));
803 }
804 }
805 }
806 }
807
808 fn transition(self) -> Result<(Self, DecodeResult<()>), ParquetError> {
813 let data_ready = DecodeResult::Data(());
815 match self {
816 Self::ReadingRowGroup {
817 mut remaining_row_groups,
818 } => {
819 match remaining_row_groups.try_next_reader()? {
820 DecodeResult::Data(record_batch_reader) => {
822 Ok((
824 Self::DecodingRowGroup {
825 record_batch_reader: Box::new(record_batch_reader),
826 remaining_row_groups,
827 },
828 data_ready,
829 ))
830 }
831 DecodeResult::NeedsData(ranges) => {
832 Ok((
835 Self::ReadingRowGroup {
836 remaining_row_groups,
837 },
838 DecodeResult::NeedsData(ranges),
839 ))
840 }
841 DecodeResult::Finished => {
843 Ok((Self::Finished, DecodeResult::Finished))
845 }
846 }
847 }
848 Self::DecodingRowGroup { .. } => Ok((self, data_ready)),
850 Self::Finished => Ok((self, DecodeResult::Finished)),
852 }
853 }
854
855 pub fn push_data(
859 self,
860 ranges: Vec<Range<u64>>,
861 data: Vec<Bytes>,
862 ) -> Result<Self, ParquetError> {
863 match self {
864 ParquetDecoderState::ReadingRowGroup {
865 mut remaining_row_groups,
866 } => {
867 remaining_row_groups.push_data(ranges, data)?;
869 Ok(ParquetDecoderState::ReadingRowGroup {
870 remaining_row_groups,
871 })
872 }
873 ParquetDecoderState::DecodingRowGroup {
875 record_batch_reader,
876 mut remaining_row_groups,
877 } => {
878 remaining_row_groups.push_data(ranges, data)?;
879 Ok(ParquetDecoderState::DecodingRowGroup {
880 record_batch_reader,
881 remaining_row_groups,
882 })
883 }
884 ParquetDecoderState::Finished => Err(ParquetError::General(
885 "Cannot push data to a finished decoder".to_string(),
886 )),
887 }
888 }
889
890 fn buffered_bytes(&self) -> u64 {
892 match self {
893 ParquetDecoderState::ReadingRowGroup {
894 remaining_row_groups,
895 } => remaining_row_groups.buffered_bytes(),
896 ParquetDecoderState::DecodingRowGroup {
897 record_batch_reader: _,
898 remaining_row_groups,
899 } => remaining_row_groups.buffered_bytes(),
900 ParquetDecoderState::Finished => 0,
901 }
902 }
903
904 fn clear_all_ranges(&mut self) {
906 match self {
907 ParquetDecoderState::ReadingRowGroup {
908 remaining_row_groups,
909 } => remaining_row_groups.clear_all_ranges(),
910 ParquetDecoderState::DecodingRowGroup {
911 record_batch_reader: _,
912 remaining_row_groups,
913 } => remaining_row_groups.clear_all_ranges(),
914 ParquetDecoderState::Finished => {}
915 }
916 }
917
918 fn is_at_row_group_boundary(&self) -> bool {
919 match self {
920 ParquetDecoderState::ReadingRowGroup {
921 remaining_row_groups,
922 } => remaining_row_groups.is_at_row_group_boundary(),
923 ParquetDecoderState::DecodingRowGroup { .. } => false,
927 ParquetDecoderState::Finished => false,
928 }
929 }
930
931 fn scan_plan(&self) -> ScanPlan {
933 match self {
934 ParquetDecoderState::ReadingRowGroup {
935 remaining_row_groups,
936 }
937 | ParquetDecoderState::DecodingRowGroup {
938 remaining_row_groups,
939 ..
940 } => remaining_row_groups.scan_plan(),
941 ParquetDecoderState::Finished => ScanPlan::empty(),
942 }
943 }
944
945 fn row_groups_remaining(&self) -> usize {
946 match self {
947 ParquetDecoderState::ReadingRowGroup {
948 remaining_row_groups,
949 } => remaining_row_groups.row_groups_remaining(),
950 ParquetDecoderState::DecodingRowGroup {
951 remaining_row_groups,
952 ..
953 } => remaining_row_groups.row_groups_remaining(),
954 ParquetDecoderState::Finished => 0,
955 }
956 }
957
958 fn peek_next_row_group(&self) -> Result<Option<usize>, ParquetError> {
966 match self {
967 ParquetDecoderState::ReadingRowGroup {
968 remaining_row_groups,
969 } => remaining_row_groups.peek_next_row_group(),
970 ParquetDecoderState::DecodingRowGroup {
971 remaining_row_groups,
972 ..
973 } => remaining_row_groups.peek_next_row_group(),
974 ParquetDecoderState::Finished => Ok(None),
975 }
976 }
977
978 fn into_builder(self) -> Result<ParquetPushDecoderBuilder, ParquetError> {
979 let remaining_row_groups = match self {
980 ParquetDecoderState::ReadingRowGroup {
981 remaining_row_groups,
982 } => remaining_row_groups,
983 ParquetDecoderState::DecodingRowGroup { .. } => {
984 return Err(ParquetError::General(
985 "into_builder called while a row group is being decoded; \
986 check is_at_row_group_boundary() first"
987 .to_string(),
988 ));
989 }
990 ParquetDecoderState::Finished => {
991 return Err(ParquetError::General(
992 "into_builder called on a finished decoder".to_string(),
993 ));
994 }
995 };
996 if !remaining_row_groups.is_at_row_group_boundary() {
997 return Err(ParquetError::General(
998 "into_builder called mid-row-group; check is_at_row_group_boundary() first"
999 .to_string(),
1000 ));
1001 }
1002 Ok(builder_from_remaining(remaining_row_groups.into_parts()))
1003 }
1004}
1005
1006#[cfg(test)]
1007mod test {
1008 use super::*;
1009 use crate::DecodeResult;
1010 use crate::arrow::arrow_reader::{ArrowPredicateFn, RowFilter, RowSelection, RowSelector};
1011 use crate::arrow::push_decoder::reader_builder::column_chunk_range;
1012 use crate::arrow::push_decoder::{
1013 ParquetPushDecoder, ParquetPushDecoderBuilder, RowGroupSelection,
1014 };
1015 use crate::arrow::{ArrowWriter, ProjectionMask};
1016 use crate::errors::ParquetError;
1017 use crate::file::metadata::page_index::PageIndexBuilder;
1018 use crate::file::metadata::{PageIndexPolicy, ParquetMetaDataPushDecoder};
1019 use crate::file::properties::WriterProperties;
1020 use arrow::compute::kernels::cmp::{gt, lt};
1021 use arrow_array::cast::AsArray;
1022 use arrow_array::types::Int64Type;
1023 use arrow_array::{ArrayRef, Int64Array, RecordBatch, StringViewArray};
1024 use arrow_buffer::BooleanBuffer;
1025 use arrow_select::concat::concat_batches;
1026 use bytes::Bytes;
1027 use std::fmt::Debug;
1028 use std::ops::Range;
1029 use std::sync::{Arc, LazyLock};
1030
1031 #[test]
1034 fn test_decoder_size() {
1035 assert_eq!(std::mem::size_of::<ParquetDecoderState>(), 24);
1036 }
1037
1038 #[test]
1041 fn test_decoder_all_data() {
1042 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
1043 .unwrap()
1044 .build()
1045 .unwrap();
1046
1047 decoder
1048 .push_range(test_file_range(), TEST_FILE_DATA.clone())
1049 .unwrap();
1050
1051 let results = vec![
1052 expect_data(decoder.try_decode()),
1054 expect_data(decoder.try_decode()),
1056 ];
1057 expect_finished(decoder.try_decode());
1058
1059 let all_output = concat_batches(&TEST_BATCH.schema(), &results).unwrap();
1060 assert_eq!(all_output, *TEST_BATCH);
1062 }
1063
1064 #[test]
1067 fn test_decoder_incremental() {
1068 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
1069 .unwrap()
1070 .build()
1071 .unwrap();
1072
1073 let mut results = vec![];
1074
1075 let ranges = expect_needs_data(decoder.try_decode());
1077 let num_bytes_requested: u64 = ranges.iter().map(|r| r.end - r.start).sum();
1078 push_ranges_to_decoder(&mut decoder, ranges);
1079 assert_eq!(decoder.buffered_bytes(), num_bytes_requested);
1081 results.push(expect_data(decoder.try_decode()));
1082 assert_eq!(decoder.buffered_bytes(), 0);
1084
1085 let ranges = expect_needs_data(decoder.try_decode());
1087 let num_bytes_requested: u64 = ranges.iter().map(|r| r.end - r.start).sum();
1088 push_ranges_to_decoder(&mut decoder, ranges);
1089 assert_eq!(decoder.buffered_bytes(), num_bytes_requested);
1091 results.push(expect_data(decoder.try_decode()));
1092 assert_eq!(decoder.buffered_bytes(), 0);
1094 expect_finished(decoder.try_decode());
1095
1096 let all_output = concat_batches(&TEST_BATCH.schema(), &results).unwrap();
1098 assert_eq!(all_output, *TEST_BATCH);
1099 }
1100
1101 #[test]
1104 fn test_decoder_clear_all_ranges() {
1105 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
1106 .unwrap()
1107 .with_batch_size(100)
1108 .build()
1109 .unwrap();
1110
1111 decoder
1112 .push_range(test_file_range(), TEST_FILE_DATA.clone())
1113 .unwrap();
1114 assert_eq!(decoder.buffered_bytes(), test_file_len());
1115
1116 let batch1 = expect_data(decoder.try_decode());
1120 assert_eq!(batch1, TEST_BATCH.slice(0, 100));
1121 assert_eq!(
1122 decoder.buffered_bytes(),
1123 test_file_len() - test_file_row_group_bytes(0)
1124 );
1125
1126 decoder.clear_all_ranges();
1128 assert_eq!(decoder.buffered_bytes(), 0);
1129
1130 let batch2 = expect_data(decoder.try_decode());
1133 assert_eq!(batch2, TEST_BATCH.slice(100, 100));
1134 assert_eq!(decoder.buffered_bytes(), 0);
1135
1136 let ranges = expect_needs_data(decoder.try_decode());
1139 let num_bytes_requested: u64 = ranges.iter().map(|r| r.end - r.start).sum();
1140 push_ranges_to_decoder(&mut decoder, ranges);
1141 assert_eq!(decoder.buffered_bytes(), num_bytes_requested);
1142
1143 let batch3 = expect_data(decoder.try_decode());
1144 assert_eq!(batch3, TEST_BATCH.slice(200, 100));
1145 assert_eq!(decoder.buffered_bytes(), 0);
1146
1147 let batch4 = expect_data(decoder.try_decode());
1148 assert_eq!(batch4, TEST_BATCH.slice(300, 100));
1149 assert_eq!(decoder.buffered_bytes(), 0);
1150
1151 expect_finished(decoder.try_decode());
1152 }
1153
1154 #[test]
1156 fn test_decoder_partial() {
1157 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
1158 .unwrap()
1159 .build()
1160 .unwrap();
1161
1162 let ranges = expect_needs_data(decoder.try_decode());
1164 push_ranges_to_decoder(&mut decoder, ranges);
1165
1166 let batch1 = expect_data(decoder.try_decode());
1167 let expected1 = TEST_BATCH.slice(0, 200);
1168 assert_eq!(batch1, expected1);
1169
1170 let ranges = expect_needs_data(decoder.try_decode());
1172 let (ranges1, ranges2) = ranges.split_at(ranges.len() / 2);
1173 assert!(!ranges1.is_empty());
1174 assert!(!ranges2.is_empty());
1175 push_ranges_to_decoder(&mut decoder, ranges1.to_vec());
1177
1178 let ranges = expect_needs_data(decoder.try_decode());
1180 assert_eq!(ranges, ranges2); push_ranges_to_decoder(&mut decoder, vec![]);
1183 let ranges = expect_needs_data(decoder.try_decode());
1184 assert_eq!(ranges, ranges2); push_ranges_to_decoder(&mut decoder, ranges);
1186
1187 let batch2 = expect_data(decoder.try_decode());
1188 let expected2 = TEST_BATCH.slice(200, 200);
1189 assert_eq!(batch2, expected2);
1190
1191 expect_finished(decoder.try_decode());
1192 }
1193
1194 #[test]
1198 fn test_decoder_first_pages_only() {
1199 let metadata = test_file_parquet_metadata_with_offset_index();
1200 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(Arc::clone(&metadata))
1201 .unwrap()
1202 .with_batch_size(100)
1205 .build()
1206 .unwrap();
1207
1208 let ranges = expect_needs_data(decoder.try_decode());
1211 assert_eq!(ranges, vec![4..1860, 1860..3716, 3716..11062]);
1212
1213 let page_index = metadata.page_index_for_row_group(0);
1216 let row_group = metadata.row_group(0);
1217 let mut first_page_ranges = vec![];
1218 let mut second_page_ranges = vec![];
1219 for (idx, column) in row_group.columns().iter().enumerate() {
1220 let (start, len) = column.byte_range();
1221 let locations = page_index.page_locations(idx).unwrap();
1222 assert_eq!(locations.len(), 2, "expected 2 data pages per column chunk");
1223 let second_page_start = locations[1].offset as u64;
1224 first_page_ranges.push(start..second_page_start);
1225 second_page_ranges.push(second_page_start..start + len);
1226 }
1227 assert_eq!(first_page_ranges, vec![4..1734, 1860..3590, 3716..10936]);
1229 assert_eq!(
1230 second_page_ranges,
1231 vec![1734..1860, 3590..3716, 10936..11062]
1232 );
1233
1234 push_ranges_to_decoder(&mut decoder, first_page_ranges);
1237
1238 let ranges = expect_needs_data(decoder.try_decode());
1244 assert_eq!(ranges, vec![4..1860, 1860..3716, 3716..11062]);
1245
1246 push_ranges_to_decoder(&mut decoder, second_page_ranges);
1250 let ranges = expect_needs_data(decoder.try_decode());
1251 assert_eq!(ranges, vec![4..1860, 1860..3716, 3716..11062]);
1252
1253 push_ranges_to_decoder(&mut decoder, ranges);
1256 let batch = expect_data(decoder.try_decode());
1257 assert_eq!(batch, TEST_BATCH.slice(0, 100));
1258 let batch = expect_data(decoder.try_decode());
1259 assert_eq!(batch, TEST_BATCH.slice(100, 100));
1260 }
1261
1262 fn decode_with_coalesced_pushes(mut decoder: ParquetPushDecoder) -> Vec<RecordBatch> {
1268 let mut batches = vec![];
1269 loop {
1270 match decoder.try_decode().unwrap() {
1271 DecodeResult::NeedsData(ranges) => {
1272 let start = ranges.iter().map(|r| r.start).min().unwrap();
1273 let end = ranges.iter().map(|r| r.end).max().unwrap();
1274 let coalesced = start..end;
1275 push_ranges_to_decoder(&mut decoder, vec![coalesced]);
1276 }
1277 DecodeResult::Data(batch) => {
1278 assert_eq!(decoder.buffered_bytes(), 0);
1280 batches.push(batch);
1281 }
1282 DecodeResult::Finished => return batches,
1283 }
1284 }
1285 }
1286
1287 #[test]
1290 fn test_decoder_releases_coalesced_push() {
1291 let decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
1292 .unwrap()
1293 .build()
1294 .unwrap();
1295 let batches = decode_with_coalesced_pushes(decoder);
1296 let all_output = concat_batches(&TEST_BATCH.schema(), &batches).unwrap();
1297 assert_eq!(all_output, *TEST_BATCH);
1298 }
1299
1300 #[test]
1303 fn test_decoder_releases_unrequested_bytes_of_row_group() {
1304 let builder =
1305 ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1306 let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1307 let decoder = builder
1308 .with_projection(ProjectionMask::columns(&schema_descr, ["a", "c"]))
1309 .build()
1310 .unwrap();
1311 let batches = decode_with_coalesced_pushes(decoder);
1312 let all_output = concat_batches(&batches[0].schema(), &batches).unwrap();
1313 assert_eq!(all_output, TEST_BATCH.project(&[0, 2]).unwrap());
1314 }
1315
1316 #[test]
1319 fn test_decoder_keeps_bytes_of_a_row_group_that_is_read_again() {
1320 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
1321 .unwrap()
1322 .with_row_groups(vec![0, 1, 0])
1323 .build()
1324 .unwrap();
1325 prefetch_test_file(&mut decoder);
1326 assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(0, 200));
1328 assert_eq!(decoder.buffered_bytes(), test_file_len());
1329
1330 assert_eq!(
1331 expect_data(decoder.try_decode()),
1332 TEST_BATCH.slice(200, 200)
1333 );
1334 assert_eq!(
1335 decoder.buffered_bytes(),
1336 test_file_len() - test_file_row_group_bytes(1)
1337 );
1338
1339 assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(0, 200));
1340 assert_eq!(
1341 decoder.buffered_bytes(),
1342 test_file_len() - test_file_row_group_bytes(0) - test_file_row_group_bytes(1)
1343 );
1344 expect_finished(decoder.try_decode());
1345 }
1346
1347 #[test]
1350 fn test_decoder_releases_coalesced_push_with_predicate() {
1351 let builder =
1352 ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1353 let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1354 let row_filter_a = ArrowPredicateFn::new(
1355 ProjectionMask::columns(&schema_descr, ["a"]),
1356 |batch: RecordBatch| {
1357 let scalar_250 = Int64Array::new_scalar(250);
1358 let column = batch.column(0).as_primitive::<Int64Type>();
1359 gt(column, &scalar_250)
1360 },
1361 );
1362 let decoder = builder
1363 .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
1364 .build()
1365 .unwrap();
1366 let batches = decode_with_coalesced_pushes(decoder);
1367 let all_output = concat_batches(&TEST_BATCH.schema(), &batches).unwrap();
1368 assert_eq!(all_output, TEST_BATCH.slice(251, 149));
1369 }
1370
1371 #[test]
1374 fn test_decoder_selection_does_one_request() {
1375 let builder =
1376 ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1377
1378 let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1379
1380 let mut decoder = builder
1381 .with_projection(
1382 ProjectionMask::columns(&schema_descr, ["a", "b"]), )
1384 .build()
1385 .unwrap();
1386
1387 let ranges = expect_needs_data(decoder.try_decode());
1389 push_ranges_to_decoder(&mut decoder, ranges);
1390
1391 let batch1 = expect_data(decoder.try_decode());
1392 let expected1 = TEST_BATCH.slice(0, 200).project(&[0, 1]).unwrap();
1393 assert_eq!(batch1, expected1);
1394
1395 let ranges = expect_needs_data(decoder.try_decode());
1397 push_ranges_to_decoder(&mut decoder, ranges);
1398
1399 let batch2 = expect_data(decoder.try_decode());
1400 let expected2 = TEST_BATCH.slice(200, 200).project(&[0, 1]).unwrap();
1401 assert_eq!(batch2, expected2);
1402
1403 expect_finished(decoder.try_decode());
1404 }
1405
1406 #[test]
1409 fn test_decoder_single_filter_partial() {
1410 let builder =
1411 ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1412
1413 let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1416
1417 let row_filter_a = ArrowPredicateFn::new(
1419 ProjectionMask::columns(&schema_descr, ["a", "b"]),
1421 |batch: RecordBatch| {
1422 let scalar_250 = Int64Array::new_scalar(250);
1423 let column = batch.column(0).as_primitive::<Int64Type>();
1424 gt(column, &scalar_250)
1425 },
1426 );
1427
1428 let mut decoder = builder
1429 .with_projection(
1430 ProjectionMask::columns(&schema_descr, ["a"]), )
1433 .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
1434 .build()
1435 .unwrap();
1436
1437 let ranges = expect_needs_data(decoder.try_decode());
1439 let (ranges1, ranges2) = ranges.split_at(ranges.len() / 2);
1441 assert!(!ranges1.is_empty());
1442 assert!(!ranges2.is_empty());
1443 push_ranges_to_decoder(&mut decoder, ranges1.to_vec());
1444 let ranges = expect_needs_data(decoder.try_decode());
1446 assert_eq!(ranges, ranges2); let ranges = expect_needs_data(decoder.try_decode());
1448 assert_eq!(ranges, ranges2); push_ranges_to_decoder(&mut decoder, ranges2.to_vec());
1450
1451 let ranges = expect_needs_data(decoder.try_decode());
1456 push_ranges_to_decoder(&mut decoder, ranges);
1457
1458 let batch = expect_data(decoder.try_decode());
1459 let expected = TEST_BATCH.slice(251, 149).project(&[0]).unwrap();
1460 assert_eq!(batch, expected);
1461
1462 expect_finished(decoder.try_decode());
1463 }
1464
1465 #[test]
1467 fn test_decoder_single_filter_and_row_selection() {
1468 let builder =
1469 ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1470
1471 let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1474
1475 let row_filter_a = ArrowPredicateFn::new(
1477 ProjectionMask::columns(&schema_descr, ["a"]),
1478 |batch: RecordBatch| {
1479 let scalar_250 = Int64Array::new_scalar(250);
1480 let column = batch.column(0).as_primitive::<Int64Type>();
1481 gt(column, &scalar_250)
1482 },
1483 );
1484
1485 let mut decoder = builder
1486 .with_projection(
1487 ProjectionMask::columns(&schema_descr, ["b"]), )
1490 .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
1491 .with_row_selection(RowSelection::from(vec![
1492 RowSelector::skip(200), RowSelector::select(100), RowSelector::skip(100),
1495 ]))
1496 .build()
1497 .unwrap();
1498
1499 let ranges = expect_needs_data(decoder.try_decode());
1503 push_ranges_to_decoder(&mut decoder, ranges);
1504
1505 let ranges = expect_needs_data(decoder.try_decode());
1507 push_ranges_to_decoder(&mut decoder, ranges);
1508
1509 let batch = expect_data(decoder.try_decode());
1510 let expected = TEST_BATCH.slice(251, 49).project(&[1]).unwrap();
1511 assert_eq!(batch, expected);
1512
1513 expect_finished(decoder.try_decode());
1514 }
1515
1516 #[test]
1518 fn test_decoder_multi_filters() {
1519 let builder =
1521 ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1522
1523 let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1528
1529 let row_filter_a = ArrowPredicateFn::new(
1531 ProjectionMask::columns(&schema_descr, ["a"]),
1532 |batch: RecordBatch| {
1533 let scalar_175 = Int64Array::new_scalar(175);
1534 let column = batch.column(0).as_primitive::<Int64Type>();
1535 gt(column, &scalar_175)
1536 },
1537 );
1538
1539 let row_filter_b = ArrowPredicateFn::new(
1541 ProjectionMask::columns(&schema_descr, ["b"]),
1542 |batch: RecordBatch| {
1543 let scalar_625 = Int64Array::new_scalar(625);
1544 let column = batch.column(0).as_primitive::<Int64Type>();
1545 lt(column, &scalar_625)
1546 },
1547 );
1548
1549 let mut decoder = builder
1550 .with_projection(
1551 ProjectionMask::columns(&schema_descr, ["c"]), )
1553 .with_row_filter(RowFilter::new(vec![
1554 Box::new(row_filter_a),
1555 Box::new(row_filter_b),
1556 ]))
1557 .build()
1558 .unwrap();
1559
1560 let ranges = expect_needs_data(decoder.try_decode());
1562 push_ranges_to_decoder(&mut decoder, ranges);
1563
1564 let ranges = expect_needs_data(decoder.try_decode());
1566 push_ranges_to_decoder(&mut decoder, ranges);
1567
1568 let ranges = expect_needs_data(decoder.try_decode());
1570 push_ranges_to_decoder(&mut decoder, ranges);
1571
1572 let batch1 = expect_data(decoder.try_decode());
1574 let expected1 = TEST_BATCH.slice(176, 24).project(&[2]).unwrap();
1575 assert_eq!(batch1, expected1);
1576
1577 let ranges = expect_needs_data(decoder.try_decode());
1579 push_ranges_to_decoder(&mut decoder, ranges);
1580
1581 let ranges = expect_needs_data(decoder.try_decode());
1583 push_ranges_to_decoder(&mut decoder, ranges);
1584
1585 let ranges = expect_needs_data(decoder.try_decode());
1587 push_ranges_to_decoder(&mut decoder, ranges);
1588
1589 let batch2 = expect_data(decoder.try_decode());
1591 let expected2 = TEST_BATCH.slice(200, 25).project(&[2]).unwrap();
1592 assert_eq!(batch2, expected2);
1593
1594 expect_finished(decoder.try_decode());
1595 }
1596
1597 #[test]
1599 fn test_decoder_same_projection_filters() {
1600 let builder =
1601 ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1602 let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1603 let projection_a = ProjectionMask::columns(&schema_descr, ["a"]);
1604
1605 let row_filter_gt = ArrowPredicateFn::new(projection_a.clone(), |batch: RecordBatch| {
1606 let column = batch.column(0).as_primitive::<Int64Type>();
1607 gt(column, &Int64Array::new_scalar(175))
1608 });
1609 let row_filter_lt = ArrowPredicateFn::new(projection_a, |batch: RecordBatch| {
1610 let column = batch.column(0).as_primitive::<Int64Type>();
1611 assert!(column.iter().flatten().all(|value| value > 175));
1612 lt(column, &Int64Array::new_scalar(190))
1613 });
1614
1615 let mut decoder = builder
1616 .with_projection(ProjectionMask::columns(&schema_descr, ["c"]))
1617 .with_row_filter(RowFilter::new(vec![
1618 Box::new(row_filter_gt),
1619 Box::new(row_filter_lt),
1620 ]))
1621 .with_batch_size(50)
1622 .build()
1623 .unwrap();
1624
1625 let ranges = expect_needs_data(decoder.try_decode());
1627 push_ranges_to_decoder(&mut decoder, ranges);
1628 let ranges = expect_needs_data(decoder.try_decode());
1629 push_ranges_to_decoder(&mut decoder, ranges);
1630
1631 let batch = expect_data(decoder.try_decode());
1632 let expected = TEST_BATCH.slice(176, 14).project(&[2]).unwrap();
1633 assert_eq!(batch, expected);
1634
1635 let ranges = expect_needs_data(decoder.try_decode());
1638 push_ranges_to_decoder(&mut decoder, ranges);
1639 expect_finished(decoder.try_decode());
1640 }
1641
1642 #[test]
1645 fn test_decoder_same_projection_filters_with_limit() {
1646 use std::sync::atomic::{AtomicUsize, Ordering};
1647
1648 let builder =
1649 ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1650 let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1651 let projection_a = ProjectionMask::columns(&schema_descr, ["a"]);
1652
1653 let rows_evaluated = Arc::new(AtomicUsize::new(0));
1654 let rows_evaluated_for_predicate = Arc::clone(&rows_evaluated);
1655 let row_filter_gt =
1656 ArrowPredicateFn::new(projection_a.clone(), move |batch: RecordBatch| {
1657 rows_evaluated_for_predicate.fetch_add(batch.num_rows(), Ordering::Relaxed);
1658 let column = batch.column(0).as_primitive::<Int64Type>();
1659 gt(column, &Int64Array::new_scalar(175))
1660 });
1661 let row_filter_lt = ArrowPredicateFn::new(projection_a, |batch: RecordBatch| {
1662 let column = batch.column(0).as_primitive::<Int64Type>();
1663 lt(column, &Int64Array::new_scalar(190))
1664 });
1665
1666 let mut decoder = builder
1667 .with_projection(ProjectionMask::columns(&schema_descr, ["c"]))
1668 .with_row_filter(RowFilter::new(vec![
1669 Box::new(row_filter_gt),
1670 Box::new(row_filter_lt),
1671 ]))
1672 .with_batch_size(10)
1673 .with_limit(5)
1674 .build()
1675 .unwrap();
1676
1677 let ranges = expect_needs_data(decoder.try_decode());
1679 push_ranges_to_decoder(&mut decoder, ranges);
1680 let ranges = expect_needs_data(decoder.try_decode());
1681 push_ranges_to_decoder(&mut decoder, ranges);
1682
1683 let batch = expect_data(decoder.try_decode());
1684 let expected = TEST_BATCH.slice(176, 5).project(&[2]).unwrap();
1685 assert_eq!(batch, expected);
1686
1687 expect_finished(decoder.try_decode());
1689
1690 let evaluated = rows_evaluated.load(Ordering::Relaxed);
1693 assert!(evaluated <= 190, "evaluated {evaluated} rows");
1694 }
1695
1696 #[test]
1699 fn test_decoder_reuses_filter_pages() {
1700 let builder =
1702 ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1703
1704 let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1707
1708 let row_filter_a = ArrowPredicateFn::new(
1710 ProjectionMask::columns(&schema_descr, ["a"]),
1711 |batch: RecordBatch| {
1712 let scalar_250 = Int64Array::new_scalar(250);
1713 let column = batch.column(0).as_primitive::<Int64Type>();
1714 gt(column, &scalar_250)
1715 },
1716 );
1717
1718 let mut decoder = builder
1719 .with_projection(
1720 ProjectionMask::columns(&schema_descr, ["a"]), )
1723 .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
1724 .build()
1725 .unwrap();
1726
1727 let ranges = expect_needs_data(decoder.try_decode());
1729 push_ranges_to_decoder(&mut decoder, ranges);
1730
1731 let ranges = expect_needs_data(decoder.try_decode());
1735 push_ranges_to_decoder(&mut decoder, ranges);
1736
1737 let batch = expect_data(decoder.try_decode());
1741 let expected = TEST_BATCH.slice(251, 149).project(&[0]).unwrap();
1742 assert_eq!(batch, expected);
1743
1744 expect_finished(decoder.try_decode());
1745 }
1746
1747 #[test]
1748 fn test_decoder_empty_filters() {
1749 let builder =
1750 ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1751 let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1752
1753 let mut decoder = builder
1755 .with_projection(
1756 ProjectionMask::columns(&schema_descr, ["c"]), )
1758 .with_row_filter(RowFilter::new(vec![
1759 ]))
1761 .build()
1762 .unwrap();
1763
1764 let ranges = expect_needs_data(decoder.try_decode());
1766 push_ranges_to_decoder(&mut decoder, ranges);
1767
1768 let batch1 = expect_data(decoder.try_decode());
1770 let expected1 = TEST_BATCH.slice(0, 200).project(&[2]).unwrap();
1771 assert_eq!(batch1, expected1);
1772
1773 let ranges = expect_needs_data(decoder.try_decode());
1775 push_ranges_to_decoder(&mut decoder, ranges);
1776
1777 let batch2 = expect_data(decoder.try_decode());
1779 let expected2 = TEST_BATCH.slice(200, 200).project(&[2]).unwrap();
1780
1781 assert_eq!(batch2, expected2);
1782
1783 expect_finished(decoder.try_decode());
1784 }
1785
1786 #[test]
1794 fn test_decoder_filter_with_limit_short_circuits_within_row_group() {
1795 use std::sync::atomic::{AtomicUsize, Ordering};
1796
1797 let builder =
1798 ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1799 let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1800
1801 let rows_filtered = Arc::new(AtomicUsize::new(0));
1802 let rows_filtered_for_predicate = Arc::clone(&rows_filtered);
1803
1804 let row_filter_a = ArrowPredicateFn::new(
1805 ProjectionMask::columns(&schema_descr, ["a"]),
1806 move |batch: RecordBatch| {
1807 rows_filtered_for_predicate.fetch_add(batch.num_rows(), Ordering::Relaxed);
1808 let scalar_175 = Int64Array::new_scalar(175);
1809 let column = batch.column(0).as_primitive::<Int64Type>();
1810 gt(column, &scalar_175)
1811 },
1812 );
1813
1814 let mut decoder = builder
1821 .with_projection(ProjectionMask::columns(&schema_descr, ["a"]))
1822 .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
1823 .with_batch_size(10)
1824 .with_limit(10)
1825 .build()
1826 .unwrap();
1827
1828 let ranges = expect_needs_data(decoder.try_decode());
1830 push_ranges_to_decoder(&mut decoder, ranges);
1831
1832 let batch = expect_data(decoder.try_decode());
1834 let expected = TEST_BATCH.slice(176, 10).project(&[0]).unwrap();
1835 assert_eq!(batch, expected);
1836
1837 expect_finished(decoder.try_decode());
1841
1842 let evaluated = rows_filtered.load(Ordering::Relaxed);
1846 assert!(
1847 evaluated <= 190,
1848 "predicate evaluated {evaluated} rows; expected ≤ 190 (stop within batch containing 11th match)"
1849 );
1850 }
1851
1852 #[test]
1856 fn test_decoder_filter_with_limit_skips_later_row_groups() {
1857 let builder =
1858 ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1859 let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1860
1861 let row_filter_a = ArrowPredicateFn::new(
1865 ProjectionMask::columns(&schema_descr, ["a"]),
1866 |batch: RecordBatch| {
1867 let scalar_175 = Int64Array::new_scalar(175);
1868 let column = batch.column(0).as_primitive::<Int64Type>();
1869 gt(column, &scalar_175)
1870 },
1871 );
1872
1873 let mut decoder = builder
1874 .with_projection(ProjectionMask::columns(&schema_descr, ["a"]))
1875 .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
1876 .with_limit(5)
1877 .build()
1878 .unwrap();
1879
1880 let ranges = expect_needs_data(decoder.try_decode());
1882 push_ranges_to_decoder(&mut decoder, ranges);
1883
1884 let batch = expect_data(decoder.try_decode());
1886 let expected = TEST_BATCH.slice(176, 5).project(&[0]).unwrap();
1887 assert_eq!(batch, expected);
1888
1889 expect_finished(decoder.try_decode());
1892 }
1893
1894 #[test]
1905 fn test_decoder_filter_with_offset_and_limit() {
1906 let builder =
1907 ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1908 let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1909
1910 let row_filter_a = ArrowPredicateFn::new(
1911 ProjectionMask::columns(&schema_descr, ["a"]),
1912 |batch: RecordBatch| {
1913 let scalar_175 = Int64Array::new_scalar(175);
1914 let column = batch.column(0).as_primitive::<Int64Type>();
1915 gt(column, &scalar_175)
1916 },
1917 );
1918
1919 let mut decoder = builder
1920 .with_projection(ProjectionMask::columns(&schema_descr, ["a"]))
1921 .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
1922 .with_offset(10)
1923 .with_limit(5)
1924 .build()
1925 .unwrap();
1926
1927 let ranges = expect_needs_data(decoder.try_decode());
1928 push_ranges_to_decoder(&mut decoder, ranges);
1929
1930 let batch = expect_data(decoder.try_decode());
1931 let expected = TEST_BATCH.slice(186, 5).project(&[0]).unwrap();
1932 assert_eq!(batch, expected);
1933
1934 expect_finished(decoder.try_decode());
1935 }
1936
1937 #[test]
1946 fn test_decoder_multi_filters_with_limit() {
1947 use std::sync::atomic::{AtomicUsize, Ordering};
1948
1949 let builder =
1950 ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
1951 let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
1952
1953 let first_predicate_rows = Arc::new(AtomicUsize::new(0));
1954 let second_predicate_rows = Arc::new(AtomicUsize::new(0));
1955
1956 let first_predicate_rows_for_filter = Arc::clone(&first_predicate_rows);
1957 let row_filter_a = ArrowPredicateFn::new(
1958 ProjectionMask::columns(&schema_descr, ["a"]),
1959 move |batch: RecordBatch| {
1960 first_predicate_rows_for_filter.fetch_add(batch.num_rows(), Ordering::Relaxed);
1961 let scalar_175 = Int64Array::new_scalar(175);
1962 let column = batch.column(0).as_primitive::<Int64Type>();
1963 gt(column, &scalar_175)
1964 },
1965 );
1966
1967 let second_predicate_rows_for_filter = Arc::clone(&second_predicate_rows);
1968 let row_filter_b = ArrowPredicateFn::new(
1969 ProjectionMask::columns(&schema_descr, ["b"]),
1970 move |batch: RecordBatch| {
1971 second_predicate_rows_for_filter.fetch_add(batch.num_rows(), Ordering::Relaxed);
1972 let scalar_625 = Int64Array::new_scalar(625);
1973 let column = batch.column(0).as_primitive::<Int64Type>();
1974 lt(column, &scalar_625)
1975 },
1976 );
1977
1978 let mut decoder = builder
1979 .with_projection(ProjectionMask::columns(&schema_descr, ["c"]))
1980 .with_row_filter(RowFilter::new(vec![
1981 Box::new(row_filter_a),
1982 Box::new(row_filter_b),
1983 ]))
1984 .with_batch_size(10)
1985 .with_limit(10)
1986 .build()
1987 .unwrap();
1988
1989 let ranges = expect_needs_data(decoder.try_decode());
1991 push_ranges_to_decoder(&mut decoder, ranges);
1992
1993 let ranges = expect_needs_data(decoder.try_decode());
1995 push_ranges_to_decoder(&mut decoder, ranges);
1996
1997 let ranges = expect_needs_data(decoder.try_decode());
1999 push_ranges_to_decoder(&mut decoder, ranges);
2000
2001 let batch = expect_data(decoder.try_decode());
2002 let expected = TEST_BATCH.slice(176, 10).project(&[2]).unwrap();
2003 assert_eq!(batch, expected);
2004
2005 expect_finished(decoder.try_decode());
2007
2008 assert_eq!(first_predicate_rows.load(Ordering::Relaxed), 200);
2009 assert!(
2010 second_predicate_rows.load(Ordering::Relaxed) < 24,
2011 "final predicate should short-circuit before consuming all 24 rows selected by the first predicate"
2012 );
2013 }
2014
2015 #[test]
2023 fn test_decoder_filter_with_row_selection_and_limit() {
2024 use std::sync::atomic::{AtomicUsize, Ordering};
2025
2026 let builder =
2027 ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
2028 let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
2029
2030 let rows_filtered = Arc::new(AtomicUsize::new(0));
2031 let rows_filtered_for_predicate = Arc::clone(&rows_filtered);
2032
2033 let row_filter_a = ArrowPredicateFn::new(
2034 ProjectionMask::columns(&schema_descr, ["a"]),
2035 move |batch: RecordBatch| {
2036 rows_filtered_for_predicate.fetch_add(batch.num_rows(), Ordering::Relaxed);
2037 let scalar_175 = Int64Array::new_scalar(175);
2038 let column = batch.column(0).as_primitive::<Int64Type>();
2039 gt(column, &scalar_175)
2040 },
2041 );
2042
2043 let mut decoder = builder
2044 .with_projection(ProjectionMask::columns(&schema_descr, ["a"]))
2045 .with_row_selection(RowSelection::from(vec![
2046 RowSelector::skip(150),
2047 RowSelector::select(50),
2048 ]))
2049 .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
2050 .with_batch_size(10)
2051 .with_limit(10)
2052 .build()
2053 .unwrap();
2054
2055 let ranges = expect_needs_data(decoder.try_decode());
2056 push_ranges_to_decoder(&mut decoder, ranges);
2057
2058 let batch = expect_data(decoder.try_decode());
2059 let expected = TEST_BATCH.slice(176, 10).project(&[0]).unwrap();
2060 assert_eq!(batch, expected);
2061
2062 expect_finished(decoder.try_decode());
2063
2064 assert!(
2065 rows_filtered.load(Ordering::Relaxed) < 50,
2066 "predicate should short-circuit before consuming all 50 rows from the explicit row selection"
2067 );
2068 }
2069
2070 #[test]
2071 fn test_decoder_offset_limit() {
2072 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2073 .unwrap()
2074 .with_offset(225)
2076 .with_limit(20)
2078 .build()
2079 .unwrap();
2080
2081 let ranges = expect_needs_data(decoder.try_decode());
2085 push_ranges_to_decoder(&mut decoder, ranges);
2086
2087 let batch1 = expect_data(decoder.try_decode());
2089 let expected1 = TEST_BATCH.slice(225, 20);
2090 assert_eq!(batch1, expected1);
2091
2092 expect_finished(decoder.try_decode());
2093 }
2094
2095 #[test]
2096 fn test_decoder_try_next_reader_offset_limit() {
2097 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2098 .unwrap()
2099 .with_offset(225)
2100 .with_limit(20)
2101 .build()
2102 .unwrap();
2103
2104 let ranges = expect_needs_data(decoder.try_next_reader());
2105 push_ranges_to_decoder(&mut decoder, ranges);
2106
2107 let reader = expect_data(decoder.try_next_reader());
2108 let batches = reader
2109 .map(|batch| batch.expect("expected decoded batch"))
2110 .collect::<Vec<_>>();
2111 let output = concat_batches(&TEST_BATCH.schema(), &batches).unwrap();
2112 assert_eq!(output, TEST_BATCH.slice(225, 20));
2113
2114 expect_finished(decoder.try_next_reader());
2115 }
2116
2117 #[test]
2118 fn test_decoder_row_group_selection() {
2119 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2121 .unwrap()
2122 .with_row_groups(vec![1])
2123 .build()
2124 .unwrap();
2125
2126 let ranges = expect_needs_data(decoder.try_decode());
2130 push_ranges_to_decoder(&mut decoder, ranges);
2131
2132 let batch1 = expect_data(decoder.try_decode());
2134 let expected1 = TEST_BATCH.slice(200, 200);
2135 assert_eq!(batch1, expected1);
2136
2137 expect_finished(decoder.try_decode());
2138 }
2139
2140 #[test]
2141 fn test_decoder_row_selection() {
2142 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2144 .unwrap()
2145 .with_row_selection(RowSelection::from(vec![
2146 RowSelector::skip(225), RowSelector::select(20), ]))
2149 .build()
2150 .unwrap();
2151
2152 let ranges = expect_needs_data(decoder.try_decode());
2156 push_ranges_to_decoder(&mut decoder, ranges);
2157
2158 let batch1 = expect_data(decoder.try_decode());
2160 let expected1 = TEST_BATCH.slice(225, 20);
2161 assert_eq!(batch1, expected1);
2162
2163 expect_finished(decoder.try_decode());
2164 }
2165
2166 #[test]
2167 fn test_decoder_row_group_local_selections() {
2168 let bitmap_selection = RowSelection::from_boolean_buffer(BooleanBuffer::from(
2169 (0..45)
2170 .map(|row_idx| (25..45).contains(&row_idx))
2171 .collect::<Vec<_>>(),
2172 ));
2173 let rle_selection =
2174 RowSelection::from(vec![RowSelector::skip(190), RowSelector::select(10)]);
2175
2176 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2177 .unwrap()
2178 .with_row_group_selections(vec![
2179 RowGroupSelection::new(1, Some(bitmap_selection)),
2180 RowGroupSelection::new(0, Some(rle_selection)),
2181 ])
2182 .build()
2183 .unwrap();
2184 prefetch_test_file(&mut decoder);
2185
2186 assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(225, 20));
2190 assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(190, 10));
2191 expect_finished(decoder.try_decode());
2192 }
2193
2194 #[test]
2195 fn test_row_group_local_selections_respect_offset_and_limit() {
2196 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2197 .unwrap()
2198 .with_row_group_selections(vec![
2199 RowGroupSelection::new(1, None),
2200 RowGroupSelection::new(0, None),
2201 ])
2202 .with_offset(195)
2203 .with_limit(10)
2204 .build()
2205 .unwrap();
2206 prefetch_test_file(&mut decoder);
2207
2208 assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(395, 5));
2209 assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(0, 5));
2210 expect_finished(decoder.try_decode());
2211 }
2212
2213 #[test]
2214 fn test_row_group_local_selections_allow_duplicate_row_groups() {
2215 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2216 .unwrap()
2217 .with_row_group_selections(vec![
2218 RowGroupSelection::new(0, None),
2219 RowGroupSelection::new(0, None),
2220 ])
2221 .build()
2222 .unwrap();
2223 prefetch_test_file(&mut decoder);
2224
2225 let expected = TEST_BATCH.slice(0, 200);
2226 assert_eq!(expect_data(decoder.try_decode()), expected);
2227 assert_eq!(expect_data(decoder.try_decode()), expected);
2228 expect_finished(decoder.try_decode());
2229 }
2230
2231 #[test]
2232 fn test_short_row_group_local_selection_skips_trailing_rows() {
2233 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2234 .unwrap()
2235 .with_row_group_selections(vec![RowGroupSelection::new(
2236 1,
2237 Some(RowSelection::from(vec![
2238 RowSelector::skip(5),
2239 RowSelector::select(3),
2240 ])),
2241 )])
2242 .build()
2243 .unwrap();
2244 prefetch_test_file(&mut decoder);
2245
2246 assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(205, 3));
2247 expect_finished(decoder.try_decode());
2248 }
2249
2250 #[test]
2251 fn test_empty_row_group_local_selections_read_nothing() {
2252 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2253 .unwrap()
2254 .with_row_group_selections(vec![])
2255 .build()
2256 .unwrap();
2257
2258 expect_finished(decoder.try_decode());
2259 }
2260
2261 fn row_group_local_selections_with_filter() -> ParquetPushDecoderBuilder {
2270 let builder =
2271 ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
2272 let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
2273
2274 let row_filter_a = ArrowPredicateFn::new(
2276 ProjectionMask::columns(&schema_descr, ["a"]),
2277 |batch: RecordBatch| {
2278 let scalar_195 = Int64Array::new_scalar(195);
2279 let column = batch.column(0).as_primitive::<Int64Type>();
2280 gt(column, &scalar_195)
2281 },
2282 );
2283
2284 builder
2285 .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
2286 .with_row_group_selections(vec![
2287 RowGroupSelection::new(
2288 1,
2289 Some(RowSelection::from(vec![
2290 RowSelector::skip(10),
2291 RowSelector::select(10),
2292 ])),
2293 ),
2294 RowGroupSelection::new(
2295 0,
2296 Some(RowSelection::from(vec![
2297 RowSelector::skip(190),
2298 RowSelector::select(10),
2299 ])),
2300 ),
2301 ])
2302 }
2303
2304 #[test]
2305 fn test_row_group_local_selections_with_row_filter() {
2306 let mut decoder = row_group_local_selections_with_filter().build().unwrap();
2307 prefetch_test_file(&mut decoder);
2308
2309 assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(210, 10));
2310 assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(196, 4));
2311 expect_finished(decoder.try_decode());
2312 }
2313
2314 #[test]
2318 fn test_into_builder_preserves_local_selections_with_row_filter() {
2319 let mut decoder = row_group_local_selections_with_filter().build().unwrap();
2320 prefetch_test_file(&mut decoder);
2321
2322 let reader1 = expect_data(decoder.try_next_reader());
2323 let batches1: Vec<_> = reader1.collect::<Result<_, _>>().unwrap();
2324 let batch1 = concat_batches(&TEST_BATCH.schema(), &batches1).unwrap();
2325 assert_eq!(batch1, TEST_BATCH.slice(210, 10));
2326
2327 assert!(decoder.is_at_row_group_boundary());
2329 assert_eq!(decoder.row_groups_remaining(), 1);
2330 let mut decoder = decoder.into_builder().unwrap().build().unwrap();
2331
2332 let reader0 = expect_data(decoder.try_next_reader());
2333 let batches0: Vec<_> = reader0.collect::<Result<_, _>>().unwrap();
2334 let batch0 = concat_batches(&TEST_BATCH.schema(), &batches0).unwrap();
2335 assert_eq!(batch0, TEST_BATCH.slice(196, 4));
2336 expect_finished(decoder.try_next_reader());
2337 }
2338
2339 #[test]
2340 fn test_row_group_local_selection_replaces_previous_configuration() {
2341 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2342 .unwrap()
2343 .with_row_group_selections(vec![RowGroupSelection::new(0, None)])
2344 .with_row_group_selections(vec![RowGroupSelection::new(1, None)])
2345 .build()
2346 .unwrap();
2347 prefetch_test_file(&mut decoder);
2348
2349 assert_eq!(
2352 expect_data(decoder.try_decode()),
2353 TEST_BATCH.slice(200, 200)
2354 );
2355 expect_finished(decoder.try_decode());
2356 }
2357
2358 #[test]
2359 fn test_legacy_row_groups_and_row_selection_remain_composable() {
2360 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2361 .unwrap()
2362 .with_row_groups(vec![1])
2363 .with_row_selection(RowSelection::from(vec![
2364 RowSelector::skip(25),
2365 RowSelector::select(20),
2366 ]))
2367 .build()
2368 .unwrap();
2369 prefetch_test_file(&mut decoder);
2370
2371 assert_eq!(expect_data(decoder.try_decode()), TEST_BATCH.slice(225, 20));
2372 expect_finished(decoder.try_decode());
2373 }
2374
2375 #[test]
2376 fn test_row_group_local_selection_is_mutually_exclusive_with_legacy_configuration() {
2377 let metadata = test_file_parquet_metadata();
2378 let new_builder =
2379 || ParquetPushDecoderBuilder::try_new_decoder(Arc::clone(&metadata)).unwrap();
2380 let local_selection = || vec![RowGroupSelection::new(0, None)];
2381 let global_selection = || RowSelection::from(vec![RowSelector::select(1)]);
2382 let assert_conflict = |builder: ParquetPushDecoderBuilder| {
2383 let error = builder.build().unwrap_err();
2384 assert_eq!(
2385 error.to_string(),
2386 "Parquet error: with_row_group_selections cannot be combined with with_row_groups or with_row_selection"
2387 );
2388 };
2389
2390 assert_conflict(
2391 new_builder()
2392 .with_row_groups(vec![0])
2393 .with_row_group_selections(local_selection()),
2394 );
2395 assert_conflict(
2396 new_builder()
2397 .with_row_group_selections(local_selection())
2398 .with_row_groups(vec![0]),
2399 );
2400 assert_conflict(
2401 new_builder()
2402 .with_row_selection(global_selection())
2403 .with_row_group_selections(local_selection()),
2404 );
2405 assert_conflict(
2406 new_builder()
2407 .with_row_group_selections(local_selection())
2408 .with_row_selection(global_selection()),
2409 );
2410 }
2411
2412 #[test]
2413 fn test_row_group_local_selection_validates_row_group_and_length_at_build() {
2414 let metadata = test_file_parquet_metadata();
2415
2416 let error = ParquetPushDecoderBuilder::try_new_decoder(Arc::clone(&metadata))
2417 .unwrap()
2418 .with_row_group_selections(vec![RowGroupSelection::new(
2419 0,
2420 Some(RowSelection::from(vec![RowSelector::select(201)])),
2421 )])
2422 .build()
2423 .unwrap_err();
2424 assert!(
2425 error.to_string().contains(
2426 "Row selection for row group 0 contains 201 rows, but the row group has 200"
2427 ),
2428 "unexpected error: {error}"
2429 );
2430
2431 let error = ParquetPushDecoderBuilder::try_new_decoder(metadata)
2432 .unwrap()
2433 .with_row_group_selections(vec![RowGroupSelection::new(2, None)])
2434 .build()
2435 .unwrap_err();
2436 assert!(
2437 error
2438 .to_string()
2439 .contains("Row group index 2 out of bounds for file with 2 row groups"),
2440 "unexpected error: {error}"
2441 );
2442 }
2443
2444 #[test]
2448 fn test_peek_next_row_group_basic() {
2449 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2450 .unwrap()
2451 .build()
2452 .unwrap();
2453
2454 assert_eq!(decoder.peek_next_row_group().unwrap(), Some(0));
2457 assert!(decoder.is_at_row_group_boundary());
2458
2459 let ranges = expect_needs_data(decoder.try_next_reader());
2460 push_ranges_to_decoder(&mut decoder, ranges);
2461 let reader = expect_data(decoder.try_next_reader());
2462 assert!(decoder.is_at_row_group_boundary());
2466 assert_eq!(decoder.peek_next_row_group().unwrap(), Some(1));
2467
2468 for batch in reader {
2470 let _ = batch.unwrap();
2471 }
2472 let ranges = expect_needs_data(decoder.try_next_reader());
2473 push_ranges_to_decoder(&mut decoder, ranges);
2474 let reader = expect_data(decoder.try_next_reader());
2475 for batch in reader {
2476 let _ = batch.unwrap();
2477 }
2478
2479 assert_eq!(decoder.peek_next_row_group().unwrap(), None);
2481 }
2482
2483 #[test]
2487 fn test_peek_next_row_group_respects_with_row_groups() {
2488 let decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2489 .unwrap()
2490 .with_row_groups(vec![1])
2491 .build()
2492 .unwrap();
2493
2494 assert_eq!(decoder.peek_next_row_group().unwrap(), Some(1));
2495 }
2496
2497 #[test]
2502 fn test_peek_next_row_group_skips_empty_selection() {
2503 let decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2506 .unwrap()
2507 .with_row_selection(RowSelection::from(vec![
2508 RowSelector::skip(250),
2509 RowSelector::select(100),
2510 ]))
2511 .build()
2512 .unwrap();
2513
2514 assert_eq!(decoder.peek_next_row_group().unwrap(), Some(1));
2515 }
2516
2517 #[test]
2519 fn test_peek_next_row_group_finished() {
2520 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2521 .unwrap()
2522 .with_row_groups(vec![])
2523 .build()
2524 .unwrap();
2525
2526 expect_finished(decoder.try_next_reader());
2528 assert_eq!(decoder.peek_next_row_group().unwrap(), None);
2529 }
2530
2531 #[test]
2536 fn test_peek_next_row_group_skips_for_budget() {
2537 let decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2540 .unwrap()
2541 .with_offset(200)
2542 .build()
2543 .unwrap();
2544
2545 assert_eq!(decoder.peek_next_row_group().unwrap(), Some(1));
2546 }
2547
2548 #[test]
2552 fn test_peek_next_row_group_budget_drains_all() {
2553 let decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2556 .unwrap()
2557 .with_offset(500)
2558 .build()
2559 .unwrap();
2560
2561 assert_eq!(decoder.peek_next_row_group().unwrap(), None);
2562 }
2563
2564 #[test]
2570 fn test_peek_next_row_group_during_decoding_row_group() {
2571 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2575 .unwrap()
2576 .with_batch_size(100)
2577 .build()
2578 .unwrap();
2579 decoder
2580 .push_range(test_file_range(), TEST_FILE_DATA.clone())
2581 .unwrap();
2582
2583 let _batch0 = expect_data(decoder.try_decode());
2588 assert!(!decoder.is_at_row_group_boundary());
2589 assert_eq!(decoder.peek_next_row_group().unwrap(), Some(1));
2590
2591 let _batch1 = expect_data(decoder.try_decode());
2593 assert_eq!(decoder.peek_next_row_group().unwrap(), Some(1));
2594
2595 let _batch2 = expect_data(decoder.try_decode());
2597 assert!(!decoder.is_at_row_group_boundary());
2598 assert_eq!(decoder.peek_next_row_group().unwrap(), None);
2599
2600 let _batch3 = expect_data(decoder.try_decode());
2601 expect_finished(decoder.try_decode());
2602 }
2603
2604 #[test]
2609 fn test_peek_next_row_group_does_not_mutate_state() {
2610 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2611 .unwrap()
2612 .build()
2613 .unwrap();
2614
2615 for expected_rg in [0usize, 1usize] {
2617 assert!(decoder.is_at_row_group_boundary());
2618
2619 let first_peek = decoder.peek_next_row_group().unwrap();
2622 let second_peek = decoder.peek_next_row_group().unwrap();
2623 let third_peek = decoder.peek_next_row_group().unwrap();
2624 assert_eq!(first_peek, Some(expected_rg));
2625 assert_eq!(first_peek, second_peek);
2626 assert_eq!(second_peek, third_peek);
2627
2628 let ranges = expect_needs_data(decoder.try_next_reader());
2631 push_ranges_to_decoder(&mut decoder, ranges);
2632 let reader = expect_data(decoder.try_next_reader());
2633 for batch in reader {
2634 let _ = batch.unwrap();
2635 }
2636 }
2637
2638 assert_eq!(decoder.peek_next_row_group().unwrap(), None);
2641 expect_finished(decoder.try_next_reader());
2642 }
2643
2644 #[test]
2652 fn test_into_builder_installs_filter_between_row_groups() {
2653 let schema_descr = test_file_parquet_metadata()
2654 .file_metadata()
2655 .schema_descr_ptr();
2656 let mut decoder = prefetched_decoder(1024);
2657
2658 let reader0 = expect_data(decoder.try_next_reader());
2660 let batches0: Vec<_> = reader0.collect::<Result<_, _>>().unwrap();
2661 let batch0 = concat_batches(&TEST_BATCH.schema(), &batches0).unwrap();
2662 assert_eq!(batch0, TEST_BATCH.slice(0, 200));
2663
2664 assert!(decoder.is_at_row_group_boundary());
2666 assert_eq!(decoder.row_groups_remaining(), 1);
2667 let filter =
2668 ArrowPredicateFn::new(ProjectionMask::columns(&schema_descr, ["a"]), |batch| {
2669 gt(batch.column(0), &Int64Array::new_scalar(250))
2670 });
2671 let mut decoder = decoder
2672 .into_builder()
2673 .unwrap()
2674 .with_row_filter(RowFilter::new(vec![Box::new(filter)]))
2675 .build()
2676 .unwrap();
2677
2678 let reader1 = expect_data(decoder.try_next_reader());
2683 let batches1: Vec<_> = reader1.collect::<Result<_, _>>().unwrap();
2684 let batch1 = concat_batches(&TEST_BATCH.schema(), &batches1).unwrap();
2685 assert_eq!(batch1, TEST_BATCH.slice(251, 149));
2686 expect_finished(decoder.try_next_reader());
2687 }
2688
2689 #[test]
2693 fn test_into_builder_rejected_mid_row_group() {
2694 let mut decoder = prefetched_decoder(50);
2695
2696 expect_data(decoder.try_decode());
2699 assert!(!decoder.is_at_row_group_boundary());
2700
2701 let err = decoder.into_builder().unwrap_err();
2702 let err_msg = format!("{err}");
2703 assert!(
2704 err_msg.contains("is_at_row_group_boundary"),
2705 "unexpected error: {err_msg}"
2706 );
2707 }
2708
2709 #[test]
2711 fn test_into_builder_rejected_on_finished_decoder() {
2712 let mut decoder = prefetched_decoder(1024);
2713 expect_data(decoder.try_decode());
2714 expect_data(decoder.try_decode());
2715 expect_finished(decoder.try_decode());
2716 assert!(!decoder.is_at_row_group_boundary());
2717
2718 let err = decoder.into_builder().unwrap_err();
2719 assert!(
2720 format!("{err}").contains("finished"),
2721 "unexpected error: {err}"
2722 );
2723 }
2724
2725 #[test]
2731 fn test_into_builder_allowed_while_iterating_handed_off_reader() {
2732 let mut decoder = prefetched_decoder(1024);
2733
2734 let reader0 = expect_data(decoder.try_next_reader());
2735 assert!(decoder.is_at_row_group_boundary());
2738 let _builder = decoder.into_builder().unwrap();
2741 let batches: Vec<_> = reader0.collect::<Result<_, _>>().unwrap();
2742 let batch0 = concat_batches(&TEST_BATCH.schema(), &batches).unwrap();
2743 assert_eq!(batch0, TEST_BATCH.slice(0, 200));
2744 }
2745
2746 #[test]
2750 fn test_into_builder_resumes_remaining_budget() {
2751 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2754 .unwrap()
2755 .with_batch_size(1024)
2756 .with_limit(250)
2757 .build()
2758 .unwrap();
2759 prefetch_test_file(&mut decoder);
2760
2761 let reader0 = expect_data(decoder.try_next_reader());
2763 let batches0: Vec<_> = reader0.collect::<Result<_, _>>().unwrap();
2764 let batch0 = concat_batches(&TEST_BATCH.schema(), &batches0).unwrap();
2765 assert_eq!(batch0, TEST_BATCH.slice(0, 200));
2766
2767 assert!(decoder.is_at_row_group_boundary());
2771 let mut decoder = decoder.into_builder().unwrap().build().unwrap();
2772
2773 let reader1 = expect_data(decoder.try_next_reader());
2774 let batches1: Vec<_> = reader1.collect::<Result<_, _>>().unwrap();
2775 let batch1 = concat_batches(&TEST_BATCH.schema(), &batches1).unwrap();
2776 assert_eq!(batch1, TEST_BATCH.slice(200, 50));
2778 expect_finished(decoder.try_next_reader());
2779 }
2780
2781 #[test]
2785 fn test_into_builder_preserves_buffered_bytes() {
2786 let mut decoder = prefetched_decoder(1024);
2787 assert_eq!(decoder.buffered_bytes(), test_file_len());
2788
2789 let reader0 = expect_data(decoder.try_next_reader());
2791 let _: Vec<_> = reader0.collect::<Result<_, _>>().unwrap();
2792 let buffered = decoder.buffered_bytes();
2794 assert!(buffered > 0);
2795
2796 let mut decoder = decoder.into_builder().unwrap().build().unwrap();
2799 assert!(decoder.buffered_bytes() < buffered);
2800 assert_eq!(decoder.buffered_bytes(), test_file_row_group_bytes(1));
2801
2802 let reader1 = expect_data(decoder.try_next_reader());
2805 let batches1: Vec<_> = reader1.collect::<Result<_, _>>().unwrap();
2806 let batch1 = concat_batches(&TEST_BATCH.schema(), &batches1).unwrap();
2807 assert_eq!(batch1, TEST_BATCH.slice(200, 200));
2808 expect_finished(decoder.try_next_reader());
2809 }
2810
2811 #[test]
2814 fn test_into_builder_releases_bytes_of_skipped_row_groups() {
2815 let builder =
2816 ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
2817 let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
2818 let mut decoder = builder
2819 .with_row_groups(vec![1])
2820 .with_projection(ProjectionMask::columns(&schema_descr, ["a", "b"]))
2821 .build()
2822 .unwrap();
2823 prefetch_test_file(&mut decoder);
2824 assert!(decoder.is_at_row_group_boundary());
2825
2826 let mut decoder = decoder
2828 .into_builder()
2829 .unwrap()
2830 .with_row_groups(vec![0])
2831 .with_projection(ProjectionMask::all())
2832 .build()
2833 .unwrap();
2834 assert_eq!(decoder.buffered_bytes(), test_file_row_group_bytes(0));
2835
2836 let reader = expect_data(decoder.try_next_reader());
2837 let batches: Vec<_> = reader.collect::<Result<_, _>>().unwrap();
2838 let batch = concat_batches(&TEST_BATCH.schema(), &batches).unwrap();
2839 assert_eq!(batch, TEST_BATCH.slice(0, 200));
2840 expect_finished(decoder.try_next_reader());
2841 }
2842
2843 #[test]
2846 fn test_into_builder_keeps_bytes_of_predicate_columns() {
2847 let builder =
2848 ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()).unwrap();
2849 let schema_descr = builder.metadata().file_metadata().schema_descr_ptr();
2850 let row_filter_a = ArrowPredicateFn::new(
2851 ProjectionMask::columns(&schema_descr, ["a"]),
2852 |batch: RecordBatch| {
2853 let scalar_250 = Int64Array::new_scalar(250);
2854 let column = batch.column(0).as_primitive::<Int64Type>();
2855 gt(column, &scalar_250)
2856 },
2857 );
2858 let mut decoder = builder
2859 .with_row_groups(vec![1])
2860 .with_projection(ProjectionMask::columns(&schema_descr, ["b"]))
2861 .with_row_filter(RowFilter::new(vec![Box::new(row_filter_a)]))
2862 .build()
2863 .unwrap();
2864 prefetch_test_file(&mut decoder);
2865
2866 let mut decoder = decoder.into_builder().unwrap().build().unwrap();
2869 let column_bytes = |column_idx: usize| {
2870 let range =
2871 column_chunk_range(test_file_parquet_metadata().row_group(1).column(column_idx));
2872 range.end - range.start
2873 };
2874 assert_eq!(decoder.buffered_bytes(), column_bytes(0) + column_bytes(1));
2875
2876 let reader = expect_data(decoder.try_next_reader());
2877 let batches: Vec<_> = reader.collect::<Result<_, _>>().unwrap();
2878 let batch = concat_batches(&batches[0].schema(), &batches).unwrap();
2879 assert_eq!(batch, TEST_BATCH.slice(251, 149).project(&[1]).unwrap());
2880 expect_finished(decoder.try_next_reader());
2881 }
2882
2883 #[test]
2884 fn test_into_builder_preserves_remaining_row_group_local_selections() {
2885 let bitmap_selection = RowSelection::from_boolean_buffer(BooleanBuffer::from(
2886 (0..45)
2887 .map(|row_idx| (25..45).contains(&row_idx))
2888 .collect::<Vec<_>>(),
2889 ));
2890 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
2891 .unwrap()
2892 .with_row_group_selections(vec![
2893 RowGroupSelection::new(
2894 0,
2895 Some(RowSelection::from(vec![
2896 RowSelector::skip(190),
2897 RowSelector::select(10),
2898 ])),
2899 ),
2900 RowGroupSelection::new(1, Some(bitmap_selection)),
2901 ])
2902 .build()
2903 .unwrap();
2904 prefetch_test_file(&mut decoder);
2905
2906 let reader0 = expect_data(decoder.try_next_reader());
2907 let batches0: Vec<_> = reader0.collect::<Result<_, _>>().unwrap();
2908 let batch0 = concat_batches(&TEST_BATCH.schema(), &batches0).unwrap();
2909 assert_eq!(batch0, TEST_BATCH.slice(190, 10));
2910
2911 assert!(decoder.is_at_row_group_boundary());
2913 assert_eq!(decoder.row_groups_remaining(), 1);
2914 let mut decoder = decoder.into_builder().unwrap().build().unwrap();
2915
2916 let reader1 = expect_data(decoder.try_next_reader());
2917 let batches1: Vec<_> = reader1.collect::<Result<_, _>>().unwrap();
2918 let batch1 = concat_batches(&TEST_BATCH.schema(), &batches1).unwrap();
2919 assert_eq!(batch1, TEST_BATCH.slice(225, 20));
2920 expect_finished(decoder.try_next_reader());
2921 }
2922
2923 #[test]
2931 fn test_into_builder_expand_projection_requests_new_bytes() {
2932 let metadata = test_file_parquet_metadata();
2933 let schema_descr = metadata.file_metadata().schema_descr_ptr();
2934
2935 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(metadata)
2936 .unwrap()
2937 .with_batch_size(1024)
2938 .with_projection(ProjectionMask::columns(&schema_descr, ["a"]))
2939 .build()
2940 .unwrap();
2941
2942 let ranges_rg0 = expect_needs_data(decoder.try_next_reader());
2945 assert_eq!(ranges_rg0, vec![4..1860]);
2946 push_ranges_to_decoder(&mut decoder, ranges_rg0);
2947
2948 let reader0 = expect_data(decoder.try_next_reader());
2949 let batches0: Vec<_> = reader0.collect::<Result<_, _>>().unwrap();
2950 let batch0 = concat_batches(&batches0[0].schema(), &batches0).unwrap();
2951 assert_eq!(batch0, TEST_BATCH.slice(0, 200).project(&[0]).unwrap());
2952
2953 assert!(decoder.is_at_row_group_boundary());
2955 let mut decoder = decoder
2956 .into_builder()
2957 .unwrap()
2958 .with_projection(ProjectionMask::columns(&schema_descr, ["a", "b", "c"]))
2959 .build()
2960 .unwrap();
2961
2962 let ranges_rg1 = expect_needs_data(decoder.try_next_reader());
2965 assert_eq!(ranges_rg1, vec![11062..12918, 12918..14774, 14774..22230]);
2966 push_ranges_to_decoder(&mut decoder, ranges_rg1);
2967
2968 let reader1 = expect_data(decoder.try_next_reader());
2969 let batches1: Vec<_> = reader1.collect::<Result<_, _>>().unwrap();
2970 let batch1 = concat_batches(&TEST_BATCH.schema(), &batches1).unwrap();
2971 assert_eq!(batch1, TEST_BATCH.slice(200, 200));
2972 expect_finished(decoder.try_next_reader());
2973 }
2974
2975 #[test]
2981 fn test_into_builder_narrow_projection_requests_fewer_bytes() {
2982 let metadata = test_file_parquet_metadata();
2983 let schema_descr = metadata.file_metadata().schema_descr_ptr();
2984
2985 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(metadata)
2986 .unwrap()
2987 .with_batch_size(1024)
2988 .build()
2989 .unwrap();
2990
2991 let ranges_rg0 = expect_needs_data(decoder.try_next_reader());
2993 assert_eq!(ranges_rg0, vec![4..1860, 1860..3716, 3716..11062]);
2994 push_ranges_to_decoder(&mut decoder, ranges_rg0);
2995
2996 let reader0 = expect_data(decoder.try_next_reader());
2997 let batches0: Vec<_> = reader0.collect::<Result<_, _>>().unwrap();
2998 let batch0 = concat_batches(&TEST_BATCH.schema(), &batches0).unwrap();
2999 assert_eq!(batch0, TEST_BATCH.slice(0, 200));
3000
3001 assert!(decoder.is_at_row_group_boundary());
3003 let mut decoder = decoder
3004 .into_builder()
3005 .unwrap()
3006 .with_projection(ProjectionMask::columns(&schema_descr, ["a"]))
3007 .build()
3008 .unwrap();
3009
3010 let ranges_rg1 = expect_needs_data(decoder.try_next_reader());
3012 assert_eq!(ranges_rg1, vec![11062..12918]);
3013 push_ranges_to_decoder(&mut decoder, ranges_rg1);
3014
3015 let reader1 = expect_data(decoder.try_next_reader());
3016 let batches1: Vec<_> = reader1.collect::<Result<_, _>>().unwrap();
3017 let batch1 = concat_batches(&batches1[0].schema(), &batches1).unwrap();
3018 assert_eq!(batch1, TEST_BATCH.slice(200, 200).project(&[0]).unwrap());
3019 expect_finished(decoder.try_next_reader());
3020 }
3021
3022 static TEST_BATCH: LazyLock<RecordBatch> = LazyLock::new(|| {
3026 let a: ArrayRef = Arc::new(Int64Array::from_iter_values(0..400));
3027 let b: ArrayRef = Arc::new(Int64Array::from_iter_values(400..800));
3028 let c: ArrayRef = Arc::new(StringViewArray::from_iter_values((0..400).map(|i| {
3029 if i % 2 == 0 {
3030 format!("string_{i}")
3031 } else {
3032 format!("A string larger than 12 bytes and thus not inlined {i}")
3033 }
3034 })));
3035
3036 RecordBatch::try_from_iter(vec![("a", a), ("b", b), ("c", c)]).unwrap()
3037 });
3038
3039 static TEST_FILE_DATA: LazyLock<Bytes> = LazyLock::new(|| {
3065 let input_batch = &TEST_BATCH;
3066 let mut output = Vec::new();
3067
3068 let writer_options = WriterProperties::builder()
3069 .set_max_row_group_row_count(Some(200))
3070 .set_data_page_row_count_limit(100)
3071 .build();
3072 let mut writer =
3073 ArrowWriter::try_new(&mut output, input_batch.schema(), Some(writer_options)).unwrap();
3074
3075 let mut row_remain = input_batch.num_rows();
3078 while row_remain > 0 {
3079 let chunk_size = row_remain.min(50);
3080 let chunk = input_batch.slice(input_batch.num_rows() - row_remain, chunk_size);
3081 writer.write(&chunk).unwrap();
3082 row_remain -= chunk_size;
3083 }
3084 writer.close().unwrap();
3085 Bytes::from(output)
3086 });
3087
3088 fn test_file_len() -> u64 {
3090 TEST_FILE_DATA.len() as u64
3091 }
3092
3093 fn test_file_range() -> Range<u64> {
3095 0..test_file_len()
3096 }
3097
3098 pub fn test_file_slice(range: Range<u64>) -> Bytes {
3100 let start: usize = range.start.try_into().unwrap();
3101 let end: usize = range.end.try_into().unwrap();
3102 TEST_FILE_DATA.slice(start..end)
3103 }
3104
3105 pub fn test_file_parquet_metadata() -> Arc<ParquetMetaData> {
3107 let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(test_file_len()).unwrap();
3108 push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![test_file_range()]);
3109 let metadata = metadata_decoder.try_decode().unwrap();
3110 let DecodeResult::Data(metadata) = metadata else {
3111 panic!("Expected metadata to be decoded successfully");
3112 };
3113 Arc::new(metadata)
3114 }
3115
3116 #[test]
3119 fn test_decoder_offset_index_page_past_column_chunk() {
3120 let metadata = Arc::unwrap_or_clone(test_file_parquet_metadata_with_offset_index());
3121 let num_columns = metadata.row_group(0).num_columns();
3122 let mut page_index = PageIndexBuilder::new(metadata.num_row_groups(), num_columns);
3123 for row_group in 0..metadata.num_row_groups() {
3124 let row_group_index = metadata.page_index_for_row_group(row_group);
3125 for column in 0..num_columns {
3126 let mut offset_index = row_group_index.offset_index(column).unwrap().clone();
3127 if (row_group, column) == (0, 0) {
3128 let last = offset_index.page_locations.last_mut().unwrap();
3129 last.compressed_page_size += 1;
3130 }
3131 page_index
3132 .try_put_offset_index(offset_index, row_group, column)
3133 .unwrap();
3134 }
3135 }
3136 let metadata = metadata
3137 .into_builder()
3138 .set_page_index(Some(Arc::new(page_index.build())))
3139 .build();
3140
3141 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(Arc::new(metadata))
3142 .unwrap()
3143 .with_row_groups(vec![0])
3144 .build()
3145 .unwrap();
3146 let ranges = expect_needs_data(decoder.try_decode());
3147 push_ranges_to_decoder(&mut decoder, ranges);
3148 let err = loop {
3149 match decoder.try_decode() {
3150 Ok(DecodeResult::Data(_)) => {}
3151 Ok(other) => panic!("expected an error, got {other:?}"),
3152 Err(e) => break e.to_string(),
3153 }
3154 };
3155 assert!(err.contains("has 126 bytes, expected 127"), "{err}");
3156 }
3157
3158 fn test_file_parquet_metadata_with_offset_index() -> Arc<ParquetMetaData> {
3160 let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(test_file_len())
3161 .unwrap()
3162 .with_offset_index_policy(PageIndexPolicy::Required);
3163 push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![test_file_range()]);
3164 let metadata = metadata_decoder.try_decode().unwrap();
3165 let DecodeResult::Data(metadata) = metadata else {
3166 panic!("Expected metadata to be decoded successfully");
3167 };
3168 Arc::new(metadata)
3169 }
3170
3171 fn push_ranges_to_metadata_decoder(
3173 metadata_decoder: &mut ParquetMetaDataPushDecoder,
3174 ranges: Vec<Range<u64>>,
3175 ) {
3176 let data = ranges
3177 .iter()
3178 .map(|range| test_file_slice(range.clone()))
3179 .collect::<Vec<_>>();
3180 metadata_decoder.push_ranges(ranges, data).unwrap();
3181 }
3182
3183 fn prefetch_test_file(decoder: &mut ParquetPushDecoder) {
3185 decoder
3186 .push_range(test_file_range(), TEST_FILE_DATA.clone())
3187 .unwrap();
3188 }
3189
3190 fn prefetched_decoder(batch_size: usize) -> ParquetPushDecoder {
3193 let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata())
3194 .unwrap()
3195 .with_batch_size(batch_size)
3196 .build()
3197 .unwrap();
3198 prefetch_test_file(&mut decoder);
3199 decoder
3200 }
3201
3202 fn test_file_row_group_bytes(row_group_idx: usize) -> u64 {
3205 test_file_parquet_metadata()
3206 .row_group(row_group_idx)
3207 .columns()
3208 .iter()
3209 .map(|column| {
3210 let range = column_chunk_range(column);
3211 range.end - range.start
3212 })
3213 .sum()
3214 }
3215
3216 fn push_ranges_to_decoder(decoder: &mut ParquetPushDecoder, ranges: Vec<Range<u64>>) {
3217 let data = ranges
3218 .iter()
3219 .map(|range| test_file_slice(range.clone()))
3220 .collect::<Vec<_>>();
3221 decoder.push_ranges(ranges, data).unwrap();
3222 }
3223
3224 fn expect_data<T: Debug>(result: Result<DecodeResult<T>, ParquetError>) -> T {
3226 match result.expect("Expected Ok(DecodeResult::Data(T))") {
3227 DecodeResult::Data(data) => data,
3228 result => panic!("Expected DecodeResult::Data, got {result:?}"),
3229 }
3230 }
3231
3232 fn expect_needs_data<T: Debug>(
3234 result: Result<DecodeResult<T>, ParquetError>,
3235 ) -> Vec<Range<u64>> {
3236 match result.expect("Expected Ok(DecodeResult::NeedsData(ranges))") {
3237 DecodeResult::NeedsData(ranges) => ranges,
3238 result => panic!("Expected DecodeResult::NeedsData, got {result:?}"),
3239 }
3240 }
3241
3242 fn expect_finished<T: Debug>(result: Result<DecodeResult<T>, ParquetError>) {
3243 match result.expect("Expected Ok(DecodeResult::Finished)") {
3244 DecodeResult::Finished => {}
3245 result => panic!("Expected DecodeResult::Finished, got {result:?}"),
3246 }
3247 }
3248}