1use crate::DecodeResult;
6#[cfg(feature = "encryption")]
7use crate::encryption::decrypt::FileDecryptionProperties;
8use crate::errors::{ParquetError, Result};
9use crate::file::FOOTER_SIZE;
10use crate::file::metadata::page_index::PageIndex;
11use crate::file::metadata::parser::{MetadataParser, parse_page_index};
12use crate::file::metadata::{
13 ColumnChunkMask, FooterTail, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions,
14};
15use crate::file::reader::ChunkReader;
16use bytes::Bytes;
17use std::ops::Range;
18use std::sync::Arc;
19
20#[cfg_attr(
41 feature = "arrow",
42 doc = r#"
43```rust
44# use std::ops::Range;
45# use bytes::Bytes;
46# use arrow_array::record_batch;
47# use parquet::DecodeResult;
48# use parquet::arrow::ArrowWriter;
49# use parquet::errors::ParquetError;
50# use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataPushDecoder};
51#
52# fn decode_metadata() -> Result<ParquetMetaData, ParquetError> {
53# let file_bytes = {
54# let mut buffer = vec![0];
55# let batch = record_batch!(("a", Int32, [1, 2, 3])).unwrap();
56# let mut writer = ArrowWriter::try_new(&mut buffer, batch.schema(), None).unwrap();
57# writer.write(&batch).unwrap();
58# writer.close().unwrap();
59# Bytes::from(buffer)
60# };
61# // mimic IO by returning a function that returns the bytes for a given range
62# let get_range = |range: &Range<u64>| -> Bytes {
63# let start = range.start as usize;
64# let end = range.end as usize;
65# file_bytes.slice(start..end)
66# };
67#
68# let file_len = file_bytes.len() as u64;
69// The `ParquetMetaDataPushDecoder` needs to know the file length.
70let mut decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
71// try to decode the metadata. If more data is needed, the decoder will tell you what ranges
72loop {
73 match decoder.try_decode() {
74 Ok(DecodeResult::Data(metadata)) => { return Ok(metadata); } // decode successful
75 Ok(DecodeResult::NeedsData(ranges)) => {
76 // The decoder needs more data
77 //
78 // In this example, we call a function that returns the bytes for each given range.
79 // In a real application, you would likely read the data from a file or network.
80 let data = ranges.iter().map(|range| get_range(range)).collect();
81 // Push the data into the decoder and try to decode again on the next iteration.
82 decoder.push_ranges(ranges, data).unwrap();
83 }
84 Ok(DecodeResult::Finished) => { unreachable!("returned metadata in previous match arm") }
85 Err(e) => return Err(e),
86 }
87}
88# }
89```
90"#
91)]
92#[cfg_attr(
113 feature = "arrow",
114 doc = r#"
115```rust
116# use std::ops::Range;
117# use bytes::Bytes;
118# use arrow_array::record_batch;
119# use parquet::DecodeResult;
120# use parquet::arrow::ArrowWriter;
121# use parquet::errors::ParquetError;
122# use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataPushDecoder};
123#
124# fn decode_metadata() -> Result<ParquetMetaData, ParquetError> {
125# let file_bytes = {
126# let mut buffer = vec![0];
127# let batch = record_batch!(("a", Int32, [1, 2, 3])).unwrap();
128# let mut writer = ArrowWriter::try_new(&mut buffer, batch.schema(), None).unwrap();
129# writer.write(&batch).unwrap();
130# writer.close().unwrap();
131# Bytes::from(buffer)
132# };
133#
134let file_len = file_bytes.len() as u64;
135// For this example, we "prefetch" all the bytes which we have in memory,
136// but in a real application, you would likely read a chunk from the end
137// for example 1MB.
138let prefetched_bytes = file_bytes.clone();
139let mut decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
140// push the prefetched bytes into the decoder
141decoder.push_ranges(vec![0..file_len], vec![prefetched_bytes]).unwrap();
142// The decoder will now be able to decode the metadata. Note in a real application,
143// unless you can guarantee that the pushed data is enough to decode the metadata,
144// you still need to call `try_decode` in a loop until it returns `DecodeResult::Data`
145// as shown in the previous example
146 match decoder.try_decode() {
147 Ok(DecodeResult::Data(metadata)) => { return Ok(metadata); } // decode successful
148 other => { panic!("expected DecodeResult::Data, got: {other:?}") }
149 }
150# }
151```
152"#
153)]
154#[cfg_attr(
163 feature = "arrow",
164 doc = r#"
165```rust
166# use std::ops::Range;
167# use bytes::Bytes;
168use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt};
169# use arrow_array::record_batch;
170# use parquet::DecodeResult;
171# use parquet::arrow::ArrowWriter;
172# use parquet::errors::ParquetError;
173# use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataPushDecoder};
174#
175// This function decodes Parquet Metadata from anything that implements
176// [`AsyncRead`] and [`AsyncSeek`] such as a tokio::fs::File
177async fn decode_metadata(
178 file_len: u64,
179 mut async_source: impl AsyncRead + AsyncSeek + Unpin
180) -> Result<ParquetMetaData, ParquetError> {
181 // We need a ParquetMetaDataPushDecoder to decode the metadata.
182 let mut decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
183 loop {
184 match decoder.try_decode() {
185 Ok(DecodeResult::Data(metadata)) => { return Ok(metadata); } // decode successful
186 Ok(DecodeResult::NeedsData(ranges)) => {
187 // The decoder needs more data
188 //
189 // In this example we use the AsyncRead and AsyncSeek traits to read the
190 // required ranges from the async source.
191 let mut data = Vec::with_capacity(ranges.len());
192 for range in &ranges {
193 let mut buffer = vec![0; (range.end - range.start) as usize];
194 async_source.seek(std::io::SeekFrom::Start(range.start)).await?;
195 async_source.read_exact(&mut buffer).await?;
196 data.push(Bytes::from(buffer));
197 }
198 // Push the data into the decoder and try to decode again on the next iteration.
199 decoder.push_ranges(ranges, data).unwrap();
200 }
201 Ok(DecodeResult::Finished) => { unreachable!("returned metadata in previous match arm") }
202 Err(e) => return Err(e),
203 }
204 }
205}
206```
207"#
208)]
209#[derive(Debug)]
211pub struct ParquetMetaDataPushDecoder {
212 state: DecodeState,
214 column_index_policy: PageIndexPolicy,
216 offset_index_policy: PageIndexPolicy,
218 column_index_mask: ColumnChunkMask,
220 offset_index_mask: ColumnChunkMask,
222 pending_page_index_range: Option<Range<u64>>,
224 buffers: crate::util::push_buffers::PushBuffers,
226 metadata_parser: MetadataParser,
228}
229
230impl ParquetMetaDataPushDecoder {
231 pub fn try_new(file_len: u64) -> Result<Self> {
238 if file_len < 8 {
239 return Err(ParquetError::General(format!(
240 "Parquet files are at least 8 bytes long, but file length is {file_len}"
241 )));
242 }
243
244 Ok(Self {
245 state: DecodeState::ReadingFooter,
246 column_index_policy: PageIndexPolicy::Optional,
247 offset_index_policy: PageIndexPolicy::Optional,
248 column_index_mask: ColumnChunkMask::all(),
249 offset_index_mask: ColumnChunkMask::all(),
250 pending_page_index_range: None,
251 buffers: crate::util::push_buffers::PushBuffers::new(file_len),
252 metadata_parser: MetadataParser::new(),
253 })
254 }
255
256 pub(crate) fn try_new_with_footer_tail(file_len: u64, footer_tail: FooterTail) -> Result<Self> {
258 let mut new_self = Self::try_new(file_len)?;
259 new_self.state = DecodeState::ReadingMetadata(footer_tail);
260 Ok(new_self)
261 }
262
263 pub fn try_new_with_metadata(file_len: u64, metadata: ParquetMetaData) -> Result<Self> {
279 let mut new_self = Self::try_new(file_len)?;
280 new_self.state = DecodeState::ReadingPageIndex(Box::new(metadata));
281 Ok(new_self)
282 }
283
284 pub fn with_page_index_policy(mut self, page_index_policy: PageIndexPolicy) -> Self {
295 self.column_index_policy = page_index_policy;
296 self.offset_index_policy = page_index_policy;
297 self.pending_page_index_range = None;
298 self
299 }
300
301 pub fn with_column_index_policy(mut self, column_index_policy: PageIndexPolicy) -> Self {
303 self.column_index_policy = column_index_policy;
304 self.pending_page_index_range = None;
305 self
306 }
307
308 pub fn with_offset_index_policy(mut self, offset_index_policy: PageIndexPolicy) -> Self {
310 self.offset_index_policy = offset_index_policy;
311 self.pending_page_index_range = None;
312 self
313 }
314
315 pub fn with_page_index_mask(mut self, mask: ColumnChunkMask) -> Self {
319 self.column_index_mask = mask.clone();
320 self.offset_index_mask = mask;
321 self.pending_page_index_range = None;
322 self
323 }
324
325 pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self {
327 self.column_index_mask = mask;
328 self.pending_page_index_range = None;
329 self
330 }
331
332 pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self {
334 self.offset_index_mask = mask;
335 self.pending_page_index_range = None;
336 self
337 }
338
339 pub fn with_metadata_options(mut self, options: Option<Arc<ParquetMetaDataOptions>>) -> Self {
341 self.metadata_parser = self.metadata_parser.with_metadata_options(options);
342 self
343 }
344
345 #[cfg(feature = "encryption")]
346 pub fn with_file_decryption_properties(
348 mut self,
349 file_decryption_properties: Option<std::sync::Arc<FileDecryptionProperties>>,
350 ) -> Self {
351 self.metadata_parser = self
352 .metadata_parser
353 .with_file_decryption_properties(file_decryption_properties);
354 self
355 }
356
357 pub fn push_ranges(&mut self, ranges: Vec<Range<u64>>, buffers: Vec<Bytes>) -> Result<()> {
377 if matches!(&self.state, DecodeState::Finished) {
378 return Err(general_err!(
379 "ParquetMetaDataPushDecoder: cannot push data after decoding is finished"
380 ));
381 }
382 self.buffers.push_ranges(ranges, buffers)?;
383 Ok(())
384 }
385
386 pub fn push_range(&mut self, range: Range<u64>, buffer: Bytes) -> Result<()> {
388 if matches!(&self.state, DecodeState::Finished) {
389 return Err(general_err!(
390 "ParquetMetaDataPushDecoder: cannot push data after decoding is finished"
391 ));
392 }
393 self.buffers.push_range(range, buffer)?;
394 Ok(())
395 }
396
397 pub fn clear_all_ranges(&mut self) {
399 self.buffers.clear_all_ranges();
400 }
401
402 pub fn try_decode(&mut self) -> Result<DecodeResult<ParquetMetaData>> {
405 let file_len = self.buffers.file_len();
406 let footer_len = FOOTER_SIZE as u64;
407 loop {
408 match std::mem::replace(&mut self.state, DecodeState::Intermediate) {
409 DecodeState::ReadingFooter => {
410 let footer_start = file_len.saturating_sub(footer_len);
412 let footer_range = footer_start..file_len;
413
414 if !self.buffers.has_range(&footer_range) {
415 self.state = DecodeState::ReadingFooter;
416 return Ok(needs_range(footer_range));
417 }
418 let footer_bytes = self.get_bytes(&footer_range)?;
419 let footer_tail = FooterTail::try_from(footer_bytes.as_ref())?;
420
421 self.state = DecodeState::ReadingMetadata(footer_tail);
422 }
423
424 DecodeState::ReadingMetadata(footer_tail) => {
425 let metadata_len: u64 = footer_tail.metadata_length() as u64;
426 let needed = footer_len + metadata_len;
428 let Some(metadata_start) = file_len.checked_sub(needed) else {
429 return Err(eof_err!(
430 "Parquet file too small. Size is {file_len} but need {needed}"
431 ));
432 };
433 let metadata_end = metadata_start + metadata_len;
434 let metadata_range = metadata_start..metadata_end;
435
436 if !self.buffers.has_range(&metadata_range) {
437 self.state = DecodeState::ReadingMetadata(footer_tail);
438 return Ok(needs_range(metadata_range));
439 }
440
441 let metadata = self.metadata_parser.decode_metadata(
442 &self.get_bytes(&metadata_range)?,
443 footer_tail.is_encrypted_footer(),
444 )?;
445 self.state = DecodeState::ReadingPageIndex(Box::new(metadata));
448 }
449
450 DecodeState::ReadingPageIndex(metadata) => {
451 match self.try_decode_page_index_for_metadata(metadata)? {
452 DecodeResult::NeedsData(ranges) => {
453 return Ok(DecodeResult::NeedsData(ranges));
454 }
455 DecodeResult::Data((mut metadata, page_index)) => {
456 if let Some(page_index) = page_index {
459 metadata.set_page_index(Some(Arc::new(page_index)));
460 } else {
461 metadata.set_page_index(None);
462 }
463 return Ok(DecodeResult::Data(*metadata));
464 }
465 DecodeResult::Finished => return Ok(DecodeResult::Finished),
466 }
467 }
468
469 DecodeState::Finished => return Ok(DecodeResult::Finished),
470 DecodeState::Intermediate => {
471 return Err(general_err!(
472 "ParquetMetaDataPushDecoder: internal error, invalid state"
473 ));
474 }
475 }
476 }
477 }
478
479 pub fn try_decode_page_index(
516 &mut self,
517 ) -> Result<DecodeResult<(ParquetMetaData, Option<PageIndex>)>> {
518 match std::mem::replace(&mut self.state, DecodeState::Intermediate) {
519 DecodeState::ReadingPageIndex(metadata) => {
520 Ok(match self.try_decode_page_index_for_metadata(metadata)? {
521 DecodeResult::NeedsData(ranges) => DecodeResult::NeedsData(ranges),
522 DecodeResult::Data((metadata, page_index)) => {
523 DecodeResult::Data((*metadata, page_index))
524 }
525 DecodeResult::Finished => DecodeResult::Finished,
526 })
527 }
528 DecodeState::Finished => {
529 self.state = DecodeState::Finished;
530 Ok(DecodeResult::Finished)
531 }
532 other => {
533 self.state = other;
535 Err(general_err!(
536 "ParquetMetaDataPushDecoder: try_decode_page_index requires a decoder created with try_new_with_metadata"
537 ))
538 }
539 }
540 }
541
542 fn try_decode_page_index_for_metadata(
543 &mut self,
544 metadata: Box<ParquetMetaData>,
545 ) -> Result<DecodeResult<(Box<ParquetMetaData>, Option<PageIndex>)>> {
546 let range = self.pending_page_index_range.take().or_else(|| {
548 range_for_page_index(
549 &metadata,
550 self.column_index_policy,
551 self.offset_index_policy,
552 &self.column_index_mask,
553 &self.offset_index_mask,
554 )
555 });
556
557 let Some(page_index_range) = range else {
558 self.state = DecodeState::Finished;
559 return Ok(DecodeResult::Data((metadata, None)));
560 };
561
562 if !self.buffers.has_range(&page_index_range) {
563 self.pending_page_index_range = Some(page_index_range.clone());
564 self.state = DecodeState::ReadingPageIndex(metadata);
565 return Ok(DecodeResult::NeedsData(vec![page_index_range]));
566 }
567
568 let buffer = match self.get_bytes(&page_index_range) {
569 Ok(buffer) => buffer,
570 Err(error) => {
571 self.state = DecodeState::ReadingPageIndex(metadata);
572 return Err(error);
573 }
574 };
575 let page_index = match parse_page_index(
576 &metadata,
577 self.column_index_policy,
578 self.offset_index_policy,
579 &self.column_index_mask,
580 &self.offset_index_mask,
581 &buffer,
582 page_index_range.start,
583 ) {
584 Ok(page_index) => page_index,
585 Err(error) => {
586 self.state = DecodeState::ReadingPageIndex(metadata);
587 return Err(error);
588 }
589 };
590
591 self.state = DecodeState::Finished;
592 Ok(DecodeResult::Data((metadata, page_index)))
593 }
594
595 fn get_bytes(&self, range: &Range<u64>) -> Result<Bytes> {
597 let start = range.start;
598 let raw_len = range.end - range.start;
599 let len: usize = raw_len.try_into().map_err(|_| {
600 ParquetError::General(format!(
601 "ParquetMetaDataPushDecoder: Range length too large to fit in usize: {raw_len}",
602 ))
603 })?;
604 self.buffers.get_bytes(start, len)
605 }
606}
607
608fn needs_range<T: std::fmt::Debug>(range: Range<u64>) -> DecodeResult<T> {
610 DecodeResult::NeedsData(vec![range])
611}
612
613#[derive(Debug)]
615enum DecodeState {
616 ReadingFooter,
618 ReadingMetadata(FooterTail),
620 ReadingPageIndex(Box<ParquetMetaData>),
622 Finished,
624 Intermediate,
627}
628
629pub(crate) fn range_for_page_index(
634 metadata: &ParquetMetaData,
635 column_index_policy: PageIndexPolicy,
636 offset_index_policy: PageIndexPolicy,
637 column_index_mask: &ColumnChunkMask,
638 offset_index_mask: &ColumnChunkMask,
639) -> Option<Range<u64>> {
640 let mut range: Option<Range<u64>> = None;
641 for (policy, mask, column_index) in [
642 (column_index_policy, column_index_mask, true),
643 (offset_index_policy, offset_index_mask, false),
644 ] {
645 if policy == PageIndexPolicy::Skip {
646 continue;
647 }
648 for row_group in mask.row_group_indices(metadata.num_row_groups()) {
649 for column in mask.column_indices(metadata.row_group(row_group).num_columns()) {
650 let column = metadata.row_group(row_group).column(column);
651 let index = if column_index {
652 column.column_index_range()
653 } else {
654 column.offset_index_range()
655 };
656 if let Some(index) = index {
657 range = Some(match range {
658 Some(range) => range.start.min(index.start)..range.end.max(index.end),
659 None => index,
660 });
661 }
662 }
663 }
664 }
665 range
666}
667
668#[cfg(all(test, feature = "arrow"))]
671mod tests {
672 use super::*;
673 use crate::arrow::ArrowWriter;
674 use crate::file::metadata::page_index::PageIndexProvider;
675 use crate::file::properties::WriterProperties;
676 use arrow_array::{ArrayRef, Int64Array, RecordBatch, StringViewArray};
677 use bytes::Bytes;
678 use std::fmt::Debug;
679 use std::ops::Range;
680 use std::sync::{Arc, LazyLock};
681
682 #[test]
684 fn test_metadata_decoder_all_data() {
685 let file_len = test_file_len();
686 let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
687 push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![test_file_range()]);
689
690 let metadata = expect_data(metadata_decoder.try_decode());
692
693 assert_eq!(metadata.num_row_groups(), 2);
694 assert_eq!(metadata.row_group(0).num_rows(), 200);
695 assert_eq!(metadata.row_group(1).num_rows(), 200);
696 assert!(metadata.page_index().is_some_and(|idx| idx.is_complete()));
697 }
698
699 #[test]
701 fn test_metadata_decoder_metadata_length_exceeds_file() {
702 let mut file = vec![0u8; 4];
704 file.extend_from_slice(&1000u32.to_le_bytes());
705 file.extend_from_slice(b"PAR1");
706 let file_len = file.len() as u64;
707
708 let mut decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
709 decoder.push_range(0..file_len, Bytes::from(file)).unwrap();
710 let err = decoder.try_decode().unwrap_err();
711 assert_eq!(
712 err.to_string(),
713 "EOF: Parquet file too small. Size is 12 but need 1008"
714 );
715 }
716
717 #[test]
718 fn test_range_for_page_index_respects_column_mask() {
719 let mut decoder = ParquetMetaDataPushDecoder::try_new(test_file_len()).unwrap();
720 push_ranges_to_metadata_decoder(&mut decoder, vec![test_file_range()]);
721 let metadata = expect_data(decoder.try_decode());
722
723 let all = range_for_page_index(
724 &metadata,
725 PageIndexPolicy::Required,
726 PageIndexPolicy::Skip,
727 &ColumnChunkMask::all(),
728 &ColumnChunkMask::all(),
729 )
730 .unwrap();
731 let masked = range_for_page_index(
732 &metadata,
733 PageIndexPolicy::Required,
734 PageIndexPolicy::Skip,
735 &ColumnChunkMask::columns([0]),
736 &ColumnChunkMask::all(),
737 )
738 .unwrap();
739
740 assert!(all.start <= masked.start && masked.end <= all.end);
741 assert!(masked.end - masked.start < all.end - all.start);
742 }
743
744 #[test]
745 fn test_decode_page_index_after_metadata() {
746 let metadata = test_metadata_without_page_index();
747 assert!(metadata.page_index().is_none());
748
749 let mut decoder =
750 ParquetMetaDataPushDecoder::try_new_with_metadata(test_file_len(), metadata)
751 .unwrap()
752 .with_column_index_policy(PageIndexPolicy::Required)
753 .with_offset_index_policy(PageIndexPolicy::Required)
754 .with_column_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2]))
755 .with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([0], [1]));
756
757 let ranges = expect_needs_data(decoder.try_decode_page_index());
758 assert_eq!(ranges.len(), 1);
759 push_ranges_to_metadata_decoder(&mut decoder, ranges);
760
761 let (metadata, page_index) = expect_data(decoder.try_decode_page_index());
762 assert!(metadata.page_index().is_none());
763 let page_index = page_index.unwrap();
764 expect_finished(decoder.try_decode_page_index());
765
766 for row_group_idx in 0..2 {
767 for column_idx in 0..3 {
768 assert_eq!(
769 page_index.column_index(row_group_idx, column_idx).is_some(),
770 row_group_idx == 1 && matches!(column_idx, 0 | 2)
771 );
772 assert_eq!(
773 page_index.offset_index(row_group_idx, column_idx).is_some(),
774 row_group_idx == 0 && column_idx == 1
775 );
776 }
777 }
778 }
779
780 #[test]
781 fn test_change_page_index_selection_while_waiting() {
782 type Decoder = ParquetMetaDataPushDecoder;
783 let metadata = test_metadata_without_page_index();
784 let check = |initial: fn(Decoder) -> Decoder, change: fn(Decoder) -> Decoder| {
785 let new_decoder = || {
786 initial(Decoder::try_new_with_metadata(test_file_len(), metadata.clone()).unwrap())
787 };
788 let mut decoder = new_decoder();
789 let original = expect_needs_data(decoder.try_decode());
790 decoder = change(decoder);
791
792 let mut fresh = change(new_decoder());
793 let expected = expect_needs_data(fresh.try_decode());
794 assert_ne!(original, expected);
795 assert_eq!(expect_needs_data(decoder.try_decode()), expected);
796 push_ranges_to_metadata_decoder(&mut decoder, expected.clone());
797 push_ranges_to_metadata_decoder(&mut fresh, expected);
798 assert_eq!(
799 expect_data(decoder.try_decode()),
800 expect_data(fresh.try_decode())
801 );
802 };
803
804 let all = |decoder| decoder;
805 let one_chunk = |decoder: Decoder| {
806 decoder.with_page_index_mask(ColumnChunkMask::row_groups_and_columns([0], [1]))
807 };
808 check(all, |d| d.with_column_index_policy(PageIndexPolicy::Skip));
809 check(all, |d| d.with_offset_index_policy(PageIndexPolicy::Skip));
810 check(all, one_chunk);
811 check(one_chunk, |d| {
812 d.with_column_index_mask(ColumnChunkMask::all())
813 });
814 check(one_chunk, |d| {
815 d.with_offset_index_mask(ColumnChunkMask::all())
816 });
817 }
818
819 #[test]
820 fn test_missing_required_offset_index_after_pending_request() {
821 let mut metadata = test_metadata_without_page_index().into_builder();
822 let mut groups = metadata.take_row_groups();
823 let column = &mut groups[0].columns_mut()[0];
824 *column = column
825 .clone()
826 .into_builder()
827 .set_offset_index_offset(None)
828 .set_offset_index_length(None)
829 .build()
830 .unwrap();
831 let metadata = metadata.set_row_groups(groups).build();
832 let mut decoder =
833 ParquetMetaDataPushDecoder::try_new_with_metadata(test_file_len(), metadata).unwrap();
834
835 expect_needs_data(decoder.try_decode());
837 decoder = decoder.with_offset_index_policy(PageIndexPolicy::Required);
838 let ranges = expect_needs_data(decoder.try_decode());
839 push_ranges_to_metadata_decoder(&mut decoder, ranges);
840 let error = decoder.try_decode().unwrap_err();
841 assert!(
842 matches!(error, ParquetError::General(message) if message == "missing offset index")
843 );
844 assert!(decoder.pending_page_index_range.is_none());
845 }
846
847 #[test]
848 fn test_page_index_decoder_no_indexes_requested() {
849 let metadata = test_metadata_without_page_index();
850 let mut decoder =
851 ParquetMetaDataPushDecoder::try_new_with_metadata(test_file_len(), metadata).unwrap();
852 expect_needs_data(decoder.try_decode_page_index());
853 decoder = decoder.with_page_index_policy(PageIndexPolicy::Skip);
854 assert!(expect_data(decoder.try_decode_page_index()).1.is_none());
855 expect_finished(decoder.try_decode_page_index());
856
857 let metadata = test_metadata_without_page_index();
858 let mut decoder =
859 ParquetMetaDataPushDecoder::try_new_with_metadata(test_file_len(), metadata)
860 .unwrap()
861 .with_page_index_mask(ColumnChunkMask::none());
862
863 assert!(expect_data(decoder.try_decode_page_index()).1.is_none());
864 expect_finished(decoder.try_decode_page_index());
865 }
866
867 #[test]
868 fn test_page_index_decoder_requires_metadata() {
869 let mut decoder = ParquetMetaDataPushDecoder::try_new(test_file_len()).unwrap();
870 let err = decoder.try_decode_page_index().unwrap_err();
871 assert!(
872 err.to_string()
873 .contains("requires a decoder created with try_new_with_metadata"),
874 "unexpected error: {err}"
875 );
876 }
877
878 #[test]
879 fn test_page_index_decoder_restores_state_after_decode_error() {
880 let metadata = test_metadata_without_page_index();
881 let mut decoder =
882 ParquetMetaDataPushDecoder::try_new_with_metadata(test_file_len(), metadata)
883 .unwrap()
884 .with_page_index_policy(PageIndexPolicy::Required);
885
886 let ranges = expect_needs_data(decoder.try_decode_page_index());
887 let invalid_data = ranges
888 .iter()
889 .map(|range| Bytes::from(vec![0; (range.end - range.start) as usize]))
890 .collect();
891 decoder.push_ranges(ranges.clone(), invalid_data).unwrap();
892 decoder.try_decode_page_index().unwrap_err();
893 assert!(decoder.pending_page_index_range.is_none());
894
895 decoder.clear_all_ranges();
896 push_ranges_to_metadata_decoder(&mut decoder, ranges);
897 let (metadata, page_index) = expect_data(decoder.try_decode_page_index());
898 assert!(metadata.page_index().is_none());
899 assert!(page_index.is_some_and(|index| index.is_complete()));
900 }
901
902 #[test]
906 fn test_decode_after_failed_page_index_decode() {
907 let file_len = test_file_len();
908 let mut decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
909
910 decoder.try_decode_page_index().unwrap_err();
912 let footer_ranges = expect_needs_data(decoder.try_decode());
913 assert_eq!(footer_ranges.len(), 1);
914 push_ranges_to_metadata_decoder(&mut decoder, footer_ranges);
915
916 decoder.try_decode_page_index().unwrap_err();
918 let metadata_ranges = expect_needs_data(decoder.try_decode());
919 assert_eq!(metadata_ranges.len(), 1);
920 push_ranges_to_metadata_decoder(&mut decoder, metadata_ranges);
921
922 decoder.try_decode_page_index().unwrap_err();
924 let index_ranges = expect_needs_data(decoder.try_decode());
925 assert_eq!(index_ranges.len(), 1);
926
927 let index_ranges_again = expect_needs_data(decoder.try_decode_page_index());
930 assert_eq!(index_ranges, index_ranges_again);
931 assert_eq!(
932 decoder.pending_page_index_range.as_ref(),
933 index_ranges.first()
934 );
935 push_ranges_to_metadata_decoder(&mut decoder, index_ranges.clone());
936 decoder.clear_all_ranges();
937 assert_eq!(expect_needs_data(decoder.try_decode()), index_ranges);
938 push_ranges_to_metadata_decoder(&mut decoder, index_ranges);
939
940 let metadata = expect_data(decoder.try_decode());
942 assert!(decoder.pending_page_index_range.is_none());
943 assert_eq!(metadata.num_row_groups(), 2);
944 assert!(metadata.page_index().is_some_and(|idx| idx.is_complete()));
945
946 expect_finished(decoder.try_decode_page_index());
948 expect_finished(decoder.try_decode_page_index());
949 expect_finished(decoder.try_decode());
950 }
951
952 #[test]
955 fn test_metadata_decoder_prefetch_success() {
956 let file_len = test_file_len();
957 let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
958 let prefetch_range = (file_len - 2 * 1024)..file_len;
960 push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![prefetch_range]);
961
962 let metadata = expect_data(metadata_decoder.try_decode());
964 expect_finished(metadata_decoder.try_decode());
965 assert_eq!(metadata.num_row_groups(), 2);
966 assert_eq!(metadata.row_group(0).num_rows(), 200);
967 assert_eq!(metadata.row_group(1).num_rows(), 200);
968 assert!(metadata.page_index().is_some_and(|idx| idx.is_complete()));
969 }
970
971 #[test]
974 fn test_metadata_decoder_prefetch_retry() {
975 let file_len = test_file_len();
976 let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
977 let prefetch_range = (file_len - 1500)..file_len;
980 push_ranges_to_metadata_decoder(&mut metadata_decoder, vec![prefetch_range]);
981
982 let ranges = expect_needs_data(metadata_decoder.try_decode());
986 push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges);
987
988 let metadata = expect_data(metadata_decoder.try_decode());
990 expect_finished(metadata_decoder.try_decode());
991
992 assert_eq!(metadata.num_row_groups(), 2);
993 assert_eq!(metadata.row_group(0).num_rows(), 200);
994 assert_eq!(metadata.row_group(1).num_rows(), 200);
995 assert!(metadata.page_index().is_some_and(|idx| idx.is_complete()));
996 }
997
998 #[test]
999 fn test_metadata_decoder_clear_all_ranges() {
1000 let file_len = test_file_len();
1001 let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
1002
1003 metadata_decoder
1004 .push_range(test_file_range(), TEST_FILE_DATA.clone())
1005 .unwrap();
1006 assert_eq!(metadata_decoder.buffers.buffered_bytes(), test_file_len());
1007
1008 metadata_decoder.clear_all_ranges();
1009 assert_eq!(metadata_decoder.buffers.buffered_bytes(), 0);
1010
1011 let ranges = expect_needs_data(metadata_decoder.try_decode());
1012 assert_eq!(ranges, vec![test_file_len() - 8..test_file_len()]);
1013 }
1014
1015 #[test]
1018 fn test_metadata_decoder_incremental() {
1019 let file_len = TEST_FILE_DATA.len() as u64;
1020 let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len).unwrap();
1021 let ranges = expect_needs_data(metadata_decoder.try_decode());
1022 assert_eq!(ranges.len(), 1);
1023 assert_eq!(ranges[0], test_file_len() - 8..test_file_len());
1024 push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges);
1025
1026 let ranges = expect_needs_data(metadata_decoder.try_decode());
1028 push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges);
1029
1030 let ranges = expect_needs_data(metadata_decoder.try_decode());
1032 assert_eq!(ranges.len(), 1);
1034 push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges);
1035
1036 let metadata = expect_data(metadata_decoder.try_decode());
1038 expect_finished(metadata_decoder.try_decode());
1039
1040 assert_eq!(metadata.num_row_groups(), 2);
1041 assert_eq!(metadata.row_group(0).num_rows(), 200);
1042 assert_eq!(metadata.row_group(1).num_rows(), 200);
1043 assert!(metadata.page_index().is_some_and(|idx| idx.is_complete()));
1044 }
1045
1046 #[test]
1049 fn test_metadata_decoder_incremental_no_page_index() {
1050 let file_len = TEST_FILE_DATA.len() as u64;
1051 let mut metadata_decoder = ParquetMetaDataPushDecoder::try_new(file_len)
1052 .unwrap()
1053 .with_page_index_policy(PageIndexPolicy::Skip);
1054 let ranges = expect_needs_data(metadata_decoder.try_decode());
1055 assert_eq!(ranges.len(), 1);
1056 assert_eq!(ranges[0], test_file_len() - 8..test_file_len());
1057 push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges);
1058
1059 let ranges = expect_needs_data(metadata_decoder.try_decode());
1061 push_ranges_to_metadata_decoder(&mut metadata_decoder, ranges);
1062
1063 let metadata = expect_data(metadata_decoder.try_decode());
1065 expect_finished(metadata_decoder.try_decode());
1066
1067 assert_eq!(metadata.num_row_groups(), 2);
1068 assert_eq!(metadata.row_group(0).num_rows(), 200);
1069 assert_eq!(metadata.row_group(1).num_rows(), 200);
1070 assert!(metadata.page_index().is_none()); }
1072
1073 static TEST_BATCH: LazyLock<RecordBatch> = LazyLock::new(|| {
1074 let a: ArrayRef = Arc::new(Int64Array::from_iter_values(0..400));
1077 let b: ArrayRef = Arc::new(Int64Array::from_iter_values(400..800));
1078 let c: ArrayRef = Arc::new(StringViewArray::from_iter_values((0..400).map(|i| {
1079 if i % 2 == 0 {
1080 format!("string_{i}")
1081 } else {
1082 format!("A string larger than 12 bytes and thus not inlined {i}")
1083 }
1084 })));
1085
1086 RecordBatch::try_from_iter(vec![("a", a), ("b", b), ("c", c)]).unwrap()
1087 });
1088
1089 static TEST_FILE_DATA: LazyLock<Bytes> = LazyLock::new(|| {
1091 let input_batch = &TEST_BATCH;
1092 let mut output = Vec::new();
1093
1094 let writer_options = WriterProperties::builder()
1095 .set_max_row_group_row_count(Some(200))
1096 .set_data_page_row_count_limit(100)
1097 .build();
1098 let mut writer =
1099 ArrowWriter::try_new(&mut output, input_batch.schema(), Some(writer_options)).unwrap();
1100
1101 let mut row_remain = input_batch.num_rows();
1104 while row_remain > 0 {
1105 let chunk_size = row_remain.min(50);
1106 let chunk = input_batch.slice(input_batch.num_rows() - row_remain, chunk_size);
1107 writer.write(&chunk).unwrap();
1108 row_remain -= chunk_size;
1109 }
1110 writer.close().unwrap();
1111 Bytes::from(output)
1112 });
1113
1114 fn test_file_len() -> u64 {
1116 TEST_FILE_DATA.len() as u64
1117 }
1118
1119 fn test_file_range() -> Range<u64> {
1121 0..test_file_len()
1122 }
1123
1124 pub fn test_file_slice(range: Range<u64>) -> Bytes {
1126 let start: usize = range.start.try_into().unwrap();
1127 let end: usize = range.end.try_into().unwrap();
1128 TEST_FILE_DATA.slice(start..end)
1129 }
1130
1131 fn push_ranges_to_metadata_decoder(
1133 metadata_decoder: &mut ParquetMetaDataPushDecoder,
1134 ranges: Vec<Range<u64>>,
1135 ) {
1136 let data = ranges
1137 .iter()
1138 .map(|range| test_file_slice(range.clone()))
1139 .collect::<Vec<_>>();
1140 metadata_decoder.push_ranges(ranges, data).unwrap();
1141 }
1142
1143 fn test_metadata_without_page_index() -> ParquetMetaData {
1144 let mut decoder = ParquetMetaDataPushDecoder::try_new(test_file_len())
1145 .unwrap()
1146 .with_page_index_policy(PageIndexPolicy::Skip);
1147 push_ranges_to_metadata_decoder(&mut decoder, vec![test_file_range()]);
1148 let metadata = expect_data(decoder.try_decode());
1149 assert!(metadata.page_index().is_none());
1150 metadata
1151 }
1152
1153 fn expect_data<T: Debug>(result: Result<DecodeResult<T>>) -> T {
1155 match result.expect("Expected Ok(DecodeResult::Data(T))") {
1156 DecodeResult::Data(data) => data,
1157 result => panic!("Expected DecodeResult::Data, got {result:?}"),
1158 }
1159 }
1160
1161 fn expect_needs_data<T: Debug>(result: Result<DecodeResult<T>>) -> Vec<Range<u64>> {
1163 match result.expect("Expected Ok(DecodeResult::NeedsData(ranges))") {
1164 DecodeResult::NeedsData(ranges) => ranges,
1165 result => panic!("Expected DecodeResult::NeedsData, got {result:?}"),
1166 }
1167 }
1168
1169 fn expect_finished<T: Debug>(result: Result<DecodeResult<T>>) {
1170 match result.expect("Expected Ok(DecodeResult::Finished)") {
1171 DecodeResult::Finished => {}
1172 result => panic!("Expected DecodeResult::Finished, got {result:?}"),
1173 }
1174 }
1175}