1use crate::basic::{PageType, Type};
9use crate::bloom_filter::Sbbf;
10use crate::column::page::{Page, PageMetadata, PageReader};
11use crate::compression::{Codec, create_codec};
12#[cfg(feature = "encryption")]
13use crate::encryption::decrypt::{CryptoContext, read_and_decrypt};
14use crate::errors::{ParquetError, Result};
15use crate::file::metadata::page_index::RowGroupPageIndex;
16use crate::file::metadata::thrift::PageHeader;
17use crate::file::page_index::offset_index::PageLocation;
18use crate::file::statistics;
19use crate::file::{
20 metadata::*,
21 properties::{ReaderProperties, ReaderPropertiesPtr},
22 reader::*,
23};
24#[cfg(feature = "encryption")]
25use crate::parquet_thrift::ThriftSliceInputProtocol;
26use crate::parquet_thrift::{ReadThrift, ThriftReadInputProtocol};
27use crate::record::Row;
28use crate::record::reader::RowIter;
29use crate::schema::types::{SchemaDescPtr, Type as SchemaType};
30use bytes::Bytes;
31use std::collections::VecDeque;
32use std::{fs::File, io::Read, path::Path, sync::Arc};
33
34impl TryFrom<File> for SerializedFileReader<File> {
35 type Error = ParquetError;
36
37 fn try_from(file: File) -> Result<Self> {
38 Self::new(file)
39 }
40}
41
42impl TryFrom<&Path> for SerializedFileReader<File> {
43 type Error = ParquetError;
44
45 fn try_from(path: &Path) -> Result<Self> {
46 let file = File::open(path)?;
47 Self::try_from(file)
48 }
49}
50
51impl TryFrom<String> for SerializedFileReader<File> {
52 type Error = ParquetError;
53
54 fn try_from(path: String) -> Result<Self> {
55 Self::try_from(Path::new(&path))
56 }
57}
58
59impl TryFrom<&str> for SerializedFileReader<File> {
60 type Error = ParquetError;
61
62 fn try_from(path: &str) -> Result<Self> {
63 Self::try_from(Path::new(&path))
64 }
65}
66
67impl IntoIterator for SerializedFileReader<File> {
70 type Item = Result<Row>;
71 type IntoIter = RowIter<'static>;
72
73 fn into_iter(self) -> Self::IntoIter {
74 RowIter::from_file_into(Box::new(self))
75 }
76}
77
78pub struct SerializedFileReader<R: ChunkReader> {
83 chunk_reader: Arc<R>,
84 metadata: Arc<ParquetMetaData>,
85 props: ReaderPropertiesPtr,
86}
87
88pub type ReadGroupPredicate = Box<dyn FnMut(&RowGroupMetaData, usize) -> bool>;
92
93#[derive(Default)]
97pub struct ReadOptionsBuilder {
98 predicates: Vec<ReadGroupPredicate>,
99 enable_page_index: bool,
100 props: Option<ReaderProperties>,
101 metadata_options: ParquetMetaDataOptions,
102}
103
104impl ReadOptionsBuilder {
105 pub fn new() -> Self {
107 Self::default()
108 }
109
110 pub fn with_predicate(mut self, predicate: ReadGroupPredicate) -> Self {
113 self.predicates.push(predicate);
114 self
115 }
116
117 pub fn with_range(mut self, start: i64, end: i64) -> Self {
124 assert!(start < end);
125 let predicate = move |rg: &RowGroupMetaData, _: usize| {
126 let mid = get_midpoint_offset(rg);
127 mid >= start && mid < end
128 };
129 self.predicates.push(Box::new(predicate));
130 self
131 }
132
133 pub fn with_page_index(mut self) -> Self {
138 self.enable_page_index = true;
139 self
140 }
141
142 pub fn with_reader_properties(mut self, properties: ReaderProperties) -> Self {
144 self.props = Some(properties);
145 self
146 }
147
148 pub fn with_parquet_schema(mut self, schema: SchemaDescPtr) -> Self {
151 self.metadata_options.set_schema(schema);
152 self
153 }
154
155 pub fn with_encoding_stats_as_mask(mut self, val: bool) -> Self {
164 self.metadata_options.set_encoding_stats_as_mask(val);
165 self
166 }
167
168 pub fn with_encoding_stats_policy(mut self, policy: ParquetStatisticsPolicy) -> Self {
173 self.metadata_options.set_encoding_stats_policy(policy);
174 self
175 }
176
177 pub fn with_column_stats_policy(mut self, policy: ParquetStatisticsPolicy) -> Self {
182 self.metadata_options.set_column_stats_policy(policy);
183 self
184 }
185
186 pub fn with_size_stats_policy(mut self, policy: ParquetStatisticsPolicy) -> Self {
191 self.metadata_options.set_size_stats_policy(policy);
192 self
193 }
194
195 pub fn build(self) -> ReadOptions {
197 let props = self
198 .props
199 .unwrap_or_else(|| ReaderProperties::builder().build());
200 ReadOptions {
201 predicates: self.predicates,
202 enable_page_index: self.enable_page_index,
203 props,
204 metadata_options: self.metadata_options,
205 }
206 }
207}
208
209pub struct ReadOptions {
214 predicates: Vec<ReadGroupPredicate>,
215 enable_page_index: bool,
216 props: ReaderProperties,
217 metadata_options: ParquetMetaDataOptions,
218}
219
220impl<R: 'static + ChunkReader> SerializedFileReader<R> {
221 pub fn new(chunk_reader: R) -> Result<Self> {
224 let metadata = ParquetMetaDataReader::new().parse_and_finish(&chunk_reader)?;
225 let props = Arc::new(ReaderProperties::builder().build());
226 Ok(Self {
227 chunk_reader: Arc::new(chunk_reader),
228 metadata: Arc::new(metadata),
229 props,
230 })
231 }
232
233 pub fn new_with_options(chunk_reader: R, options: ReadOptions) -> Result<Self> {
236 let mut metadata = ParquetMetaDataReader::new()
237 .with_metadata_options(Some(options.metadata_options.clone()))
238 .parse_and_finish(&chunk_reader)?;
239 let mut predicates = options.predicates;
240
241 if !predicates.is_empty() {
242 let row_groups = metadata.row_groups();
243 let mut filtered = None;
244
245 for (i, rg_meta) in row_groups.iter().enumerate() {
247 let keep = predicates.iter_mut().all(|predicate| predicate(rg_meta, i));
248 if !keep {
249 filtered.get_or_insert_with(|| row_groups[..i].to_vec());
250 } else if let Some(kept) = &mut filtered {
251 kept.push(rg_meta.clone());
252 }
253 }
254
255 if let Some(row_groups) = filtered {
256 metadata = metadata.into_builder().set_row_groups(row_groups).build();
257 }
258 }
259
260 if options.enable_page_index {
262 let mut reader = ParquetMetaDataReader::new_with_metadata(metadata)
263 .with_page_index_policy(PageIndexPolicy::Required);
264 reader.read_page_indexes(&chunk_reader)?;
265 metadata = reader.finish()?;
266 }
267
268 Ok(Self {
269 chunk_reader: Arc::new(chunk_reader),
270 metadata: Arc::new(metadata),
271 props: Arc::new(options.props),
272 })
273 }
274}
275
276fn get_midpoint_offset(meta: &RowGroupMetaData) -> i64 {
278 let col = meta.column(0);
279 let mut offset = col.data_page_offset();
280 if let Some(dic_offset) = col.dictionary_page_offset()
281 && offset > dic_offset
282 {
283 offset = dic_offset
284 }
285 offset + meta.compressed_size() / 2
286}
287
288impl<R: 'static + ChunkReader> FileReader for SerializedFileReader<R> {
289 fn metadata(&self) -> &ParquetMetaData {
290 &self.metadata
291 }
292
293 fn num_row_groups(&self) -> usize {
294 self.metadata.num_row_groups()
295 }
296
297 fn get_row_group(&self, i: usize) -> Result<Box<dyn RowGroupReader + '_>> {
298 let row_group_metadata = self.metadata.row_group(i);
299 let props = Arc::clone(&self.props);
301 let f = Arc::clone(&self.chunk_reader);
302 let page_index = self.metadata.page_index_for_row_group(i);
303 Ok(Box::new(SerializedRowGroupReader::new(
304 f,
305 row_group_metadata,
306 page_index,
307 props,
308 )?))
309 }
310
311 fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter<'_>> {
312 RowIter::from_file(projection, self)
313 }
314}
315
316pub struct SerializedRowGroupReader<'a, R: ChunkReader> {
318 chunk_reader: Arc<R>,
319 metadata: &'a RowGroupMetaData,
320 page_index: RowGroupPageIndex,
321 props: ReaderPropertiesPtr,
322 bloom_filters: Vec<Option<Sbbf>>,
323}
324
325impl<'a, R: ChunkReader> SerializedRowGroupReader<'a, R> {
326 pub fn new(
328 chunk_reader: Arc<R>,
329 metadata: &'a RowGroupMetaData,
330 page_index: RowGroupPageIndex,
331 props: ReaderPropertiesPtr,
332 ) -> Result<Self> {
333 let bloom_filters = if props.read_bloom_filter() {
334 metadata
335 .columns()
336 .iter()
337 .map(|col| Sbbf::read_from_column_chunk(col, &*chunk_reader))
338 .collect::<Result<Vec<_>>>()?
339 } else {
340 std::iter::repeat_n(None, metadata.columns().len()).collect()
341 };
342 Ok(Self {
343 chunk_reader,
344 metadata,
345 page_index,
346 props,
347 bloom_filters,
348 })
349 }
350}
351
352impl<R: 'static + ChunkReader> RowGroupReader for SerializedRowGroupReader<'_, R> {
353 fn metadata(&self) -> &RowGroupMetaData {
354 self.metadata
355 }
356
357 fn num_columns(&self) -> usize {
358 self.metadata.num_columns()
359 }
360
361 fn get_column_page_reader(&self, i: usize) -> Result<Box<dyn PageReader>> {
363 let col = self.metadata.column(i);
364
365 let page_locations = self.page_index.page_locations(i).cloned();
367
368 let props = Arc::clone(&self.props);
369 Ok(Box::new(SerializedPageReader::new_with_properties(
370 Arc::clone(&self.chunk_reader),
371 col,
372 usize::try_from(self.metadata.num_rows())?,
373 page_locations,
374 props,
375 )?))
376 }
377
378 fn get_column_bloom_filter(&self, i: usize) -> Option<&Sbbf> {
380 self.bloom_filters[i].as_ref()
381 }
382
383 fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter<'_>> {
384 RowIter::from_row_group(projection, self)
385 }
386}
387
388pub(crate) fn decode_page(
390 page_header: PageHeader,
391 buffer: Bytes,
392 physical_type: Type,
393 decompressor: Option<&mut Box<dyn Codec>>,
394) -> Result<Page> {
395 #[cfg(feature = "crc")]
397 if let Some(expected_crc) = page_header.crc {
398 let crc = crc32fast::hash(&buffer);
399 if crc != expected_crc as u32 {
400 return Err(general_err!("Page CRC checksum mismatch"));
401 }
402 }
403
404 let (offset, can_decompress): (usize, bool) = match page_header.data_page_header_v2 {
411 Some(ref header_v2) => {
412 if header_v2.definition_levels_byte_length < 0
413 || header_v2.repetition_levels_byte_length < 0
414 || header_v2.definition_levels_byte_length + header_v2.repetition_levels_byte_length
415 > page_header.uncompressed_page_size
416 {
417 return Err(general_err!(
418 "DataPage v2 header contains implausible values \
419 for definition_levels_byte_length ({}) \
420 and repetition_levels_byte_length ({}) \
421 given DataPage header provides uncompressed_page_size ({})",
422 header_v2.definition_levels_byte_length,
423 header_v2.repetition_levels_byte_length,
424 page_header.uncompressed_page_size
425 ));
426 }
427 (
428 usize::try_from(
429 header_v2.definition_levels_byte_length
430 + header_v2.repetition_levels_byte_length,
431 )?,
432 header_v2.is_compressed.unwrap_or(true),
434 )
435 }
436 None => (0, true),
437 };
438
439 let buffer = match decompressor {
440 Some(decompressor) if can_decompress => {
441 let uncompressed_page_size = usize::try_from(page_header.uncompressed_page_size)?;
442 if offset > buffer.len() || offset > uncompressed_page_size {
443 return Err(general_err!("Invalid page header"));
444 }
445 let decompressed_size = uncompressed_page_size - offset;
446 let mut decompressed = Vec::with_capacity(uncompressed_page_size);
447 decompressed.extend_from_slice(&buffer[..offset]);
448 if decompressed_size > 0 {
451 let compressed = &buffer[offset..];
452 decompressor.decompress(compressed, &mut decompressed, Some(decompressed_size))?;
453 }
454
455 if decompressed.len() != uncompressed_page_size {
456 return Err(general_err!(
457 "Actual decompressed size doesn't match the expected one ({} vs {})",
458 decompressed.len(),
459 uncompressed_page_size
460 ));
461 }
462
463 Bytes::from(decompressed)
464 }
465 _ => buffer,
466 };
467
468 let result = match page_header.r#type {
469 PageType::DICTIONARY_PAGE => {
470 let dict_header = page_header.dictionary_page_header.as_ref().ok_or_else(|| {
471 ParquetError::General("Missing dictionary page header".to_string())
472 })?;
473 let is_sorted = dict_header.is_sorted.unwrap_or(false);
474 Page::DictionaryPage {
475 buf: buffer,
476 num_values: dict_header.num_values.try_into()?,
477 encoding: dict_header.encoding,
478 is_sorted,
479 }
480 }
481 PageType::DATA_PAGE => {
482 let header = page_header
483 .data_page_header
484 .ok_or_else(|| ParquetError::General("Missing V1 data page header".to_string()))?;
485 Page::DataPage {
486 buf: buffer,
487 num_values: header.num_values.try_into()?,
488 encoding: header.encoding,
489 def_level_encoding: header.definition_level_encoding,
490 rep_level_encoding: header.repetition_level_encoding,
491 statistics: statistics::from_thrift_page_stats(physical_type, header.statistics)?,
492 }
493 }
494 PageType::DATA_PAGE_V2 => {
495 let header = page_header
496 .data_page_header_v2
497 .ok_or_else(|| ParquetError::General("Missing V2 data page header".to_string()))?;
498 let is_compressed = header.is_compressed.unwrap_or(true);
499 Page::DataPageV2 {
500 buf: buffer,
501 num_values: header.num_values.try_into()?,
502 encoding: header.encoding,
503 num_nulls: header.num_nulls.try_into()?,
504 num_rows: header.num_rows.try_into()?,
505 def_levels_byte_len: header.definition_levels_byte_length.try_into()?,
506 rep_levels_byte_len: header.repetition_levels_byte_length.try_into()?,
507 is_compressed,
508 statistics: statistics::from_thrift_page_stats(physical_type, header.statistics)?,
509 }
510 }
511 PageType::INDEX_PAGE => {
512 return Err(general_err!(
514 "Page type {:?} is not supported",
515 page_header.r#type
516 ));
517 }
518 };
519
520 Ok(result)
521}
522
523enum DeferredDictionaryPage {
545 Header {
548 data_start: u64,
549 header: Box<PageHeader>,
550 },
551 Location(PageLocation),
554}
555
556enum SerializedPageReaderState {
557 Values {
558 offset: u64,
561
562 remaining_bytes: u64,
565
566 next_page_header: Option<Box<PageHeader>>,
568
569 page_index: usize,
571
572 require_dictionary: bool,
574 },
575 Pages {
576 page_locations: VecDeque<PageLocation>,
578 dictionary_page: Option<PageLocation>,
580 total_rows: usize,
582 page_index: usize,
584 },
585}
586
587#[derive(Default)]
588pub(crate) struct SerializedPageReaderContext {
589 pub(crate) read_stats: bool,
591 #[cfg(feature = "encryption")]
593 pub(crate) crypto_context: Option<Arc<CryptoContext>>,
594}
595
596pub struct SerializedPageReader<R: ChunkReader> {
598 reader: Arc<R>,
600
601 deferred_dictionary: Option<DeferredDictionaryPage>,
604
605 decompressor: Option<Box<dyn Codec>>,
607
608 physical_type: Type,
610
611 state: SerializedPageReaderState,
612
613 context: SerializedPageReaderContext,
614}
615
616impl<R: ChunkReader> SerializedPageReader<R> {
617 pub fn new(
619 reader: Arc<R>,
620 column_chunk_metadata: &ColumnChunkMetaData,
621 total_rows: usize,
622 page_locations: Option<Vec<PageLocation>>,
623 ) -> Result<Self> {
624 let props = Arc::new(ReaderProperties::builder().build());
625 SerializedPageReader::new_with_properties(
626 reader,
627 column_chunk_metadata,
628 total_rows,
629 page_locations,
630 props,
631 )
632 }
633
634 #[cfg(all(feature = "arrow", not(feature = "encryption")))]
636 pub(crate) fn add_crypto_context(
637 self,
638 _rg_idx: usize,
639 _column_idx: usize,
640 _parquet_meta_data: &ParquetMetaData,
641 _column_chunk_metadata: &ColumnChunkMetaData,
642 ) -> Result<SerializedPageReader<R>> {
643 Ok(self)
644 }
645
646 #[cfg(feature = "encryption")]
648 pub(crate) fn add_crypto_context(
649 mut self,
650 rg_idx: usize,
651 column_idx: usize,
652 parquet_meta_data: &ParquetMetaData,
653 column_chunk_metadata: &ColumnChunkMetaData,
654 ) -> Result<SerializedPageReader<R>> {
655 let Some(file_decryptor) = parquet_meta_data.file_decryptor() else {
656 return Ok(self);
657 };
658 let Some(crypto_metadata) = column_chunk_metadata.crypto_metadata() else {
659 return Ok(self);
660 };
661 let crypto_context =
662 CryptoContext::for_column(file_decryptor, crypto_metadata, rg_idx, column_idx)?;
663 self.context.crypto_context = Some(Arc::new(crypto_context));
664 Ok(self)
665 }
666
667 pub fn new_with_properties(
669 reader: Arc<R>,
670 meta: &ColumnChunkMetaData,
671 total_rows: usize,
672 page_locations: Option<Vec<PageLocation>>,
673 props: ReaderPropertiesPtr,
674 ) -> Result<Self> {
675 let decompressor = create_codec(meta.compression(), props.codec_options())?;
676 let (start, len) = meta.byte_range();
677
678 let state = match page_locations {
679 Some(locations) => {
680 let dictionary_page = match locations.first() {
683 Some(dict_offset) if dict_offset.offset as u64 != start => Some(PageLocation {
684 offset: start as i64,
685 compressed_page_size: (dict_offset.offset as u64 - start) as i32,
686 first_row_index: 0,
687 }),
688 _ => None,
689 };
690
691 SerializedPageReaderState::Pages {
692 page_locations: locations.into(),
693 dictionary_page,
694 total_rows,
695 page_index: 0,
696 }
697 }
698 None => SerializedPageReaderState::Values {
699 offset: start,
700 remaining_bytes: len,
701 next_page_header: None,
702 page_index: 0,
703 require_dictionary: meta.dictionary_page_offset().is_some(),
704 },
705 };
706 let mut context = SerializedPageReaderContext::default();
707 if props.read_page_stats() {
708 context.read_stats = true;
709 }
710 Ok(Self {
711 reader,
712 deferred_dictionary: None,
713 decompressor,
714 state,
715 physical_type: meta.column_type(),
716 context,
717 })
718 }
719
720 #[cfg(test)]
726 fn peek_next_page_offset(&mut self) -> Result<Option<u64>> {
727 match &mut self.state {
728 SerializedPageReaderState::Values {
729 offset,
730 remaining_bytes,
731 next_page_header,
732 page_index,
733 require_dictionary,
734 } => {
735 loop {
736 if *remaining_bytes == 0 {
737 return Ok(None);
738 }
739 return if let Some(header) = next_page_header.as_ref() {
740 if let Ok(_page_meta) = PageMetadata::try_from(&**header) {
741 Ok(Some(*offset))
742 } else {
743 *next_page_header = None;
745 continue;
746 }
747 } else {
748 let mut read = self.reader.get_read(*offset)?;
749 let (header_len, header) = Self::read_page_header_len(
750 &self.context,
751 &mut read,
752 *page_index,
753 *require_dictionary,
754 )?;
755 *offset += header_len as u64;
756 *remaining_bytes -= header_len as u64;
757 let page_meta = if let Ok(_page_meta) = PageMetadata::try_from(&header) {
758 Ok(Some(*offset))
759 } else {
760 continue;
762 };
763 *next_page_header = Some(Box::new(header));
764 page_meta
765 };
766 }
767 }
768 SerializedPageReaderState::Pages {
769 page_locations,
770 dictionary_page,
771 ..
772 } => {
773 if let Some(page) = dictionary_page {
774 Ok(Some(page.offset as u64))
775 } else if let Some(page) = page_locations.front() {
776 Ok(Some(page.offset as u64))
777 } else {
778 Ok(None)
779 }
780 }
781 }
782 }
783
784 fn read_page_header_len<T: Read>(
785 context: &SerializedPageReaderContext,
786 input: &mut T,
787 page_index: usize,
788 dictionary_page: bool,
789 ) -> Result<(usize, PageHeader)> {
790 struct TrackedRead<R> {
792 inner: R,
793 bytes_read: usize,
794 }
795
796 impl<R: Read> Read for TrackedRead<R> {
797 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
798 let v = self.inner.read(buf)?;
799 self.bytes_read += v;
800 Ok(v)
801 }
802 }
803
804 let mut tracked = TrackedRead {
805 inner: input,
806 bytes_read: 0,
807 };
808 let header = context.read_page_header(&mut tracked, page_index, dictionary_page)?;
809 Ok((tracked.bytes_read, header))
810 }
811}
812
813pub(crate) fn read_page_header_len_from_bytes(
821 context: &SerializedPageReaderContext,
822 buffer: &[u8],
823 page_index: usize,
824 dictionary_page: bool,
825) -> Result<(usize, PageHeader)> {
826 let mut input = std::io::Cursor::new(buffer);
827 let header = context.read_page_header(&mut input, page_index, dictionary_page)?;
828 let header_len = input.position() as usize;
829 Ok((header_len, header))
830}
831
832#[cfg(not(feature = "encryption"))]
833impl SerializedPageReaderContext {
834 pub(crate) fn read_page_header<T: Read>(
835 &self,
836 input: &mut T,
837 _page_index: usize,
838 _dictionary_page: bool,
839 ) -> Result<PageHeader> {
840 let mut prot = ThriftReadInputProtocol::new(input);
841 if self.read_stats {
842 Ok(PageHeader::read_thrift(&mut prot)?)
843 } else {
844 Ok(PageHeader::read_thrift_without_stats(&mut prot)?)
845 }
846 }
847
848 pub(crate) fn decrypt_page_data<T>(
849 &self,
850 buffer: T,
851 _page_index: usize,
852 _dictionary_page: bool,
853 ) -> Result<T> {
854 Ok(buffer)
855 }
856}
857
858#[cfg(feature = "encryption")]
859impl SerializedPageReaderContext {
860 pub(crate) fn read_page_header<T: Read>(
861 &self,
862 input: &mut T,
863 page_index: usize,
864 dictionary_page: bool,
865 ) -> Result<PageHeader> {
866 match self.page_crypto_context(page_index, dictionary_page) {
867 None => {
868 let mut prot = ThriftReadInputProtocol::new(input);
869 if self.read_stats {
870 Ok(PageHeader::read_thrift(&mut prot)?)
871 } else {
872 use crate::file::metadata::thrift::PageHeader;
873
874 Ok(PageHeader::read_thrift_without_stats(&mut prot)?)
875 }
876 }
877 Some(page_crypto_context) => {
878 let data_decryptor = page_crypto_context.data_decryptor();
879 let aad = page_crypto_context.create_page_header_aad()?;
880
881 let buf = read_and_decrypt(data_decryptor, input, aad.as_ref()).map_err(|_| {
882 ParquetError::General(format!(
883 "Error decrypting page header for column {}, decryption key may be wrong",
884 page_crypto_context.column_ordinal
885 ))
886 })?;
887
888 let mut prot = ThriftSliceInputProtocol::new(buf.as_slice());
889 if self.read_stats {
890 Ok(PageHeader::read_thrift(&mut prot)?)
891 } else {
892 Ok(PageHeader::read_thrift_without_stats(&mut prot)?)
893 }
894 }
895 }
896 }
897
898 pub(crate) fn decrypt_page_data<T>(
899 &self,
900 buffer: T,
901 page_index: usize,
902 dictionary_page: bool,
903 ) -> Result<T>
904 where
905 T: AsRef<[u8]>,
906 T: From<Vec<u8>>,
907 {
908 let page_crypto_context = self.page_crypto_context(page_index, dictionary_page);
909 if let Some(page_crypto_context) = page_crypto_context {
910 let decryptor = page_crypto_context.data_decryptor();
911 let aad = page_crypto_context.create_page_aad()?;
912 let decrypted = decryptor.decrypt(buffer.as_ref(), &aad)?;
913 Ok(T::from(decrypted))
914 } else {
915 Ok(buffer)
916 }
917 }
918
919 fn page_crypto_context(
920 &self,
921 page_index: usize,
922 dictionary_page: bool,
923 ) -> Option<Arc<CryptoContext>> {
924 self.crypto_context.as_ref().map(|c| {
925 Arc::new(if dictionary_page {
926 c.for_dictionary_page()
927 } else {
928 c.with_page_ordinal(page_index)
929 })
930 })
931 }
932}
933
934impl<R: ChunkReader> Iterator for SerializedPageReader<R> {
935 type Item = Result<Page>;
936
937 fn next(&mut self) -> Option<Self::Item> {
938 self.get_next_page().transpose()
939 }
940}
941
942fn verify_page_header_len(header_len: usize, remaining_bytes: u64) -> Result<()> {
943 if header_len as u64 > remaining_bytes {
944 return Err(eof_err!("Invalid page header"));
945 }
946 Ok(())
947}
948
949pub(crate) fn verify_page_size(
950 compressed_size: i32,
951 uncompressed_size: i32,
952 remaining_bytes: u64,
953) -> Result<()> {
954 if compressed_size < 0 || compressed_size as u64 > remaining_bytes || uncompressed_size < 0 {
958 return Err(eof_err!("Invalid page header"));
959 }
960 Ok(())
961}
962
963impl<R: ChunkReader> PageReader for SerializedPageReader<R> {
964 fn get_next_page(&mut self) -> Result<Option<Page>> {
965 loop {
966 let page = match &mut self.state {
967 SerializedPageReaderState::Values {
968 offset,
969 remaining_bytes: remaining,
970 next_page_header,
971 page_index,
972 require_dictionary,
973 } => {
974 if *remaining == 0 {
975 return Ok(None);
976 }
977
978 let mut read = self.reader.get_read(*offset)?;
979 let header = if let Some(header) = next_page_header.take() {
980 *header
981 } else {
982 let (header_len, header) = Self::read_page_header_len(
983 &self.context,
984 &mut read,
985 *page_index,
986 *require_dictionary,
987 )?;
988 verify_page_header_len(header_len, *remaining)?;
989 *offset += header_len as u64;
990 *remaining -= header_len as u64;
991 header
992 };
993 verify_page_size(
994 header.compressed_page_size,
995 header.uncompressed_page_size,
996 *remaining,
997 )?;
998 let data_len = header.compressed_page_size as usize;
999 let data_start = *offset;
1000 *offset += data_len as u64;
1001 *remaining -= data_len as u64;
1002
1003 if header.r#type == PageType::INDEX_PAGE {
1004 continue;
1005 }
1006
1007 let buffer = self.reader.get_bytes(data_start, data_len)?;
1008
1009 let buffer =
1010 self.context
1011 .decrypt_page_data(buffer, *page_index, *require_dictionary)?;
1012
1013 let page = decode_page(
1014 header,
1015 buffer,
1016 self.physical_type,
1017 self.decompressor.as_mut(),
1018 )?;
1019 if page.is_data_page() {
1020 *page_index += 1;
1021 } else if page.is_dictionary_page() {
1022 *require_dictionary = false;
1023 }
1024 page
1025 }
1026 SerializedPageReaderState::Pages {
1027 page_locations,
1028 dictionary_page,
1029 page_index,
1030 ..
1031 } => {
1032 let (front, is_dictionary_page) = match dictionary_page.take() {
1033 Some(front) => (front, true),
1034 None => match page_locations.pop_front() {
1035 Some(front) => (front, false),
1036 None => return Ok(None),
1037 },
1038 };
1039
1040 let page_len = usize::try_from(front.compressed_page_size)?;
1041 let buffer = self.reader.get_bytes(front.offset as u64, page_len)?;
1042
1043 let (offset, header) = read_page_header_len_from_bytes(
1044 &self.context,
1045 buffer.as_ref(),
1046 *page_index,
1047 is_dictionary_page,
1048 )?;
1049 let bytes = buffer.slice(offset..);
1050 let bytes =
1051 self.context
1052 .decrypt_page_data(bytes, *page_index, is_dictionary_page)?;
1053
1054 if !is_dictionary_page {
1055 *page_index += 1;
1056 }
1057 decode_page(
1058 header,
1059 bytes,
1060 self.physical_type,
1061 self.decompressor.as_mut(),
1062 )?
1063 }
1064 };
1065
1066 return Ok(Some(page));
1067 }
1068 }
1069
1070 fn peek_next_page(&mut self) -> Result<Option<PageMetadata>> {
1071 match &mut self.state {
1072 SerializedPageReaderState::Values {
1073 offset,
1074 remaining_bytes,
1075 next_page_header,
1076 page_index,
1077 require_dictionary,
1078 } => {
1079 loop {
1080 if *remaining_bytes == 0 {
1081 return Ok(None);
1082 }
1083 return if let Some(header) = next_page_header.as_ref() {
1084 if let Ok(page_meta) = (&**header).try_into() {
1085 Ok(Some(page_meta))
1086 } else {
1087 *next_page_header = None;
1089 continue;
1090 }
1091 } else {
1092 let mut read = self.reader.get_read(*offset)?;
1093 let (header_len, header) = Self::read_page_header_len(
1094 &self.context,
1095 &mut read,
1096 *page_index,
1097 *require_dictionary,
1098 )?;
1099 verify_page_header_len(header_len, *remaining_bytes)?;
1100 *offset += header_len as u64;
1101 *remaining_bytes -= header_len as u64;
1102 let page_meta = if let Ok(page_meta) = (&header).try_into() {
1103 Ok(Some(page_meta))
1104 } else {
1105 continue;
1107 };
1108 *next_page_header = Some(Box::new(header));
1109 page_meta
1110 };
1111 }
1112 }
1113 SerializedPageReaderState::Pages {
1114 page_locations,
1115 dictionary_page,
1116 total_rows,
1117 page_index: _,
1118 } => {
1119 if dictionary_page.is_some() {
1120 Ok(Some(PageMetadata {
1121 num_rows: None,
1122 num_levels: None,
1123 is_dict: true,
1124 }))
1125 } else if let Some(page) = page_locations.front() {
1126 let next_rows = page_locations
1127 .get(1)
1128 .map(|x| x.first_row_index as usize)
1129 .unwrap_or(*total_rows);
1130
1131 Ok(Some(PageMetadata {
1132 num_rows: Some(next_rows - page.first_row_index as usize),
1133 num_levels: None,
1134 is_dict: false,
1135 }))
1136 } else {
1137 Ok(None)
1138 }
1139 }
1140 }
1141 }
1142
1143 fn skip_next_page(&mut self) -> Result<()> {
1144 let deferred = match &mut self.state {
1147 SerializedPageReaderState::Values {
1148 offset,
1149 remaining_bytes,
1150 next_page_header,
1151 page_index,
1152 require_dictionary,
1153 } => {
1154 let mut deferred = None;
1155 if let Some(buffered_header) = next_page_header.take() {
1156 verify_page_size(
1157 buffered_header.compressed_page_size,
1158 buffered_header.uncompressed_page_size,
1159 *remaining_bytes,
1160 )?;
1161 let body_len = buffered_header.compressed_page_size as u64;
1164 if buffered_header.r#type == PageType::DICTIONARY_PAGE {
1165 deferred = Some(DeferredDictionaryPage::Header {
1166 data_start: *offset,
1167 header: buffered_header,
1168 });
1169 }
1170 *offset += body_len;
1171 *remaining_bytes -= body_len;
1172 } else {
1173 let mut read = self.reader.get_read(*offset)?;
1174 let (header_len, header) = Self::read_page_header_len(
1175 &self.context,
1176 &mut read,
1177 *page_index,
1178 *require_dictionary,
1179 )?;
1180 verify_page_header_len(header_len, *remaining_bytes)?;
1181 verify_page_size(
1182 header.compressed_page_size,
1183 header.uncompressed_page_size,
1184 *remaining_bytes,
1185 )?;
1186 let data_page_size = header.compressed_page_size as u64;
1187 let is_dict = header.r#type == PageType::DICTIONARY_PAGE;
1188 if is_dict {
1189 deferred = Some(DeferredDictionaryPage::Header {
1190 data_start: *offset + header_len as u64,
1191 header: Box::new(header),
1192 });
1193 }
1194 *offset += header_len as u64 + data_page_size;
1195 *remaining_bytes -= header_len as u64 + data_page_size;
1196 }
1197 if deferred.is_some() {
1198 *require_dictionary = false;
1199 } else {
1200 *page_index += 1;
1201 }
1202 deferred
1203 }
1204 SerializedPageReaderState::Pages {
1205 page_locations,
1206 dictionary_page,
1207 page_index,
1208 ..
1209 } => {
1210 if dictionary_page.is_some() {
1211 dictionary_page.take().map(DeferredDictionaryPage::Location)
1213 } else {
1214 if page_locations.pop_front().is_some() {
1216 *page_index += 1;
1217 }
1218 None
1219 }
1220 }
1221 };
1222 if deferred.is_some() {
1223 self.deferred_dictionary = deferred;
1224 }
1225 Ok(())
1226 }
1227
1228 fn supports_deferred_dictionary(&self) -> bool {
1229 match &self.state {
1230 SerializedPageReaderState::Values { .. } => true,
1232 SerializedPageReaderState::Pages {
1236 dictionary_page, ..
1237 } => dictionary_page.is_some(),
1238 }
1239 }
1240
1241 fn take_deferred_dictionary(&mut self) -> Result<Option<Page>> {
1242 let Some(deferred) = self.deferred_dictionary.take() else {
1243 return Ok(None);
1244 };
1245 let page = match deferred {
1246 DeferredDictionaryPage::Header { data_start, header } => {
1247 let data_len = header.compressed_page_size as usize;
1248 let buffer = self.reader.get_bytes(data_start, data_len)?;
1249 let buffer = self.context.decrypt_page_data(buffer, 0, true)?;
1250 decode_page(
1251 *header,
1252 buffer,
1253 self.physical_type,
1254 self.decompressor.as_mut(),
1255 )?
1256 }
1257 DeferredDictionaryPage::Location(front) => {
1258 let page_len = usize::try_from(front.compressed_page_size)?;
1259 let buffer = self.reader.get_bytes(front.offset as u64, page_len)?;
1260 let (offset, header) =
1261 read_page_header_len_from_bytes(&self.context, buffer.as_ref(), 0, true)?;
1262 let bytes = buffer.slice(offset..);
1263 let bytes = self.context.decrypt_page_data(bytes, 0, true)?;
1264 decode_page(
1265 header,
1266 bytes,
1267 self.physical_type,
1268 self.decompressor.as_mut(),
1269 )?
1270 }
1271 };
1272 Ok(Some(page))
1273 }
1274
1275 fn at_record_boundary(&mut self) -> Result<bool> {
1276 match &mut self.state {
1277 SerializedPageReaderState::Values { .. } => match self.peek_next_page()? {
1278 None => Ok(true),
1279 Some(metadata) => Ok(metadata.num_rows.is_some()),
1282 },
1283 SerializedPageReaderState::Pages { .. } => Ok(true),
1284 }
1285 }
1286}
1287
1288#[cfg(test)]
1289mod tests {
1290 use std::cell::RefCell;
1291 use std::collections::HashSet;
1292 use std::rc::Rc;
1293
1294 use bytes::Buf;
1295
1296 use crate::file::page_index::column_index::{
1297 ByteArrayColumnIndex, ColumnIndexMetaData, PrimitiveColumnIndex,
1298 };
1299 use crate::file::properties::{EnabledStatistics, WriterProperties};
1300
1301 use crate::basic::{self, BoundaryOrder, ColumnOrder, Compression, Encoding, SortOrder};
1302 use crate::column::reader::ColumnReader;
1303 use crate::data_type::private::ParquetValueType;
1304 use crate::data_type::{AsBytes, FixedLenByteArrayType, Int32Type};
1305 use crate::file::metadata::thrift::DataPageHeaderV2;
1306 use crate::file::writer::SerializedFileWriter;
1307 use crate::parquet_thrift::{ReadThrift, ThriftSliceInputProtocol};
1308 use crate::record::RowAccessor;
1309 use crate::schema::parser::parse_message_type;
1310 use crate::util::test_common::file_util::{get_test_file, get_test_path};
1311
1312 use super::*;
1313
1314 #[test]
1315 fn test_decode_page_invalid_offset() {
1316 let page_header = PageHeader {
1317 r#type: PageType::DATA_PAGE_V2,
1318 uncompressed_page_size: 10,
1319 compressed_page_size: 10,
1320 data_page_header: None,
1321 index_page_header: None,
1322 dictionary_page_header: None,
1323 crc: None,
1324 data_page_header_v2: Some(DataPageHeaderV2 {
1325 num_nulls: 0,
1326 num_rows: 0,
1327 num_values: 0,
1328 encoding: Encoding::PLAIN,
1329 definition_levels_byte_length: 11,
1330 repetition_levels_byte_length: 0,
1331 is_compressed: None,
1332 statistics: None,
1333 }),
1334 };
1335
1336 let buffer = Bytes::new();
1337 let err = decode_page(page_header, buffer, Type::INT32, None).unwrap_err();
1338 assert!(
1339 err.to_string()
1340 .contains("DataPage v2 header contains implausible values")
1341 );
1342 }
1343
1344 #[test]
1345 fn test_decode_unsupported_page() {
1346 let mut page_header = PageHeader {
1347 r#type: PageType::INDEX_PAGE,
1348 uncompressed_page_size: 10,
1349 compressed_page_size: 10,
1350 data_page_header: None,
1351 index_page_header: None,
1352 dictionary_page_header: None,
1353 crc: None,
1354 data_page_header_v2: None,
1355 };
1356 let buffer = Bytes::new();
1357 let err = decode_page(page_header.clone(), buffer.clone(), Type::INT32, None).unwrap_err();
1358 assert_eq!(
1359 err.to_string(),
1360 "Parquet error: Page type INDEX_PAGE is not supported"
1361 );
1362
1363 page_header.data_page_header_v2 = Some(DataPageHeaderV2 {
1364 num_nulls: 0,
1365 num_rows: 0,
1366 num_values: 0,
1367 encoding: Encoding::PLAIN,
1368 definition_levels_byte_length: 11,
1369 repetition_levels_byte_length: 0,
1370 is_compressed: None,
1371 statistics: None,
1372 });
1373 let err = decode_page(page_header, buffer, Type::INT32, None).unwrap_err();
1374 assert!(
1375 err.to_string()
1376 .contains("DataPage v2 header contains implausible values")
1377 );
1378 }
1379
1380 #[test]
1381 fn test_cursor_and_file_has_the_same_behaviour() {
1382 let mut buf: Vec<u8> = Vec::new();
1383 get_test_file("alltypes_plain.parquet")
1384 .read_to_end(&mut buf)
1385 .unwrap();
1386 let cursor = Bytes::from(buf);
1387 let read_from_cursor = SerializedFileReader::new(cursor).unwrap();
1388
1389 let test_file = get_test_file("alltypes_plain.parquet");
1390 let read_from_file = SerializedFileReader::new(test_file).unwrap();
1391
1392 let file_iter = read_from_file.get_row_iter(None).unwrap();
1393 let cursor_iter = read_from_cursor.get_row_iter(None).unwrap();
1394
1395 for (a, b) in file_iter.zip(cursor_iter) {
1396 assert_eq!(a.unwrap(), b.unwrap())
1397 }
1398 }
1399
1400 #[test]
1401 fn test_file_reader_try_from() {
1402 let test_file = get_test_file("alltypes_plain.parquet");
1404 let test_path_buf = get_test_path("alltypes_plain.parquet");
1405 let test_path = test_path_buf.as_path();
1406 let test_path_str = test_path.to_str().unwrap();
1407
1408 let reader = SerializedFileReader::try_from(test_file);
1409 assert!(reader.is_ok());
1410
1411 let reader = SerializedFileReader::try_from(test_path);
1412 assert!(reader.is_ok());
1413
1414 let reader = SerializedFileReader::try_from(test_path_str);
1415 assert!(reader.is_ok());
1416
1417 let reader = SerializedFileReader::try_from(test_path_str.to_string());
1418 assert!(reader.is_ok());
1419
1420 let test_path = Path::new("invalid.parquet");
1422 let test_path_str = test_path.to_str().unwrap();
1423
1424 let reader = SerializedFileReader::try_from(test_path);
1425 assert!(reader.is_err());
1426
1427 let reader = SerializedFileReader::try_from(test_path_str);
1428 assert!(reader.is_err());
1429
1430 let reader = SerializedFileReader::try_from(test_path_str.to_string());
1431 assert!(reader.is_err());
1432 }
1433
1434 #[test]
1435 fn test_file_reader_into_iter() {
1436 let path = get_test_path("alltypes_plain.parquet");
1437 let reader = SerializedFileReader::try_from(path.as_path()).unwrap();
1438 let iter = reader.into_iter();
1439 let values: Vec<_> = iter.flat_map(|x| x.unwrap().get_int(0)).collect();
1440
1441 assert_eq!(values, &[4, 5, 6, 7, 2, 3, 0, 1]);
1442 }
1443
1444 #[test]
1445 fn test_file_reader_into_iter_project() {
1446 let path = get_test_path("alltypes_plain.parquet");
1447 let reader = SerializedFileReader::try_from(path.as_path()).unwrap();
1448 let schema = "message schema { OPTIONAL INT32 id; }";
1449 let proj = parse_message_type(schema).ok();
1450 let iter = reader.into_iter().project(proj).unwrap();
1451 let values: Vec<_> = iter.flat_map(|x| x.unwrap().get_int(0)).collect();
1452
1453 assert_eq!(values, &[4, 5, 6, 7, 2, 3, 0, 1]);
1454 }
1455
1456 #[test]
1457 fn test_reuse_file_chunk() {
1458 let test_file = get_test_file("alltypes_plain.parquet");
1462 let reader = SerializedFileReader::new(test_file).unwrap();
1463 let row_group = reader.get_row_group(0).unwrap();
1464
1465 let mut page_readers = Vec::new();
1466 for i in 0..row_group.num_columns() {
1467 page_readers.push(row_group.get_column_page_reader(i).unwrap());
1468 }
1469
1470 for mut page_reader in page_readers {
1473 assert!(page_reader.get_next_page().is_ok());
1474 }
1475 }
1476
1477 #[test]
1478 fn test_file_reader() {
1479 let test_file = get_test_file("alltypes_plain.parquet");
1480 let reader_result = SerializedFileReader::new(test_file);
1481 assert!(reader_result.is_ok());
1482 let reader = reader_result.unwrap();
1483
1484 let metadata = reader.metadata();
1486 assert_eq!(metadata.num_row_groups(), 1);
1487
1488 let file_metadata = metadata.file_metadata();
1490 assert!(file_metadata.created_by().is_some());
1491 assert_eq!(
1492 file_metadata.created_by().unwrap(),
1493 "impala version 1.3.0-INTERNAL (build 8a48ddb1eff84592b3fc06bc6f51ec120e1fffc9)"
1494 );
1495 assert!(file_metadata.key_value_metadata().is_none());
1496 assert_eq!(file_metadata.num_rows(), 8);
1497 assert_eq!(file_metadata.version(), 1);
1498 assert_eq!(file_metadata.column_orders(), None);
1499
1500 let row_group_metadata = metadata.row_group(0);
1502 assert_eq!(row_group_metadata.num_columns(), 11);
1503 assert_eq!(row_group_metadata.num_rows(), 8);
1504 assert_eq!(row_group_metadata.total_byte_size(), 671);
1505 for i in 0..row_group_metadata.num_columns() {
1507 assert_eq!(file_metadata.column_order(i), ColumnOrder::UNDEFINED);
1508 }
1509
1510 let row_group_reader_result = reader.get_row_group(0);
1512 assert!(row_group_reader_result.is_ok());
1513 let row_group_reader: Box<dyn RowGroupReader> = row_group_reader_result.unwrap();
1514 assert_eq!(
1515 row_group_reader.num_columns(),
1516 row_group_metadata.num_columns()
1517 );
1518 assert_eq!(
1519 row_group_reader.metadata().total_byte_size(),
1520 row_group_metadata.total_byte_size()
1521 );
1522
1523 let page_reader_0_result = row_group_reader.get_column_page_reader(0);
1526 assert!(page_reader_0_result.is_ok());
1527 let mut page_reader_0: Box<dyn PageReader> = page_reader_0_result.unwrap();
1528 let mut page_count = 0;
1529 while let Some(page) = page_reader_0.get_next_page().unwrap() {
1530 let is_expected_page = match page {
1531 Page::DictionaryPage {
1532 buf,
1533 num_values,
1534 encoding,
1535 is_sorted,
1536 } => {
1537 assert_eq!(buf.len(), 32);
1538 assert_eq!(num_values, 8);
1539 assert_eq!(encoding, Encoding::PLAIN_DICTIONARY);
1540 assert!(!is_sorted);
1541 true
1542 }
1543 Page::DataPage {
1544 buf,
1545 num_values,
1546 encoding,
1547 def_level_encoding,
1548 rep_level_encoding,
1549 statistics,
1550 } => {
1551 assert_eq!(buf.len(), 11);
1552 assert_eq!(num_values, 8);
1553 assert_eq!(encoding, Encoding::PLAIN_DICTIONARY);
1554 assert_eq!(def_level_encoding, Encoding::RLE);
1555 #[expect(deprecated)]
1556 let expected_rep_level_encoding = Encoding::BIT_PACKED;
1557 assert_eq!(rep_level_encoding, expected_rep_level_encoding);
1558 assert!(statistics.is_none());
1559 true
1560 }
1561 Page::DataPageV2 { .. } => false,
1562 };
1563 assert!(is_expected_page);
1564 page_count += 1;
1565 }
1566 assert_eq!(page_count, 2);
1567 }
1568
1569 #[test]
1570 fn test_file_reader_datapage_v2() {
1571 let test_file = get_test_file("datapage_v2.snappy.parquet");
1572 let reader_result = SerializedFileReader::new(test_file);
1573 assert!(reader_result.is_ok());
1574 let reader = reader_result.unwrap();
1575
1576 let metadata = reader.metadata();
1578 assert_eq!(metadata.num_row_groups(), 1);
1579
1580 let file_metadata = metadata.file_metadata();
1582 assert!(file_metadata.created_by().is_some());
1583 assert_eq!(
1584 file_metadata.created_by().unwrap(),
1585 "parquet-mr version 1.8.1 (build 4aba4dae7bb0d4edbcf7923ae1339f28fd3f7fcf)"
1586 );
1587 assert!(file_metadata.key_value_metadata().is_some());
1588 assert_eq!(file_metadata.key_value_metadata().unwrap().len(), 1);
1589
1590 assert_eq!(file_metadata.num_rows(), 5);
1591 assert_eq!(file_metadata.version(), 1);
1592 assert_eq!(file_metadata.column_orders(), None);
1593
1594 let row_group_metadata = metadata.row_group(0);
1595
1596 for i in 0..row_group_metadata.num_columns() {
1598 assert_eq!(file_metadata.column_order(i), ColumnOrder::UNDEFINED);
1599 }
1600
1601 let row_group_reader_result = reader.get_row_group(0);
1603 assert!(row_group_reader_result.is_ok());
1604 let row_group_reader: Box<dyn RowGroupReader> = row_group_reader_result.unwrap();
1605 assert_eq!(
1606 row_group_reader.num_columns(),
1607 row_group_metadata.num_columns()
1608 );
1609 assert_eq!(
1610 row_group_reader.metadata().total_byte_size(),
1611 row_group_metadata.total_byte_size()
1612 );
1613
1614 let page_reader_0_result = row_group_reader.get_column_page_reader(0);
1617 assert!(page_reader_0_result.is_ok());
1618 let mut page_reader_0: Box<dyn PageReader> = page_reader_0_result.unwrap();
1619 let mut page_count = 0;
1620 while let Some(page) = page_reader_0.get_next_page().unwrap() {
1621 let is_expected_page = match page {
1622 Page::DictionaryPage {
1623 buf,
1624 num_values,
1625 encoding,
1626 is_sorted,
1627 } => {
1628 assert_eq!(buf.len(), 7);
1629 assert_eq!(num_values, 1);
1630 assert_eq!(encoding, Encoding::PLAIN);
1631 assert!(!is_sorted);
1632 true
1633 }
1634 Page::DataPageV2 {
1635 buf,
1636 num_values,
1637 encoding,
1638 num_nulls,
1639 num_rows,
1640 def_levels_byte_len,
1641 rep_levels_byte_len,
1642 is_compressed,
1643 statistics,
1644 } => {
1645 assert_eq!(buf.len(), 4);
1646 assert_eq!(num_values, 5);
1647 assert_eq!(encoding, Encoding::RLE_DICTIONARY);
1648 assert_eq!(num_nulls, 1);
1649 assert_eq!(num_rows, 5);
1650 assert_eq!(def_levels_byte_len, 2);
1651 assert_eq!(rep_levels_byte_len, 0);
1652 assert!(is_compressed);
1653 assert!(statistics.is_none()); true
1655 }
1656 Page::DataPage { .. } => false,
1657 };
1658 assert!(is_expected_page);
1659 page_count += 1;
1660 }
1661 assert_eq!(page_count, 2);
1662 }
1663
1664 #[cfg_attr(miri, ignore)] #[test]
1666 fn test_file_reader_empty_compressed_datapage_v2() {
1667 let test_file = get_test_file("page_v2_empty_compressed.parquet");
1669 let reader_result = SerializedFileReader::new(test_file);
1670 assert!(reader_result.is_ok());
1671 let reader = reader_result.unwrap();
1672
1673 let metadata = reader.metadata();
1675 assert_eq!(metadata.num_row_groups(), 1);
1676
1677 let file_metadata = metadata.file_metadata();
1679 assert!(file_metadata.created_by().is_some());
1680 assert_eq!(
1681 file_metadata.created_by().unwrap(),
1682 "parquet-cpp-arrow version 14.0.2"
1683 );
1684 assert!(file_metadata.key_value_metadata().is_some());
1685 assert_eq!(file_metadata.key_value_metadata().unwrap().len(), 1);
1686
1687 assert_eq!(file_metadata.num_rows(), 10);
1688 assert_eq!(file_metadata.version(), 2);
1689 let expected_order = ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::SIGNED);
1690 assert_eq!(
1691 file_metadata.column_orders(),
1692 Some(vec![expected_order].as_ref())
1693 );
1694
1695 let row_group_metadata = metadata.row_group(0);
1696
1697 for i in 0..row_group_metadata.num_columns() {
1699 assert_eq!(file_metadata.column_order(i), expected_order);
1700 }
1701
1702 let row_group_reader_result = reader.get_row_group(0);
1704 assert!(row_group_reader_result.is_ok());
1705 let row_group_reader: Box<dyn RowGroupReader> = row_group_reader_result.unwrap();
1706 assert_eq!(
1707 row_group_reader.num_columns(),
1708 row_group_metadata.num_columns()
1709 );
1710 assert_eq!(
1711 row_group_reader.metadata().total_byte_size(),
1712 row_group_metadata.total_byte_size()
1713 );
1714
1715 let page_reader_0_result = row_group_reader.get_column_page_reader(0);
1717 assert!(page_reader_0_result.is_ok());
1718 let mut page_reader_0: Box<dyn PageReader> = page_reader_0_result.unwrap();
1719 let mut page_count = 0;
1720 while let Some(page) = page_reader_0.get_next_page().unwrap() {
1721 let is_expected_page = match page {
1722 Page::DictionaryPage {
1723 buf,
1724 num_values,
1725 encoding,
1726 is_sorted,
1727 } => {
1728 assert_eq!(buf.len(), 0);
1729 assert_eq!(num_values, 0);
1730 assert_eq!(encoding, Encoding::PLAIN);
1731 assert!(!is_sorted);
1732 true
1733 }
1734 Page::DataPageV2 {
1735 buf,
1736 num_values,
1737 encoding,
1738 num_nulls,
1739 num_rows,
1740 def_levels_byte_len,
1741 rep_levels_byte_len,
1742 is_compressed,
1743 statistics,
1744 } => {
1745 assert_eq!(buf.len(), 3);
1746 assert_eq!(num_values, 10);
1747 assert_eq!(encoding, Encoding::RLE_DICTIONARY);
1748 assert_eq!(num_nulls, 10);
1749 assert_eq!(num_rows, 10);
1750 assert_eq!(def_levels_byte_len, 2);
1751 assert_eq!(rep_levels_byte_len, 0);
1752 assert!(is_compressed);
1753 assert!(statistics.is_none()); true
1755 }
1756 Page::DataPage { .. } => false,
1757 };
1758 assert!(is_expected_page);
1759 page_count += 1;
1760 }
1761 assert_eq!(page_count, 2);
1762 }
1763
1764 #[test]
1765 fn test_file_reader_empty_datapage_v2() {
1766 let test_file = get_test_file("datapage_v2_empty_datapage.snappy.parquet");
1768 let reader_result = SerializedFileReader::new(test_file);
1769 assert!(reader_result.is_ok());
1770 let reader = reader_result.unwrap();
1771
1772 let metadata = reader.metadata();
1774 assert_eq!(metadata.num_row_groups(), 1);
1775
1776 let file_metadata = metadata.file_metadata();
1778 assert!(file_metadata.created_by().is_some());
1779 assert_eq!(
1780 file_metadata.created_by().unwrap(),
1781 "parquet-mr version 1.13.1 (build db4183109d5b734ec5930d870cdae161e408ddba)"
1782 );
1783 assert!(file_metadata.key_value_metadata().is_some());
1784 assert_eq!(file_metadata.key_value_metadata().unwrap().len(), 2);
1785
1786 assert_eq!(file_metadata.num_rows(), 1);
1787 assert_eq!(file_metadata.version(), 1);
1788 let expected_order = ColumnOrder::TYPE_DEFINED_ORDER(SortOrder::SIGNED);
1789 assert_eq!(
1790 file_metadata.column_orders(),
1791 Some(vec![expected_order].as_ref())
1792 );
1793
1794 let row_group_metadata = metadata.row_group(0);
1795
1796 for i in 0..row_group_metadata.num_columns() {
1798 assert_eq!(file_metadata.column_order(i), expected_order);
1799 }
1800
1801 let row_group_reader_result = reader.get_row_group(0);
1803 assert!(row_group_reader_result.is_ok());
1804 let row_group_reader: Box<dyn RowGroupReader> = row_group_reader_result.unwrap();
1805 assert_eq!(
1806 row_group_reader.num_columns(),
1807 row_group_metadata.num_columns()
1808 );
1809 assert_eq!(
1810 row_group_reader.metadata().total_byte_size(),
1811 row_group_metadata.total_byte_size()
1812 );
1813
1814 let page_reader_0_result = row_group_reader.get_column_page_reader(0);
1816 assert!(page_reader_0_result.is_ok());
1817 let mut page_reader_0: Box<dyn PageReader> = page_reader_0_result.unwrap();
1818 let mut page_count = 0;
1819 while let Some(page) = page_reader_0.get_next_page().unwrap() {
1820 let is_expected_page = match page {
1821 Page::DataPageV2 {
1822 buf,
1823 num_values,
1824 encoding,
1825 num_nulls,
1826 num_rows,
1827 def_levels_byte_len,
1828 rep_levels_byte_len,
1829 is_compressed,
1830 statistics,
1831 } => {
1832 assert_eq!(buf.len(), 2);
1833 assert_eq!(num_values, 1);
1834 assert_eq!(encoding, Encoding::PLAIN);
1835 assert_eq!(num_nulls, 1);
1836 assert_eq!(num_rows, 1);
1837 assert_eq!(def_levels_byte_len, 2);
1838 assert_eq!(rep_levels_byte_len, 0);
1839 assert!(is_compressed);
1840 assert!(statistics.is_none());
1841 true
1842 }
1843 _ => false,
1844 };
1845 assert!(is_expected_page);
1846 page_count += 1;
1847 }
1848 assert_eq!(page_count, 1);
1849 }
1850
1851 fn get_serialized_page_reader<R: ChunkReader>(
1852 file_reader: &SerializedFileReader<R>,
1853 row_group_idx: usize,
1854 column: usize,
1855 ) -> Result<SerializedPageReader<R>> {
1856 let row_group = {
1857 let row_group_metadata = file_reader.metadata.row_group(row_group_idx);
1858 let props = Arc::clone(&file_reader.props);
1859 let f = Arc::clone(&file_reader.chunk_reader);
1860 let page_index = file_reader.metadata.page_index_for_row_group(row_group_idx);
1861 SerializedRowGroupReader::new(f, row_group_metadata, page_index, props)?
1862 };
1863
1864 let col = row_group.metadata.column(column);
1865 let page_locations = file_reader
1866 .metadata
1867 .page_index()
1868 .map(|pi| pi.page_locations(row_group_idx, column))
1869 .unwrap_or(None)
1870 .cloned();
1871
1872 let props = Arc::clone(&row_group.props);
1873 SerializedPageReader::new_with_properties(
1874 Arc::clone(&row_group.chunk_reader),
1875 col,
1876 usize::try_from(row_group.metadata.num_rows())?,
1877 page_locations,
1878 props,
1879 )
1880 }
1881
1882 #[test]
1883 fn test_peek_next_page_offset_matches_actual() -> Result<()> {
1884 let test_file = get_test_file("alltypes_plain.parquet");
1885 let reader = SerializedFileReader::new(test_file)?;
1886
1887 let mut offset_set = HashSet::new();
1888 let num_row_groups = reader.metadata.num_row_groups();
1889 for row_group in 0..num_row_groups {
1890 let num_columns = reader.metadata.row_group(row_group).num_columns();
1891 for column in 0..num_columns {
1892 let mut page_reader = get_serialized_page_reader(&reader, row_group, column)?;
1893
1894 while let Ok(Some(page_offset)) = page_reader.peek_next_page_offset() {
1895 match &page_reader.state {
1896 SerializedPageReaderState::Pages {
1897 page_locations,
1898 dictionary_page,
1899 ..
1900 } => {
1901 if let Some(page) = dictionary_page {
1902 assert_eq!(page.offset as u64, page_offset);
1903 } else if let Some(page) = page_locations.front() {
1904 assert_eq!(page.offset as u64, page_offset);
1905 } else {
1906 unreachable!()
1907 }
1908 }
1909 SerializedPageReaderState::Values {
1910 offset,
1911 next_page_header,
1912 ..
1913 } => {
1914 assert!(next_page_header.is_some());
1915 assert_eq!(*offset, page_offset);
1916 }
1917 }
1918 let page = page_reader.get_next_page()?;
1919 assert!(page.is_some());
1920 let newly_inserted = offset_set.insert(page_offset);
1921 assert!(newly_inserted);
1922 }
1923 }
1924 }
1925
1926 Ok(())
1927 }
1928
1929 #[test]
1930 fn test_page_iterator() {
1931 let file = get_test_file("alltypes_plain.parquet");
1932 let file_reader = Arc::new(SerializedFileReader::new(file).unwrap());
1933
1934 let mut page_iterator = FilePageIterator::new(0, file_reader.clone()).unwrap();
1935
1936 let page = page_iterator.next();
1938 assert!(page.is_some());
1939 assert!(page.unwrap().is_ok());
1940
1941 let page = page_iterator.next();
1943 assert!(page.is_none());
1944
1945 let row_group_indices = Box::new(0..1);
1946 let mut page_iterator =
1947 FilePageIterator::with_row_groups(0, row_group_indices, file_reader).unwrap();
1948
1949 let page = page_iterator.next();
1951 assert!(page.is_some());
1952 assert!(page.unwrap().is_ok());
1953
1954 let page = page_iterator.next();
1956 assert!(page.is_none());
1957 }
1958
1959 #[test]
1960 fn test_file_reader_key_value_metadata() {
1961 let file = get_test_file("binary.parquet");
1962 let file_reader = Arc::new(SerializedFileReader::new(file).unwrap());
1963
1964 let metadata = file_reader
1965 .metadata
1966 .file_metadata()
1967 .key_value_metadata()
1968 .unwrap();
1969
1970 assert_eq!(metadata.len(), 3);
1971
1972 assert_eq!(metadata[0].key, "parquet.proto.descriptor");
1973
1974 assert_eq!(metadata[1].key, "writer.model.name");
1975 assert_eq!(metadata[1].value, Some("protobuf".to_owned()));
1976
1977 assert_eq!(metadata[2].key, "parquet.proto.class");
1978 assert_eq!(metadata[2].value, Some("foo.baz.Foobaz$Event".to_owned()));
1979 }
1980
1981 #[test]
1982 fn test_file_reader_optional_metadata() {
1983 let file = get_test_file("data_index_bloom_encoding_stats.parquet");
1985 let options = ReadOptionsBuilder::new()
1986 .with_encoding_stats_as_mask(false)
1987 .build();
1988 let file_reader = Arc::new(SerializedFileReader::new_with_options(file, options).unwrap());
1989
1990 let row_group_metadata = file_reader.metadata.row_group(0);
1991 let col0_metadata = row_group_metadata.column(0);
1992
1993 assert_eq!(col0_metadata.bloom_filter_offset().unwrap(), 192);
1995
1996 let page_encoding_stats = &col0_metadata.page_encoding_stats().unwrap()[0];
1998
1999 assert_eq!(page_encoding_stats.page_type, basic::PageType::DATA_PAGE);
2000 assert_eq!(page_encoding_stats.encoding, Encoding::PLAIN);
2001 assert_eq!(page_encoding_stats.count, 1);
2002
2003 assert_eq!(col0_metadata.column_index_offset().unwrap(), 156);
2005 assert_eq!(col0_metadata.column_index_length().unwrap(), 25);
2006
2007 assert_eq!(col0_metadata.offset_index_offset().unwrap(), 181);
2009 assert_eq!(col0_metadata.offset_index_length().unwrap(), 11);
2010 }
2011
2012 #[test]
2013 fn test_file_reader_page_stats_mask() {
2014 let file = get_test_file("alltypes_tiny_pages.parquet");
2015 let options = ReadOptionsBuilder::new()
2016 .with_encoding_stats_as_mask(true)
2017 .build();
2018 let file_reader = Arc::new(SerializedFileReader::new_with_options(file, options).unwrap());
2019
2020 let row_group_metadata = file_reader.metadata.row_group(0);
2021
2022 let page_encoding_stats = row_group_metadata
2024 .column(0)
2025 .page_encoding_stats_mask()
2026 .unwrap();
2027 assert!(page_encoding_stats.is_only(Encoding::PLAIN));
2028 let page_encoding_stats = row_group_metadata
2029 .column(2)
2030 .page_encoding_stats_mask()
2031 .unwrap();
2032 assert!(page_encoding_stats.is_only(Encoding::PLAIN_DICTIONARY));
2033 }
2034
2035 #[test]
2036 fn test_file_reader_page_stats_skipped() {
2037 let file = get_test_file("alltypes_tiny_pages.parquet");
2038
2039 let options = ReadOptionsBuilder::new()
2041 .with_encoding_stats_policy(ParquetStatisticsPolicy::SkipAll)
2042 .with_column_stats_policy(ParquetStatisticsPolicy::SkipAll)
2043 .build();
2044 let file_reader = Arc::new(
2045 SerializedFileReader::new_with_options(file.try_clone().unwrap(), options).unwrap(),
2046 );
2047
2048 let row_group_metadata = file_reader.metadata.row_group(0);
2049 for column in row_group_metadata.columns() {
2050 assert!(column.page_encoding_stats().is_none());
2051 assert!(column.page_encoding_stats_mask().is_none());
2052 assert!(column.statistics().is_none());
2053 }
2054
2055 let options = ReadOptionsBuilder::new()
2057 .with_encoding_stats_as_mask(true)
2058 .with_encoding_stats_policy(ParquetStatisticsPolicy::skip_except(&[0]))
2059 .with_column_stats_policy(ParquetStatisticsPolicy::skip_except(&[0]))
2060 .build();
2061 let file_reader = Arc::new(
2062 SerializedFileReader::new_with_options(file.try_clone().unwrap(), options).unwrap(),
2063 );
2064
2065 let row_group_metadata = file_reader.metadata.row_group(0);
2066 for (idx, column) in row_group_metadata.columns().iter().enumerate() {
2067 assert!(column.page_encoding_stats().is_none());
2068 assert_eq!(column.page_encoding_stats_mask().is_some(), idx == 0);
2069 assert_eq!(column.statistics().is_some(), idx == 0);
2070 }
2071 }
2072
2073 #[test]
2074 fn test_file_reader_size_stats_skipped() {
2075 let file = get_test_file("repeated_primitive_no_list.parquet");
2076
2077 let options = ReadOptionsBuilder::new()
2079 .with_size_stats_policy(ParquetStatisticsPolicy::SkipAll)
2080 .build();
2081 let file_reader = Arc::new(
2082 SerializedFileReader::new_with_options(file.try_clone().unwrap(), options).unwrap(),
2083 );
2084
2085 let row_group_metadata = file_reader.metadata.row_group(0);
2086 for column in row_group_metadata.columns() {
2087 assert!(column.repetition_level_histogram().is_none());
2088 assert!(column.definition_level_histogram().is_none());
2089 assert!(column.unencoded_byte_array_data_bytes().is_none());
2090 }
2091
2092 let options = ReadOptionsBuilder::new()
2094 .with_encoding_stats_as_mask(true)
2095 .with_size_stats_policy(ParquetStatisticsPolicy::skip_except(&[1]))
2096 .build();
2097 let file_reader = Arc::new(
2098 SerializedFileReader::new_with_options(file.try_clone().unwrap(), options).unwrap(),
2099 );
2100
2101 let row_group_metadata = file_reader.metadata.row_group(0);
2102 for (idx, column) in row_group_metadata.columns().iter().enumerate() {
2103 assert_eq!(column.repetition_level_histogram().is_some(), idx == 1);
2104 assert_eq!(column.definition_level_histogram().is_some(), idx == 1);
2105 assert_eq!(column.unencoded_byte_array_data_bytes().is_some(), idx == 1);
2106 }
2107 }
2108
2109 #[test]
2110 fn test_file_reader_with_no_filter() -> Result<()> {
2111 let test_file = get_test_file("alltypes_plain.parquet");
2112 let origin_reader = SerializedFileReader::new(test_file)?;
2113 let metadata = origin_reader.metadata();
2115 assert_eq!(metadata.num_row_groups(), 1);
2116 Ok(())
2117 }
2118
2119 #[test]
2120 fn test_file_reader_filter_row_groups_with_predicate() -> Result<()> {
2121 let test_file = get_test_file("alltypes_plain.parquet");
2122 let read_options = ReadOptionsBuilder::new()
2123 .with_predicate(Box::new(|_, _| false))
2124 .build();
2125 let reader = SerializedFileReader::new_with_options(test_file, read_options)?;
2126 let metadata = reader.metadata();
2127 assert_eq!(metadata.num_row_groups(), 0);
2128 Ok(())
2129 }
2130
2131 #[test]
2132 fn test_file_reader_filter_row_groups_with_range() -> Result<()> {
2133 let test_file = get_test_file("alltypes_plain.parquet");
2134 let origin_reader = SerializedFileReader::new(test_file)?;
2135 let metadata = origin_reader.metadata();
2137 assert_eq!(metadata.num_row_groups(), 1);
2138 let mid = get_midpoint_offset(metadata.row_group(0));
2139
2140 let test_file = get_test_file("alltypes_plain.parquet");
2141 let read_options = ReadOptionsBuilder::new().with_range(0, mid + 1).build();
2142 let reader = SerializedFileReader::new_with_options(test_file, read_options)?;
2143 let metadata = reader.metadata();
2144 assert_eq!(metadata.num_row_groups(), 1);
2145
2146 let test_file = get_test_file("alltypes_plain.parquet");
2147 let read_options = ReadOptionsBuilder::new().with_range(0, mid).build();
2148 let reader = SerializedFileReader::new_with_options(test_file, read_options)?;
2149 let metadata = reader.metadata();
2150 assert_eq!(metadata.num_row_groups(), 0);
2151 Ok(())
2152 }
2153
2154 #[test]
2155 #[cfg_attr(miri, ignore)] fn test_file_reader_filter_row_groups_and_range() -> Result<()> {
2157 let test_file = get_test_file("alltypes_tiny_pages.parquet");
2158 let origin_reader = SerializedFileReader::new(test_file)?;
2159 let metadata = origin_reader.metadata();
2160 let mid = get_midpoint_offset(metadata.row_group(0));
2161
2162 let test_file = get_test_file("alltypes_tiny_pages.parquet");
2164 let read_options = ReadOptionsBuilder::new()
2165 .with_page_index()
2166 .with_predicate(Box::new(|_, _| true))
2167 .with_range(mid, mid + 1)
2168 .build();
2169 let reader = SerializedFileReader::new_with_options(test_file, read_options)?;
2170 let metadata = reader.metadata();
2171 assert_eq!(metadata.num_row_groups(), 1);
2172 assert!(metadata.page_index().is_some());
2173
2174 let test_file = get_test_file("alltypes_tiny_pages.parquet");
2176 let read_options = ReadOptionsBuilder::new()
2177 .with_page_index()
2178 .with_predicate(Box::new(|_, _| true))
2179 .with_range(0, mid)
2180 .build();
2181 let reader = SerializedFileReader::new_with_options(test_file, read_options)?;
2182 let metadata = reader.metadata();
2183 assert_eq!(metadata.num_row_groups(), 0);
2184 assert!(metadata.page_index().is_none());
2185
2186 let test_file = get_test_file("alltypes_tiny_pages.parquet");
2188 let read_options = ReadOptionsBuilder::new()
2189 .with_page_index()
2190 .with_predicate(Box::new(|_, _| false))
2191 .with_range(mid, mid + 1)
2192 .build();
2193 let reader = SerializedFileReader::new_with_options(test_file, read_options)?;
2194 let metadata = reader.metadata();
2195 assert_eq!(metadata.num_row_groups(), 0);
2196 assert!(metadata.page_index().is_none());
2197
2198 let test_file = get_test_file("alltypes_tiny_pages.parquet");
2200 let read_options = ReadOptionsBuilder::new()
2201 .with_page_index()
2202 .with_predicate(Box::new(|_, _| false))
2203 .with_range(0, mid)
2204 .build();
2205 let reader = SerializedFileReader::new_with_options(test_file, read_options)?;
2206 let metadata = reader.metadata();
2207 assert_eq!(metadata.num_row_groups(), 0);
2208 assert!(metadata.page_index().is_none());
2209 Ok(())
2210 }
2211
2212 #[test]
2213 fn test_file_reader_invalid_metadata() {
2214 let data = [
2215 255, 172, 1, 0, 50, 82, 65, 73, 1, 0, 0, 0, 169, 168, 168, 162, 87, 255, 16, 0, 0, 0,
2216 80, 65, 82, 49,
2217 ];
2218 let ret = SerializedFileReader::new(Bytes::copy_from_slice(&data));
2219 assert_eq!(
2220 ret.err().unwrap().to_string(),
2221 "Parquet error: Expected list element type of Struct but got List"
2222 );
2223 }
2224
2225 #[test]
2226 fn test_file_reader_empty_row_groups_with_zero_list_header() {
2227 let schema = parse_message_type("message test_schema { OPTIONAL INT32 k; }").unwrap();
2228 let mut out = Vec::new();
2229 SerializedFileWriter::new(&mut out, Arc::new(schema), Default::default())
2230 .unwrap()
2231 .close()
2232 .unwrap();
2233
2234 let pos = out
2238 .windows(4)
2239 .position(|w| w == [0x16, 0x00, 0x19, 0x0C])
2240 .expect("empty row_groups list header not found in footer");
2241 out[pos + 3] = 0x00;
2242
2243 let reader = SerializedFileReader::new(Bytes::from(out)).unwrap();
2244 assert_eq!(reader.metadata().num_row_groups(), 0);
2245 assert_eq!(reader.metadata().file_metadata().num_rows(), 0);
2246 }
2247
2248 #[test]
2249 fn test_page_index_reader() {
2266 let test_file = get_test_file("data_index_bloom_encoding_stats.parquet");
2267 let builder = ReadOptionsBuilder::new();
2268 let options = builder.with_page_index().build();
2270 let reader_result = SerializedFileReader::new_with_options(test_file, options);
2271 let reader = reader_result.unwrap();
2272
2273 let metadata = reader.metadata();
2275 assert_eq!(metadata.num_row_groups(), 1);
2276
2277 let page_index = metadata.page_index().expect("page index should be present");
2278
2279 let Some(ColumnIndexMetaData::BYTE_ARRAY(index)) = page_index.column_index(0, 0) else {
2281 unreachable!()
2282 };
2283
2284 assert_eq!(index.boundary_order, BoundaryOrder::ASCENDING);
2285
2286 assert_eq!(index.num_pages(), 1);
2288
2289 let min = index.min_value(0).unwrap();
2290 let max = index.max_value(0).unwrap();
2291 assert_eq!(b"Hello", min.as_bytes());
2292 assert_eq!(b"today", max.as_bytes());
2293
2294 let offset_index = page_index
2296 .offset_index(0, 0)
2297 .expect("offset index should be present");
2298 let page_offset = offset_index
2299 .page_locations()
2300 .first()
2301 .expect("offset index too small");
2302
2303 assert_eq!(4, page_offset.offset);
2304 assert_eq!(152, page_offset.compressed_page_size);
2305 assert_eq!(0, page_offset.first_row_index);
2306 }
2307
2308 #[test]
2309 #[cfg_attr(miri, ignore)] fn test_page_index_reader_all_type() {
2311 let test_file = get_test_file("alltypes_tiny_pages_plain.parquet");
2312 let builder = ReadOptionsBuilder::new();
2313 let options = builder.with_page_index().build();
2315 let reader_result = SerializedFileReader::new_with_options(test_file, options);
2316 let reader = reader_result.unwrap();
2317
2318 let metadata = reader.metadata();
2320 assert_eq!(metadata.num_row_groups(), 1);
2321
2322 let page_index = metadata.page_index().unwrap();
2323 let row_group_offset_indexes = metadata.page_index_for_row_group(0);
2324
2325 let row_group_metadata = metadata.row_group(0);
2327
2328 let ci = page_index.column_index(0, 0).unwrap();
2330 assert!(!ci.is_sorted());
2331 assert!(matches!(
2332 ci.get_boundary_order(),
2333 Some(BoundaryOrder::UNORDERED)
2334 ));
2335 if let ColumnIndexMetaData::INT32(index) = ci {
2336 check_native_page_index(
2337 index,
2338 325,
2339 get_row_group_min_max_bytes(row_group_metadata, 0),
2340 BoundaryOrder::UNORDERED,
2341 );
2342 assert_eq!(
2343 row_group_offset_indexes.page_locations(0).unwrap().len(),
2344 325
2345 );
2346 } else {
2347 unreachable!()
2348 }
2349 let ci = page_index.column_index(0, 1).unwrap();
2351 assert!(ci.is_sorted());
2352 if let ColumnIndexMetaData::BOOLEAN(index) = ci {
2353 assert_eq!(index.num_pages(), 82);
2354 assert_eq!(
2355 row_group_offset_indexes.page_locations(1).unwrap().len(),
2356 82
2357 );
2358 } else {
2359 unreachable!()
2360 }
2361 let ci = page_index.column_index(0, 2).unwrap();
2363 assert!(ci.is_sorted());
2364 if let ColumnIndexMetaData::INT32(index) = ci {
2365 check_native_page_index(
2366 index,
2367 325,
2368 get_row_group_min_max_bytes(row_group_metadata, 2),
2369 BoundaryOrder::ASCENDING,
2370 );
2371 assert_eq!(
2372 row_group_offset_indexes.page_locations(2).unwrap().len(),
2373 325
2374 );
2375 } else {
2376 unreachable!()
2377 }
2378 let ci = page_index.column_index(0, 3).unwrap();
2380 assert!(ci.is_sorted());
2381 if let ColumnIndexMetaData::INT32(index) = ci {
2382 check_native_page_index(
2383 index,
2384 325,
2385 get_row_group_min_max_bytes(row_group_metadata, 3),
2386 BoundaryOrder::ASCENDING,
2387 );
2388 assert_eq!(
2389 row_group_offset_indexes.page_locations(3).unwrap().len(),
2390 325
2391 );
2392 } else {
2393 unreachable!()
2394 }
2395 let ci = page_index.column_index(0, 4).unwrap();
2397 assert!(ci.is_sorted());
2398 if let ColumnIndexMetaData::INT32(index) = ci {
2399 check_native_page_index(
2400 index,
2401 325,
2402 get_row_group_min_max_bytes(row_group_metadata, 4),
2403 BoundaryOrder::ASCENDING,
2404 );
2405 assert_eq!(
2406 row_group_offset_indexes.page_locations(4).unwrap().len(),
2407 325
2408 );
2409 } else {
2410 unreachable!()
2411 }
2412 let ci = page_index.column_index(0, 5).unwrap();
2414 assert!(!ci.is_sorted());
2415 if let ColumnIndexMetaData::INT64(index) = ci {
2416 check_native_page_index(
2417 index,
2418 528,
2419 get_row_group_min_max_bytes(row_group_metadata, 5),
2420 BoundaryOrder::UNORDERED,
2421 );
2422 assert_eq!(
2423 row_group_offset_indexes.page_locations(5).unwrap().len(),
2424 528
2425 );
2426 } else {
2427 unreachable!()
2428 }
2429 let ci = page_index.column_index(0, 6).unwrap();
2431 assert!(ci.is_sorted());
2432 if let ColumnIndexMetaData::FLOAT(index) = ci {
2433 check_native_page_index(
2434 index,
2435 325,
2436 get_row_group_min_max_bytes(row_group_metadata, 6),
2437 BoundaryOrder::ASCENDING,
2438 );
2439 assert_eq!(
2440 row_group_offset_indexes.page_locations(6).unwrap().len(),
2441 325
2442 );
2443 } else {
2444 unreachable!()
2445 }
2446 let ci = page_index.column_index(0, 7).unwrap();
2448 assert!(!ci.is_sorted());
2449 if let ColumnIndexMetaData::DOUBLE(index) = ci {
2450 check_native_page_index(
2451 index,
2452 528,
2453 get_row_group_min_max_bytes(row_group_metadata, 7),
2454 BoundaryOrder::UNORDERED,
2455 );
2456 assert_eq!(
2457 row_group_offset_indexes.page_locations(7).unwrap().len(),
2458 528
2459 );
2460 } else {
2461 unreachable!()
2462 }
2463 let ci = page_index.column_index(0, 8).unwrap();
2465 assert!(!ci.is_sorted());
2466 if let ColumnIndexMetaData::BYTE_ARRAY(index) = ci {
2467 check_byte_array_page_index(
2468 index,
2469 974,
2470 get_row_group_min_max_bytes(row_group_metadata, 8),
2471 BoundaryOrder::UNORDERED,
2472 );
2473 assert_eq!(
2474 row_group_offset_indexes.page_locations(8).unwrap().len(),
2475 974
2476 );
2477 } else {
2478 unreachable!()
2479 }
2480 let ci = page_index.column_index(0, 9).unwrap();
2482 assert!(ci.is_sorted());
2483 if let ColumnIndexMetaData::BYTE_ARRAY(index) = ci {
2484 check_byte_array_page_index(
2485 index,
2486 352,
2487 get_row_group_min_max_bytes(row_group_metadata, 9),
2488 BoundaryOrder::ASCENDING,
2489 );
2490 assert_eq!(
2491 row_group_offset_indexes.page_locations(9).unwrap().len(),
2492 352
2493 );
2494 } else {
2495 unreachable!()
2496 }
2497 assert!(page_index.column_index(0, 10).is_none());
2500 let ci = page_index.column_index(0, 11).unwrap();
2502 assert!(ci.is_sorted());
2503 if let ColumnIndexMetaData::INT32(index) = ci {
2504 check_native_page_index(
2505 index,
2506 325,
2507 get_row_group_min_max_bytes(row_group_metadata, 11),
2508 BoundaryOrder::ASCENDING,
2509 );
2510 assert_eq!(
2511 row_group_offset_indexes.page_locations(11).unwrap().len(),
2512 325
2513 );
2514 } else {
2515 unreachable!()
2516 }
2517 let ci = page_index.column_index(0, 12).unwrap();
2519 assert!(!ci.is_sorted());
2520 if let ColumnIndexMetaData::INT32(index) = ci {
2521 check_native_page_index(
2522 index,
2523 325,
2524 get_row_group_min_max_bytes(row_group_metadata, 12),
2525 BoundaryOrder::UNORDERED,
2526 );
2527 assert_eq!(
2528 row_group_offset_indexes.page_locations(12).unwrap().len(),
2529 325
2530 );
2531 } else {
2532 unreachable!()
2533 }
2534 }
2535
2536 fn check_native_page_index<T: ParquetValueType>(
2537 row_group_index: &PrimitiveColumnIndex<T>,
2538 page_size: usize,
2539 min_max: (&[u8], &[u8]),
2540 boundary_order: BoundaryOrder,
2541 ) {
2542 assert_eq!(row_group_index.num_pages() as usize, page_size);
2543 assert_eq!(row_group_index.boundary_order, boundary_order);
2544 assert!(row_group_index.min_values().iter().all(|x| {
2545 x >= &T::try_from_le_slice(min_max.0).unwrap()
2546 && x <= &T::try_from_le_slice(min_max.1).unwrap()
2547 }));
2548 }
2549
2550 fn check_byte_array_page_index(
2551 row_group_index: &ByteArrayColumnIndex,
2552 page_size: usize,
2553 min_max: (&[u8], &[u8]),
2554 boundary_order: BoundaryOrder,
2555 ) {
2556 assert_eq!(row_group_index.num_pages() as usize, page_size);
2557 assert_eq!(row_group_index.boundary_order, boundary_order);
2558 for i in 0..row_group_index.num_pages() as usize {
2559 let x = row_group_index.min_value(i).unwrap();
2560 assert!(x >= min_max.0 && x <= min_max.1);
2561 }
2562 }
2563
2564 fn get_row_group_min_max_bytes(r: &RowGroupMetaData, col_num: usize) -> (&[u8], &[u8]) {
2565 let statistics = r.column(col_num).statistics().unwrap();
2566 (
2567 statistics.min_bytes_opt().unwrap_or_default(),
2568 statistics.max_bytes_opt().unwrap_or_default(),
2569 )
2570 }
2571
2572 #[test]
2573 #[cfg_attr(miri, ignore)] fn test_skip_next_page_with_dictionary_page() {
2575 let test_file = get_test_file("alltypes_tiny_pages.parquet");
2576 let builder = ReadOptionsBuilder::new();
2577 let options = builder.with_page_index().build();
2579 let reader_result = SerializedFileReader::new_with_options(test_file, options);
2580 let reader = reader_result.unwrap();
2581
2582 let row_group_reader = reader.get_row_group(0).unwrap();
2583
2584 let mut column_page_reader = row_group_reader.get_column_page_reader(9).unwrap();
2586
2587 let mut vec = vec![];
2588
2589 let meta = column_page_reader.peek_next_page().unwrap().unwrap();
2591 assert!(meta.is_dict);
2592
2593 column_page_reader.skip_next_page().unwrap();
2595
2596 let page = column_page_reader.get_next_page().unwrap().unwrap();
2598 assert!(matches!(page.page_type(), basic::PageType::DATA_PAGE));
2599
2600 for _i in 0..351 {
2602 let meta = column_page_reader.peek_next_page().unwrap().unwrap();
2604 assert!(!meta.is_dict); vec.push(meta);
2606
2607 let page = column_page_reader.get_next_page().unwrap().unwrap();
2608 assert!(matches!(page.page_type(), basic::PageType::DATA_PAGE));
2609 }
2610
2611 assert!(column_page_reader.peek_next_page().unwrap().is_none());
2613 assert!(column_page_reader.get_next_page().unwrap().is_none());
2614
2615 assert_eq!(vec.len(), 351);
2617 }
2618
2619 #[test]
2620 #[cfg_attr(miri, ignore)] fn test_skip_page_with_offset_index() {
2622 let test_file = get_test_file("alltypes_tiny_pages_plain.parquet");
2623 let builder = ReadOptionsBuilder::new();
2624 let options = builder.with_page_index().build();
2626 let reader_result = SerializedFileReader::new_with_options(test_file, options);
2627 let reader = reader_result.unwrap();
2628
2629 let row_group_reader = reader.get_row_group(0).unwrap();
2630
2631 let mut column_page_reader = row_group_reader.get_column_page_reader(4).unwrap();
2633
2634 let mut vec = vec![];
2635
2636 for i in 0..325 {
2637 if i % 2 == 0 {
2638 vec.push(column_page_reader.get_next_page().unwrap().unwrap());
2639 } else {
2640 column_page_reader.skip_next_page().unwrap();
2641 }
2642 }
2643 assert!(column_page_reader.peek_next_page().unwrap().is_none());
2645 assert!(column_page_reader.get_next_page().unwrap().is_none());
2646
2647 assert_eq!(vec.len(), 163);
2648 }
2649
2650 #[test]
2651 fn test_skip_page_without_offset_index() {
2652 let test_file = get_test_file("alltypes_tiny_pages_plain.parquet");
2653
2654 let reader_result = SerializedFileReader::new(test_file);
2656 let reader = reader_result.unwrap();
2657
2658 let row_group_reader = reader.get_row_group(0).unwrap();
2659
2660 let mut column_page_reader = row_group_reader.get_column_page_reader(4).unwrap();
2662
2663 let mut vec = vec![];
2664
2665 for i in 0..325 {
2666 if i % 2 == 0 {
2667 vec.push(column_page_reader.get_next_page().unwrap().unwrap());
2668 } else {
2669 column_page_reader.peek_next_page().unwrap().unwrap();
2670 column_page_reader.skip_next_page().unwrap();
2671 }
2672 }
2673 assert!(column_page_reader.peek_next_page().unwrap().is_none());
2675 assert!(column_page_reader.get_next_page().unwrap().is_none());
2676
2677 assert_eq!(vec.len(), 163);
2678 }
2679
2680 #[test]
2681 #[cfg_attr(miri, ignore)] fn test_peek_page_with_dictionary_page() {
2683 let test_file = get_test_file("alltypes_tiny_pages.parquet");
2684 let builder = ReadOptionsBuilder::new();
2685 let options = builder.with_page_index().build();
2687 let reader_result = SerializedFileReader::new_with_options(test_file, options);
2688 let reader = reader_result.unwrap();
2689 let row_group_reader = reader.get_row_group(0).unwrap();
2690
2691 let mut column_page_reader = row_group_reader.get_column_page_reader(9).unwrap();
2693
2694 let mut vec = vec![];
2695
2696 let meta = column_page_reader.peek_next_page().unwrap().unwrap();
2697 assert!(meta.is_dict);
2698 let page = column_page_reader.get_next_page().unwrap().unwrap();
2699 assert!(matches!(page.page_type(), basic::PageType::DICTIONARY_PAGE));
2700
2701 for i in 0..352 {
2702 let meta = column_page_reader.peek_next_page().unwrap().unwrap();
2703 if i != 351 {
2706 assert!((meta.num_rows == Some(21)) || (meta.num_rows == Some(20)));
2707 } else {
2708 assert_eq!(meta.num_rows, Some(10));
2711 }
2712 assert!(!meta.is_dict);
2713 vec.push(meta);
2714 let page = column_page_reader.get_next_page().unwrap().unwrap();
2715 assert!(matches!(page.page_type(), basic::PageType::DATA_PAGE));
2716 }
2717
2718 assert!(column_page_reader.peek_next_page().unwrap().is_none());
2720 assert!(column_page_reader.get_next_page().unwrap().is_none());
2721
2722 assert_eq!(vec.len(), 352);
2723 }
2724
2725 #[test]
2726 fn test_peek_page_with_dictionary_page_without_offset_index() {
2727 let test_file = get_test_file("alltypes_tiny_pages.parquet");
2728
2729 let reader_result = SerializedFileReader::new(test_file);
2730 let reader = reader_result.unwrap();
2731 let row_group_reader = reader.get_row_group(0).unwrap();
2732
2733 let mut column_page_reader = row_group_reader.get_column_page_reader(9).unwrap();
2735
2736 let mut vec = vec![];
2737
2738 let meta = column_page_reader.peek_next_page().unwrap().unwrap();
2739 assert!(meta.is_dict);
2740 let page = column_page_reader.get_next_page().unwrap().unwrap();
2741 assert!(matches!(page.page_type(), basic::PageType::DICTIONARY_PAGE));
2742
2743 for i in 0..352 {
2744 let meta = column_page_reader.peek_next_page().unwrap().unwrap();
2745 if i != 351 {
2748 assert!((meta.num_levels == Some(21)) || (meta.num_levels == Some(20)));
2749 } else {
2750 assert_eq!(meta.num_levels, Some(10));
2753 }
2754 assert!(!meta.is_dict);
2755 vec.push(meta);
2756 let page = column_page_reader.get_next_page().unwrap().unwrap();
2757 assert!(matches!(page.page_type(), basic::PageType::DATA_PAGE));
2758 }
2759
2760 assert!(column_page_reader.peek_next_page().unwrap().is_none());
2762 assert!(column_page_reader.get_next_page().unwrap().is_none());
2763
2764 assert_eq!(vec.len(), 352);
2765 }
2766
2767 #[test]
2768 fn test_fixed_length_index() {
2769 let message_type = "
2770 message test_schema {
2771 OPTIONAL FIXED_LEN_BYTE_ARRAY (11) value (DECIMAL(25,2));
2772 }
2773 ";
2774
2775 let schema = parse_message_type(message_type).unwrap();
2776 let mut out = Vec::with_capacity(1024);
2777 let mut writer =
2778 SerializedFileWriter::new(&mut out, Arc::new(schema), Default::default()).unwrap();
2779
2780 let mut r = writer.next_row_group().unwrap();
2781 let mut c = r.next_column().unwrap().unwrap();
2782 c.typed::<FixedLenByteArrayType>()
2783 .write_batch(
2784 &[vec![0; 11].into(), vec![5; 11].into(), vec![3; 11].into()],
2785 Some(&[1, 1, 0, 1]),
2786 None,
2787 )
2788 .unwrap();
2789 c.close().unwrap();
2790 r.close().unwrap();
2791 writer.close().unwrap();
2792
2793 let b = Bytes::from(out);
2794 let options = ReadOptionsBuilder::new().with_page_index().build();
2795 let reader = SerializedFileReader::new_with_options(b, options).unwrap();
2796 let page_index = reader.metadata().page_index().unwrap();
2797
2798 match page_index.column_index(0, 0) {
2799 Some(ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(v)) => {
2800 assert_eq!(v.num_pages(), 1);
2801 assert_eq!(v.null_count(0).unwrap(), 1);
2802 assert_eq!(v.min_value(0).unwrap(), &[0; 11]);
2803 assert_eq!(v.max_value(0).unwrap(), &[5; 11]);
2804 }
2805 _ => unreachable!(),
2806 }
2807 }
2808
2809 #[test]
2810 fn test_multi_gz() {
2811 let file = get_test_file("concatenated_gzip_members.parquet");
2812 let reader = SerializedFileReader::new(file).unwrap();
2813 let row_group_reader = reader.get_row_group(0).unwrap();
2814 match row_group_reader.get_column_reader(0).unwrap() {
2815 ColumnReader::Int64ColumnReader(mut reader) => {
2816 let mut buffer = Vec::with_capacity(1024);
2817 let mut def_levels = Vec::with_capacity(1024);
2818 let (num_records, num_values, num_levels) = reader
2819 .read_records(1024, Some(&mut def_levels), None, &mut buffer)
2820 .unwrap();
2821
2822 assert_eq!(num_records, 513);
2823 assert_eq!(num_values, 513);
2824 assert_eq!(num_levels, 513);
2825
2826 let expected: Vec<i64> = (1..514).collect();
2827 assert_eq!(&buffer, &expected);
2828 }
2829 _ => unreachable!(),
2830 }
2831 }
2832
2833 #[test]
2834 #[cfg_attr(miri, ignore)] fn test_byte_stream_split_extended() {
2836 let path = format!(
2837 "{}/byte_stream_split_extended.gzip.parquet",
2838 arrow::util::test_util::parquet_test_data(),
2839 );
2840 let file = File::open(path).unwrap();
2841 let reader = Box::new(SerializedFileReader::new(file).expect("Failed to create reader"));
2842
2843 let mut iter = reader
2845 .get_row_iter(None)
2846 .expect("Failed to create row iterator");
2847
2848 let mut start = 0;
2849 let end = reader.metadata().file_metadata().num_rows();
2850
2851 let check_row = |row: Result<Row, ParquetError>| {
2852 assert!(row.is_ok());
2853 let r = row.unwrap();
2854 assert_eq!(r.get_float16(0).unwrap(), r.get_float16(1).unwrap());
2855 assert_eq!(r.get_float(2).unwrap(), r.get_float(3).unwrap());
2856 assert_eq!(r.get_double(4).unwrap(), r.get_double(5).unwrap());
2857 assert_eq!(r.get_int(6).unwrap(), r.get_int(7).unwrap());
2858 assert_eq!(r.get_long(8).unwrap(), r.get_long(9).unwrap());
2859 assert_eq!(r.get_bytes(10).unwrap(), r.get_bytes(11).unwrap());
2860 assert_eq!(r.get_decimal(12).unwrap(), r.get_decimal(13).unwrap());
2861 };
2862
2863 while start < end {
2864 match iter.next() {
2865 Some(row) => check_row(row),
2866 None => break,
2867 }
2868 start += 1;
2869 }
2870 }
2871
2872 #[test]
2873 fn test_filtered_rowgroup_metadata() {
2874 let message_type = "
2875 message test_schema {
2876 REQUIRED INT32 a;
2877 }
2878 ";
2879 let schema = Arc::new(parse_message_type(message_type).unwrap());
2880 let props = Arc::new(
2881 WriterProperties::builder()
2882 .set_statistics_enabled(EnabledStatistics::Page)
2883 .build(),
2884 );
2885 let mut file: File = tempfile::tempfile().unwrap();
2886 let mut file_writer = SerializedFileWriter::new(&mut file, schema, props).unwrap();
2887 let data = [1, 2, 3, 4, 5];
2888
2889 for idx in 0..5 {
2891 let data_i: Vec<i32> = data.iter().map(|x| x * (idx + 1)).collect();
2892 let mut row_group_writer = file_writer.next_row_group().unwrap();
2893 if let Some(mut writer) = row_group_writer.next_column().unwrap() {
2894 writer
2895 .typed::<Int32Type>()
2896 .write_batch(data_i.as_slice(), None, None)
2897 .unwrap();
2898 writer.close().unwrap();
2899 }
2900 row_group_writer.close().unwrap();
2901 file_writer.flushed_row_groups();
2902 }
2903 let file_metadata = file_writer.close().unwrap();
2904
2905 assert_eq!(file_metadata.file_metadata().num_rows(), 25);
2906 assert_eq!(file_metadata.num_row_groups(), 5);
2907
2908 for filter in [false, true] {
2911 let calls = Rc::new(RefCell::new(Vec::new()));
2912 let first_calls = Rc::clone(&calls);
2913 let second_calls = Rc::clone(&calls);
2914 let mut next_index = 0;
2915 let read_options = ReadOptionsBuilder::new()
2916 .with_page_index()
2917 .with_predicate(Box::new(move |rg_meta, i| {
2918 assert_eq!(i, next_index);
2919 assert_eq!(rg_meta.ordinal(), Some(i as i32));
2920 next_index += 1;
2921 first_calls.borrow_mut().push((i, 0));
2922 !filter || i != 1
2923 }))
2924 .with_predicate(Box::new(move |_, i| {
2925 second_calls.borrow_mut().push((i, 1));
2926 !filter || i != 3
2927 }))
2928 .build();
2929 let reader =
2930 SerializedFileReader::new_with_options(file.try_clone().unwrap(), read_options)
2931 .unwrap();
2932 let metadata = reader.metadata();
2933 let expected_ordinals = if filter {
2934 vec![0, 2, 4]
2935 } else {
2936 vec![0, 1, 2, 3, 4]
2937 };
2938 let ordinals: Vec<_> = metadata
2939 .row_groups()
2940 .iter()
2941 .map(|rg| rg.ordinal().unwrap())
2942 .collect();
2943 assert_eq!(ordinals, expected_ordinals);
2944 assert!(metadata.page_index().is_some_and(|idx| idx.is_complete()));
2945
2946 let expected_calls: Vec<_> = (0..5)
2947 .flat_map(|i| {
2948 if filter && i == 1 {
2949 vec![(i, 0)]
2950 } else {
2951 vec![(i, 0), (i, 1)]
2952 }
2953 })
2954 .collect();
2955 assert_eq!(*calls.borrow(), expected_calls);
2956 }
2957
2958 let read_options = ReadOptionsBuilder::new()
2960 .with_page_index()
2961 .with_predicate(Box::new(|rgmeta, _| rgmeta.ordinal().unwrap_or(0) == 2))
2962 .build();
2963 let reader =
2964 SerializedFileReader::new_with_options(file.try_clone().unwrap(), read_options)
2965 .unwrap();
2966 let metadata = reader.metadata();
2967
2968 assert_eq!(metadata.num_row_groups(), 1);
2970 assert_eq!(metadata.row_group(0).ordinal(), Some(2));
2971
2972 assert!(metadata.page_index().is_some_and(|idx| idx.is_complete()));
2974 let page_index = metadata.page_index().unwrap();
2975
2976 let col_stats = metadata.row_group(0).column(0).statistics().unwrap();
2977 let pg_idx = page_index.column_index(0, 0);
2978 let off_idx_i = page_index.offset_index(0, 0);
2979
2980 match pg_idx {
2982 Some(ColumnIndexMetaData::INT32(int_idx)) => {
2983 let min = col_stats.min_bytes_opt().unwrap().get_i32_le();
2984 let max = col_stats.max_bytes_opt().unwrap().get_i32_le();
2985 assert_eq!(int_idx.min_value(0), Some(min).as_ref());
2986 assert_eq!(int_idx.max_value(0), Some(max).as_ref());
2987 }
2988 _ => panic!("wrong stats type"),
2989 }
2990
2991 assert_eq!(
2993 off_idx_i.as_ref().unwrap().page_locations[0].offset,
2994 metadata.row_group(0).column(0).data_page_offset()
2995 );
2996
2997 let read_options = ReadOptionsBuilder::new()
2999 .with_page_index()
3000 .with_predicate(Box::new(|rgmeta, _| rgmeta.ordinal().unwrap_or(0) % 2 == 1))
3001 .build();
3002 let reader =
3003 SerializedFileReader::new_with_options(file.try_clone().unwrap(), read_options)
3004 .unwrap();
3005 let metadata = reader.metadata();
3006
3007 assert_eq!(metadata.num_row_groups(), 2);
3009 assert_eq!(metadata.row_group(0).ordinal(), Some(1));
3010 assert_eq!(metadata.row_group(1).ordinal(), Some(3));
3011
3012 assert!(metadata.page_index().is_some_and(|idx| idx.is_complete()));
3014
3015 let page_index = metadata.page_index().unwrap();
3016
3017 for rg_idx in 0..metadata.num_row_groups() {
3018 let col_stats = metadata.row_group(rg_idx).column(0).statistics().unwrap();
3019 let pg_idx = page_index.column_index(rg_idx, 0);
3020 let off_idx_i = page_index.offset_index(rg_idx, 0);
3021
3022 match pg_idx {
3024 Some(ColumnIndexMetaData::INT32(int_idx)) => {
3025 let min = col_stats.min_bytes_opt().unwrap().get_i32_le();
3026 let max = col_stats.max_bytes_opt().unwrap().get_i32_le();
3027 assert_eq!(int_idx.min_value(0), Some(min).as_ref());
3028 assert_eq!(int_idx.max_value(0), Some(max).as_ref());
3029 }
3030 _ => panic!("wrong stats type"),
3031 }
3032
3033 assert_eq!(
3035 off_idx_i.as_ref().unwrap().page_locations[0].offset,
3036 metadata.row_group(rg_idx).column(0).data_page_offset()
3037 );
3038 }
3039 }
3040
3041 #[test]
3042 fn test_reuse_schema() {
3043 let file = get_test_file("alltypes_plain.parquet");
3044 let file_reader = SerializedFileReader::new(file.try_clone().unwrap()).unwrap();
3045 let schema = file_reader.metadata().file_metadata().schema_descr_ptr();
3046 let expected = file_reader.metadata;
3047
3048 let options = ReadOptionsBuilder::new()
3049 .with_parquet_schema(schema)
3050 .build();
3051 let file_reader = SerializedFileReader::new_with_options(file, options).unwrap();
3052
3053 assert_eq!(expected.as_ref(), file_reader.metadata.as_ref());
3054 assert!(Arc::ptr_eq(
3056 &expected.file_metadata().schema_descr_ptr(),
3057 &file_reader.metadata.file_metadata().schema_descr_ptr()
3058 ));
3059 }
3060
3061 #[test]
3062 fn test_read_unknown_logical_type() {
3063 let file = get_test_file("unknown-logical-type.parquet");
3064 let reader = SerializedFileReader::new(file).expect("Error opening file");
3065
3066 let schema = reader.metadata().file_metadata().schema_descr();
3067 assert_eq!(
3068 schema.column(0).logical_type_ref(),
3069 Some(&basic::LogicalType::String)
3070 );
3071 assert_eq!(
3072 schema.column(1).logical_type_ref(),
3073 Some(&basic::LogicalType::_Unknown { field_id: 2555 })
3074 );
3075 assert_eq!(schema.column(1).physical_type(), Type::BYTE_ARRAY);
3076
3077 let mut iter = reader
3078 .get_row_iter(None)
3079 .expect("Failed to create row iterator");
3080
3081 let mut num_rows = 0;
3082 while iter.next().is_some() {
3083 num_rows += 1;
3084 }
3085 assert_eq!(num_rows, reader.metadata().file_metadata().num_rows());
3086 }
3087
3088 fn dict_column_file() -> (Bytes, std::ops::Range<usize>) {
3094 let message_type = "
3095 message test_schema {
3096 REQUIRED INT32 a;
3097 }
3098 ";
3099 let schema = Arc::new(parse_message_type(message_type).unwrap());
3100 let props = Arc::new(
3101 WriterProperties::builder()
3102 .set_compression(Compression::SNAPPY)
3103 .build(),
3104 );
3105 let mut out = Vec::new();
3106 let mut writer = SerializedFileWriter::new(&mut out, schema, props).unwrap();
3107 let mut rg = writer.next_row_group().unwrap();
3108 let mut col = rg.next_column().unwrap().unwrap();
3109 col.typed::<Int32Type>()
3110 .write_batch(&[1, 2, 3, 4, 5], None, None)
3111 .unwrap();
3112 col.close().unwrap();
3113 rg.close().unwrap();
3114 writer.close().unwrap();
3115 let bytes = Bytes::from(out);
3116
3117 let reader = SerializedFileReader::new(bytes.clone()).unwrap();
3118 let col_meta = reader.metadata().row_group(0).column(0).clone();
3119 let dict_offset = col_meta.dictionary_page_offset().unwrap() as usize;
3120 let data_offset = col_meta.data_page_offset() as usize;
3121 let mut prot = ThriftSliceInputProtocol::new(&bytes[dict_offset..data_offset]);
3123 let _header = PageHeader::read_thrift(&mut prot).unwrap();
3124 let header_len = (data_offset - dict_offset) - prot.as_slice().len();
3125 (bytes, dict_offset + header_len..data_offset)
3126 }
3127
3128 fn corrupt_dictionary_body(bytes: &Bytes, body: &std::ops::Range<usize>) -> Bytes {
3134 let mut v = bytes.to_vec();
3135 v[body.clone()].fill(0);
3136 Bytes::from(v)
3137 }
3138
3139 fn int32_reader(bytes: Bytes) -> crate::column::reader::ColumnReaderImpl<Int32Type> {
3140 let reader = SerializedFileReader::new(bytes.clone()).unwrap();
3141 let meta = reader.metadata();
3142 let descr = meta.file_metadata().schema_descr().column(0);
3143 let col_meta = meta.row_group(0).column(0).clone();
3144 let total_rows = meta.row_group(0).num_rows() as usize;
3145 let pages =
3146 SerializedPageReader::new(Arc::new(bytes), &col_meta, total_rows, None).unwrap();
3147 crate::column::reader::ColumnReaderImpl::<Int32Type>::new(descr, Box::new(pages))
3148 }
3149
3150 #[test]
3155 fn skipping_a_whole_chunk_never_decodes_the_dictionary() {
3156 let (bytes, body) = dict_column_file();
3157 let corrupted = corrupt_dictionary_body(&bytes, &body);
3158 let mut reader = int32_reader(corrupted);
3159 let skipped = reader.skip_records(5).unwrap();
3160 assert_eq!(skipped, 5);
3161 }
3162
3163 #[test]
3167 fn a_partial_skip_pays_the_deferred_dictionary() {
3168 let (bytes, body) = dict_column_file();
3169 let corrupted = corrupt_dictionary_body(&bytes, &body);
3170 let mut reader = int32_reader(corrupted);
3171 let err = reader.skip_records(2).unwrap_err().to_string();
3172 assert!(
3173 err.contains("corrupt") || err.contains("size"),
3174 "expected the deferred dictionary decode to fail on \
3175 decompression, got: {err}"
3176 );
3177 }
3178
3179 #[test]
3182 fn skip_then_read_decodes_correct_values() {
3183 let (bytes, _) = dict_column_file();
3184 let mut reader = int32_reader(bytes);
3185 assert_eq!(reader.skip_records(2).unwrap(), 2);
3186 let mut values = Vec::new();
3187 let (records, _, read) = reader.read_records(3, None, None, &mut values).unwrap();
3188 assert_eq!(records, 3);
3189 assert_eq!(read, 3);
3190 assert_eq!(values, vec![3, 4, 5]);
3191 }
3192
3193 #[test]
3196 fn plain_read_is_unchanged() {
3197 let (bytes, _) = dict_column_file();
3198 let mut reader = int32_reader(bytes);
3199 let mut values = Vec::new();
3200 let (records, _, read) = reader.read_records(5, None, None, &mut values).unwrap();
3201 assert_eq!((records, read), (5, 5));
3202 assert_eq!(values, vec![1, 2, 3, 4, 5]);
3203 }
3204
3205 struct NonDeferringPageReader(SerializedPageReader<Bytes>);
3211
3212 impl Iterator for NonDeferringPageReader {
3213 type Item = Result<Page>;
3214
3215 fn next(&mut self) -> Option<Self::Item> {
3216 self.get_next_page().transpose()
3217 }
3218 }
3219
3220 impl PageReader for NonDeferringPageReader {
3221 fn get_next_page(&mut self) -> Result<Option<Page>> {
3222 self.0.get_next_page()
3223 }
3224
3225 fn peek_next_page(&mut self) -> Result<Option<PageMetadata>> {
3226 self.0.peek_next_page()
3227 }
3228
3229 fn skip_next_page(&mut self) -> Result<()> {
3230 self.0.skip_next_page()
3231 }
3232 }
3233
3234 fn non_deferring_int32_reader(
3235 bytes: Bytes,
3236 ) -> crate::column::reader::ColumnReaderImpl<Int32Type> {
3237 let reader = SerializedFileReader::new(bytes.clone()).unwrap();
3238 let meta = reader.metadata();
3239 let descr = meta.file_metadata().schema_descr().column(0);
3240 let col_meta = meta.row_group(0).column(0).clone();
3241 let total_rows = meta.row_group(0).num_rows() as usize;
3242 let pages =
3243 SerializedPageReader::new(Arc::new(bytes), &col_meta, total_rows, None).unwrap();
3244 crate::column::reader::ColumnReaderImpl::<Int32Type>::new(
3245 descr,
3246 Box::new(NonDeferringPageReader(pages)),
3247 )
3248 }
3249
3250 #[test]
3258 fn a_reader_that_cannot_defer_still_gets_its_dictionary() {
3259 let (bytes, _) = dict_column_file();
3260 let mut reader = non_deferring_int32_reader(bytes);
3261
3262 assert_eq!(reader.skip_records(2).unwrap(), 2);
3263
3264 let mut values = Vec::new();
3265 let (records, _, read) = reader.read_records(3, None, None, &mut values).unwrap();
3266 assert_eq!(records, 3);
3267 assert_eq!(read, 3);
3268 assert_eq!(values, vec![3, 4, 5]);
3269 }
3270
3271 #[test]
3275 fn a_reader_that_cannot_defer_survives_a_whole_chunk_skip() {
3276 let (bytes, _) = dict_column_file();
3277 let mut reader = non_deferring_int32_reader(bytes);
3278
3279 assert_eq!(reader.skip_records(5).unwrap(), 5);
3280
3281 let mut values = Vec::new();
3282 let (records, _, _) = reader.read_records(1, None, None, &mut values).unwrap();
3283 assert_eq!(records, 0, "chunk is exhausted");
3284 assert!(values.is_empty());
3285 }
3286
3287 #[test]
3291 fn values_state_supports_deferral() {
3292 let (bytes, _) = dict_column_file();
3293 let reader = SerializedFileReader::new(bytes.clone()).unwrap();
3294 let meta = reader.metadata();
3295 let col_meta = meta.row_group(0).column(0).clone();
3296 let total_rows = meta.row_group(0).num_rows() as usize;
3297 let pages =
3298 SerializedPageReader::new(Arc::new(bytes), &col_meta, total_rows, None).unwrap();
3299
3300 assert!(pages.supports_deferred_dictionary());
3301 }
3302
3303 #[test]
3307 fn offset_index_state_with_a_dictionary_offset_supports_deferral() {
3308 let (bytes, _) = dict_column_file();
3309 let reader = SerializedFileReader::new(bytes.clone()).unwrap();
3310 let meta = reader.metadata();
3311 let col_meta = meta.row_group(0).column(0).clone();
3312 let total_rows = meta.row_group(0).num_rows() as usize;
3313 let (dict_offset, data_offset) = (
3314 col_meta.dictionary_page_offset().unwrap(),
3315 col_meta.data_page_offset(),
3316 );
3317 assert_ne!(
3318 dict_offset, data_offset,
3319 "fixture must record a dictionary offset"
3320 );
3321
3322 let locations = vec![PageLocation {
3323 offset: data_offset,
3324 compressed_page_size: 0,
3325 first_row_index: 0,
3326 }];
3327 let pages =
3328 SerializedPageReader::new(Arc::new(bytes), &col_meta, total_rows, Some(locations))
3329 .unwrap();
3330
3331 assert!(pages.supports_deferred_dictionary());
3332 }
3333
3334 #[test]
3340 fn offset_index_state_without_a_dictionary_offset_declines_deferral() {
3341 let (bytes, _) = dict_column_file();
3342 let reader = SerializedFileReader::new(bytes.clone()).unwrap();
3343 let meta = reader.metadata();
3344 let total_rows = meta.row_group(0).num_rows() as usize;
3345
3346 let col_meta = meta.row_group(0).column(0).clone();
3349 let stripped = col_meta
3350 .clone()
3351 .into_builder()
3352 .set_dictionary_page_offset(None)
3353 .build()
3354 .unwrap();
3355
3356 let locations = vec![PageLocation {
3357 offset: stripped.data_page_offset(),
3358 compressed_page_size: 0,
3359 first_row_index: 0,
3360 }];
3361 let pages =
3362 SerializedPageReader::new(Arc::new(bytes), &stripped, total_rows, Some(locations))
3363 .unwrap();
3364
3365 assert!(
3366 !pages.supports_deferred_dictionary(),
3367 "an offset-less dictionary cannot be recovered from the offset-index state"
3368 );
3369 }
3374
3375 #[test]
3380 fn alternating_skip_and_read_decodes_correct_values() {
3381 let (bytes, _) = dict_column_file();
3382 let mut reader = int32_reader(bytes);
3383
3384 assert_eq!(reader.skip_records(1).unwrap(), 1);
3385 let mut first = Vec::new();
3386 let (records, _, _) = reader.read_records(2, None, None, &mut first).unwrap();
3387 assert_eq!(records, 2);
3388 assert_eq!(first, vec![2, 3]);
3389
3390 assert_eq!(reader.skip_records(1).unwrap(), 1);
3391 let mut second = Vec::new();
3392 let (records, _, _) = reader.read_records(1, None, None, &mut second).unwrap();
3393 assert_eq!(records, 1);
3394 assert_eq!(second, vec![5]);
3395 }
3396
3397 struct HalfForwardingPageReader(SerializedPageReader<Bytes>);
3401
3402 impl Iterator for HalfForwardingPageReader {
3403 type Item = Result<Page>;
3404
3405 fn next(&mut self) -> Option<Self::Item> {
3406 self.get_next_page().transpose()
3407 }
3408 }
3409
3410 impl PageReader for HalfForwardingPageReader {
3411 fn get_next_page(&mut self) -> Result<Option<Page>> {
3412 self.0.get_next_page()
3413 }
3414
3415 fn peek_next_page(&mut self) -> Result<Option<PageMetadata>> {
3416 self.0.peek_next_page()
3417 }
3418
3419 fn skip_next_page(&mut self) -> Result<()> {
3420 self.0.skip_next_page()
3421 }
3422
3423 fn take_deferred_dictionary(&mut self) -> Result<Option<Page>> {
3424 self.0.take_deferred_dictionary()
3425 }
3426 }
3427
3428 #[test]
3433 fn forwarding_only_take_deferred_dictionary_is_still_correct() {
3434 let (bytes, _) = dict_column_file();
3435 let reader = SerializedFileReader::new(bytes.clone()).unwrap();
3436 let meta = reader.metadata();
3437 let descr = meta.file_metadata().schema_descr().column(0);
3438 let col_meta = meta.row_group(0).column(0).clone();
3439 let total_rows = meta.row_group(0).num_rows() as usize;
3440 let pages =
3441 SerializedPageReader::new(Arc::new(bytes), &col_meta, total_rows, None).unwrap();
3442 let mut reader = crate::column::reader::ColumnReaderImpl::<Int32Type>::new(
3443 descr,
3444 Box::new(HalfForwardingPageReader(pages)),
3445 );
3446
3447 assert_eq!(reader.skip_records(2).unwrap(), 2);
3448 let mut values = Vec::new();
3449 let (records, _, _) = reader.read_records(3, None, None, &mut values).unwrap();
3450 assert_eq!(records, 3);
3451 assert_eq!(values, vec![3, 4, 5]);
3452 }
3453}