1use std::{collections::HashMap, fmt, sync::Arc};
9
10use crate::basic::{ConvertedType, Repetition};
11use crate::errors::{ParquetError, Result};
12use crate::file::reader::{FileReader, RowGroupReader};
13use crate::record::{
14 api::{Field, Row, make_list, make_map},
15 triplet::TripletIter,
16};
17use crate::schema::types::{ColumnPath, SchemaDescPtr, SchemaDescriptor, Type, TypePtr};
18
19const DEFAULT_BATCH_SIZE: usize = 1024;
21
22pub struct TreeBuilder {
26 batch_size: usize,
28}
29
30impl Default for TreeBuilder {
31 fn default() -> Self {
32 Self::new()
33 }
34}
35
36impl TreeBuilder {
37 pub fn new() -> Self {
39 Self {
40 batch_size: DEFAULT_BATCH_SIZE,
41 }
42 }
43
44 pub fn with_batch_size(mut self, batch_size: usize) -> Self {
46 self.batch_size = batch_size;
47 self
48 }
49
50 pub fn build(
52 &self,
53 descr: SchemaDescPtr,
54 row_group_reader: &dyn RowGroupReader,
55 ) -> Result<Reader> {
56 let mut paths: HashMap<ColumnPath, usize> = HashMap::new();
59 let row_group_metadata = row_group_reader.metadata();
60
61 for col_index in 0..row_group_reader.num_columns() {
62 let col_meta = row_group_metadata.column(col_index);
63 let col_path = col_meta.column_path().clone();
64 paths.insert(col_path, col_index);
65 }
66
67 let mut readers = Vec::new();
69 let mut path = Vec::new();
70
71 for field in descr.root_schema().get_fields() {
72 let reader =
73 self.reader_tree(field.clone(), &mut path, 0, 0, &paths, row_group_reader)?;
74 readers.push(reader);
75 }
76
77 Ok(Reader::GroupReader(None, 0, readers))
80 }
81
82 pub fn as_iter(
84 &self,
85 descr: SchemaDescPtr,
86 row_group_reader: &dyn RowGroupReader,
87 ) -> Result<ReaderIter> {
88 let num_records = row_group_reader.metadata().num_rows() as usize;
89 ReaderIter::new(self.build(descr, row_group_reader)?, num_records)
90 }
91
92 fn reader_tree(
94 &self,
95 field: TypePtr,
96 path: &mut Vec<String>,
97 mut curr_def_level: i16,
98 mut curr_rep_level: i16,
99 paths: &HashMap<ColumnPath, usize>,
100 row_group_reader: &dyn RowGroupReader,
101 ) -> Result<Reader> {
102 assert!(field.get_basic_info().has_repetition());
103 let repetition = field.get_basic_info().repetition();
105 match repetition {
106 Repetition::OPTIONAL => {
107 curr_def_level += 1;
108 }
109 Repetition::REPEATED => {
110 curr_def_level += 1;
111 curr_rep_level += 1;
112 }
113 Repetition::REQUIRED => {}
114 }
115
116 path.push(String::from(field.name()));
117 let reader = if field.is_primitive() {
118 let col_path = ColumnPath::new(path.clone());
119 let orig_index = *paths
120 .get(&col_path)
121 .ok_or(general_err!("Path {:?} not found", col_path))?;
122 let col_descr = row_group_reader
123 .metadata()
124 .column(orig_index)
125 .column_descr_ptr();
126 let col_reader = row_group_reader.get_column_reader(orig_index)?;
127 let column = TripletIter::new(col_descr, col_reader, self.batch_size);
128 let reader = Reader::PrimitiveReader(field.clone(), Box::new(column));
129 if repetition == Repetition::REPEATED {
130 Reader::RepeatedReader(
131 field,
132 curr_def_level - 1,
133 curr_rep_level - 1,
134 Box::new(reader),
135 )
136 } else {
137 reader
138 }
139 } else {
140 match field.get_basic_info().converted_type() {
141 ConvertedType::LIST => {
143 assert_eq!(field.get_fields().len(), 1, "Invalid list type {field:?}");
144
145 let repeated_field = field.get_fields()[0].clone();
146 assert_eq!(
147 repeated_field.get_basic_info().repetition(),
148 Repetition::REPEATED,
149 "Invalid list type {field:?}"
150 );
151
152 if Reader::is_element_type(&repeated_field) {
153 let reader = self.reader_tree(
155 repeated_field,
156 path,
157 curr_def_level,
158 curr_rep_level,
159 paths,
160 row_group_reader,
161 )?;
162
163 Reader::RepeatedReader(
164 field,
165 curr_def_level,
166 curr_rep_level,
167 Box::new(reader),
168 )
169 } else {
170 let child_field = repeated_field.get_fields()[0].clone();
171
172 path.push(String::from(repeated_field.name()));
173
174 let reader = self.reader_tree(
175 child_field,
176 path,
177 curr_def_level + 1,
178 curr_rep_level + 1,
179 paths,
180 row_group_reader,
181 )?;
182
183 path.pop();
184
185 Reader::RepeatedReader(
186 field,
187 curr_def_level,
188 curr_rep_level,
189 Box::new(reader),
190 )
191 }
192 }
193 ConvertedType::MAP | ConvertedType::MAP_KEY_VALUE => {
195 assert_eq!(field.get_fields().len(), 1, "Invalid map type: {field:?}");
196 assert!(
197 !field.get_fields()[0].is_primitive(),
198 "Invalid map type: {field:?}"
199 );
200
201 let key_value_type = field.get_fields()[0].clone();
202 assert_eq!(
203 key_value_type.get_basic_info().repetition(),
204 Repetition::REPEATED,
205 "Invalid map type: {field:?}"
206 );
207 if key_value_type.get_fields().len() != 1 {
209 assert_eq!(
211 key_value_type.get_fields().len(),
212 2,
213 "Invalid map type: {field:?}"
214 );
215 }
216
217 path.push(String::from(key_value_type.name()));
218
219 let key_type = &key_value_type.get_fields()[0];
220 let key_reader = self.reader_tree(
221 key_type.clone(),
222 path,
223 curr_def_level + 1,
224 curr_rep_level + 1,
225 paths,
226 row_group_reader,
227 )?;
228
229 if key_value_type.get_fields().len() == 1 {
230 path.pop();
231 Reader::RepeatedReader(
232 field,
233 curr_def_level,
234 curr_rep_level,
235 Box::new(key_reader),
236 )
237 } else {
238 let value_type = &key_value_type.get_fields()[1];
239 let value_reader = self.reader_tree(
240 value_type.clone(),
241 path,
242 curr_def_level + 1,
243 curr_rep_level + 1,
244 paths,
245 row_group_reader,
246 )?;
247
248 path.pop();
249
250 Reader::KeyValueReader(
251 field,
252 curr_def_level,
253 curr_rep_level,
254 Box::new(key_reader),
255 Box::new(value_reader),
256 )
257 }
258 }
259 _ if repetition == Repetition::REPEATED => {
264 let required_field = Type::group_type_builder(field.name())
265 .with_repetition(Repetition::REQUIRED)
266 .with_converted_type(field.get_basic_info().converted_type())
267 .with_fields(field.get_fields().to_vec())
268 .build()?;
269
270 path.pop();
271
272 let reader = self.reader_tree(
273 Arc::new(required_field),
274 path,
275 curr_def_level,
276 curr_rep_level,
277 paths,
278 row_group_reader,
279 )?;
280
281 return Ok(Reader::RepeatedReader(
282 field,
283 curr_def_level - 1,
284 curr_rep_level - 1,
285 Box::new(reader),
286 ));
287 }
288 _ => {
290 let mut readers = Vec::new();
291 for child in field.get_fields() {
292 let reader = self.reader_tree(
293 child.clone(),
294 path,
295 curr_def_level,
296 curr_rep_level,
297 paths,
298 row_group_reader,
299 )?;
300 readers.push(reader);
301 }
302 Reader::GroupReader(Some(field), curr_def_level, readers)
303 }
304 }
305 };
306 path.pop();
307
308 Ok(Reader::option(repetition, curr_def_level, reader))
309 }
310}
311
312pub enum Reader {
314 PrimitiveReader(TypePtr, Box<TripletIter>),
316 OptionReader(i16, Box<Reader>),
318 GroupReader(Option<TypePtr>, i16, Vec<Reader>),
321 RepeatedReader(TypePtr, i16, i16, Box<Reader>),
324 KeyValueReader(TypePtr, i16, i16, Box<Reader>, Box<Reader>),
327}
328
329impl Reader {
330 fn option(repetition: Repetition, def_level: i16, reader: Reader) -> Self {
332 if repetition == Repetition::OPTIONAL {
333 Reader::OptionReader(def_level - 1, Box::new(reader))
334 } else {
335 reader
336 }
337 }
338
339 fn is_element_type(repeated_type: &Type) -> bool {
345 if repeated_type.is_list() || repeated_type.has_single_repeated_child() {
355 return false;
356 }
357
358 repeated_type.is_primitive() ||
366 repeated_type.is_group() && repeated_type.get_fields().len() > 1 ||
378 repeated_type.name() == "array" ||
389 repeated_type.name().ends_with("_tuple")
399 }
400
401 fn read(&mut self) -> Result<Row> {
406 match *self {
407 Reader::GroupReader(_, _, ref mut readers) => {
408 let mut fields = Vec::new();
409 for reader in readers {
410 fields.push((String::from(reader.field_name()), reader.read_field()?));
411 }
412 Ok(Row::new(fields))
413 }
414 _ => panic!("Cannot call read() on {self}"),
415 }
416 }
417
418 fn read_field(&mut self) -> Result<Field> {
421 let field = match *self {
422 Reader::PrimitiveReader(_, ref mut column) => {
423 if !column.has_next() {
424 return Err(general_err!("Unexpected end of column data"));
425 }
426 let value = column.current_value()?;
427 column.read_next()?;
428 value
429 }
430 Reader::OptionReader(def_level, ref mut reader) => {
431 if !reader.has_next() {
432 return Err(general_err!("Unexpected end of column data"));
433 }
434 if reader.current_def_level() > def_level {
435 reader.read_field()?
436 } else {
437 reader.advance_columns()?;
438 Field::Null
439 }
440 }
441 Reader::GroupReader(_, def_level, ref mut readers) => {
442 let mut fields = Vec::new();
443 for reader in readers {
444 if reader.repetition() != Repetition::OPTIONAL
445 || reader.current_def_level() > def_level
446 {
447 fields.push((String::from(reader.field_name()), reader.read_field()?));
448 } else {
449 reader.advance_columns()?;
450 fields.push((String::from(reader.field_name()), Field::Null));
451 }
452 }
453 let row = Row::new(fields);
454 Field::Group(row)
455 }
456 Reader::RepeatedReader(_, def_level, rep_level, ref mut reader) => {
457 if !reader.has_next() {
458 return Err(general_err!("Unexpected end of column data"));
459 }
460 let mut elements = Vec::new();
461 loop {
462 if reader.current_def_level() > def_level {
463 elements.push(reader.read_field()?);
464 } else {
465 reader.advance_columns()?;
466 break;
471 }
472
473 if !reader.has_next() || reader.current_rep_level() <= rep_level {
477 break;
478 }
479 }
480 Field::ListInternal(make_list(elements))
481 }
482 Reader::KeyValueReader(_, def_level, rep_level, ref mut keys, ref mut values) => {
483 if !keys.has_next() {
484 return Err(general_err!("Unexpected end of column data"));
485 }
486 let mut pairs = Vec::new();
487 loop {
488 if keys.current_def_level() > def_level {
489 pairs.push((keys.read_field()?, values.read_field()?));
490 } else {
491 keys.advance_columns()?;
492 values.advance_columns()?;
493 break;
498 }
499
500 if !keys.has_next() || keys.current_rep_level() <= rep_level {
504 break;
505 }
506 }
507
508 Field::MapInternal(make_map(pairs))
509 }
510 };
511 Ok(field)
512 }
513
514 fn field_name(&self) -> &str {
516 match *self {
517 Reader::PrimitiveReader(ref field, _) => field.name(),
518 Reader::OptionReader(_, ref reader) => reader.field_name(),
519 Reader::GroupReader(ref opt, ..) => match opt {
520 Some(field) => field.name(),
521 None => panic!("Field is None for group reader"),
522 },
523 Reader::RepeatedReader(ref field, ..) => field.name(),
524 Reader::KeyValueReader(ref field, ..) => field.name(),
525 }
526 }
527
528 fn repetition(&self) -> Repetition {
530 match *self {
531 Reader::PrimitiveReader(ref field, _) => field.get_basic_info().repetition(),
532 Reader::OptionReader(_, ref reader) => reader.repetition(),
533 Reader::GroupReader(ref opt, ..) => match opt {
534 Some(field) => field.get_basic_info().repetition(),
535 None => panic!("Field is None for group reader"),
536 },
537 Reader::RepeatedReader(ref field, ..) => field.get_basic_info().repetition(),
538 Reader::KeyValueReader(ref field, ..) => field.get_basic_info().repetition(),
539 }
540 }
541
542 fn has_next(&self) -> bool {
545 match *self {
546 Reader::PrimitiveReader(_, ref column) => column.has_next(),
547 Reader::OptionReader(_, ref reader) => reader.has_next(),
548 Reader::GroupReader(_, _, ref readers) => readers.first().unwrap().has_next(),
549 Reader::RepeatedReader(_, _, _, ref reader) => reader.has_next(),
550 Reader::KeyValueReader(_, _, _, ref keys, _) => keys.has_next(),
551 }
552 }
553
554 fn current_def_level(&self) -> i16 {
557 match *self {
558 Reader::PrimitiveReader(_, ref column) => column.current_def_level(),
559 Reader::OptionReader(_, ref reader) => reader.current_def_level(),
560 Reader::GroupReader(_, _, ref readers) => match readers.first() {
561 Some(reader) => reader.current_def_level(),
562 None => panic!("Current definition level: empty group reader"),
563 },
564 Reader::RepeatedReader(_, _, _, ref reader) => reader.current_def_level(),
565 Reader::KeyValueReader(_, _, _, ref keys, _) => keys.current_def_level(),
566 }
567 }
568
569 fn current_rep_level(&self) -> i16 {
572 match *self {
573 Reader::PrimitiveReader(_, ref column) => column.current_rep_level(),
574 Reader::OptionReader(_, ref reader) => reader.current_rep_level(),
575 Reader::GroupReader(_, _, ref readers) => match readers.first() {
576 Some(reader) => reader.current_rep_level(),
577 None => panic!("Current repetition level: empty group reader"),
578 },
579 Reader::RepeatedReader(_, _, _, ref reader) => reader.current_rep_level(),
580 Reader::KeyValueReader(_, _, _, ref keys, _) => keys.current_rep_level(),
581 }
582 }
583
584 fn advance_columns(&mut self) -> Result<()> {
586 match *self {
587 Reader::PrimitiveReader(_, ref mut column) => column.read_next().map(|_| ()),
588 Reader::OptionReader(_, ref mut reader) => reader.advance_columns(),
589 Reader::GroupReader(_, _, ref mut readers) => {
590 for reader in readers {
591 reader.advance_columns()?;
592 }
593 Ok(())
594 }
595 Reader::RepeatedReader(_, _, _, ref mut reader) => reader.advance_columns(),
596 Reader::KeyValueReader(_, _, _, ref mut keys, ref mut values) => {
597 keys.advance_columns()?;
598 values.advance_columns()
599 }
600 }
601 }
602}
603
604impl fmt::Display for Reader {
605 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
606 let s = match self {
607 Reader::PrimitiveReader(..) => "PrimitiveReader",
608 Reader::OptionReader(..) => "OptionReader",
609 Reader::GroupReader(..) => "GroupReader",
610 Reader::RepeatedReader(..) => "RepeatedReader",
611 Reader::KeyValueReader(..) => "KeyValueReader",
612 };
613 write!(f, "{s}")
614 }
615}
616
617enum Either<'a> {
623 Left(&'a dyn FileReader),
624 Right(Box<dyn FileReader>),
625}
626
627impl Either<'_> {
628 fn reader(&self) -> &dyn FileReader {
629 match *self {
630 Either::Left(r) => r,
631 Either::Right(ref r) => &**r,
632 }
633 }
634}
635
636pub struct RowIter<'a> {
651 descr: SchemaDescPtr,
652 tree_builder: TreeBuilder,
653 file_reader: Option<Either<'a>>,
654 current_row_group: usize,
655 num_row_groups: usize,
656 row_iter: Option<ReaderIter>,
657}
658
659impl<'a> RowIter<'a> {
660 fn new(
662 file_reader: Option<Either<'a>>,
663 row_iter: Option<ReaderIter>,
664 descr: SchemaDescPtr,
665 ) -> Self {
666 let tree_builder = Self::tree_builder();
667 let num_row_groups = match file_reader {
668 Some(ref r) => r.reader().num_row_groups(),
669 None => 0,
670 };
671
672 Self {
673 descr,
674 file_reader,
675 tree_builder,
676 num_row_groups,
677 row_iter,
678 current_row_group: 0,
679 }
680 }
681
682 pub fn from_file(proj: Option<Type>, reader: &'a dyn FileReader) -> Result<Self> {
685 let either = Either::Left(reader);
686 let descr =
687 Self::get_proj_descr(proj, reader.metadata().file_metadata().schema_descr_ptr())?;
688
689 Ok(Self::new(Some(either), None, descr))
690 }
691
692 pub fn from_row_group(proj: Option<Type>, reader: &'a dyn RowGroupReader) -> Result<Self> {
694 let descr = Self::get_proj_descr(proj, reader.metadata().schema_descr_ptr())?;
695 let tree_builder = Self::tree_builder();
696 let row_iter = tree_builder.as_iter(descr.clone(), reader)?;
697
698 Ok(Self::new(None, Some(row_iter), descr))
701 }
702
703 pub fn from_file_into(reader: Box<dyn FileReader>) -> Self {
705 let either = Either::Right(reader);
706 let descr = either
707 .reader()
708 .metadata()
709 .file_metadata()
710 .schema_descr_ptr();
711
712 Self::new(Some(either), None, descr)
713 }
714
715 pub fn project(self, proj: Option<Type>) -> Result<Self> {
721 match self.file_reader {
722 Some(ref either) => {
723 let schema = either
724 .reader()
725 .metadata()
726 .file_metadata()
727 .schema_descr_ptr();
728 let descr = Self::get_proj_descr(proj, schema)?;
729
730 Ok(Self::new(self.file_reader, None, descr))
731 }
732 None => Err(general_err!("File reader is required to use projections")),
733 }
734 }
735
736 #[inline]
739 fn get_proj_descr(proj: Option<Type>, root_descr: SchemaDescPtr) -> Result<SchemaDescPtr> {
740 match proj {
741 Some(projection) => {
742 let root_schema = root_descr.root_schema();
744 if !root_schema.check_contains(&projection) {
745 return Err(general_err!("Root schema does not contain projection"));
746 }
747 Ok(Arc::new(SchemaDescriptor::new(Arc::new(projection))))
748 }
749 None => Ok(root_descr),
750 }
751 }
752
753 pub fn with_batch_size(mut self, batch_size: usize) -> Self {
755 self.tree_builder = self.tree_builder.with_batch_size(batch_size);
756 self
757 }
758
759 #[inline]
762 fn tree_builder() -> TreeBuilder {
763 TreeBuilder::new()
764 }
765}
766
767impl Iterator for RowIter<'_> {
768 type Item = Result<Row>;
769
770 fn next(&mut self) -> Option<Result<Row>> {
771 let mut row = self.row_iter.as_mut().and_then(|iter| iter.next());
772
773 while row.is_none() && self.current_row_group < self.num_row_groups {
774 if let Some(ref either) = self.file_reader {
777 let file_reader = either.reader();
778 let row_group_reader = &*file_reader
779 .get_row_group(self.current_row_group)
780 .expect("Row group is required to advance");
781
782 match self
783 .tree_builder
784 .as_iter(self.descr.clone(), row_group_reader)
785 {
786 Ok(mut iter) => {
787 row = iter.next();
788
789 self.current_row_group += 1;
790 self.row_iter = Some(iter);
791 }
792 Err(e) => return Some(Err(e)),
793 }
794 }
795 }
796
797 row
798 }
799}
800
801pub struct ReaderIter {
803 root_reader: Reader,
804 records_left: usize,
805}
806
807impl ReaderIter {
808 fn new(mut root_reader: Reader, num_records: usize) -> Result<Self> {
809 root_reader.advance_columns()?;
811 Ok(Self {
812 root_reader,
813 records_left: num_records,
814 })
815 }
816}
817
818impl Iterator for ReaderIter {
819 type Item = Result<Row>;
820
821 fn next(&mut self) -> Option<Result<Row>> {
822 if self.records_left > 0 {
823 self.records_left -= 1;
824 Some(self.root_reader.read())
825 } else {
826 None
827 }
828 }
829}
830
831#[cfg(test)]
832mod tests {
833 use super::*;
834
835 use crate::data_type::Int64Type;
836 use crate::file::reader::SerializedFileReader;
837 use crate::file::writer::SerializedFileWriter;
838 use crate::record::api::RowAccessor;
839 use crate::schema::parser::parse_message_type;
840 use crate::util::test_common::file_util::{get_test_file, get_test_path};
841 use bytes::Bytes;
842
843 macro_rules! row {
846 ($($e:tt)*) => {
847 {
848 Row::new(vec![$($e)*])
849 }
850 }
851 }
852
853 macro_rules! list {
854 ($($e:tt)*) => {
855 {
856 Field::ListInternal(make_list(vec![$($e)*]))
857 }
858 }
859 }
860
861 macro_rules! map {
862 ($($e:tt)*) => {
863 {
864 Field::MapInternal(make_map(vec![$($e)*]))
865 }
866 }
867 }
868
869 macro_rules! group {
870 ( $( $e:expr ), * ) => {
871 {
872 Field::Group(row!($( $e ), *))
873 }
874 }
875 }
876
877 #[test]
878 fn test_file_reader_rows_nulls() {
879 let rows = test_file_reader_rows("nulls.snappy.parquet", None).unwrap();
880 let expected_rows = vec![
881 row![(
882 "b_struct".to_string(),
883 group![("b_c_int".to_string(), Field::Null)]
884 )],
885 row![(
886 "b_struct".to_string(),
887 group![("b_c_int".to_string(), Field::Null)]
888 )],
889 row![(
890 "b_struct".to_string(),
891 group![("b_c_int".to_string(), Field::Null)]
892 )],
893 row![(
894 "b_struct".to_string(),
895 group![("b_c_int".to_string(), Field::Null)]
896 )],
897 row![(
898 "b_struct".to_string(),
899 group![("b_c_int".to_string(), Field::Null)]
900 )],
901 row![(
902 "b_struct".to_string(),
903 group![("b_c_int".to_string(), Field::Null)]
904 )],
905 row![(
906 "b_struct".to_string(),
907 group![("b_c_int".to_string(), Field::Null)]
908 )],
909 row![(
910 "b_struct".to_string(),
911 group![("b_c_int".to_string(), Field::Null)]
912 )],
913 ];
914 assert_eq!(rows, expected_rows);
915 }
916
917 #[test]
918 #[cfg_attr(miri, ignore)] fn test_file_reader_rows_nonnullable() {
920 let rows = test_file_reader_rows("nonnullable.impala.parquet", None).unwrap();
921 let expected_rows = vec![row![
922 ("ID".to_string(), Field::Long(8)),
923 ("Int_Array".to_string(), list![Field::Int(-1)]),
924 (
925 "int_array_array".to_string(),
926 list![list![Field::Int(-1), Field::Int(-2)], list![]]
927 ),
928 (
929 "Int_Map".to_string(),
930 map![(Field::Str("k1".to_string()), Field::Int(-1))]
931 ),
932 (
933 "int_map_array".to_string(),
934 list![
935 map![],
936 map![(Field::Str("k1".to_string()), Field::Int(1))],
937 map![],
938 map![]
939 ]
940 ),
941 (
942 "nested_Struct".to_string(),
943 group![
944 ("a".to_string(), Field::Int(-1)),
945 ("B".to_string(), list![Field::Int(-1)]),
946 (
947 "c".to_string(),
948 group![(
949 "D".to_string(),
950 list![list![group![
951 ("e".to_string(), Field::Int(-1)),
952 ("f".to_string(), Field::Str("nonnullable".to_string()))
953 ]]]
954 )]
955 ),
956 ("G".to_string(), map![])
957 ]
958 )
959 ]];
960 assert_eq!(rows, expected_rows);
961 }
962
963 #[test]
964 #[cfg_attr(miri, ignore)] fn test_file_reader_rows_nullable() {
966 let rows = test_file_reader_rows("nullable.impala.parquet", None).unwrap();
967 let expected_rows = vec![
968 row![
969 ("id".to_string(), Field::Long(1)),
970 (
971 "int_array".to_string(),
972 list![Field::Int(1), Field::Int(2), Field::Int(3)]
973 ),
974 (
975 "int_array_Array".to_string(),
976 list![
977 list![Field::Int(1), Field::Int(2)],
978 list![Field::Int(3), Field::Int(4)]
979 ]
980 ),
981 (
982 "int_map".to_string(),
983 map![
984 (Field::Str("k1".to_string()), Field::Int(1)),
985 (Field::Str("k2".to_string()), Field::Int(100))
986 ]
987 ),
988 (
989 "int_Map_Array".to_string(),
990 list![map![(Field::Str("k1".to_string()), Field::Int(1))]]
991 ),
992 (
993 "nested_struct".to_string(),
994 group![
995 ("A".to_string(), Field::Int(1)),
996 ("b".to_string(), list![Field::Int(1)]),
997 (
998 "C".to_string(),
999 group![(
1000 "d".to_string(),
1001 list![
1002 list![
1003 group![
1004 ("E".to_string(), Field::Int(10)),
1005 ("F".to_string(), Field::Str("aaa".to_string()))
1006 ],
1007 group![
1008 ("E".to_string(), Field::Int(-10)),
1009 ("F".to_string(), Field::Str("bbb".to_string()))
1010 ]
1011 ],
1012 list![group![
1013 ("E".to_string(), Field::Int(11)),
1014 ("F".to_string(), Field::Str("c".to_string()))
1015 ]]
1016 ]
1017 )]
1018 ),
1019 (
1020 "g".to_string(),
1021 map![(
1022 Field::Str("foo".to_string()),
1023 group![(
1024 "H".to_string(),
1025 group![("i".to_string(), list![Field::Double(1.1)])]
1026 )]
1027 )]
1028 )
1029 ]
1030 )
1031 ],
1032 row![
1033 ("id".to_string(), Field::Long(2)),
1034 (
1035 "int_array".to_string(),
1036 list![
1037 Field::Null,
1038 Field::Int(1),
1039 Field::Int(2),
1040 Field::Null,
1041 Field::Int(3),
1042 Field::Null
1043 ]
1044 ),
1045 (
1046 "int_array_Array".to_string(),
1047 list![
1048 list![Field::Null, Field::Int(1), Field::Int(2), Field::Null],
1049 list![Field::Int(3), Field::Null, Field::Int(4)],
1050 list![],
1051 Field::Null
1052 ]
1053 ),
1054 (
1055 "int_map".to_string(),
1056 map![
1057 (Field::Str("k1".to_string()), Field::Int(2)),
1058 (Field::Str("k2".to_string()), Field::Null)
1059 ]
1060 ),
1061 (
1062 "int_Map_Array".to_string(),
1063 list![
1064 map![
1065 (Field::Str("k3".to_string()), Field::Null),
1066 (Field::Str("k1".to_string()), Field::Int(1))
1067 ],
1068 Field::Null,
1069 map![]
1070 ]
1071 ),
1072 (
1073 "nested_struct".to_string(),
1074 group![
1075 ("A".to_string(), Field::Null),
1076 ("b".to_string(), list![Field::Null]),
1077 (
1078 "C".to_string(),
1079 group![(
1080 "d".to_string(),
1081 list![
1082 list![
1083 group![
1084 ("E".to_string(), Field::Null),
1085 ("F".to_string(), Field::Null)
1086 ],
1087 group![
1088 ("E".to_string(), Field::Int(10)),
1089 ("F".to_string(), Field::Str("aaa".to_string()))
1090 ],
1091 group![
1092 ("E".to_string(), Field::Null),
1093 ("F".to_string(), Field::Null)
1094 ],
1095 group![
1096 ("E".to_string(), Field::Int(-10)),
1097 ("F".to_string(), Field::Str("bbb".to_string()))
1098 ],
1099 group![
1100 ("E".to_string(), Field::Null),
1101 ("F".to_string(), Field::Null)
1102 ]
1103 ],
1104 list![
1105 group![
1106 ("E".to_string(), Field::Int(11)),
1107 ("F".to_string(), Field::Str("c".to_string()))
1108 ],
1109 Field::Null
1110 ],
1111 list![],
1112 Field::Null
1113 ]
1114 )]
1115 ),
1116 (
1117 "g".to_string(),
1118 map![
1119 (
1120 Field::Str("g1".to_string()),
1121 group![(
1122 "H".to_string(),
1123 group![(
1124 "i".to_string(),
1125 list![Field::Double(2.2), Field::Null]
1126 )]
1127 )]
1128 ),
1129 (
1130 Field::Str("g2".to_string()),
1131 group![("H".to_string(), group![("i".to_string(), list![])])]
1132 ),
1133 (Field::Str("g3".to_string()), Field::Null),
1134 (
1135 Field::Str("g4".to_string()),
1136 group![(
1137 "H".to_string(),
1138 group![("i".to_string(), Field::Null)]
1139 )]
1140 ),
1141 (
1142 Field::Str("g5".to_string()),
1143 group![("H".to_string(), Field::Null)]
1144 )
1145 ]
1146 )
1147 ]
1148 )
1149 ],
1150 row![
1151 ("id".to_string(), Field::Long(3)),
1152 ("int_array".to_string(), list![]),
1153 ("int_array_Array".to_string(), list![Field::Null]),
1154 ("int_map".to_string(), map![]),
1155 ("int_Map_Array".to_string(), list![Field::Null, Field::Null]),
1156 (
1157 "nested_struct".to_string(),
1158 group![
1159 ("A".to_string(), Field::Null),
1160 ("b".to_string(), Field::Null),
1161 ("C".to_string(), group![("d".to_string(), list![])]),
1162 ("g".to_string(), map![])
1163 ]
1164 )
1165 ],
1166 row![
1167 ("id".to_string(), Field::Long(4)),
1168 ("int_array".to_string(), Field::Null),
1169 ("int_array_Array".to_string(), list![]),
1170 ("int_map".to_string(), map![]),
1171 ("int_Map_Array".to_string(), list![]),
1172 (
1173 "nested_struct".to_string(),
1174 group![
1175 ("A".to_string(), Field::Null),
1176 ("b".to_string(), Field::Null),
1177 ("C".to_string(), group![("d".to_string(), Field::Null)]),
1178 ("g".to_string(), Field::Null)
1179 ]
1180 )
1181 ],
1182 row![
1183 ("id".to_string(), Field::Long(5)),
1184 ("int_array".to_string(), Field::Null),
1185 ("int_array_Array".to_string(), Field::Null),
1186 ("int_map".to_string(), map![]),
1187 ("int_Map_Array".to_string(), Field::Null),
1188 (
1189 "nested_struct".to_string(),
1190 group![
1191 ("A".to_string(), Field::Null),
1192 ("b".to_string(), Field::Null),
1193 ("C".to_string(), Field::Null),
1194 (
1195 "g".to_string(),
1196 map![(
1197 Field::Str("foo".to_string()),
1198 group![(
1199 "H".to_string(),
1200 group![(
1201 "i".to_string(),
1202 list![Field::Double(2.2), Field::Double(3.3)]
1203 )]
1204 )]
1205 )]
1206 )
1207 ]
1208 )
1209 ],
1210 row![
1211 ("id".to_string(), Field::Long(6)),
1212 ("int_array".to_string(), Field::Null),
1213 ("int_array_Array".to_string(), Field::Null),
1214 ("int_map".to_string(), Field::Null),
1215 ("int_Map_Array".to_string(), Field::Null),
1216 ("nested_struct".to_string(), Field::Null)
1217 ],
1218 row![
1219 ("id".to_string(), Field::Long(7)),
1220 ("int_array".to_string(), Field::Null),
1221 (
1222 "int_array_Array".to_string(),
1223 list![Field::Null, list![Field::Int(5), Field::Int(6)]]
1224 ),
1225 (
1226 "int_map".to_string(),
1227 map![
1228 (Field::Str("k1".to_string()), Field::Null),
1229 (Field::Str("k3".to_string()), Field::Null)
1230 ]
1231 ),
1232 ("int_Map_Array".to_string(), Field::Null),
1233 (
1234 "nested_struct".to_string(),
1235 group![
1236 ("A".to_string(), Field::Int(7)),
1237 (
1238 "b".to_string(),
1239 list![Field::Int(2), Field::Int(3), Field::Null]
1240 ),
1241 (
1242 "C".to_string(),
1243 group![(
1244 "d".to_string(),
1245 list![list![], list![Field::Null], Field::Null]
1246 )]
1247 ),
1248 ("g".to_string(), Field::Null)
1249 ]
1250 )
1251 ],
1252 ];
1253 assert_eq!(rows, expected_rows);
1254 }
1255
1256 #[test]
1257 fn test_file_reader_rows_projection() {
1258 let schema = "
1259 message spark_schema {
1260 REQUIRED DOUBLE c;
1261 REQUIRED INT32 b;
1262 }
1263 ";
1264 let schema = parse_message_type(schema).unwrap();
1265 let rows = test_file_reader_rows("nested_maps.snappy.parquet", Some(schema)).unwrap();
1266 let expected_rows = vec![
1267 row![
1268 ("c".to_string(), Field::Double(1.0)),
1269 ("b".to_string(), Field::Int(1))
1270 ],
1271 row![
1272 ("c".to_string(), Field::Double(1.0)),
1273 ("b".to_string(), Field::Int(1))
1274 ],
1275 row![
1276 ("c".to_string(), Field::Double(1.0)),
1277 ("b".to_string(), Field::Int(1))
1278 ],
1279 row![
1280 ("c".to_string(), Field::Double(1.0)),
1281 ("b".to_string(), Field::Int(1))
1282 ],
1283 row![
1284 ("c".to_string(), Field::Double(1.0)),
1285 ("b".to_string(), Field::Int(1))
1286 ],
1287 row![
1288 ("c".to_string(), Field::Double(1.0)),
1289 ("b".to_string(), Field::Int(1))
1290 ],
1291 ];
1292 assert_eq!(rows, expected_rows);
1293 }
1294
1295 #[test]
1296 fn test_iter_columns_in_row() {
1297 let r = row![
1298 ("c".to_string(), Field::Double(1.0)),
1299 ("b".to_string(), Field::Int(1))
1300 ];
1301 let mut result = Vec::new();
1302 for (name, record) in r.get_column_iter() {
1303 result.push((name, record));
1304 }
1305 assert_eq!(
1306 vec![
1307 (&"c".to_string(), &Field::Double(1.0)),
1308 (&"b".to_string(), &Field::Int(1))
1309 ],
1310 result
1311 );
1312 }
1313
1314 #[test]
1315 fn test_into_columns_in_row() {
1316 let r = row![
1317 ("a".to_string(), Field::Str("My string".to_owned())),
1318 ("b".to_string(), Field::Int(1))
1319 ];
1320 assert_eq!(
1321 r.into_columns(),
1322 vec![
1323 ("a".to_string(), Field::Str("My string".to_owned())),
1324 ("b".to_string(), Field::Int(1)),
1325 ]
1326 );
1327 }
1328
1329 #[test]
1330 fn test_file_reader_rows_projection_map() {
1331 let schema = "
1332 message spark_schema {
1333 OPTIONAL group a (MAP) {
1334 REPEATED group key_value {
1335 REQUIRED BYTE_ARRAY key (UTF8);
1336 OPTIONAL group value (MAP) {
1337 REPEATED group key_value {
1338 REQUIRED INT32 key;
1339 REQUIRED BOOLEAN value;
1340 }
1341 }
1342 }
1343 }
1344 }
1345 ";
1346 let schema = parse_message_type(schema).unwrap();
1347 let rows = test_file_reader_rows("nested_maps.snappy.parquet", Some(schema)).unwrap();
1348 let expected_rows = vec![
1349 row![(
1350 "a".to_string(),
1351 map![(
1352 Field::Str("a".to_string()),
1353 map![
1354 (Field::Int(1), Field::Bool(true)),
1355 (Field::Int(2), Field::Bool(false))
1356 ]
1357 )]
1358 )],
1359 row![(
1360 "a".to_string(),
1361 map![(
1362 Field::Str("b".to_string()),
1363 map![(Field::Int(1), Field::Bool(true))]
1364 )]
1365 )],
1366 row![(
1367 "a".to_string(),
1368 map![(Field::Str("c".to_string()), Field::Null)]
1369 )],
1370 row![("a".to_string(), map![(Field::Str("d".to_string()), map![])])],
1371 row![(
1372 "a".to_string(),
1373 map![(
1374 Field::Str("e".to_string()),
1375 map![(Field::Int(1), Field::Bool(true))]
1376 )]
1377 )],
1378 row![(
1379 "a".to_string(),
1380 map![(
1381 Field::Str("f".to_string()),
1382 map![
1383 (Field::Int(3), Field::Bool(true)),
1384 (Field::Int(4), Field::Bool(false)),
1385 (Field::Int(5), Field::Bool(true))
1386 ]
1387 )]
1388 )],
1389 ];
1390 assert_eq!(rows, expected_rows);
1391 }
1392
1393 #[test]
1394 fn test_file_reader_rows_projection_list() {
1395 let schema = "
1396 message spark_schema {
1397 OPTIONAL group a (LIST) {
1398 REPEATED group list {
1399 OPTIONAL group element (LIST) {
1400 REPEATED group list {
1401 OPTIONAL group element (LIST) {
1402 REPEATED group list {
1403 OPTIONAL BYTE_ARRAY element (UTF8);
1404 }
1405 }
1406 }
1407 }
1408 }
1409 }
1410 }
1411 ";
1412 let schema = parse_message_type(schema).unwrap();
1413 let rows = test_file_reader_rows("nested_lists.snappy.parquet", Some(schema)).unwrap();
1414 let expected_rows = vec![
1415 row![(
1416 "a".to_string(),
1417 list![
1418 list![
1419 list![Field::Str("a".to_string()), Field::Str("b".to_string())],
1420 list![Field::Str("c".to_string())]
1421 ],
1422 list![Field::Null, list![Field::Str("d".to_string())]]
1423 ]
1424 )],
1425 row![(
1426 "a".to_string(),
1427 list![
1428 list![
1429 list![Field::Str("a".to_string()), Field::Str("b".to_string())],
1430 list![Field::Str("c".to_string()), Field::Str("d".to_string())]
1431 ],
1432 list![Field::Null, list![Field::Str("e".to_string())]]
1433 ]
1434 )],
1435 row![(
1436 "a".to_string(),
1437 list![
1438 list![
1439 list![Field::Str("a".to_string()), Field::Str("b".to_string())],
1440 list![Field::Str("c".to_string()), Field::Str("d".to_string())],
1441 list![Field::Str("e".to_string())]
1442 ],
1443 list![Field::Null, list![Field::Str("f".to_string())]]
1444 ]
1445 )],
1446 ];
1447 assert_eq!(rows, expected_rows);
1448 }
1449
1450 #[test]
1451 fn test_file_reader_rows_invalid_projection() {
1452 let schema = "
1453 message spark_schema {
1454 REQUIRED INT32 key;
1455 REQUIRED BOOLEAN value;
1456 }
1457 ";
1458 let schema = parse_message_type(schema).unwrap();
1459 let res = test_file_reader_rows("nested_maps.snappy.parquet", Some(schema));
1460 assert_eq!(
1461 res.unwrap_err().to_string(),
1462 "Parquet error: Root schema does not contain projection"
1463 );
1464 }
1465
1466 #[test]
1467 fn test_row_group_rows_invalid_projection() {
1468 let schema = "
1469 message spark_schema {
1470 REQUIRED INT32 key;
1471 REQUIRED BOOLEAN value;
1472 }
1473 ";
1474 let schema = parse_message_type(schema).unwrap();
1475 let res = test_row_group_rows("nested_maps.snappy.parquet", Some(schema));
1476 assert_eq!(
1477 res.unwrap_err().to_string(),
1478 "Parquet error: Root schema does not contain projection"
1479 );
1480 }
1481
1482 #[test]
1483 fn test_file_reader_rows_nested_map_type() {
1484 let schema = "
1485 message spark_schema {
1486 OPTIONAL group a (MAP) {
1487 REPEATED group key_value {
1488 REQUIRED BYTE_ARRAY key (UTF8);
1489 OPTIONAL group value (MAP) {
1490 REPEATED group key_value {
1491 REQUIRED INT32 key;
1492 }
1493 }
1494 }
1495 }
1496 }
1497 ";
1498 let schema = parse_message_type(schema).unwrap();
1499 test_file_reader_rows("nested_maps.snappy.parquet", Some(schema)).unwrap();
1500 }
1501
1502 #[test]
1503 fn test_file_reader_iter() {
1504 let path = get_test_path("alltypes_plain.parquet");
1505 let reader = SerializedFileReader::try_from(path.as_path()).unwrap();
1506 let iter = RowIter::from_file_into(Box::new(reader));
1507
1508 let values: Vec<_> = iter.flat_map(|r| r.unwrap().get_int(0)).collect();
1509 assert_eq!(values, &[4, 5, 6, 7, 2, 3, 0, 1]);
1510 }
1511
1512 #[test]
1513 fn test_file_reader_iter_projection() {
1514 let path = get_test_path("alltypes_plain.parquet");
1515 let reader = SerializedFileReader::try_from(path.as_path()).unwrap();
1516 let schema = "message schema { OPTIONAL INT32 id; }";
1517 let proj = parse_message_type(schema).ok();
1518
1519 let iter = RowIter::from_file_into(Box::new(reader))
1520 .project(proj)
1521 .unwrap();
1522 let values: Vec<_> = iter.flat_map(|r| r.unwrap().get_int(0)).collect();
1523
1524 assert_eq!(values, &[4, 5, 6, 7, 2, 3, 0, 1]);
1525 }
1526
1527 #[test]
1528 fn test_file_reader_iter_projection_err() {
1529 let schema = "
1530 message spark_schema {
1531 REQUIRED INT32 key;
1532 REQUIRED BOOLEAN value;
1533 }
1534 ";
1535 let proj = parse_message_type(schema).ok();
1536 let path = get_test_path("nested_maps.snappy.parquet");
1537 let reader = SerializedFileReader::try_from(path.as_path()).unwrap();
1538 let res = RowIter::from_file_into(Box::new(reader)).project(proj);
1539
1540 assert_eq!(
1541 res.err().unwrap().to_string(),
1542 "Parquet error: Root schema does not contain projection"
1543 );
1544 }
1545
1546 #[test]
1547 fn test_tree_reader_handle_repeated_fields_with_no_annotation() {
1548 let rows = test_file_reader_rows("repeated_no_annotation.parquet", None).unwrap();
1551 let expected_rows = vec![
1552 row![
1553 ("id".to_string(), Field::Int(1)),
1554 ("phoneNumbers".to_string(), Field::Null)
1555 ],
1556 row![
1557 ("id".to_string(), Field::Int(2)),
1558 ("phoneNumbers".to_string(), Field::Null)
1559 ],
1560 row![
1561 ("id".to_string(), Field::Int(3)),
1562 (
1563 "phoneNumbers".to_string(),
1564 group![("phone".to_string(), list![])]
1565 )
1566 ],
1567 row![
1568 ("id".to_string(), Field::Int(4)),
1569 (
1570 "phoneNumbers".to_string(),
1571 group![(
1572 "phone".to_string(),
1573 list![group![
1574 ("number".to_string(), Field::Long(5555555555)),
1575 ("kind".to_string(), Field::Null)
1576 ]]
1577 )]
1578 )
1579 ],
1580 row![
1581 ("id".to_string(), Field::Int(5)),
1582 (
1583 "phoneNumbers".to_string(),
1584 group![(
1585 "phone".to_string(),
1586 list![group![
1587 ("number".to_string(), Field::Long(1111111111)),
1588 ("kind".to_string(), Field::Str("home".to_string()))
1589 ]]
1590 )]
1591 )
1592 ],
1593 row![
1594 ("id".to_string(), Field::Int(6)),
1595 (
1596 "phoneNumbers".to_string(),
1597 group![(
1598 "phone".to_string(),
1599 list![
1600 group![
1601 ("number".to_string(), Field::Long(1111111111)),
1602 ("kind".to_string(), Field::Str("home".to_string()))
1603 ],
1604 group![
1605 ("number".to_string(), Field::Long(2222222222)),
1606 ("kind".to_string(), Field::Null)
1607 ],
1608 group![
1609 ("number".to_string(), Field::Long(3333333333)),
1610 ("kind".to_string(), Field::Str("mobile".to_string()))
1611 ]
1612 ]
1613 )]
1614 )
1615 ],
1616 ];
1617
1618 assert_eq!(rows, expected_rows);
1619 }
1620
1621 #[test]
1622 fn test_tree_reader_handle_nested_repeated_fields_with_no_annotation() {
1623 let schema = Arc::new(
1625 parse_message_type(
1626 "
1627 message schema {
1628 REPEATED group level1 {
1629 REPEATED group level2 {
1630 REQUIRED group level3 {
1631 REQUIRED INT64 value3;
1632 }
1633 }
1634 REQUIRED INT64 value1;
1635 }
1636 }",
1637 )
1638 .unwrap(),
1639 );
1640
1641 let mut buffer: Vec<u8> = Vec::new();
1643 let mut file_writer =
1644 SerializedFileWriter::new(&mut buffer, schema, Default::default()).unwrap();
1645 let mut row_group_writer = file_writer.next_row_group().unwrap();
1646
1647 let mut column_writer = row_group_writer.next_column().unwrap().unwrap();
1649 column_writer
1650 .typed::<Int64Type>()
1651 .write_batch(&[30, 31, 32], Some(&[2, 2, 2]), Some(&[0, 0, 0]))
1652 .unwrap();
1653 column_writer.close().unwrap();
1654
1655 let mut column_writer = row_group_writer.next_column().unwrap().unwrap();
1657 column_writer
1658 .typed::<Int64Type>()
1659 .write_batch(&[10, 11, 12], Some(&[1, 1, 1]), Some(&[0, 0, 0]))
1660 .unwrap();
1661 column_writer.close().unwrap();
1662
1663 row_group_writer.close().unwrap();
1665 file_writer.close().unwrap();
1666 assert_eq!(&buffer[0..4], b"PAR1");
1667
1668 let file_reader = SerializedFileReader::new(Bytes::from(buffer)).unwrap();
1670 let rows: Vec<_> = file_reader
1671 .get_row_iter(None)
1672 .unwrap()
1673 .map(|row| row.unwrap())
1674 .collect();
1675
1676 let expected_rows = vec![
1677 row![(
1678 "level1".to_string(),
1679 list![group![
1680 (
1681 "level2".to_string(),
1682 list![group![(
1683 "level3".to_string(),
1684 group![("value3".to_string(), Field::Long(30))]
1685 )]]
1686 ),
1687 ("value1".to_string(), Field::Long(10))
1688 ]]
1689 )],
1690 row![(
1691 "level1".to_string(),
1692 list![group![
1693 (
1694 "level2".to_string(),
1695 list![group![(
1696 "level3".to_string(),
1697 group![("value3".to_string(), Field::Long(31))]
1698 )]]
1699 ),
1700 ("value1".to_string(), Field::Long(11))
1701 ]]
1702 )],
1703 row![(
1704 "level1".to_string(),
1705 list![group![
1706 (
1707 "level2".to_string(),
1708 list![group![(
1709 "level3".to_string(),
1710 group![("value3".to_string(), Field::Long(32))]
1711 )]]
1712 ),
1713 ("value1".to_string(), Field::Long(12))
1714 ]]
1715 )],
1716 ];
1717
1718 assert_eq!(rows, expected_rows);
1719 }
1720
1721 #[test]
1722 fn test_tree_reader_handle_primitive_repeated_fields_with_no_annotation() {
1723 let rows = test_file_reader_rows("repeated_primitive_no_list.parquet", None).unwrap();
1725 let expected_rows = vec![
1726 row![
1727 (
1728 "Int32_list".to_string(),
1729 Field::ListInternal(make_list([0, 1, 2, 3].map(Field::Int).to_vec()))
1730 ),
1731 (
1732 "String_list".to_string(),
1733 Field::ListInternal(make_list(
1734 ["foo", "zero", "one", "two"]
1735 .map(|s| Field::Str(s.to_string()))
1736 .to_vec()
1737 ))
1738 ),
1739 (
1740 "group_of_lists".to_string(),
1741 group![
1742 (
1743 "Int32_list_in_group".to_string(),
1744 Field::ListInternal(make_list([0, 1, 2, 3].map(Field::Int).to_vec()))
1745 ),
1746 (
1747 "String_list_in_group".to_string(),
1748 Field::ListInternal(make_list(
1749 ["foo", "zero", "one", "two"]
1750 .map(|s| Field::Str(s.to_string()))
1751 .to_vec()
1752 ))
1753 )
1754 ]
1755 )
1756 ],
1757 row![
1758 (
1759 "Int32_list".to_string(),
1760 Field::ListInternal(make_list(vec![]))
1761 ),
1762 (
1763 "String_list".to_string(),
1764 Field::ListInternal(make_list(
1765 ["three"].map(|s| Field::Str(s.to_string())).to_vec()
1766 ))
1767 ),
1768 (
1769 "group_of_lists".to_string(),
1770 group![
1771 (
1772 "Int32_list_in_group".to_string(),
1773 Field::ListInternal(make_list(vec![]))
1774 ),
1775 (
1776 "String_list_in_group".to_string(),
1777 Field::ListInternal(make_list(
1778 ["three"].map(|s| Field::Str(s.to_string())).to_vec()
1779 ))
1780 )
1781 ]
1782 )
1783 ],
1784 row![
1785 (
1786 "Int32_list".to_string(),
1787 Field::ListInternal(make_list(vec![Field::Int(4)]))
1788 ),
1789 (
1790 "String_list".to_string(),
1791 Field::ListInternal(make_list(
1792 ["four"].map(|s| Field::Str(s.to_string())).to_vec()
1793 ))
1794 ),
1795 (
1796 "group_of_lists".to_string(),
1797 group![
1798 (
1799 "Int32_list_in_group".to_string(),
1800 Field::ListInternal(make_list(vec![Field::Int(4)]))
1801 ),
1802 (
1803 "String_list_in_group".to_string(),
1804 Field::ListInternal(make_list(
1805 ["four"].map(|s| Field::Str(s.to_string())).to_vec()
1806 ))
1807 )
1808 ]
1809 )
1810 ],
1811 row![
1812 (
1813 "Int32_list".to_string(),
1814 Field::ListInternal(make_list([5, 6, 7, 8].map(Field::Int).to_vec()))
1815 ),
1816 (
1817 "String_list".to_string(),
1818 Field::ListInternal(make_list(
1819 ["five", "six", "seven", "eight"]
1820 .map(|s| Field::Str(s.to_string()))
1821 .to_vec()
1822 ))
1823 ),
1824 (
1825 "group_of_lists".to_string(),
1826 group![
1827 (
1828 "Int32_list_in_group".to_string(),
1829 Field::ListInternal(make_list([5, 6, 7, 8].map(Field::Int).to_vec()))
1830 ),
1831 (
1832 "String_list_in_group".to_string(),
1833 Field::ListInternal(make_list(
1834 ["five", "six", "seven", "eight"]
1835 .map(|s| Field::Str(s.to_string()))
1836 .to_vec()
1837 ))
1838 )
1839 ]
1840 )
1841 ],
1842 ];
1843 assert_eq!(rows, expected_rows);
1844 }
1845
1846 #[test]
1847 fn test_map_no_value() {
1848 let rows = test_file_reader_rows("map_no_value.parquet", None).unwrap();
1868
1869 for row in rows {
1871 let cols = row.into_columns();
1872 assert_eq!(cols[1].1, cols[2].1);
1873 }
1874 }
1875
1876 #[test]
1877 #[cfg(feature = "arrow")]
1878 fn test_compound_map_key() {
1879 use crate::arrow::ArrowWriter;
1880 use arrow_array::builder::{Int32Builder, ListBuilder};
1881 use arrow_array::{
1882 Array, ArrayRef, Int32Array, MapArray, RecordBatch, StringArray, StructArray,
1883 };
1884 use arrow_buffer::OffsetBuffer;
1885 use arrow_schema::{DataType, Field as ArrowField, Fields};
1886
1887 let map_array = |keys: ArrayRef, offsets: Vec<i32>, values: Vec<i32>| {
1888 let entries = StructArray::from(vec![
1889 (
1890 Arc::new(ArrowField::new("key", keys.data_type().clone(), false)),
1891 keys,
1892 ),
1893 (
1894 Arc::new(ArrowField::new("value", DataType::Int32, true)),
1895 Arc::new(Int32Array::from(values)) as ArrayRef,
1896 ),
1897 ]);
1898 let entries_field = ArrowField::new("key_value", entries.data_type().clone(), false);
1899 let map = MapArray::try_new(
1900 Arc::new(entries_field),
1901 OffsetBuffer::new(offsets.into()),
1902 entries,
1903 None,
1904 false,
1905 )
1906 .unwrap();
1907 Arc::new(map) as ArrayRef
1908 };
1909
1910 let struct_keys = StructArray::new(
1912 Fields::from(vec![
1913 ArrowField::new("id", DataType::Int32, false),
1914 ArrowField::new("name", DataType::Utf8, false),
1915 ]),
1916 vec![
1917 Arc::new(Int32Array::from(vec![1, 2, 3])),
1918 Arc::new(StringArray::from(vec!["a", "b", "c"])),
1919 ],
1920 None,
1921 );
1922 let struct_map = map_array(Arc::new(struct_keys), vec![0, 2, 2, 3], vec![10, 20, 30]);
1923
1924 let mut list_keys = ListBuilder::new(Int32Builder::new());
1926 list_keys.append_value([Some(1), Some(2)]);
1927 list_keys.append_value([Some(3)]);
1928 list_keys.append_value([]);
1929 let list_map = map_array(
1930 Arc::new(list_keys.finish()),
1931 vec![0, 2, 2, 3],
1932 vec![10, 20, 30],
1933 );
1934
1935 let batch =
1936 RecordBatch::try_from_iter([("struct_map", struct_map), ("list_map", list_map)])
1937 .unwrap();
1938 let mut buffer = Vec::new();
1939 let mut writer = ArrowWriter::try_new(&mut buffer, batch.schema(), None).unwrap();
1940 writer.write(&batch).unwrap();
1941 writer.close().unwrap();
1942
1943 let reader = SerializedFileReader::new(Bytes::from(buffer)).unwrap();
1944 let rows: Vec<Row> = reader
1945 .get_row_iter(None)
1946 .unwrap()
1947 .map(|row| row.unwrap())
1948 .collect();
1949
1950 let key = |id: i32, name: &str| {
1951 group![
1952 ("id".to_string(), Field::Int(id)),
1953 ("name".to_string(), Field::Str(name.to_string()))
1954 ]
1955 };
1956 let expected_rows = vec![
1957 row![
1958 (
1959 "struct_map".to_string(),
1960 map![(key(1, "a"), Field::Int(10)), (key(2, "b"), Field::Int(20))]
1961 ),
1962 (
1963 "list_map".to_string(),
1964 map![
1965 (list![Field::Int(1), Field::Int(2)], Field::Int(10)),
1966 (list![Field::Int(3)], Field::Int(20))
1967 ]
1968 ),
1969 ],
1970 row![
1971 ("struct_map".to_string(), map![]),
1972 ("list_map".to_string(), map![]),
1973 ],
1974 row![
1975 (
1976 "struct_map".to_string(),
1977 map![(key(3, "c"), Field::Int(30))]
1978 ),
1979 ("list_map".to_string(), map![(list![], Field::Int(30))]),
1980 ],
1981 ];
1982 assert_eq!(rows, expected_rows);
1983 }
1984
1985 fn test_file_reader_rows(file_name: &str, schema: Option<Type>) -> Result<Vec<Row>> {
1986 let file = get_test_file(file_name);
1987 let file_reader: Box<dyn FileReader> = Box::new(SerializedFileReader::new(file)?);
1988 let iter = file_reader.get_row_iter(schema)?;
1989 Ok(iter.map(|row| row.unwrap()).collect())
1990 }
1991
1992 fn test_row_group_rows(file_name: &str, schema: Option<Type>) -> Result<Vec<Row>> {
1993 let file = get_test_file(file_name);
1994 let file_reader: Box<dyn FileReader> = Box::new(SerializedFileReader::new(file)?);
1995 let row_group_reader = file_reader.get_row_group(0).unwrap();
1998 let iter = row_group_reader.get_row_iter(schema)?;
1999 Ok(iter.map(|row| row.unwrap()).collect())
2000 }
2001
2002 #[test]
2003 fn test_read_old_nested_list() {
2004 let rows = test_file_reader_rows("old_list_structure.parquet", None).unwrap();
2005 let expected_rows = vec![row![(
2006 "a".to_string(),
2007 Field::ListInternal(make_list(
2008 [
2009 make_list([1, 2].map(Field::Int).to_vec()),
2010 make_list([3, 4].map(Field::Int).to_vec())
2011 ]
2012 .map(Field::ListInternal)
2013 .to_vec()
2014 ))
2015 ),]];
2016 assert_eq!(rows, expected_rows);
2017 }
2018
2019 fn assert_err_on_overcount(file_name: &str, proj_schema: Option<Type>) {
2020 let file = get_test_file(file_name);
2021 let file_reader = SerializedFileReader::new(file).unwrap();
2022 let metadata = file_reader.metadata();
2023 let row_group_reader = file_reader.get_row_group(0).unwrap();
2024 let actual_rows = row_group_reader.metadata().num_rows() as usize;
2025
2026 let descr = match proj_schema {
2027 Some(schema) => Arc::new(SchemaDescriptor::new(Arc::new(schema))),
2028 None => metadata.file_metadata().schema_descr_ptr(),
2029 };
2030 let reader = TreeBuilder::new().build(descr, &*row_group_reader).unwrap();
2031 let iter = ReaderIter::new(reader, actual_rows + 1).unwrap();
2032
2033 let rows: Vec<Result<Row>> = iter.collect();
2034 assert_eq!(rows.len(), actual_rows + 1);
2035 for row in &rows[..actual_rows] {
2036 assert!(row.is_ok(), "Expected Ok row, got: {row:?}");
2037 }
2038 let err = rows[actual_rows].as_ref().unwrap_err();
2039 assert!(
2040 err.to_string().contains("Unexpected end of column data"),
2041 "Unexpected error message: {err}"
2042 );
2043 }
2044
2045 #[test]
2046 fn test_reader_iter_returns_error_when_num_records_exceeds_data() {
2047 assert_err_on_overcount("nulls.snappy.parquet", None);
2048 }
2049
2050 #[test]
2051 fn test_reader_iter_returns_error_for_repeated_field_when_num_records_exceeds_data() {
2052 assert_err_on_overcount("repeated_primitive_no_list.parquet", None);
2053 }
2054
2055 #[test]
2056 fn test_reader_iter_returns_error_for_map_field_when_num_records_exceeds_data() {
2057 let schema = parse_message_type(
2058 "message schema {
2059 REQUIRED group my_map (MAP) {
2060 REPEATED group key_value {
2061 REQUIRED INT32 key;
2062 OPTIONAL INT32 value;
2063 }
2064 }
2065 }",
2066 )
2067 .unwrap();
2068 assert_err_on_overcount("map_no_value.parquet", Some(schema));
2069 }
2070}