Skip to main content

parquet/record/
reader.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Contains implementation of record assembly and converting Parquet types into
6//! [`Row`]s.
7
8use std::{collections::HashMap, fmt, sync::Arc};
9
10use crate::basic::{ConvertedType, Repetition};
11use crate::errors::{ParquetError, Result};
12use crate::file::reader::{FileReader, RowGroupReader};
13use crate::record::{
14    api::{Field, Row, make_list, make_map},
15    triplet::TripletIter,
16};
17use crate::schema::types::{ColumnPath, SchemaDescPtr, SchemaDescriptor, Type, TypePtr};
18
19/// Default batch size for a reader
20const DEFAULT_BATCH_SIZE: usize = 1024;
21
22/// Tree builder for `Reader` enum.
23/// Serves as a container of options for building a reader tree and a builder, and
24/// accessing a records iterator [`RowIter`].
25pub struct TreeBuilder {
26    // Batch size (>= 1) for triplet iterators
27    batch_size: usize,
28}
29
30impl Default for TreeBuilder {
31    fn default() -> Self {
32        Self::new()
33    }
34}
35
36impl TreeBuilder {
37    /// Creates new tree builder with default parameters.
38    pub fn new() -> Self {
39        Self {
40            batch_size: DEFAULT_BATCH_SIZE,
41        }
42    }
43
44    /// Sets batch size for this tree builder.
45    pub fn with_batch_size(mut self, batch_size: usize) -> Self {
46        self.batch_size = batch_size;
47        self
48    }
49
50    /// Creates new root reader for provided schema and row group.
51    pub fn build(
52        &self,
53        descr: SchemaDescPtr,
54        row_group_reader: &dyn RowGroupReader,
55    ) -> Result<Reader> {
56        // Prepare lookup table of column path -> original column index
57        // This allows to prune columns and map schema leaf nodes to the column readers
58        let mut paths: HashMap<ColumnPath, usize> = HashMap::new();
59        let row_group_metadata = row_group_reader.metadata();
60
61        for col_index in 0..row_group_reader.num_columns() {
62            let col_meta = row_group_metadata.column(col_index);
63            let col_path = col_meta.column_path().clone();
64            paths.insert(col_path, col_index);
65        }
66
67        // Build child readers for the message type
68        let mut readers = Vec::new();
69        let mut path = Vec::new();
70
71        for field in descr.root_schema().get_fields() {
72            let reader =
73                self.reader_tree(field.clone(), &mut path, 0, 0, &paths, row_group_reader)?;
74            readers.push(reader);
75        }
76
77        // Return group reader for message type,
78        // it is always required with definition level 0
79        Ok(Reader::GroupReader(None, 0, readers))
80    }
81
82    /// Creates iterator of `Row`s directly from schema descriptor and row group.
83    pub fn as_iter(
84        &self,
85        descr: SchemaDescPtr,
86        row_group_reader: &dyn RowGroupReader,
87    ) -> Result<ReaderIter> {
88        let num_records = row_group_reader.metadata().num_rows() as usize;
89        ReaderIter::new(self.build(descr, row_group_reader)?, num_records)
90    }
91
92    /// Builds tree of readers for the current schema recursively.
93    fn reader_tree(
94        &self,
95        field: TypePtr,
96        path: &mut Vec<String>,
97        mut curr_def_level: i16,
98        mut curr_rep_level: i16,
99        paths: &HashMap<ColumnPath, usize>,
100        row_group_reader: &dyn RowGroupReader,
101    ) -> Result<Reader> {
102        assert!(field.get_basic_info().has_repetition());
103        // Update current definition and repetition levels for this type
104        let repetition = field.get_basic_info().repetition();
105        match repetition {
106            Repetition::OPTIONAL => {
107                curr_def_level += 1;
108            }
109            Repetition::REPEATED => {
110                curr_def_level += 1;
111                curr_rep_level += 1;
112            }
113            Repetition::REQUIRED => {}
114        }
115
116        path.push(String::from(field.name()));
117        let reader = if field.is_primitive() {
118            let col_path = ColumnPath::new(path.clone());
119            let orig_index = *paths
120                .get(&col_path)
121                .ok_or(general_err!("Path {:?} not found", col_path))?;
122            let col_descr = row_group_reader
123                .metadata()
124                .column(orig_index)
125                .column_descr_ptr();
126            let col_reader = row_group_reader.get_column_reader(orig_index)?;
127            let column = TripletIter::new(col_descr, col_reader, self.batch_size);
128            let reader = Reader::PrimitiveReader(field.clone(), Box::new(column));
129            if repetition == Repetition::REPEATED {
130                Reader::RepeatedReader(
131                    field,
132                    curr_def_level - 1,
133                    curr_rep_level - 1,
134                    Box::new(reader),
135                )
136            } else {
137                reader
138            }
139        } else {
140            match field.get_basic_info().converted_type() {
141                // List types
142                ConvertedType::LIST => {
143                    assert_eq!(field.get_fields().len(), 1, "Invalid list type {field:?}");
144
145                    let repeated_field = field.get_fields()[0].clone();
146                    assert_eq!(
147                        repeated_field.get_basic_info().repetition(),
148                        Repetition::REPEATED,
149                        "Invalid list type {field:?}"
150                    );
151
152                    if Reader::is_element_type(&repeated_field) {
153                        // Support for backward compatible lists
154                        let reader = self.reader_tree(
155                            repeated_field,
156                            path,
157                            curr_def_level,
158                            curr_rep_level,
159                            paths,
160                            row_group_reader,
161                        )?;
162
163                        Reader::RepeatedReader(
164                            field,
165                            curr_def_level,
166                            curr_rep_level,
167                            Box::new(reader),
168                        )
169                    } else {
170                        let child_field = repeated_field.get_fields()[0].clone();
171
172                        path.push(String::from(repeated_field.name()));
173
174                        let reader = self.reader_tree(
175                            child_field,
176                            path,
177                            curr_def_level + 1,
178                            curr_rep_level + 1,
179                            paths,
180                            row_group_reader,
181                        )?;
182
183                        path.pop();
184
185                        Reader::RepeatedReader(
186                            field,
187                            curr_def_level,
188                            curr_rep_level,
189                            Box::new(reader),
190                        )
191                    }
192                }
193                // Map types (key-value pairs)
194                ConvertedType::MAP | ConvertedType::MAP_KEY_VALUE => {
195                    assert_eq!(field.get_fields().len(), 1, "Invalid map type: {field:?}");
196                    assert!(
197                        !field.get_fields()[0].is_primitive(),
198                        "Invalid map type: {field:?}"
199                    );
200
201                    let key_value_type = field.get_fields()[0].clone();
202                    assert_eq!(
203                        key_value_type.get_basic_info().repetition(),
204                        Repetition::REPEATED,
205                        "Invalid map type: {field:?}"
206                    );
207                    // Parquet spec allows no value. In that case treat as a list. #1642
208                    if key_value_type.get_fields().len() != 1 {
209                        // If not a list, then there can only be 2 fields in the struct
210                        assert_eq!(
211                            key_value_type.get_fields().len(),
212                            2,
213                            "Invalid map type: {field:?}"
214                        );
215                    }
216
217                    path.push(String::from(key_value_type.name()));
218
219                    let key_type = &key_value_type.get_fields()[0];
220                    let key_reader = self.reader_tree(
221                        key_type.clone(),
222                        path,
223                        curr_def_level + 1,
224                        curr_rep_level + 1,
225                        paths,
226                        row_group_reader,
227                    )?;
228
229                    if key_value_type.get_fields().len() == 1 {
230                        path.pop();
231                        Reader::RepeatedReader(
232                            field,
233                            curr_def_level,
234                            curr_rep_level,
235                            Box::new(key_reader),
236                        )
237                    } else {
238                        let value_type = &key_value_type.get_fields()[1];
239                        let value_reader = self.reader_tree(
240                            value_type.clone(),
241                            path,
242                            curr_def_level + 1,
243                            curr_rep_level + 1,
244                            paths,
245                            row_group_reader,
246                        )?;
247
248                        path.pop();
249
250                        Reader::KeyValueReader(
251                            field,
252                            curr_def_level,
253                            curr_rep_level,
254                            Box::new(key_reader),
255                            Box::new(value_reader),
256                        )
257                    }
258                }
259                // A repeated field that is neither contained by a `LIST`- or
260                // `MAP`-annotated group nor annotated by `LIST` or `MAP`
261                // should be interpreted as a required list of required
262                // elements where the element type is the type of the field.
263                _ if repetition == Repetition::REPEATED => {
264                    let required_field = Type::group_type_builder(field.name())
265                        .with_repetition(Repetition::REQUIRED)
266                        .with_converted_type(field.get_basic_info().converted_type())
267                        .with_fields(field.get_fields().to_vec())
268                        .build()?;
269
270                    path.pop();
271
272                    let reader = self.reader_tree(
273                        Arc::new(required_field),
274                        path,
275                        curr_def_level,
276                        curr_rep_level,
277                        paths,
278                        row_group_reader,
279                    )?;
280
281                    return Ok(Reader::RepeatedReader(
282                        field,
283                        curr_def_level - 1,
284                        curr_rep_level - 1,
285                        Box::new(reader),
286                    ));
287                }
288                // Group types (structs)
289                _ => {
290                    let mut readers = Vec::new();
291                    for child in field.get_fields() {
292                        let reader = self.reader_tree(
293                            child.clone(),
294                            path,
295                            curr_def_level,
296                            curr_rep_level,
297                            paths,
298                            row_group_reader,
299                        )?;
300                        readers.push(reader);
301                    }
302                    Reader::GroupReader(Some(field), curr_def_level, readers)
303                }
304            }
305        };
306        path.pop();
307
308        Ok(Reader::option(repetition, curr_def_level, reader))
309    }
310}
311
312/// Reader tree for record assembly
313pub enum Reader {
314    /// Primitive reader with type information and triplet iterator
315    PrimitiveReader(TypePtr, Box<TripletIter>),
316    /// Optional reader with definition level of a parent and a reader
317    OptionReader(i16, Box<Reader>),
318    /// Group (struct) reader with type information, definition level and list of child
319    /// readers. When it represents message type, type information is None
320    GroupReader(Option<TypePtr>, i16, Vec<Reader>),
321    /// Reader for repeated values, e.g. lists, contains type information, definition
322    /// level, repetition level and a child reader
323    RepeatedReader(TypePtr, i16, i16, Box<Reader>),
324    /// Reader of key-value pairs, e.g. maps, contains type information, definition
325    /// level, repetition level, child reader for keys and child reader for values
326    KeyValueReader(TypePtr, i16, i16, Box<Reader>, Box<Reader>),
327}
328
329impl Reader {
330    /// Wraps reader in option reader based on repetition.
331    fn option(repetition: Repetition, def_level: i16, reader: Reader) -> Self {
332        if repetition == Repetition::OPTIONAL {
333            Reader::OptionReader(def_level - 1, Box::new(reader))
334        } else {
335            reader
336        }
337    }
338
339    /// Returns true if repeated type is an element type for the list.
340    /// Used to determine legacy list types.
341    /// This method is copied from Spark Parquet reader and is based on the reference:
342    /// <https://github.com/apache/parquet-format/blob/master/LogicalTypes.md>
343    ///   #backward-compatibility-rules
344    fn is_element_type(repeated_type: &Type) -> bool {
345        // For legacy 2-level list types whose element type is a 2-level list
346        //
347        //    // ARRAY<ARRAY<INT>> (nullable list, non-null elements)
348        //    optional group my_list (LIST) {
349        //      repeated group array (LIST) {
350        //        repeated int32 array;
351        //      };
352        //    }
353        //
354        if repeated_type.is_list() || repeated_type.has_single_repeated_child() {
355            return false;
356        }
357
358        // For legacy 2-level list types with primitive element type, e.g.:
359        //
360        //    // ARRAY<INT> (nullable list, non-null elements)
361        //    optional group my_list (LIST) {
362        //      repeated int32 element;
363        //    }
364        //
365        repeated_type.is_primitive() ||
366    // For legacy 2-level list types whose element type is a group type with 2 or more
367    // fields, e.g.:
368    //
369    //    // ARRAY<STRUCT<str: STRING, num: INT>> (nullable list, non-null elements)
370    //    optional group my_list (LIST) {
371    //      repeated group element {
372    //        required binary str (UTF8);
373    //        required int32 num;
374    //      };
375    //    }
376    //
377    repeated_type.is_group() && repeated_type.get_fields().len() > 1 ||
378    // For legacy 2-level list types generated by parquet-avro (Parquet version < 1.6.0),
379    // e.g.:
380    //
381    //    // ARRAY<STRUCT<str: STRING>> (nullable list, non-null elements)
382    //    optional group my_list (LIST) {
383    //      repeated group array {
384    //        required binary str (UTF8);
385    //      };
386    //    }
387    //
388    repeated_type.name() == "array" ||
389    // For Parquet data generated by parquet-thrift, e.g.:
390    //
391    //    // ARRAY<STRUCT<str: STRING>> (nullable list, non-null elements)
392    //    optional group my_list (LIST) {
393    //      repeated group my_list_tuple {
394    //        required binary str (UTF8);
395    //      };
396    //    }
397    //
398    repeated_type.name().ends_with("_tuple")
399    }
400
401    /// Reads current record as `Row` from the reader tree.
402    /// Automatically advances all necessary readers.
403    /// This must be called on the root level reader (i.e., for Message type).
404    /// Otherwise, it will panic.
405    fn read(&mut self) -> Result<Row> {
406        match *self {
407            Reader::GroupReader(_, _, ref mut readers) => {
408                let mut fields = Vec::new();
409                for reader in readers {
410                    fields.push((String::from(reader.field_name()), reader.read_field()?));
411                }
412                Ok(Row::new(fields))
413            }
414            _ => panic!("Cannot call read() on {self}"),
415        }
416    }
417
418    /// Reads current record as `Field` from the reader tree.
419    /// Automatically advances all necessary readers.
420    fn read_field(&mut self) -> Result<Field> {
421        let field = match *self {
422            Reader::PrimitiveReader(_, ref mut column) => {
423                if !column.has_next() {
424                    return Err(general_err!("Unexpected end of column data"));
425                }
426                let value = column.current_value()?;
427                column.read_next()?;
428                value
429            }
430            Reader::OptionReader(def_level, ref mut reader) => {
431                if !reader.has_next() {
432                    return Err(general_err!("Unexpected end of column data"));
433                }
434                if reader.current_def_level() > def_level {
435                    reader.read_field()?
436                } else {
437                    reader.advance_columns()?;
438                    Field::Null
439                }
440            }
441            Reader::GroupReader(_, def_level, ref mut readers) => {
442                let mut fields = Vec::new();
443                for reader in readers {
444                    if reader.repetition() != Repetition::OPTIONAL
445                        || reader.current_def_level() > def_level
446                    {
447                        fields.push((String::from(reader.field_name()), reader.read_field()?));
448                    } else {
449                        reader.advance_columns()?;
450                        fields.push((String::from(reader.field_name()), Field::Null));
451                    }
452                }
453                let row = Row::new(fields);
454                Field::Group(row)
455            }
456            Reader::RepeatedReader(_, def_level, rep_level, ref mut reader) => {
457                if !reader.has_next() {
458                    return Err(general_err!("Unexpected end of column data"));
459                }
460                let mut elements = Vec::new();
461                loop {
462                    if reader.current_def_level() > def_level {
463                        elements.push(reader.read_field()?);
464                    } else {
465                        reader.advance_columns()?;
466                        // If the current definition level is equal to the definition
467                        // level of this repeated type, then the
468                        // result is an empty list and the repetition level
469                        // will always be <= rl.
470                        break;
471                    }
472
473                    // This covers case when we are out of repetition levels and should
474                    // close the group, or there are no values left to
475                    // buffer.
476                    if !reader.has_next() || reader.current_rep_level() <= rep_level {
477                        break;
478                    }
479                }
480                Field::ListInternal(make_list(elements))
481            }
482            Reader::KeyValueReader(_, def_level, rep_level, ref mut keys, ref mut values) => {
483                if !keys.has_next() {
484                    return Err(general_err!("Unexpected end of column data"));
485                }
486                let mut pairs = Vec::new();
487                loop {
488                    if keys.current_def_level() > def_level {
489                        pairs.push((keys.read_field()?, values.read_field()?));
490                    } else {
491                        keys.advance_columns()?;
492                        values.advance_columns()?;
493                        // If the current definition level is equal to the definition
494                        // level of this repeated type, then the
495                        // result is an empty list and the repetition level
496                        // will always be <= rl.
497                        break;
498                    }
499
500                    // This covers case when we are out of repetition levels and should
501                    // close the group, or there are no values left to
502                    // buffer.
503                    if !keys.has_next() || keys.current_rep_level() <= rep_level {
504                        break;
505                    }
506                }
507
508                Field::MapInternal(make_map(pairs))
509            }
510        };
511        Ok(field)
512    }
513
514    /// Returns field name for the current reader.
515    fn field_name(&self) -> &str {
516        match *self {
517            Reader::PrimitiveReader(ref field, _) => field.name(),
518            Reader::OptionReader(_, ref reader) => reader.field_name(),
519            Reader::GroupReader(ref opt, ..) => match opt {
520                Some(field) => field.name(),
521                None => panic!("Field is None for group reader"),
522            },
523            Reader::RepeatedReader(ref field, ..) => field.name(),
524            Reader::KeyValueReader(ref field, ..) => field.name(),
525        }
526    }
527
528    /// Returns repetition for the current reader.
529    fn repetition(&self) -> Repetition {
530        match *self {
531            Reader::PrimitiveReader(ref field, _) => field.get_basic_info().repetition(),
532            Reader::OptionReader(_, ref reader) => reader.repetition(),
533            Reader::GroupReader(ref opt, ..) => match opt {
534                Some(field) => field.get_basic_info().repetition(),
535                None => panic!("Field is None for group reader"),
536            },
537            Reader::RepeatedReader(ref field, ..) => field.get_basic_info().repetition(),
538            Reader::KeyValueReader(ref field, ..) => field.get_basic_info().repetition(),
539        }
540    }
541
542    /// Returns true, if current reader has more values, false otherwise.
543    /// Method does not advance internal iterator.
544    fn has_next(&self) -> bool {
545        match *self {
546            Reader::PrimitiveReader(_, ref column) => column.has_next(),
547            Reader::OptionReader(_, ref reader) => reader.has_next(),
548            Reader::GroupReader(_, _, ref readers) => readers.first().unwrap().has_next(),
549            Reader::RepeatedReader(_, _, _, ref reader) => reader.has_next(),
550            Reader::KeyValueReader(_, _, _, ref keys, _) => keys.has_next(),
551        }
552    }
553
554    /// Returns current definition level,
555    /// Method does not advance internal iterator.
556    fn current_def_level(&self) -> i16 {
557        match *self {
558            Reader::PrimitiveReader(_, ref column) => column.current_def_level(),
559            Reader::OptionReader(_, ref reader) => reader.current_def_level(),
560            Reader::GroupReader(_, _, ref readers) => match readers.first() {
561                Some(reader) => reader.current_def_level(),
562                None => panic!("Current definition level: empty group reader"),
563            },
564            Reader::RepeatedReader(_, _, _, ref reader) => reader.current_def_level(),
565            Reader::KeyValueReader(_, _, _, ref keys, _) => keys.current_def_level(),
566        }
567    }
568
569    /// Returns current repetition level.
570    /// Method does not advance internal iterator.
571    fn current_rep_level(&self) -> i16 {
572        match *self {
573            Reader::PrimitiveReader(_, ref column) => column.current_rep_level(),
574            Reader::OptionReader(_, ref reader) => reader.current_rep_level(),
575            Reader::GroupReader(_, _, ref readers) => match readers.first() {
576                Some(reader) => reader.current_rep_level(),
577                None => panic!("Current repetition level: empty group reader"),
578            },
579            Reader::RepeatedReader(_, _, _, ref reader) => reader.current_rep_level(),
580            Reader::KeyValueReader(_, _, _, ref keys, _) => keys.current_rep_level(),
581        }
582    }
583
584    /// Advances leaf columns for the current reader.
585    fn advance_columns(&mut self) -> Result<()> {
586        match *self {
587            Reader::PrimitiveReader(_, ref mut column) => column.read_next().map(|_| ()),
588            Reader::OptionReader(_, ref mut reader) => reader.advance_columns(),
589            Reader::GroupReader(_, _, ref mut readers) => {
590                for reader in readers {
591                    reader.advance_columns()?;
592                }
593                Ok(())
594            }
595            Reader::RepeatedReader(_, _, _, ref mut reader) => reader.advance_columns(),
596            Reader::KeyValueReader(_, _, _, ref mut keys, ref mut values) => {
597                keys.advance_columns()?;
598                values.advance_columns()
599            }
600        }
601    }
602}
603
604impl fmt::Display for Reader {
605    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
606        let s = match self {
607            Reader::PrimitiveReader(..) => "PrimitiveReader",
608            Reader::OptionReader(..) => "OptionReader",
609            Reader::GroupReader(..) => "GroupReader",
610            Reader::RepeatedReader(..) => "RepeatedReader",
611            Reader::KeyValueReader(..) => "KeyValueReader",
612        };
613        write!(f, "{s}")
614    }
615}
616
617// ----------------------------------------------------------------------
618// Row iterators
619
620/// The enum Either with variants That represents a reference and a box of
621/// [`FileReader`].
622enum Either<'a> {
623    Left(&'a dyn FileReader),
624    Right(Box<dyn FileReader>),
625}
626
627impl Either<'_> {
628    fn reader(&self) -> &dyn FileReader {
629        match *self {
630            Either::Left(r) => r,
631            Either::Right(ref r) => &**r,
632        }
633    }
634}
635
636/// Access parquet data as an iterator of [`Row`]
637///
638/// # Caveats
639///
640/// Parquet stores data in a columnar fashion using [Dremel] encoding, and is therefore highly
641/// optimised for reading data by column, not row. As a consequence applications concerned with
642/// performance should prefer the columnar arrow or [ColumnReader] APIs.
643///
644/// Additionally the current implementation does not correctly handle repeated fields ([#2394]),
645/// and workloads looking to handle such schema should use the other APIs.
646///
647/// [#2394]: https://github.com/apache/arrow-rs/issues/2394
648/// [ColumnReader]: crate::file::reader::RowGroupReader::get_column_reader
649/// [Dremel]: https://research.google/pubs/pub36632/
650pub struct RowIter<'a> {
651    descr: SchemaDescPtr,
652    tree_builder: TreeBuilder,
653    file_reader: Option<Either<'a>>,
654    current_row_group: usize,
655    num_row_groups: usize,
656    row_iter: Option<ReaderIter>,
657}
658
659impl<'a> RowIter<'a> {
660    /// Creates a new iterator of [`Row`]s.
661    fn new(
662        file_reader: Option<Either<'a>>,
663        row_iter: Option<ReaderIter>,
664        descr: SchemaDescPtr,
665    ) -> Self {
666        let tree_builder = Self::tree_builder();
667        let num_row_groups = match file_reader {
668            Some(ref r) => r.reader().num_row_groups(),
669            None => 0,
670        };
671
672        Self {
673            descr,
674            file_reader,
675            tree_builder,
676            num_row_groups,
677            row_iter,
678            current_row_group: 0,
679        }
680    }
681
682    /// Creates iterator of [`Row`]s for all row groups in a
683    /// file.
684    pub fn from_file(proj: Option<Type>, reader: &'a dyn FileReader) -> Result<Self> {
685        let either = Either::Left(reader);
686        let descr =
687            Self::get_proj_descr(proj, reader.metadata().file_metadata().schema_descr_ptr())?;
688
689        Ok(Self::new(Some(either), None, descr))
690    }
691
692    /// Creates iterator of [`Row`]s for a specific row group.
693    pub fn from_row_group(proj: Option<Type>, reader: &'a dyn RowGroupReader) -> Result<Self> {
694        let descr = Self::get_proj_descr(proj, reader.metadata().schema_descr_ptr())?;
695        let tree_builder = Self::tree_builder();
696        let row_iter = tree_builder.as_iter(descr.clone(), reader)?;
697
698        // For row group we need to set `current_row_group` >= `num_row_groups`, because
699        // we only have one row group and can't buffer more.
700        Ok(Self::new(None, Some(row_iter), descr))
701    }
702
703    /// Creates a iterator of [`Row`]s from a [`FileReader`] using the full file schema.
704    pub fn from_file_into(reader: Box<dyn FileReader>) -> Self {
705        let either = Either::Right(reader);
706        let descr = either
707            .reader()
708            .metadata()
709            .file_metadata()
710            .schema_descr_ptr();
711
712        Self::new(Some(either), None, descr)
713    }
714
715    /// Tries to create a iterator of [`Row`]s using projections.
716    /// Returns a error if a file reader is not the source of this iterator.
717    ///
718    /// The Projected schema can be a subset of or equal to the file schema,
719    /// when it is None, full file schema is assumed.
720    pub fn project(self, proj: Option<Type>) -> Result<Self> {
721        match self.file_reader {
722            Some(ref either) => {
723                let schema = either
724                    .reader()
725                    .metadata()
726                    .file_metadata()
727                    .schema_descr_ptr();
728                let descr = Self::get_proj_descr(proj, schema)?;
729
730                Ok(Self::new(self.file_reader, None, descr))
731            }
732            None => Err(general_err!("File reader is required to use projections")),
733        }
734    }
735
736    /// Helper method to get schema descriptor for projected schema.
737    /// If projection is None, then full schema is returned.
738    #[inline]
739    fn get_proj_descr(proj: Option<Type>, root_descr: SchemaDescPtr) -> Result<SchemaDescPtr> {
740        match proj {
741            Some(projection) => {
742                // check if projection is part of file schema
743                let root_schema = root_descr.root_schema();
744                if !root_schema.check_contains(&projection) {
745                    return Err(general_err!("Root schema does not contain projection"));
746                }
747                Ok(Arc::new(SchemaDescriptor::new(Arc::new(projection))))
748            }
749            None => Ok(root_descr),
750        }
751    }
752
753    /// Sets batch size for this row iter.
754    pub fn with_batch_size(mut self, batch_size: usize) -> Self {
755        self.tree_builder = self.tree_builder.with_batch_size(batch_size);
756        self
757    }
758
759    /// Returns common tree builder, so the same settings are applied to both iterators
760    /// from file reader and row group.
761    #[inline]
762    fn tree_builder() -> TreeBuilder {
763        TreeBuilder::new()
764    }
765}
766
767impl Iterator for RowIter<'_> {
768    type Item = Result<Row>;
769
770    fn next(&mut self) -> Option<Result<Row>> {
771        let mut row = self.row_iter.as_mut().and_then(|iter| iter.next());
772
773        while row.is_none() && self.current_row_group < self.num_row_groups {
774            // We do not expect any failures when accessing a row group, and file reader
775            // must be set for selecting next row group.
776            if let Some(ref either) = self.file_reader {
777                let file_reader = either.reader();
778                let row_group_reader = &*file_reader
779                    .get_row_group(self.current_row_group)
780                    .expect("Row group is required to advance");
781
782                match self
783                    .tree_builder
784                    .as_iter(self.descr.clone(), row_group_reader)
785                {
786                    Ok(mut iter) => {
787                        row = iter.next();
788
789                        self.current_row_group += 1;
790                        self.row_iter = Some(iter);
791                    }
792                    Err(e) => return Some(Err(e)),
793                }
794            }
795        }
796
797        row
798    }
799}
800
801/// Internal iterator of [`Row`]s for a reader.
802pub struct ReaderIter {
803    root_reader: Reader,
804    records_left: usize,
805}
806
807impl ReaderIter {
808    fn new(mut root_reader: Reader, num_records: usize) -> Result<Self> {
809        // Prepare root reader by advancing all column vectors
810        root_reader.advance_columns()?;
811        Ok(Self {
812            root_reader,
813            records_left: num_records,
814        })
815    }
816}
817
818impl Iterator for ReaderIter {
819    type Item = Result<Row>;
820
821    fn next(&mut self) -> Option<Result<Row>> {
822        if self.records_left > 0 {
823            self.records_left -= 1;
824            Some(self.root_reader.read())
825        } else {
826            None
827        }
828    }
829}
830
831#[cfg(test)]
832mod tests {
833    use super::*;
834
835    use crate::data_type::Int64Type;
836    use crate::file::reader::SerializedFileReader;
837    use crate::file::writer::SerializedFileWriter;
838    use crate::record::api::RowAccessor;
839    use crate::schema::parser::parse_message_type;
840    use crate::util::test_common::file_util::{get_test_file, get_test_path};
841    use bytes::Bytes;
842
843    // Convenient macros to assemble row, list, map, and group.
844
845    macro_rules! row {
846        ($($e:tt)*) => {
847            {
848                Row::new(vec![$($e)*])
849            }
850        }
851    }
852
853    macro_rules! list {
854        ($($e:tt)*) => {
855            {
856                Field::ListInternal(make_list(vec![$($e)*]))
857            }
858        }
859    }
860
861    macro_rules! map {
862        ($($e:tt)*) => {
863            {
864                Field::MapInternal(make_map(vec![$($e)*]))
865            }
866        }
867    }
868
869    macro_rules! group {
870        ( $( $e:expr ), * ) => {
871            {
872                Field::Group(row!($( $e ), *))
873            }
874        }
875    }
876
877    #[test]
878    fn test_file_reader_rows_nulls() {
879        let rows = test_file_reader_rows("nulls.snappy.parquet", None).unwrap();
880        let expected_rows = vec![
881            row![(
882                "b_struct".to_string(),
883                group![("b_c_int".to_string(), Field::Null)]
884            )],
885            row![(
886                "b_struct".to_string(),
887                group![("b_c_int".to_string(), Field::Null)]
888            )],
889            row![(
890                "b_struct".to_string(),
891                group![("b_c_int".to_string(), Field::Null)]
892            )],
893            row![(
894                "b_struct".to_string(),
895                group![("b_c_int".to_string(), Field::Null)]
896            )],
897            row![(
898                "b_struct".to_string(),
899                group![("b_c_int".to_string(), Field::Null)]
900            )],
901            row![(
902                "b_struct".to_string(),
903                group![("b_c_int".to_string(), Field::Null)]
904            )],
905            row![(
906                "b_struct".to_string(),
907                group![("b_c_int".to_string(), Field::Null)]
908            )],
909            row![(
910                "b_struct".to_string(),
911                group![("b_c_int".to_string(), Field::Null)]
912            )],
913        ];
914        assert_eq!(rows, expected_rows);
915    }
916
917    #[test]
918    #[cfg_attr(miri, ignore)] // Takes too long
919    fn test_file_reader_rows_nonnullable() {
920        let rows = test_file_reader_rows("nonnullable.impala.parquet", None).unwrap();
921        let expected_rows = vec![row![
922            ("ID".to_string(), Field::Long(8)),
923            ("Int_Array".to_string(), list![Field::Int(-1)]),
924            (
925                "int_array_array".to_string(),
926                list![list![Field::Int(-1), Field::Int(-2)], list![]]
927            ),
928            (
929                "Int_Map".to_string(),
930                map![(Field::Str("k1".to_string()), Field::Int(-1))]
931            ),
932            (
933                "int_map_array".to_string(),
934                list![
935                    map![],
936                    map![(Field::Str("k1".to_string()), Field::Int(1))],
937                    map![],
938                    map![]
939                ]
940            ),
941            (
942                "nested_Struct".to_string(),
943                group![
944                    ("a".to_string(), Field::Int(-1)),
945                    ("B".to_string(), list![Field::Int(-1)]),
946                    (
947                        "c".to_string(),
948                        group![(
949                            "D".to_string(),
950                            list![list![group![
951                                ("e".to_string(), Field::Int(-1)),
952                                ("f".to_string(), Field::Str("nonnullable".to_string()))
953                            ]]]
954                        )]
955                    ),
956                    ("G".to_string(), map![])
957                ]
958            )
959        ]];
960        assert_eq!(rows, expected_rows);
961    }
962
963    #[test]
964    #[cfg_attr(miri, ignore)] // Takes too long
965    fn test_file_reader_rows_nullable() {
966        let rows = test_file_reader_rows("nullable.impala.parquet", None).unwrap();
967        let expected_rows = vec![
968            row![
969                ("id".to_string(), Field::Long(1)),
970                (
971                    "int_array".to_string(),
972                    list![Field::Int(1), Field::Int(2), Field::Int(3)]
973                ),
974                (
975                    "int_array_Array".to_string(),
976                    list![
977                        list![Field::Int(1), Field::Int(2)],
978                        list![Field::Int(3), Field::Int(4)]
979                    ]
980                ),
981                (
982                    "int_map".to_string(),
983                    map![
984                        (Field::Str("k1".to_string()), Field::Int(1)),
985                        (Field::Str("k2".to_string()), Field::Int(100))
986                    ]
987                ),
988                (
989                    "int_Map_Array".to_string(),
990                    list![map![(Field::Str("k1".to_string()), Field::Int(1))]]
991                ),
992                (
993                    "nested_struct".to_string(),
994                    group![
995                        ("A".to_string(), Field::Int(1)),
996                        ("b".to_string(), list![Field::Int(1)]),
997                        (
998                            "C".to_string(),
999                            group![(
1000                                "d".to_string(),
1001                                list![
1002                                    list![
1003                                        group![
1004                                            ("E".to_string(), Field::Int(10)),
1005                                            ("F".to_string(), Field::Str("aaa".to_string()))
1006                                        ],
1007                                        group![
1008                                            ("E".to_string(), Field::Int(-10)),
1009                                            ("F".to_string(), Field::Str("bbb".to_string()))
1010                                        ]
1011                                    ],
1012                                    list![group![
1013                                        ("E".to_string(), Field::Int(11)),
1014                                        ("F".to_string(), Field::Str("c".to_string()))
1015                                    ]]
1016                                ]
1017                            )]
1018                        ),
1019                        (
1020                            "g".to_string(),
1021                            map![(
1022                                Field::Str("foo".to_string()),
1023                                group![(
1024                                    "H".to_string(),
1025                                    group![("i".to_string(), list![Field::Double(1.1)])]
1026                                )]
1027                            )]
1028                        )
1029                    ]
1030                )
1031            ],
1032            row![
1033                ("id".to_string(), Field::Long(2)),
1034                (
1035                    "int_array".to_string(),
1036                    list![
1037                        Field::Null,
1038                        Field::Int(1),
1039                        Field::Int(2),
1040                        Field::Null,
1041                        Field::Int(3),
1042                        Field::Null
1043                    ]
1044                ),
1045                (
1046                    "int_array_Array".to_string(),
1047                    list![
1048                        list![Field::Null, Field::Int(1), Field::Int(2), Field::Null],
1049                        list![Field::Int(3), Field::Null, Field::Int(4)],
1050                        list![],
1051                        Field::Null
1052                    ]
1053                ),
1054                (
1055                    "int_map".to_string(),
1056                    map![
1057                        (Field::Str("k1".to_string()), Field::Int(2)),
1058                        (Field::Str("k2".to_string()), Field::Null)
1059                    ]
1060                ),
1061                (
1062                    "int_Map_Array".to_string(),
1063                    list![
1064                        map![
1065                            (Field::Str("k3".to_string()), Field::Null),
1066                            (Field::Str("k1".to_string()), Field::Int(1))
1067                        ],
1068                        Field::Null,
1069                        map![]
1070                    ]
1071                ),
1072                (
1073                    "nested_struct".to_string(),
1074                    group![
1075                        ("A".to_string(), Field::Null),
1076                        ("b".to_string(), list![Field::Null]),
1077                        (
1078                            "C".to_string(),
1079                            group![(
1080                                "d".to_string(),
1081                                list![
1082                                    list![
1083                                        group![
1084                                            ("E".to_string(), Field::Null),
1085                                            ("F".to_string(), Field::Null)
1086                                        ],
1087                                        group![
1088                                            ("E".to_string(), Field::Int(10)),
1089                                            ("F".to_string(), Field::Str("aaa".to_string()))
1090                                        ],
1091                                        group![
1092                                            ("E".to_string(), Field::Null),
1093                                            ("F".to_string(), Field::Null)
1094                                        ],
1095                                        group![
1096                                            ("E".to_string(), Field::Int(-10)),
1097                                            ("F".to_string(), Field::Str("bbb".to_string()))
1098                                        ],
1099                                        group![
1100                                            ("E".to_string(), Field::Null),
1101                                            ("F".to_string(), Field::Null)
1102                                        ]
1103                                    ],
1104                                    list![
1105                                        group![
1106                                            ("E".to_string(), Field::Int(11)),
1107                                            ("F".to_string(), Field::Str("c".to_string()))
1108                                        ],
1109                                        Field::Null
1110                                    ],
1111                                    list![],
1112                                    Field::Null
1113                                ]
1114                            )]
1115                        ),
1116                        (
1117                            "g".to_string(),
1118                            map![
1119                                (
1120                                    Field::Str("g1".to_string()),
1121                                    group![(
1122                                        "H".to_string(),
1123                                        group![(
1124                                            "i".to_string(),
1125                                            list![Field::Double(2.2), Field::Null]
1126                                        )]
1127                                    )]
1128                                ),
1129                                (
1130                                    Field::Str("g2".to_string()),
1131                                    group![("H".to_string(), group![("i".to_string(), list![])])]
1132                                ),
1133                                (Field::Str("g3".to_string()), Field::Null),
1134                                (
1135                                    Field::Str("g4".to_string()),
1136                                    group![(
1137                                        "H".to_string(),
1138                                        group![("i".to_string(), Field::Null)]
1139                                    )]
1140                                ),
1141                                (
1142                                    Field::Str("g5".to_string()),
1143                                    group![("H".to_string(), Field::Null)]
1144                                )
1145                            ]
1146                        )
1147                    ]
1148                )
1149            ],
1150            row![
1151                ("id".to_string(), Field::Long(3)),
1152                ("int_array".to_string(), list![]),
1153                ("int_array_Array".to_string(), list![Field::Null]),
1154                ("int_map".to_string(), map![]),
1155                ("int_Map_Array".to_string(), list![Field::Null, Field::Null]),
1156                (
1157                    "nested_struct".to_string(),
1158                    group![
1159                        ("A".to_string(), Field::Null),
1160                        ("b".to_string(), Field::Null),
1161                        ("C".to_string(), group![("d".to_string(), list![])]),
1162                        ("g".to_string(), map![])
1163                    ]
1164                )
1165            ],
1166            row![
1167                ("id".to_string(), Field::Long(4)),
1168                ("int_array".to_string(), Field::Null),
1169                ("int_array_Array".to_string(), list![]),
1170                ("int_map".to_string(), map![]),
1171                ("int_Map_Array".to_string(), list![]),
1172                (
1173                    "nested_struct".to_string(),
1174                    group![
1175                        ("A".to_string(), Field::Null),
1176                        ("b".to_string(), Field::Null),
1177                        ("C".to_string(), group![("d".to_string(), Field::Null)]),
1178                        ("g".to_string(), Field::Null)
1179                    ]
1180                )
1181            ],
1182            row![
1183                ("id".to_string(), Field::Long(5)),
1184                ("int_array".to_string(), Field::Null),
1185                ("int_array_Array".to_string(), Field::Null),
1186                ("int_map".to_string(), map![]),
1187                ("int_Map_Array".to_string(), Field::Null),
1188                (
1189                    "nested_struct".to_string(),
1190                    group![
1191                        ("A".to_string(), Field::Null),
1192                        ("b".to_string(), Field::Null),
1193                        ("C".to_string(), Field::Null),
1194                        (
1195                            "g".to_string(),
1196                            map![(
1197                                Field::Str("foo".to_string()),
1198                                group![(
1199                                    "H".to_string(),
1200                                    group![(
1201                                        "i".to_string(),
1202                                        list![Field::Double(2.2), Field::Double(3.3)]
1203                                    )]
1204                                )]
1205                            )]
1206                        )
1207                    ]
1208                )
1209            ],
1210            row![
1211                ("id".to_string(), Field::Long(6)),
1212                ("int_array".to_string(), Field::Null),
1213                ("int_array_Array".to_string(), Field::Null),
1214                ("int_map".to_string(), Field::Null),
1215                ("int_Map_Array".to_string(), Field::Null),
1216                ("nested_struct".to_string(), Field::Null)
1217            ],
1218            row![
1219                ("id".to_string(), Field::Long(7)),
1220                ("int_array".to_string(), Field::Null),
1221                (
1222                    "int_array_Array".to_string(),
1223                    list![Field::Null, list![Field::Int(5), Field::Int(6)]]
1224                ),
1225                (
1226                    "int_map".to_string(),
1227                    map![
1228                        (Field::Str("k1".to_string()), Field::Null),
1229                        (Field::Str("k3".to_string()), Field::Null)
1230                    ]
1231                ),
1232                ("int_Map_Array".to_string(), Field::Null),
1233                (
1234                    "nested_struct".to_string(),
1235                    group![
1236                        ("A".to_string(), Field::Int(7)),
1237                        (
1238                            "b".to_string(),
1239                            list![Field::Int(2), Field::Int(3), Field::Null]
1240                        ),
1241                        (
1242                            "C".to_string(),
1243                            group![(
1244                                "d".to_string(),
1245                                list![list![], list![Field::Null], Field::Null]
1246                            )]
1247                        ),
1248                        ("g".to_string(), Field::Null)
1249                    ]
1250                )
1251            ],
1252        ];
1253        assert_eq!(rows, expected_rows);
1254    }
1255
1256    #[test]
1257    fn test_file_reader_rows_projection() {
1258        let schema = "
1259      message spark_schema {
1260        REQUIRED DOUBLE c;
1261        REQUIRED INT32 b;
1262      }
1263    ";
1264        let schema = parse_message_type(schema).unwrap();
1265        let rows = test_file_reader_rows("nested_maps.snappy.parquet", Some(schema)).unwrap();
1266        let expected_rows = vec![
1267            row![
1268                ("c".to_string(), Field::Double(1.0)),
1269                ("b".to_string(), Field::Int(1))
1270            ],
1271            row![
1272                ("c".to_string(), Field::Double(1.0)),
1273                ("b".to_string(), Field::Int(1))
1274            ],
1275            row![
1276                ("c".to_string(), Field::Double(1.0)),
1277                ("b".to_string(), Field::Int(1))
1278            ],
1279            row![
1280                ("c".to_string(), Field::Double(1.0)),
1281                ("b".to_string(), Field::Int(1))
1282            ],
1283            row![
1284                ("c".to_string(), Field::Double(1.0)),
1285                ("b".to_string(), Field::Int(1))
1286            ],
1287            row![
1288                ("c".to_string(), Field::Double(1.0)),
1289                ("b".to_string(), Field::Int(1))
1290            ],
1291        ];
1292        assert_eq!(rows, expected_rows);
1293    }
1294
1295    #[test]
1296    fn test_iter_columns_in_row() {
1297        let r = row![
1298            ("c".to_string(), Field::Double(1.0)),
1299            ("b".to_string(), Field::Int(1))
1300        ];
1301        let mut result = Vec::new();
1302        for (name, record) in r.get_column_iter() {
1303            result.push((name, record));
1304        }
1305        assert_eq!(
1306            vec![
1307                (&"c".to_string(), &Field::Double(1.0)),
1308                (&"b".to_string(), &Field::Int(1))
1309            ],
1310            result
1311        );
1312    }
1313
1314    #[test]
1315    fn test_into_columns_in_row() {
1316        let r = row![
1317            ("a".to_string(), Field::Str("My string".to_owned())),
1318            ("b".to_string(), Field::Int(1))
1319        ];
1320        assert_eq!(
1321            r.into_columns(),
1322            vec![
1323                ("a".to_string(), Field::Str("My string".to_owned())),
1324                ("b".to_string(), Field::Int(1)),
1325            ]
1326        );
1327    }
1328
1329    #[test]
1330    fn test_file_reader_rows_projection_map() {
1331        let schema = "
1332      message spark_schema {
1333        OPTIONAL group a (MAP) {
1334          REPEATED group key_value {
1335            REQUIRED BYTE_ARRAY key (UTF8);
1336            OPTIONAL group value (MAP) {
1337              REPEATED group key_value {
1338                REQUIRED INT32 key;
1339                REQUIRED BOOLEAN value;
1340              }
1341            }
1342          }
1343        }
1344      }
1345    ";
1346        let schema = parse_message_type(schema).unwrap();
1347        let rows = test_file_reader_rows("nested_maps.snappy.parquet", Some(schema)).unwrap();
1348        let expected_rows = vec![
1349            row![(
1350                "a".to_string(),
1351                map![(
1352                    Field::Str("a".to_string()),
1353                    map![
1354                        (Field::Int(1), Field::Bool(true)),
1355                        (Field::Int(2), Field::Bool(false))
1356                    ]
1357                )]
1358            )],
1359            row![(
1360                "a".to_string(),
1361                map![(
1362                    Field::Str("b".to_string()),
1363                    map![(Field::Int(1), Field::Bool(true))]
1364                )]
1365            )],
1366            row![(
1367                "a".to_string(),
1368                map![(Field::Str("c".to_string()), Field::Null)]
1369            )],
1370            row![("a".to_string(), map![(Field::Str("d".to_string()), map![])])],
1371            row![(
1372                "a".to_string(),
1373                map![(
1374                    Field::Str("e".to_string()),
1375                    map![(Field::Int(1), Field::Bool(true))]
1376                )]
1377            )],
1378            row![(
1379                "a".to_string(),
1380                map![(
1381                    Field::Str("f".to_string()),
1382                    map![
1383                        (Field::Int(3), Field::Bool(true)),
1384                        (Field::Int(4), Field::Bool(false)),
1385                        (Field::Int(5), Field::Bool(true))
1386                    ]
1387                )]
1388            )],
1389        ];
1390        assert_eq!(rows, expected_rows);
1391    }
1392
1393    #[test]
1394    fn test_file_reader_rows_projection_list() {
1395        let schema = "
1396      message spark_schema {
1397        OPTIONAL group a (LIST) {
1398          REPEATED group list {
1399            OPTIONAL group element (LIST) {
1400              REPEATED group list {
1401                OPTIONAL group element (LIST) {
1402                  REPEATED group list {
1403                    OPTIONAL BYTE_ARRAY element (UTF8);
1404                  }
1405                }
1406              }
1407            }
1408          }
1409        }
1410      }
1411    ";
1412        let schema = parse_message_type(schema).unwrap();
1413        let rows = test_file_reader_rows("nested_lists.snappy.parquet", Some(schema)).unwrap();
1414        let expected_rows = vec![
1415            row![(
1416                "a".to_string(),
1417                list![
1418                    list![
1419                        list![Field::Str("a".to_string()), Field::Str("b".to_string())],
1420                        list![Field::Str("c".to_string())]
1421                    ],
1422                    list![Field::Null, list![Field::Str("d".to_string())]]
1423                ]
1424            )],
1425            row![(
1426                "a".to_string(),
1427                list![
1428                    list![
1429                        list![Field::Str("a".to_string()), Field::Str("b".to_string())],
1430                        list![Field::Str("c".to_string()), Field::Str("d".to_string())]
1431                    ],
1432                    list![Field::Null, list![Field::Str("e".to_string())]]
1433                ]
1434            )],
1435            row![(
1436                "a".to_string(),
1437                list![
1438                    list![
1439                        list![Field::Str("a".to_string()), Field::Str("b".to_string())],
1440                        list![Field::Str("c".to_string()), Field::Str("d".to_string())],
1441                        list![Field::Str("e".to_string())]
1442                    ],
1443                    list![Field::Null, list![Field::Str("f".to_string())]]
1444                ]
1445            )],
1446        ];
1447        assert_eq!(rows, expected_rows);
1448    }
1449
1450    #[test]
1451    fn test_file_reader_rows_invalid_projection() {
1452        let schema = "
1453      message spark_schema {
1454        REQUIRED INT32 key;
1455        REQUIRED BOOLEAN value;
1456      }
1457    ";
1458        let schema = parse_message_type(schema).unwrap();
1459        let res = test_file_reader_rows("nested_maps.snappy.parquet", Some(schema));
1460        assert_eq!(
1461            res.unwrap_err().to_string(),
1462            "Parquet error: Root schema does not contain projection"
1463        );
1464    }
1465
1466    #[test]
1467    fn test_row_group_rows_invalid_projection() {
1468        let schema = "
1469      message spark_schema {
1470        REQUIRED INT32 key;
1471        REQUIRED BOOLEAN value;
1472      }
1473    ";
1474        let schema = parse_message_type(schema).unwrap();
1475        let res = test_row_group_rows("nested_maps.snappy.parquet", Some(schema));
1476        assert_eq!(
1477            res.unwrap_err().to_string(),
1478            "Parquet error: Root schema does not contain projection"
1479        );
1480    }
1481
1482    #[test]
1483    fn test_file_reader_rows_nested_map_type() {
1484        let schema = "
1485      message spark_schema {
1486        OPTIONAL group a (MAP) {
1487          REPEATED group key_value {
1488            REQUIRED BYTE_ARRAY key (UTF8);
1489            OPTIONAL group value (MAP) {
1490              REPEATED group key_value {
1491                REQUIRED INT32 key;
1492              }
1493            }
1494          }
1495        }
1496      }
1497    ";
1498        let schema = parse_message_type(schema).unwrap();
1499        test_file_reader_rows("nested_maps.snappy.parquet", Some(schema)).unwrap();
1500    }
1501
1502    #[test]
1503    fn test_file_reader_iter() {
1504        let path = get_test_path("alltypes_plain.parquet");
1505        let reader = SerializedFileReader::try_from(path.as_path()).unwrap();
1506        let iter = RowIter::from_file_into(Box::new(reader));
1507
1508        let values: Vec<_> = iter.flat_map(|r| r.unwrap().get_int(0)).collect();
1509        assert_eq!(values, &[4, 5, 6, 7, 2, 3, 0, 1]);
1510    }
1511
1512    #[test]
1513    fn test_file_reader_iter_projection() {
1514        let path = get_test_path("alltypes_plain.parquet");
1515        let reader = SerializedFileReader::try_from(path.as_path()).unwrap();
1516        let schema = "message schema { OPTIONAL INT32 id; }";
1517        let proj = parse_message_type(schema).ok();
1518
1519        let iter = RowIter::from_file_into(Box::new(reader))
1520            .project(proj)
1521            .unwrap();
1522        let values: Vec<_> = iter.flat_map(|r| r.unwrap().get_int(0)).collect();
1523
1524        assert_eq!(values, &[4, 5, 6, 7, 2, 3, 0, 1]);
1525    }
1526
1527    #[test]
1528    fn test_file_reader_iter_projection_err() {
1529        let schema = "
1530      message spark_schema {
1531        REQUIRED INT32 key;
1532        REQUIRED BOOLEAN value;
1533      }
1534    ";
1535        let proj = parse_message_type(schema).ok();
1536        let path = get_test_path("nested_maps.snappy.parquet");
1537        let reader = SerializedFileReader::try_from(path.as_path()).unwrap();
1538        let res = RowIter::from_file_into(Box::new(reader)).project(proj);
1539
1540        assert_eq!(
1541            res.err().unwrap().to_string(),
1542            "Parquet error: Root schema does not contain projection"
1543        );
1544    }
1545
1546    #[test]
1547    fn test_tree_reader_handle_repeated_fields_with_no_annotation() {
1548        // Array field `phoneNumbers` does not contain LIST annotation.
1549        // We parse it as struct with `phone` repeated field as array.
1550        let rows = test_file_reader_rows("repeated_no_annotation.parquet", None).unwrap();
1551        let expected_rows = vec![
1552            row![
1553                ("id".to_string(), Field::Int(1)),
1554                ("phoneNumbers".to_string(), Field::Null)
1555            ],
1556            row![
1557                ("id".to_string(), Field::Int(2)),
1558                ("phoneNumbers".to_string(), Field::Null)
1559            ],
1560            row![
1561                ("id".to_string(), Field::Int(3)),
1562                (
1563                    "phoneNumbers".to_string(),
1564                    group![("phone".to_string(), list![])]
1565                )
1566            ],
1567            row![
1568                ("id".to_string(), Field::Int(4)),
1569                (
1570                    "phoneNumbers".to_string(),
1571                    group![(
1572                        "phone".to_string(),
1573                        list![group![
1574                            ("number".to_string(), Field::Long(5555555555)),
1575                            ("kind".to_string(), Field::Null)
1576                        ]]
1577                    )]
1578                )
1579            ],
1580            row![
1581                ("id".to_string(), Field::Int(5)),
1582                (
1583                    "phoneNumbers".to_string(),
1584                    group![(
1585                        "phone".to_string(),
1586                        list![group![
1587                            ("number".to_string(), Field::Long(1111111111)),
1588                            ("kind".to_string(), Field::Str("home".to_string()))
1589                        ]]
1590                    )]
1591                )
1592            ],
1593            row![
1594                ("id".to_string(), Field::Int(6)),
1595                (
1596                    "phoneNumbers".to_string(),
1597                    group![(
1598                        "phone".to_string(),
1599                        list![
1600                            group![
1601                                ("number".to_string(), Field::Long(1111111111)),
1602                                ("kind".to_string(), Field::Str("home".to_string()))
1603                            ],
1604                            group![
1605                                ("number".to_string(), Field::Long(2222222222)),
1606                                ("kind".to_string(), Field::Null)
1607                            ],
1608                            group![
1609                                ("number".to_string(), Field::Long(3333333333)),
1610                                ("kind".to_string(), Field::Str("mobile".to_string()))
1611                            ]
1612                        ]
1613                    )]
1614                )
1615            ],
1616        ];
1617
1618        assert_eq!(rows, expected_rows);
1619    }
1620
1621    #[test]
1622    fn test_tree_reader_handle_nested_repeated_fields_with_no_annotation() {
1623        // Create schema
1624        let schema = Arc::new(
1625            parse_message_type(
1626                "
1627            message schema {
1628                REPEATED group level1 {
1629                    REPEATED group level2 {
1630                        REQUIRED group level3 {
1631                            REQUIRED INT64 value3;
1632                        }
1633                    }
1634                    REQUIRED INT64 value1;
1635                }
1636            }",
1637            )
1638            .unwrap(),
1639        );
1640
1641        // Write Parquet file to buffer
1642        let mut buffer: Vec<u8> = Vec::new();
1643        let mut file_writer =
1644            SerializedFileWriter::new(&mut buffer, schema, Default::default()).unwrap();
1645        let mut row_group_writer = file_writer.next_row_group().unwrap();
1646
1647        // Write column level1.level2.level3.value3
1648        let mut column_writer = row_group_writer.next_column().unwrap().unwrap();
1649        column_writer
1650            .typed::<Int64Type>()
1651            .write_batch(&[30, 31, 32], Some(&[2, 2, 2]), Some(&[0, 0, 0]))
1652            .unwrap();
1653        column_writer.close().unwrap();
1654
1655        // Write column level1.value1
1656        let mut column_writer = row_group_writer.next_column().unwrap().unwrap();
1657        column_writer
1658            .typed::<Int64Type>()
1659            .write_batch(&[10, 11, 12], Some(&[1, 1, 1]), Some(&[0, 0, 0]))
1660            .unwrap();
1661        column_writer.close().unwrap();
1662
1663        // Finalize Parquet file
1664        row_group_writer.close().unwrap();
1665        file_writer.close().unwrap();
1666        assert_eq!(&buffer[0..4], b"PAR1");
1667
1668        // Read Parquet file from buffer
1669        let file_reader = SerializedFileReader::new(Bytes::from(buffer)).unwrap();
1670        let rows: Vec<_> = file_reader
1671            .get_row_iter(None)
1672            .unwrap()
1673            .map(|row| row.unwrap())
1674            .collect();
1675
1676        let expected_rows = vec![
1677            row![(
1678                "level1".to_string(),
1679                list![group![
1680                    (
1681                        "level2".to_string(),
1682                        list![group![(
1683                            "level3".to_string(),
1684                            group![("value3".to_string(), Field::Long(30))]
1685                        )]]
1686                    ),
1687                    ("value1".to_string(), Field::Long(10))
1688                ]]
1689            )],
1690            row![(
1691                "level1".to_string(),
1692                list![group![
1693                    (
1694                        "level2".to_string(),
1695                        list![group![(
1696                            "level3".to_string(),
1697                            group![("value3".to_string(), Field::Long(31))]
1698                        )]]
1699                    ),
1700                    ("value1".to_string(), Field::Long(11))
1701                ]]
1702            )],
1703            row![(
1704                "level1".to_string(),
1705                list![group![
1706                    (
1707                        "level2".to_string(),
1708                        list![group![(
1709                            "level3".to_string(),
1710                            group![("value3".to_string(), Field::Long(32))]
1711                        )]]
1712                    ),
1713                    ("value1".to_string(), Field::Long(12))
1714                ]]
1715            )],
1716        ];
1717
1718        assert_eq!(rows, expected_rows);
1719    }
1720
1721    #[test]
1722    fn test_tree_reader_handle_primitive_repeated_fields_with_no_annotation() {
1723        // In this test the REPEATED fields are primitives
1724        let rows = test_file_reader_rows("repeated_primitive_no_list.parquet", None).unwrap();
1725        let expected_rows = vec![
1726            row![
1727                (
1728                    "Int32_list".to_string(),
1729                    Field::ListInternal(make_list([0, 1, 2, 3].map(Field::Int).to_vec()))
1730                ),
1731                (
1732                    "String_list".to_string(),
1733                    Field::ListInternal(make_list(
1734                        ["foo", "zero", "one", "two"]
1735                            .map(|s| Field::Str(s.to_string()))
1736                            .to_vec()
1737                    ))
1738                ),
1739                (
1740                    "group_of_lists".to_string(),
1741                    group![
1742                        (
1743                            "Int32_list_in_group".to_string(),
1744                            Field::ListInternal(make_list([0, 1, 2, 3].map(Field::Int).to_vec()))
1745                        ),
1746                        (
1747                            "String_list_in_group".to_string(),
1748                            Field::ListInternal(make_list(
1749                                ["foo", "zero", "one", "two"]
1750                                    .map(|s| Field::Str(s.to_string()))
1751                                    .to_vec()
1752                            ))
1753                        )
1754                    ]
1755                )
1756            ],
1757            row![
1758                (
1759                    "Int32_list".to_string(),
1760                    Field::ListInternal(make_list(vec![]))
1761                ),
1762                (
1763                    "String_list".to_string(),
1764                    Field::ListInternal(make_list(
1765                        ["three"].map(|s| Field::Str(s.to_string())).to_vec()
1766                    ))
1767                ),
1768                (
1769                    "group_of_lists".to_string(),
1770                    group![
1771                        (
1772                            "Int32_list_in_group".to_string(),
1773                            Field::ListInternal(make_list(vec![]))
1774                        ),
1775                        (
1776                            "String_list_in_group".to_string(),
1777                            Field::ListInternal(make_list(
1778                                ["three"].map(|s| Field::Str(s.to_string())).to_vec()
1779                            ))
1780                        )
1781                    ]
1782                )
1783            ],
1784            row![
1785                (
1786                    "Int32_list".to_string(),
1787                    Field::ListInternal(make_list(vec![Field::Int(4)]))
1788                ),
1789                (
1790                    "String_list".to_string(),
1791                    Field::ListInternal(make_list(
1792                        ["four"].map(|s| Field::Str(s.to_string())).to_vec()
1793                    ))
1794                ),
1795                (
1796                    "group_of_lists".to_string(),
1797                    group![
1798                        (
1799                            "Int32_list_in_group".to_string(),
1800                            Field::ListInternal(make_list(vec![Field::Int(4)]))
1801                        ),
1802                        (
1803                            "String_list_in_group".to_string(),
1804                            Field::ListInternal(make_list(
1805                                ["four"].map(|s| Field::Str(s.to_string())).to_vec()
1806                            ))
1807                        )
1808                    ]
1809                )
1810            ],
1811            row![
1812                (
1813                    "Int32_list".to_string(),
1814                    Field::ListInternal(make_list([5, 6, 7, 8].map(Field::Int).to_vec()))
1815                ),
1816                (
1817                    "String_list".to_string(),
1818                    Field::ListInternal(make_list(
1819                        ["five", "six", "seven", "eight"]
1820                            .map(|s| Field::Str(s.to_string()))
1821                            .to_vec()
1822                    ))
1823                ),
1824                (
1825                    "group_of_lists".to_string(),
1826                    group![
1827                        (
1828                            "Int32_list_in_group".to_string(),
1829                            Field::ListInternal(make_list([5, 6, 7, 8].map(Field::Int).to_vec()))
1830                        ),
1831                        (
1832                            "String_list_in_group".to_string(),
1833                            Field::ListInternal(make_list(
1834                                ["five", "six", "seven", "eight"]
1835                                    .map(|s| Field::Str(s.to_string()))
1836                                    .to_vec()
1837                            ))
1838                        )
1839                    ]
1840                )
1841            ],
1842        ];
1843        assert_eq!(rows, expected_rows);
1844    }
1845
1846    #[test]
1847    fn test_map_no_value() {
1848        // File schema:
1849        // message schema {
1850        //   required group my_map (MAP) {
1851        //     repeated group key_value {
1852        //       required int32 key;
1853        //       optional int32 value;
1854        //     }
1855        //   }
1856        //   required group my_map_no_v (MAP) {
1857        //     repeated group key_value {
1858        //       required int32 key;
1859        //     }
1860        //   }
1861        //   required group my_list (LIST) {
1862        //     repeated group list {
1863        //       required int32 element;
1864        //     }
1865        //   }
1866        // }
1867        let rows = test_file_reader_rows("map_no_value.parquet", None).unwrap();
1868
1869        // the my_map_no_v and my_list columns should be equivalent lists by this point
1870        for row in rows {
1871            let cols = row.into_columns();
1872            assert_eq!(cols[1].1, cols[2].1);
1873        }
1874    }
1875
1876    #[test]
1877    #[cfg(feature = "arrow")]
1878    fn test_compound_map_key() {
1879        use crate::arrow::ArrowWriter;
1880        use arrow_array::builder::{Int32Builder, ListBuilder};
1881        use arrow_array::{
1882            Array, ArrayRef, Int32Array, MapArray, RecordBatch, StringArray, StructArray,
1883        };
1884        use arrow_buffer::OffsetBuffer;
1885        use arrow_schema::{DataType, Field as ArrowField, Fields};
1886
1887        let map_array = |keys: ArrayRef, offsets: Vec<i32>, values: Vec<i32>| {
1888            let entries = StructArray::from(vec![
1889                (
1890                    Arc::new(ArrowField::new("key", keys.data_type().clone(), false)),
1891                    keys,
1892                ),
1893                (
1894                    Arc::new(ArrowField::new("value", DataType::Int32, true)),
1895                    Arc::new(Int32Array::from(values)) as ArrayRef,
1896                ),
1897            ]);
1898            let entries_field = ArrowField::new("key_value", entries.data_type().clone(), false);
1899            let map = MapArray::try_new(
1900                Arc::new(entries_field),
1901                OffsetBuffer::new(offsets.into()),
1902                entries,
1903                None,
1904                false,
1905            )
1906            .unwrap();
1907            Arc::new(map) as ArrayRef
1908        };
1909
1910        // struct keys: [{(1, "a") -> 10, (2, "b") -> 20}, {}, {(3, "c") -> 30}]
1911        let struct_keys = StructArray::new(
1912            Fields::from(vec![
1913                ArrowField::new("id", DataType::Int32, false),
1914                ArrowField::new("name", DataType::Utf8, false),
1915            ]),
1916            vec![
1917                Arc::new(Int32Array::from(vec![1, 2, 3])),
1918                Arc::new(StringArray::from(vec!["a", "b", "c"])),
1919            ],
1920            None,
1921        );
1922        let struct_map = map_array(Arc::new(struct_keys), vec![0, 2, 2, 3], vec![10, 20, 30]);
1923
1924        // list keys: [{[1, 2] -> 10, [3] -> 20}, {}, {[] -> 30}]
1925        let mut list_keys = ListBuilder::new(Int32Builder::new());
1926        list_keys.append_value([Some(1), Some(2)]);
1927        list_keys.append_value([Some(3)]);
1928        list_keys.append_value([]);
1929        let list_map = map_array(
1930            Arc::new(list_keys.finish()),
1931            vec![0, 2, 2, 3],
1932            vec![10, 20, 30],
1933        );
1934
1935        let batch =
1936            RecordBatch::try_from_iter([("struct_map", struct_map), ("list_map", list_map)])
1937                .unwrap();
1938        let mut buffer = Vec::new();
1939        let mut writer = ArrowWriter::try_new(&mut buffer, batch.schema(), None).unwrap();
1940        writer.write(&batch).unwrap();
1941        writer.close().unwrap();
1942
1943        let reader = SerializedFileReader::new(Bytes::from(buffer)).unwrap();
1944        let rows: Vec<Row> = reader
1945            .get_row_iter(None)
1946            .unwrap()
1947            .map(|row| row.unwrap())
1948            .collect();
1949
1950        let key = |id: i32, name: &str| {
1951            group![
1952                ("id".to_string(), Field::Int(id)),
1953                ("name".to_string(), Field::Str(name.to_string()))
1954            ]
1955        };
1956        let expected_rows = vec![
1957            row![
1958                (
1959                    "struct_map".to_string(),
1960                    map![(key(1, "a"), Field::Int(10)), (key(2, "b"), Field::Int(20))]
1961                ),
1962                (
1963                    "list_map".to_string(),
1964                    map![
1965                        (list![Field::Int(1), Field::Int(2)], Field::Int(10)),
1966                        (list![Field::Int(3)], Field::Int(20))
1967                    ]
1968                ),
1969            ],
1970            row![
1971                ("struct_map".to_string(), map![]),
1972                ("list_map".to_string(), map![]),
1973            ],
1974            row![
1975                (
1976                    "struct_map".to_string(),
1977                    map![(key(3, "c"), Field::Int(30))]
1978                ),
1979                ("list_map".to_string(), map![(list![], Field::Int(30))]),
1980            ],
1981        ];
1982        assert_eq!(rows, expected_rows);
1983    }
1984
1985    fn test_file_reader_rows(file_name: &str, schema: Option<Type>) -> Result<Vec<Row>> {
1986        let file = get_test_file(file_name);
1987        let file_reader: Box<dyn FileReader> = Box::new(SerializedFileReader::new(file)?);
1988        let iter = file_reader.get_row_iter(schema)?;
1989        Ok(iter.map(|row| row.unwrap()).collect())
1990    }
1991
1992    fn test_row_group_rows(file_name: &str, schema: Option<Type>) -> Result<Vec<Row>> {
1993        let file = get_test_file(file_name);
1994        let file_reader: Box<dyn FileReader> = Box::new(SerializedFileReader::new(file)?);
1995        // Check the first row group only, because files will contain only single row
1996        // group
1997        let row_group_reader = file_reader.get_row_group(0).unwrap();
1998        let iter = row_group_reader.get_row_iter(schema)?;
1999        Ok(iter.map(|row| row.unwrap()).collect())
2000    }
2001
2002    #[test]
2003    fn test_read_old_nested_list() {
2004        let rows = test_file_reader_rows("old_list_structure.parquet", None).unwrap();
2005        let expected_rows = vec![row![(
2006            "a".to_string(),
2007            Field::ListInternal(make_list(
2008                [
2009                    make_list([1, 2].map(Field::Int).to_vec()),
2010                    make_list([3, 4].map(Field::Int).to_vec())
2011                ]
2012                .map(Field::ListInternal)
2013                .to_vec()
2014            ))
2015        ),]];
2016        assert_eq!(rows, expected_rows);
2017    }
2018
2019    fn assert_err_on_overcount(file_name: &str, proj_schema: Option<Type>) {
2020        let file = get_test_file(file_name);
2021        let file_reader = SerializedFileReader::new(file).unwrap();
2022        let metadata = file_reader.metadata();
2023        let row_group_reader = file_reader.get_row_group(0).unwrap();
2024        let actual_rows = row_group_reader.metadata().num_rows() as usize;
2025
2026        let descr = match proj_schema {
2027            Some(schema) => Arc::new(SchemaDescriptor::new(Arc::new(schema))),
2028            None => metadata.file_metadata().schema_descr_ptr(),
2029        };
2030        let reader = TreeBuilder::new().build(descr, &*row_group_reader).unwrap();
2031        let iter = ReaderIter::new(reader, actual_rows + 1).unwrap();
2032
2033        let rows: Vec<Result<Row>> = iter.collect();
2034        assert_eq!(rows.len(), actual_rows + 1);
2035        for row in &rows[..actual_rows] {
2036            assert!(row.is_ok(), "Expected Ok row, got: {row:?}");
2037        }
2038        let err = rows[actual_rows].as_ref().unwrap_err();
2039        assert!(
2040            err.to_string().contains("Unexpected end of column data"),
2041            "Unexpected error message: {err}"
2042        );
2043    }
2044
2045    #[test]
2046    fn test_reader_iter_returns_error_when_num_records_exceeds_data() {
2047        assert_err_on_overcount("nulls.snappy.parquet", None);
2048    }
2049
2050    #[test]
2051    fn test_reader_iter_returns_error_for_repeated_field_when_num_records_exceeds_data() {
2052        assert_err_on_overcount("repeated_primitive_no_list.parquet", None);
2053    }
2054
2055    #[test]
2056    fn test_reader_iter_returns_error_for_map_field_when_num_records_exceeds_data() {
2057        let schema = parse_message_type(
2058            "message schema {
2059               REQUIRED group my_map (MAP) {
2060                 REPEATED group key_value {
2061                   REQUIRED INT32 key;
2062                   OPTIONAL INT32 value;
2063                 }
2064               }
2065             }",
2066        )
2067        .unwrap();
2068        assert_err_on_overcount("map_no_value.parquet", Some(schema));
2069    }
2070}