parquet/arrow/schema/
complex.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::HashMap;
19use std::sync::Arc;
20
21use crate::arrow::schema::extension::try_add_extension_type;
22use crate::arrow::schema::primitive::convert_primitive;
23use crate::arrow::schema::virtual_type::{RowGroupIndex, RowNumber};
24use crate::arrow::{PARQUET_FIELD_ID_META_KEY, ProjectionMask};
25use crate::basic::{ConvertedType, Repetition};
26use crate::errors::ParquetError;
27use crate::errors::Result;
28use crate::schema::types::{SchemaDescriptor, Type, TypePtr};
29use arrow_schema::{DataType, Field, Fields, SchemaBuilder, extension::ExtensionType};
30
31fn get_repetition(t: &Type) -> Repetition {
32    let info = t.get_basic_info();
33    match info.has_repetition() {
34        true => info.repetition(),
35        false => Repetition::REQUIRED,
36    }
37}
38
39/// Representation of a parquet schema element, in terms of arrow schema elements
40#[derive(Debug, Clone)]
41pub struct ParquetField {
42    /// The level which represents an insertion into the current list
43    /// i.e. guaranteed to be > 0 for an element of list type
44    pub rep_level: i16,
45    /// The level at which this field is fully defined,
46    /// i.e. guaranteed to be > 0 for a nullable type or child of a
47    /// nullable type
48    pub def_level: i16,
49    /// Whether this field is nullable
50    pub nullable: bool,
51    /// The arrow type of the column data
52    ///
53    /// Note: In certain cases the data stored in parquet may have been coerced
54    /// to a different type and will require conversion on read (e.g. Date64 and Interval)
55    pub arrow_type: DataType,
56    /// The type of this field
57    pub field_type: ParquetFieldType,
58}
59
60impl ParquetField {
61    /// Converts `self` into an arrow list, with its current type as the field type
62    ///
63    /// This is used to convert repeated columns, into their arrow representation
64    fn into_list(self, name: &str) -> Self {
65        ParquetField {
66            rep_level: self.rep_level,
67            def_level: self.def_level,
68            nullable: false,
69            arrow_type: DataType::List(Arc::new(Field::new(name, self.arrow_type.clone(), false))),
70            field_type: ParquetFieldType::Group {
71                children: vec![self],
72            },
73        }
74    }
75
76    /// Converts `self` into an arrow list, with its current type as the field type
77    /// accept an optional `list_data_type` to specify the type of list to create
78    ///
79    /// This is used to convert [deprecated repeated columns] (not in a list), into their arrow representation
80    ///
81    /// [deprecated repeated columns]: https://github.com/apache/parquet-format/blob/9fd57b59e0ce1a82a69237dcf8977d3e72a2965d/LogicalTypes.md?plain=1#L649-L650
82    fn into_list_with_arrow_list_hint(
83        self,
84        parquet_field_type: &Type,
85        list_data_type: Option<DataType>,
86    ) -> Result<Self, ParquetError> {
87        let arrow_field = match &list_data_type {
88            Some(
89                DataType::List(field_hint)
90                | DataType::LargeList(field_hint)
91                | DataType::FixedSizeList(field_hint, _),
92            ) => Some(field_hint.as_ref()),
93            Some(_) => {
94                return Err(general_err!(
95                    "Internal error: should be validated earlier that list_data_type is only a type of list"
96                ));
97            }
98            None => None,
99        };
100
101        let arrow_field = convert_field(
102            parquet_field_type,
103            &self,
104            arrow_field,
105            // Only add the field id to the list and not to the element
106            false,
107        )?
108        .with_nullable(false);
109
110        Ok(ParquetField {
111            rep_level: self.rep_level,
112            def_level: self.def_level,
113            nullable: false,
114            arrow_type: match list_data_type {
115                Some(DataType::List(_)) => DataType::List(Arc::new(arrow_field)),
116                Some(DataType::LargeList(_)) => DataType::LargeList(Arc::new(arrow_field)),
117                Some(DataType::FixedSizeList(_, len)) => {
118                    DataType::FixedSizeList(Arc::new(arrow_field), len)
119                }
120                _ => DataType::List(Arc::new(arrow_field)),
121            },
122            field_type: ParquetFieldType::Group {
123                children: vec![self],
124            },
125        })
126    }
127
128    /// Returns a list of [`ParquetField`] children if this is a group type
129    pub fn children(&self) -> Option<&[Self]> {
130        match &self.field_type {
131            ParquetFieldType::Primitive { .. } => None,
132            ParquetFieldType::Group { children } => Some(children),
133            ParquetFieldType::Virtual(_) => None,
134        }
135    }
136}
137
138/// Types of virtual columns that can be computed at read time
139#[derive(Debug, Clone, Copy, PartialEq)]
140pub enum VirtualColumnType {
141    /// Row number within the file
142    RowNumber,
143    /// Row group index
144    RowGroupIndex,
145}
146
147#[derive(Debug, Clone)]
148pub enum ParquetFieldType {
149    Primitive {
150        /// The index of the column in parquet
151        col_idx: usize,
152        /// The type of the column in parquet
153        primitive_type: TypePtr,
154    },
155    Group {
156        children: Vec<ParquetField>,
157    },
158    /// Virtual column that doesn't exist in the parquet file
159    /// but is computed at read time (e.g., row_number)
160    Virtual(VirtualColumnType),
161}
162
163/// Encodes the context of the parent of the field currently under consideration
164struct VisitorContext {
165    rep_level: i16,
166    def_level: i16,
167    /// An optional [`DataType`] sourced from the embedded arrow schema
168    data_type: Option<DataType>,
169
170    /// Whether to treat repeated types as list from arrow types
171    /// when true, if data_type provided it should be DataType::List() (or other list type)
172    /// and the list field data type would be treated as the hint for the parquet type
173    ///
174    /// when false, if data_type provided it will be treated as the hint without unwrapping
175    ///
176    /// This is for supporting [deprecated parquet list representation][1]
177    ///
178    /// [1]: https://github.com/apache/parquet-format/blob/38818fa0e7efd54b535001a4448030a40619c2a3/LogicalTypes.md?plain=1#L718-L806
179    treat_repeated_as_list_arrow_hint: bool,
180}
181
182impl VisitorContext {
183    /// Compute the resulting definition level, repetition level and nullability
184    /// for a child field with the given [`Repetition`]
185    fn levels(&self, repetition: Repetition) -> (i16, i16, bool) {
186        match repetition {
187            Repetition::OPTIONAL => (self.def_level + 1, self.rep_level, true),
188            Repetition::REQUIRED => (self.def_level, self.rep_level, false),
189            Repetition::REPEATED => (self.def_level + 1, self.rep_level + 1, false),
190        }
191    }
192}
193
194/// Walks the parquet schema in a depth-first fashion in order to map it to arrow data structures
195///
196/// See [Logical Types] for more information on the conversion algorithm
197///
198/// [Logical Types]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md
199struct Visitor {
200    /// The column index of the next leaf column
201    next_col_idx: usize,
202
203    /// Mask of columns to include
204    mask: ProjectionMask,
205}
206
207impl Visitor {
208    fn visit_primitive(
209        &mut self,
210        primitive_type: &TypePtr,
211        context: VisitorContext,
212    ) -> Result<Option<ParquetField>> {
213        let col_idx = self.next_col_idx;
214        self.next_col_idx += 1;
215
216        if !self.mask.leaf_included(col_idx) {
217            return Ok(None);
218        }
219
220        let repetition = get_repetition(primitive_type);
221        let (def_level, rep_level, nullable) = context.levels(repetition);
222
223        let primitive_arrow_data_type = match repetition {
224            Repetition::REPEATED if context.treat_repeated_as_list_arrow_hint => {
225                let arrow_field = match &context.data_type {
226                    Some(DataType::List(f)) => Some(f.as_ref()),
227                    Some(DataType::LargeList(f)) => Some(f.as_ref()),
228                    Some(DataType::FixedSizeList(f, _)) => Some(f.as_ref()),
229                    Some(d) => {
230                        return Err(arrow_err!(
231                            "incompatible arrow schema, expected list got {} for repeated primitive field",
232                            d
233                        ));
234                    }
235                    None => None,
236                };
237
238                arrow_field.map(|f| f.data_type().clone())
239            }
240            _ => context.data_type.clone(),
241        };
242
243        let arrow_type = convert_primitive(primitive_type, primitive_arrow_data_type)?;
244
245        let primitive_field = ParquetField {
246            rep_level,
247            def_level,
248            nullable,
249            arrow_type,
250            field_type: ParquetFieldType::Primitive {
251                primitive_type: primitive_type.clone(),
252                col_idx,
253            },
254        };
255
256        Ok(Some(match repetition {
257            Repetition::REPEATED if context.treat_repeated_as_list_arrow_hint => {
258                primitive_field.into_list_with_arrow_list_hint(primitive_type, context.data_type)?
259            }
260            Repetition::REPEATED => primitive_field.into_list(primitive_type.name()),
261            _ => primitive_field,
262        }))
263    }
264
265    fn visit_struct(
266        &mut self,
267        struct_type: &TypePtr,
268        context: VisitorContext,
269    ) -> Result<Option<ParquetField>> {
270        // The root type will not have a repetition level
271        let repetition = get_repetition(struct_type);
272        let (def_level, rep_level, nullable) = context.levels(repetition);
273
274        let parquet_fields = struct_type.get_fields();
275
276        // Extract any arrow fields from the hints
277        let arrow_struct = match repetition {
278            Repetition::REPEATED if context.treat_repeated_as_list_arrow_hint => {
279                let arrow_field = match &context.data_type {
280                    Some(DataType::List(f)) => Some(f.as_ref()),
281                    Some(DataType::LargeList(f)) => Some(f.as_ref()),
282                    Some(DataType::FixedSizeList(f, _)) => Some(f.as_ref()),
283                    Some(d) => {
284                        return Err(arrow_err!(
285                            "incompatible arrow schema, expected list got {} for repeated struct field",
286                            d
287                        ));
288                    }
289                    None => None,
290                };
291
292                arrow_field.map(|f| f.data_type())
293            }
294            _ => context.data_type.as_ref(),
295        };
296
297        let arrow_fields = match &arrow_struct {
298            Some(DataType::Struct(fields)) => {
299                if fields.len() != parquet_fields.len() {
300                    return Err(arrow_err!(
301                        "incompatible arrow schema, expected {} struct fields got {}",
302                        parquet_fields.len(),
303                        fields.len()
304                    ));
305                }
306                Some(fields)
307            }
308            Some(d) => {
309                return Err(arrow_err!(
310                    "incompatible arrow schema, expected struct got {}",
311                    d
312                ));
313            }
314            None => None,
315        };
316
317        let mut child_fields = SchemaBuilder::with_capacity(parquet_fields.len());
318        let mut children = Vec::with_capacity(parquet_fields.len());
319
320        // Perform a DFS of children
321        for (idx, parquet_field) in parquet_fields.iter().enumerate() {
322            let data_type = match arrow_fields {
323                Some(fields) => {
324                    let field = &fields[idx];
325                    if field.name() != parquet_field.name() {
326                        return Err(arrow_err!(
327                            "incompatible arrow schema, expected field named {} got {}",
328                            parquet_field.name(),
329                            field.name()
330                        ));
331                    }
332                    Some(field.data_type().clone())
333                }
334                None => None,
335            };
336
337            let arrow_field = arrow_fields.map(|x| &*x[idx]);
338            let child_ctx = VisitorContext {
339                rep_level,
340                def_level,
341                data_type,
342
343                // Always true: each child is independently responsible for its own
344                // repeated-to-list conversion. The parent's flag may be false when
345                // this struct's own repetition is consumed by an outer visit_list
346                // backward-compat path, but that only applies to the struct itself,
347                // not its children. A repeated child's arrow hint will be List<...>
348                // and needs to be unwrapped accordingly.
349                treat_repeated_as_list_arrow_hint: true,
350            };
351
352            if let Some(child) = self.dispatch(parquet_field, child_ctx)? {
353                // The child type returned may be different from what is encoded in the arrow
354                // schema in the event of a mismatch or a projection
355                child_fields.push(convert_field(parquet_field, &child, arrow_field, true)?);
356                children.push(child);
357            }
358        }
359
360        if children.is_empty() {
361            return Ok(None);
362        }
363
364        let struct_field = ParquetField {
365            rep_level,
366            def_level,
367            nullable,
368            arrow_type: DataType::Struct(child_fields.finish().fields),
369            field_type: ParquetFieldType::Group { children },
370        };
371
372        Ok(Some(match repetition {
373            Repetition::REPEATED if context.treat_repeated_as_list_arrow_hint => {
374                struct_field.into_list_with_arrow_list_hint(struct_type, context.data_type)?
375            }
376            Repetition::REPEATED => struct_field.into_list(struct_type.name()),
377            _ => struct_field,
378        }))
379    }
380
381    fn visit_map(
382        &mut self,
383        map_type: &TypePtr,
384        context: VisitorContext,
385    ) -> Result<Option<ParquetField>> {
386        let rep_level = context.rep_level + 1;
387        let (def_level, nullable) = match get_repetition(map_type) {
388            Repetition::REQUIRED => (context.def_level + 1, false),
389            Repetition::OPTIONAL => (context.def_level + 2, true),
390            Repetition::REPEATED => return Err(arrow_err!("Map cannot be repeated")),
391        };
392
393        if map_type.get_fields().len() != 1 {
394            return Err(arrow_err!(
395                "Map field must have exactly one key_value child, found {}",
396                map_type.get_fields().len()
397            ));
398        }
399
400        // Add map entry (key_value) to context
401        let map_key_value = &map_type.get_fields()[0];
402        if map_key_value.get_basic_info().repetition() != Repetition::REPEATED {
403            return Err(arrow_err!("Child of map field must be repeated"));
404        }
405
406        // According to the specification the values are optional (#1642).
407        // In this case, return the keys as a list.
408        if map_key_value.get_fields().len() == 1 {
409            return self.visit_list(map_type, context);
410        }
411
412        if map_key_value.get_fields().len() != 2 {
413            return Err(arrow_err!(
414                "Child of map field must have two children, found {}",
415                map_key_value.get_fields().len()
416            ));
417        }
418
419        // Get key and value, and create context for each
420        let map_key = &map_key_value.get_fields()[0];
421        let map_value = &map_key_value.get_fields()[1];
422
423        match map_key.get_basic_info().repetition() {
424            Repetition::REPEATED => {
425                return Err(arrow_err!("Map keys cannot be repeated"));
426            }
427            Repetition::REQUIRED | Repetition::OPTIONAL => {
428                // Relaxed check for having repetition REQUIRED as there exists
429                // parquet writers and files that do not conform to this standard.
430                // This allows us to consume a broader range of existing files even
431                // if they are out of spec.
432            }
433        }
434
435        if map_value.get_basic_info().repetition() == Repetition::REPEATED {
436            return Err(arrow_err!("Map values cannot be repeated"));
437        }
438
439        // Extract the arrow fields
440        let (arrow_map, arrow_key, arrow_value, sorted) = match &context.data_type {
441            Some(DataType::Map(field, sorted)) => match field.data_type() {
442                DataType::Struct(fields) => {
443                    if fields.len() != 2 {
444                        return Err(arrow_err!(
445                            "Map data type should contain struct with two children, got {}",
446                            fields.len()
447                        ));
448                    }
449
450                    (Some(field), Some(&*fields[0]), Some(&*fields[1]), *sorted)
451                }
452                d => {
453                    return Err(arrow_err!("Map data type should contain struct got {}", d));
454                }
455            },
456            Some(d) => {
457                return Err(arrow_err!(
458                    "incompatible arrow schema, expected map got {}",
459                    d
460                ));
461            }
462            None => (None, None, None, false),
463        };
464
465        let maybe_key = {
466            let context = VisitorContext {
467                rep_level,
468                def_level,
469                data_type: arrow_key.map(|x| x.data_type().clone()),
470                // Key is not repeated
471                treat_repeated_as_list_arrow_hint: false,
472            };
473
474            self.dispatch(map_key, context)?
475        };
476
477        let maybe_value = {
478            let context = VisitorContext {
479                rep_level,
480                def_level,
481                data_type: arrow_value.map(|x| x.data_type().clone()),
482                // Value type can be repeated
483                treat_repeated_as_list_arrow_hint: true,
484            };
485
486            self.dispatch(map_value, context)?
487        };
488
489        // Need both columns to be projected
490        match (maybe_key, maybe_value) {
491            (Some(key), Some(value)) => {
492                let key_field = Arc::new(
493                    convert_field(map_key, &key, arrow_key, true)?
494                        // The key is always non-nullable (#5630)
495                        .with_nullable(false),
496                );
497                let value_field = Arc::new(convert_field(map_value, &value, arrow_value, true)?);
498                let field_metadata = arrow_map
499                    .map(|field| field.metadata().clone())
500                    .unwrap_or_default();
501
502                let map_field = Field::new_struct(
503                    map_key_value.name(),
504                    [key_field, value_field],
505                    false, // The inner map field is always non-nullable (#1697)
506                )
507                .with_metadata(field_metadata);
508
509                Ok(Some(ParquetField {
510                    rep_level,
511                    def_level,
512                    nullable,
513                    arrow_type: DataType::Map(Arc::new(map_field), sorted),
514                    field_type: ParquetFieldType::Group {
515                        children: vec![key, value],
516                    },
517                }))
518            }
519            _ => Ok(None),
520        }
521    }
522
523    fn visit_list(
524        &mut self,
525        list_type: &TypePtr,
526        context: VisitorContext,
527    ) -> Result<Option<ParquetField>> {
528        if list_type.is_primitive() {
529            return Err(arrow_err!(
530                "{:?} is a list type and can't be processed as primitive.",
531                list_type
532            ));
533        }
534
535        let fields = list_type.get_fields();
536        if fields.len() != 1 {
537            return Err(arrow_err!(
538                "list type must have a single child, found {}",
539                fields.len()
540            ));
541        }
542
543        let repeated_field = &fields[0];
544        if get_repetition(repeated_field) != Repetition::REPEATED {
545            return Err(arrow_err!("List child must be repeated"));
546        }
547
548        // If the list is nullable
549        let (def_level, nullable) = match list_type.get_basic_info().repetition() {
550            Repetition::REQUIRED => (context.def_level, false),
551            Repetition::OPTIONAL => (context.def_level + 1, true),
552            Repetition::REPEATED => return Err(arrow_err!("List type cannot be repeated")),
553        };
554
555        let arrow_field = match &context.data_type {
556            Some(DataType::List(f)) => Some(f.as_ref()),
557            Some(DataType::LargeList(f)) => Some(f.as_ref()),
558            Some(DataType::FixedSizeList(f, _)) => Some(f.as_ref()),
559            Some(DataType::ListView(f)) => Some(f.as_ref()),
560            Some(DataType::LargeListView(f)) => Some(f.as_ref()),
561            Some(d) => {
562                return Err(arrow_err!(
563                    "incompatible arrow schema, expected list got {}",
564                    d
565                ));
566            }
567            None => None,
568        };
569
570        if repeated_field.is_primitive() {
571            // If the repeated field is not a group, then its type is the element type and elements are required.
572            //
573            // required/optional group my_list (LIST) {
574            //   repeated int32 element;
575            // }
576            //
577            let context = VisitorContext {
578                rep_level: context.rep_level,
579                def_level,
580                data_type: arrow_field.map(|f| f.data_type().clone()),
581                treat_repeated_as_list_arrow_hint: false,
582            };
583
584            return match self.visit_primitive(repeated_field, context) {
585                Ok(Some(mut field)) => {
586                    // visit_primitive will infer a non-nullable list, update if necessary
587                    field.nullable = nullable;
588                    Ok(Some(field))
589                }
590                r => r,
591            };
592        }
593
594        // test to see if the repeated field is a struct or one-tuple
595        let items = repeated_field.get_fields();
596        if items.len() != 1
597            || (!repeated_field.is_list()
598                && !repeated_field.has_single_repeated_child()
599                && (repeated_field.name() == "array"
600                    || repeated_field.name() == format!("{}_tuple", list_type.name())))
601        {
602            // If the repeated field is a group with multiple fields, then its type is the element
603            // type and elements are required.
604            //
605            // If the repeated field is a group with one field and is named either array or uses
606            // the LIST-annotated group's name with _tuple appended then the repeated type is the
607            // element type and elements are required. But this rule only applies if the
608            // repeated field is not annotated, and the single child field is not `repeated`.
609            let context = VisitorContext {
610                rep_level: context.rep_level,
611                def_level,
612                data_type: arrow_field.map(|f| f.data_type().clone()),
613                treat_repeated_as_list_arrow_hint: false,
614            };
615
616            return match self.visit_struct(repeated_field, context) {
617                Ok(Some(mut field)) => {
618                    field.nullable = nullable;
619                    Ok(Some(field))
620                }
621                r => r,
622            };
623        }
624
625        // Regular list handling logic
626        let item_type = &items[0];
627        let rep_level = context.rep_level + 1;
628        let def_level = def_level + 1;
629
630        let new_context = VisitorContext {
631            def_level,
632            rep_level,
633            data_type: arrow_field.map(|f| f.data_type().clone()),
634            treat_repeated_as_list_arrow_hint: true,
635        };
636
637        match self.dispatch(item_type, new_context) {
638            Ok(Some(item)) => {
639                let item_field = Arc::new(convert_field(item_type, &item, arrow_field, true)?);
640
641                // Use arrow type as hint for index size
642                let arrow_type = match context.data_type {
643                    Some(DataType::LargeList(_)) => DataType::LargeList(item_field),
644                    Some(DataType::FixedSizeList(_, len)) => {
645                        DataType::FixedSizeList(item_field, len)
646                    }
647                    Some(DataType::ListView(_)) => DataType::ListView(item_field),
648                    Some(DataType::LargeListView(_)) => DataType::LargeListView(item_field),
649                    _ => DataType::List(item_field),
650                };
651
652                Ok(Some(ParquetField {
653                    rep_level,
654                    def_level,
655                    nullable,
656                    arrow_type,
657                    field_type: ParquetFieldType::Group {
658                        children: vec![item],
659                    },
660                }))
661            }
662            r => r,
663        }
664    }
665
666    fn dispatch(
667        &mut self,
668        cur_type: &TypePtr,
669        context: VisitorContext,
670    ) -> Result<Option<ParquetField>> {
671        if cur_type.is_primitive() {
672            self.visit_primitive(cur_type, context)
673        } else {
674            match cur_type.get_basic_info().converted_type() {
675                ConvertedType::LIST => self.visit_list(cur_type, context),
676                ConvertedType::MAP | ConvertedType::MAP_KEY_VALUE => {
677                    self.visit_map(cur_type, context)
678                }
679                _ => self.visit_struct(cur_type, context),
680            }
681        }
682    }
683}
684
685/// Converts a virtual Arrow [`Field`] to a [`ParquetField`]
686///
687/// Virtual fields don't correspond to any data in the parquet file,
688/// but are computed at read time (e.g., row_number)
689///
690/// The levels are computed based on the parent context:
691/// - If nullable: def_level = parent_def_level + 1
692/// - If required: def_level = parent_def_level
693/// - rep_level = parent_rep_level (virtual fields are not repeated)
694pub(super) fn convert_virtual_field(
695    arrow_field: &Field,
696    parent_rep_level: i16,
697    parent_def_level: i16,
698) -> Result<ParquetField> {
699    let nullable = arrow_field.is_nullable();
700    let def_level = if nullable {
701        parent_def_level + 1
702    } else {
703        parent_def_level
704    };
705
706    // Determine the virtual column type based on the extension type name
707    let extension_name = arrow_field.extension_type_name().ok_or_else(|| {
708        ParquetError::ArrowError(format!(
709            "virtual column field '{}' must have an extension type",
710            arrow_field.name()
711        ))
712    })?;
713
714    let virtual_type = match extension_name {
715        RowNumber::NAME => VirtualColumnType::RowNumber,
716        RowGroupIndex::NAME => VirtualColumnType::RowGroupIndex,
717        _ => {
718            return Err(ParquetError::ArrowError(format!(
719                "unsupported virtual column type '{}' for field '{}'",
720                extension_name,
721                arrow_field.name()
722            )));
723        }
724    };
725
726    Ok(ParquetField {
727        rep_level: parent_rep_level,
728        def_level,
729        nullable,
730        arrow_type: arrow_field.data_type().clone(),
731        field_type: ParquetFieldType::Virtual(virtual_type),
732    })
733}
734
735/// Computes the Arrow [`Field`] for a child column
736///
737/// The resulting Arrow [`Field`] will have the type dictated by the Parquet `field`, a name
738/// dictated by the `parquet_type`, and any metadata from `arrow_hint`
739fn convert_field(
740    parquet_type: &Type,
741    field: &ParquetField,
742    arrow_hint: Option<&Field>,
743    add_field_id: bool,
744) -> Result<Field, ParquetError> {
745    let name = parquet_type.name();
746    let data_type = field.arrow_type.clone();
747    let nullable = field.nullable;
748
749    match arrow_hint {
750        Some(hint) => {
751            // If the inferred type is a dictionary, preserve dictionary metadata
752            #[allow(deprecated)]
753            let field = match (&data_type, hint.dict_id(), hint.dict_is_ordered()) {
754                (DataType::Dictionary(_, _), Some(id), Some(ordered)) =>
755                {
756                    #[allow(deprecated)]
757                    Field::new_dict(name, data_type, nullable, id, ordered)
758                }
759                _ => Field::new(name, data_type, nullable),
760            };
761
762            Ok(field.with_metadata(hint.metadata().clone()))
763        }
764        None => {
765            let mut ret = Field::new(name, data_type, nullable);
766            let basic_info = parquet_type.get_basic_info();
767            if add_field_id && basic_info.has_id() {
768                let mut meta = HashMap::with_capacity(1);
769                meta.insert(
770                    PARQUET_FIELD_ID_META_KEY.to_string(),
771                    basic_info.id().to_string(),
772                );
773                ret.set_metadata(meta);
774            }
775            try_add_extension_type(ret, parquet_type)
776        }
777    }
778}
779
780/// Computes the [`ParquetField`] for the provided [`SchemaDescriptor`] with `leaf_columns` listing
781/// the indexes of leaf columns to project, and `embedded_arrow_schema` the optional
782/// [`Fields`] embedded in the parquet metadata
783///
784/// Note: This does not support out of order column projection
785pub fn convert_schema(
786    schema: &SchemaDescriptor,
787    mask: ProjectionMask,
788    embedded_arrow_schema: Option<&Fields>,
789) -> Result<Option<ParquetField>> {
790    let mut visitor = Visitor {
791        next_col_idx: 0,
792        mask,
793    };
794
795    let context = VisitorContext {
796        rep_level: 0,
797        def_level: 0,
798        data_type: embedded_arrow_schema.map(|fields| DataType::Struct(fields.clone())),
799        treat_repeated_as_list_arrow_hint: true,
800    };
801
802    visitor.dispatch(&schema.root_schema_ptr(), context)
803}
804
805/// Computes the [`ParquetField`] for the provided `parquet_type`
806pub fn convert_type(parquet_type: &TypePtr) -> Result<ParquetField> {
807    let mut visitor = Visitor {
808        next_col_idx: 0,
809        mask: ProjectionMask::all(),
810    };
811
812    let context = VisitorContext {
813        rep_level: 0,
814        def_level: 0,
815        data_type: None,
816        // We might be inside list
817        treat_repeated_as_list_arrow_hint: false,
818    };
819
820    Ok(visitor.dispatch(parquet_type, context)?.unwrap())
821}
822
823#[cfg(test)]
824mod tests {
825    use crate::arrow::schema::complex::convert_schema;
826    use crate::arrow::{PARQUET_FIELD_ID_META_KEY, ProjectionMask};
827    use crate::schema::parser::parse_message_type;
828    use crate::schema::types::SchemaDescriptor;
829    use arrow_schema::{DataType, Field, Fields};
830    use std::sync::Arc;
831
832    trait WithFieldId {
833        fn with_field_id(self, id: i32) -> Self;
834    }
835    impl WithFieldId for arrow_schema::Field {
836        fn with_field_id(self, id: i32) -> Self {
837            let mut metadata = self.metadata().clone();
838            metadata.insert(PARQUET_FIELD_ID_META_KEY.to_string(), id.to_string());
839            self.with_metadata(metadata)
840        }
841    }
842
843    fn test_roundtrip(message_type: &str) -> crate::errors::Result<()> {
844        let parsed_input_schema = Arc::new(parse_message_type(message_type)?);
845        let schema = SchemaDescriptor::new(parsed_input_schema);
846
847        let converted = convert_schema(&schema, ProjectionMask::all(), None)?.unwrap();
848
849        let DataType::Struct(schema_fields) = &converted.arrow_type else {
850            panic!("Expected struct from convert_schema");
851        };
852
853        // Should be able to convert the same thing
854        let converted_again =
855            convert_schema(&schema, ProjectionMask::all(), Some(schema_fields))?.unwrap();
856
857        // Assert that we changed to Utf8
858        assert_eq!(converted_again.arrow_type, converted.arrow_type);
859
860        Ok(())
861    }
862
863    fn test_expected_type(
864        message_type: &str,
865        expected_fields: Fields,
866    ) -> crate::errors::Result<()> {
867        test_roundtrip(message_type)?;
868
869        let parsed_input_schema = Arc::new(parse_message_type(message_type)?);
870        let schema = SchemaDescriptor::new(parsed_input_schema);
871
872        let converted = convert_schema(&schema, ProjectionMask::all(), None)?.unwrap();
873
874        let DataType::Struct(schema_fields) = &converted.arrow_type else {
875            panic!("Expected struct from convert_schema");
876        };
877
878        assert_eq!(schema_fields, &expected_fields);
879
880        Ok(())
881    }
882
883    /// Taken from the example in [Parquet Format - Nested Types - Lists - Backward-compatibility rules](https://github.com/apache/parquet-format/blob/9fd57b59e0ce1a82a69237dcf8977d3e72a2965d/LogicalTypes.md?plain=1#L766-L769)
884    #[test]
885    fn basic_backward_compatible_list_1() -> crate::errors::Result<()> {
886        test_expected_type(
887            "
888            message schema {
889                optional group my_list (LIST) {
890                  repeated int32 element;
891                }
892            }
893        ",
894            Fields::from(vec![
895                // Rule 1: List<Integer> (nullable list, non-null elements)
896                Field::new(
897                    "my_list",
898                    DataType::List(Arc::new(Field::new("element", DataType::Int32, false))),
899                    true,
900                ),
901            ]),
902        )
903    }
904
905    /// Taken from the example in [Parquet Format - Nested Types - Lists - Backward-compatibility rules](https://github.com/apache/parquet-format/blob/9fd57b59e0ce1a82a69237dcf8977d3e72a2965d/LogicalTypes.md?plain=1#L771-L777)
906    #[test]
907    fn basic_backward_compatible_list_2() -> crate::errors::Result<()> {
908        test_expected_type(
909            "
910            message schema {
911              optional group my_list (LIST) {
912                  repeated group element {
913                    required binary str (STRING);
914                    required int32 num;
915                  }
916              }
917            }
918        ",
919            Fields::from(vec![
920                // Rule 2: List<Tuple<String, Integer>> (nullable list, non-null elements)
921                Field::new(
922                    "my_list",
923                    DataType::List(Arc::new(Field::new(
924                        "element",
925                        DataType::Struct(Fields::from(vec![
926                            Field::new("str", DataType::Utf8, false),
927                            Field::new("num", DataType::Int32, false),
928                        ])),
929                        false,
930                    ))),
931                    true,
932                ),
933            ]),
934        )
935    }
936
937    /// Taken from the example in [Parquet Format - Nested Types - Lists - Backward-compatibility rules](https://github.com/apache/parquet-format/blob/9fd57b59e0ce1a82a69237dcf8977d3e72a2965d/LogicalTypes.md?plain=1#L779-L784)
938    #[test]
939    fn basic_backward_compatible_list_3() -> crate::errors::Result<()> {
940        test_expected_type(
941            "
942            message schema {
943              optional group my_list (LIST) {
944                  repeated group array (LIST) {
945                    repeated int32 array;
946                  }
947              }
948            }
949        ",
950            Fields::from(vec![
951                // Rule 3: List<List<Integer>> (nullable outer list, non-null elements)
952                Field::new(
953                    "my_list",
954                    DataType::List(Arc::new(Field::new(
955                        "array",
956                        DataType::List(Arc::new(Field::new("array", DataType::Int32, false))),
957                        false,
958                    ))),
959                    true,
960                ),
961            ]),
962        )
963    }
964
965    /// Taken from the example in [Parquet Format - Nested Types - Lists - Backward-compatibility rules](https://github.com/apache/parquet-format/blob/9fd57b59e0ce1a82a69237dcf8977d3e72a2965d/LogicalTypes.md?plain=1#L786-L791)
966    #[test]
967    fn basic_backward_compatible_list_4_1() -> crate::errors::Result<()> {
968        test_expected_type(
969            "
970            message schema {
971              optional group my_list (LIST) {
972                  repeated group array {
973                    required binary str (STRING);
974                  }
975              }
976            }
977        ",
978            Fields::from(vec![
979                // Rule 4: List<OneTuple<String>> (nullable list, non-null elements)
980                Field::new(
981                    "my_list",
982                    DataType::List(Arc::new(Field::new(
983                        "array",
984                        DataType::Struct(Fields::from(vec![Field::new(
985                            "str",
986                            DataType::Utf8,
987                            false,
988                        )])),
989                        false,
990                    ))),
991                    true,
992                ),
993            ]),
994        )
995    }
996
997    /// Taken from the example in [Parquet Format - Nested Types - Lists - Backward-compatibility rules](https://github.com/apache/parquet-format/blob/9fd57b59e0ce1a82a69237dcf8977d3e72a2965d/LogicalTypes.md?plain=1#L793-L798)
998    #[test]
999    fn basic_backward_compatible_list_4_2() -> crate::errors::Result<()> {
1000        test_expected_type(
1001            "
1002            message schema {
1003                optional group my_list (LIST) {
1004                    repeated group my_list_tuple {
1005                        required binary str (STRING);
1006                    }
1007                }
1008            }
1009        ",
1010            Fields::from(vec![
1011                // Rule 4: List<OneTuple<String>> (nullable list, non-null elements)
1012                Field::new(
1013                    "my_list",
1014                    DataType::List(Arc::new(Field::new(
1015                        "my_list_tuple",
1016                        DataType::Struct(Fields::from(vec![Field::new(
1017                            "str",
1018                            DataType::Utf8,
1019                            false,
1020                        )])),
1021                        false,
1022                    ))),
1023                    true,
1024                ),
1025            ]),
1026        )
1027    }
1028
1029    /// Taken from the example in [Parquet Format - Nested Types - Lists - Backward-compatibility rules](https://github.com/apache/parquet-format/blob/9fd57b59e0ce1a82a69237dcf8977d3e72a2965d/LogicalTypes.md?plain=1#L800-L805)
1030    #[test]
1031    fn basic_backward_compatible_list_5() -> crate::errors::Result<()> {
1032        test_expected_type(
1033            "
1034            message schema {
1035                optional group my_list (LIST) {
1036                    repeated group element {
1037                        optional binary str (STRING);
1038                    }
1039                }
1040            }
1041        ",
1042            Fields::from(vec![
1043                // Rule 5: List<String>  (nullable list, nullable elements)
1044                Field::new(
1045                    "my_list",
1046                    DataType::List(Arc::new(Field::new("str", DataType::Utf8, true))),
1047                    true,
1048                ),
1049            ]),
1050        )
1051    }
1052
1053    #[test]
1054    fn basic_backward_compatible_map_1() -> crate::errors::Result<()> {
1055        test_expected_type(
1056            "
1057            message schema {
1058                optional group my_map (MAP) {
1059                  repeated group map {
1060                    required binary str (STRING);
1061                    required int32 num;
1062                  }
1063                }
1064            }
1065        ",
1066            Fields::from(vec![
1067                // Map<String, Integer> (nullable map, non-null values)
1068                Field::new(
1069                    "my_map",
1070                    DataType::Map(
1071                        Arc::new(Field::new(
1072                            "map",
1073                            DataType::Struct(Fields::from(vec![
1074                                Field::new("str", DataType::Utf8, false),
1075                                Field::new("num", DataType::Int32, false),
1076                            ])),
1077                            false,
1078                        )),
1079                        false,
1080                    ),
1081                    true,
1082                ),
1083            ]),
1084        )
1085    }
1086
1087    #[test]
1088    fn basic_backward_compatible_map_2() -> crate::errors::Result<()> {
1089        test_expected_type(
1090            "
1091            message schema {
1092                optional group my_map (MAP_KEY_VALUE) {
1093                  repeated group map {
1094                    required binary key (STRING);
1095                    optional int32 value;
1096                  }
1097                }
1098            }
1099        ",
1100            Fields::from(vec![
1101                // Map<String, Integer> (nullable map, nullable values)
1102                Field::new(
1103                    "my_map",
1104                    DataType::Map(
1105                        Arc::new(Field::new(
1106                            "map",
1107                            DataType::Struct(Fields::from(vec![
1108                                Field::new("key", DataType::Utf8, false),
1109                                Field::new("value", DataType::Int32, true),
1110                            ])),
1111                            false,
1112                        )),
1113                        false,
1114                    ),
1115                    true,
1116                ),
1117            ]),
1118        )
1119    }
1120
1121    #[test]
1122    fn convert_schema_with_nested_list_repeated_primitive() -> crate::errors::Result<()> {
1123        test_roundtrip(
1124            "
1125            message schema {
1126                optional group f1 (LIST) {
1127                    repeated group element {
1128                        repeated int32 element;
1129                    }
1130                }
1131            }
1132        ",
1133        )
1134    }
1135
1136    #[test]
1137    fn convert_schema_with_repeated_primitive_keep_field_id() -> crate::errors::Result<()> {
1138        let message_type = "
1139    message schema {
1140      repeated BYTE_ARRAY col_1 = 1;
1141    }
1142    ";
1143
1144        let parsed_input_schema = Arc::new(parse_message_type(message_type)?);
1145        let schema = SchemaDescriptor::new(parsed_input_schema);
1146
1147        let converted = convert_schema(&schema, ProjectionMask::all(), None)?.unwrap();
1148
1149        let DataType::Struct(schema_fields) = &converted.arrow_type else {
1150            panic!("Expected struct from convert_schema");
1151        };
1152
1153        assert_eq!(schema_fields.len(), 1);
1154
1155        let expected_schema = DataType::Struct(Fields::from(vec![Arc::new(
1156            arrow_schema::Field::new(
1157                "col_1",
1158                DataType::List(Arc::new(
1159                    // No metadata on inner field
1160                    arrow_schema::Field::new("col_1", DataType::Binary, false),
1161                )),
1162                false,
1163            )
1164            // add the field id to the outer list
1165            .with_field_id(1),
1166        )]));
1167
1168        assert_eq!(converted.arrow_type, expected_schema);
1169
1170        Ok(())
1171    }
1172
1173    #[test]
1174    fn convert_schema_with_repeated_primitive_should_use_inferred_schema()
1175    -> crate::errors::Result<()> {
1176        let message_type = "
1177    message schema {
1178      repeated BYTE_ARRAY col_1 = 1;
1179    }
1180    ";
1181
1182        let parsed_input_schema = Arc::new(parse_message_type(message_type)?);
1183        let schema = SchemaDescriptor::new(parsed_input_schema);
1184
1185        let converted = convert_schema(&schema, ProjectionMask::all(), None)?.unwrap();
1186
1187        let DataType::Struct(schema_fields) = &converted.arrow_type else {
1188            panic!("Expected struct from convert_schema");
1189        };
1190
1191        assert_eq!(schema_fields.len(), 1);
1192
1193        let expected_schema = DataType::Struct(Fields::from(vec![Arc::new(
1194            arrow_schema::Field::new(
1195                "col_1",
1196                DataType::List(Arc::new(arrow_schema::Field::new(
1197                    "col_1",
1198                    DataType::Binary,
1199                    false,
1200                ))),
1201                false,
1202            )
1203            .with_metadata(schema_fields[0].metadata().clone()),
1204        )]));
1205
1206        assert_eq!(converted.arrow_type, expected_schema);
1207
1208        let utf8_instead_of_binary = Fields::from(vec![Arc::new(
1209            arrow_schema::Field::new(
1210                "col_1",
1211                DataType::List(Arc::new(arrow_schema::Field::new(
1212                    "col_1",
1213                    DataType::Utf8,
1214                    false,
1215                ))),
1216                false,
1217            )
1218            .with_metadata(schema_fields[0].metadata().clone()),
1219        )]);
1220
1221        // Should be able to convert the same thing
1222        let converted_again = convert_schema(
1223            &schema,
1224            ProjectionMask::all(),
1225            Some(&utf8_instead_of_binary),
1226        )?
1227        .unwrap();
1228
1229        // Assert that we changed to Utf8
1230        assert_eq!(
1231            converted_again.arrow_type,
1232            DataType::Struct(utf8_instead_of_binary)
1233        );
1234
1235        Ok(())
1236    }
1237
1238    #[test]
1239    fn convert_schema_with_repeated_primitive_should_use_inferred_schema_for_list_as_well()
1240    -> crate::errors::Result<()> {
1241        let message_type = "
1242    message schema {
1243      repeated BYTE_ARRAY col_1 = 1;
1244    }
1245    ";
1246
1247        let parsed_input_schema = Arc::new(parse_message_type(message_type)?);
1248        let schema = SchemaDescriptor::new(parsed_input_schema);
1249
1250        let converted = convert_schema(&schema, ProjectionMask::all(), None)?.unwrap();
1251
1252        let DataType::Struct(schema_fields) = &converted.arrow_type else {
1253            panic!("Expected struct from convert_schema");
1254        };
1255
1256        assert_eq!(schema_fields.len(), 1);
1257
1258        let expected_schema = DataType::Struct(Fields::from(vec![Arc::new(
1259            arrow_schema::Field::new(
1260                "col_1",
1261                DataType::List(Arc::new(arrow_schema::Field::new(
1262                    "col_1",
1263                    DataType::Binary,
1264                    false,
1265                ))),
1266                false,
1267            )
1268            .with_metadata(schema_fields[0].metadata().clone()),
1269        )]));
1270
1271        assert_eq!(converted.arrow_type, expected_schema);
1272
1273        let utf8_instead_of_binary = Fields::from(vec![Arc::new(
1274            arrow_schema::Field::new(
1275                "col_1",
1276                // Inferring as LargeList instead of List
1277                DataType::LargeList(Arc::new(arrow_schema::Field::new(
1278                    "col_1",
1279                    DataType::Utf8,
1280                    false,
1281                ))),
1282                false,
1283            )
1284            .with_metadata(schema_fields[0].metadata().clone()),
1285        )]);
1286
1287        // Should be able to convert the same thing
1288        let converted_again = convert_schema(
1289            &schema,
1290            ProjectionMask::all(),
1291            Some(&utf8_instead_of_binary),
1292        )?
1293        .unwrap();
1294
1295        // Assert that we changed to Utf8
1296        assert_eq!(
1297            converted_again.arrow_type,
1298            DataType::Struct(utf8_instead_of_binary)
1299        );
1300
1301        Ok(())
1302    }
1303
1304    #[test]
1305    fn convert_schema_with_repeated_struct_and_inferred_schema() -> crate::errors::Result<()> {
1306        test_roundtrip(
1307            "
1308    message schema {
1309        repeated group my_col_1 = 1 {
1310          optional binary my_col_2 = 2;
1311          optional binary my_col_3 = 3;
1312          optional group my_col_4 = 4 {
1313            optional int64 my_col_5 = 5;
1314            optional int32 my_col_6 = 6;
1315          }
1316        }
1317    }
1318    ",
1319        )
1320    }
1321
1322    #[test]
1323    fn convert_schema_with_repeated_struct_and_inferred_schema_and_field_id()
1324    -> crate::errors::Result<()> {
1325        let message_type = "
1326    message schema {
1327        repeated group my_col_1 = 1 {
1328          optional binary my_col_2 = 2;
1329          optional binary my_col_3 = 3;
1330          optional group my_col_4 = 4 {
1331            optional int64 my_col_5 = 5;
1332            optional int32 my_col_6 = 6;
1333          }
1334        }
1335    }
1336    ";
1337
1338        let parsed_input_schema = Arc::new(parse_message_type(message_type)?);
1339        let schema = SchemaDescriptor::new(parsed_input_schema);
1340
1341        let converted = convert_schema(&schema, ProjectionMask::all(), None)?.unwrap();
1342
1343        let DataType::Struct(schema_fields) = &converted.arrow_type else {
1344            panic!("Expected struct from convert_schema");
1345        };
1346
1347        assert_eq!(schema_fields.len(), 1);
1348
1349        // Should be able to convert the same thing
1350        let converted_again =
1351            convert_schema(&schema, ProjectionMask::all(), Some(schema_fields))?.unwrap();
1352
1353        // Assert that we changed to Utf8
1354        assert_eq!(converted_again.arrow_type, converted.arrow_type);
1355
1356        Ok(())
1357    }
1358
1359    #[test]
1360    fn convert_schema_with_nested_repeated_struct_and_primitives() -> crate::errors::Result<()> {
1361        let message_type = "
1362message schema {
1363    repeated group my_col_1 = 1 {
1364        optional binary my_col_2 = 2;
1365        repeated BYTE_ARRAY my_col_3 = 3;
1366        repeated group my_col_4 = 4 {
1367            optional int64 my_col_5 = 5;
1368            repeated binary my_col_6 = 6;
1369        }
1370    }
1371}
1372";
1373
1374        let parsed_input_schema = Arc::new(parse_message_type(message_type)?);
1375        let schema = SchemaDescriptor::new(parsed_input_schema);
1376
1377        let converted = convert_schema(&schema, ProjectionMask::all(), None)?.unwrap();
1378
1379        let DataType::Struct(schema_fields) = &converted.arrow_type else {
1380            panic!("Expected struct from convert_schema");
1381        };
1382
1383        assert_eq!(schema_fields.len(), 1);
1384
1385        // Build expected schema
1386        let expected_schema = DataType::Struct(Fields::from(vec![Arc::new(
1387            arrow_schema::Field::new(
1388                "my_col_1",
1389                DataType::List(Arc::new(arrow_schema::Field::new(
1390                    "my_col_1",
1391                    DataType::Struct(Fields::from(vec![
1392                        Arc::new(
1393                            arrow_schema::Field::new("my_col_2", DataType::Binary, true)
1394                                .with_field_id(2),
1395                        ),
1396                        Arc::new(
1397                            arrow_schema::Field::new(
1398                                "my_col_3",
1399                                DataType::List(Arc::new(arrow_schema::Field::new(
1400                                    "my_col_3",
1401                                    DataType::Binary,
1402                                    false,
1403                                ))),
1404                                false,
1405                            )
1406                            // add the field id to the outer list
1407                            .with_field_id(3),
1408                        ),
1409                        Arc::new(
1410                            arrow_schema::Field::new(
1411                                "my_col_4",
1412                                DataType::List(Arc::new(arrow_schema::Field::new(
1413                                    "my_col_4",
1414                                    DataType::Struct(Fields::from(vec![
1415                                        Arc::new(
1416                                            arrow_schema::Field::new(
1417                                                "my_col_5",
1418                                                DataType::Int64,
1419                                                true,
1420                                            )
1421                                            // add the field id to the outer list
1422                                            .with_field_id(5),
1423                                        ),
1424                                        Arc::new(
1425                                            arrow_schema::Field::new(
1426                                                "my_col_6",
1427                                                DataType::List(Arc::new(arrow_schema::Field::new(
1428                                                    "my_col_6",
1429                                                    DataType::Binary,
1430                                                    false,
1431                                                ))),
1432                                                false,
1433                                            )
1434                                            // add the field id to the outer list
1435                                            .with_field_id(6),
1436                                        ),
1437                                    ])),
1438                                    false,
1439                                ))),
1440                                false,
1441                            )
1442                            // add the field id to the outer list
1443                            .with_field_id(4),
1444                        ),
1445                    ])),
1446                    false,
1447                ))),
1448                false,
1449            )
1450            // add the field id to the outer list
1451            .with_field_id(1),
1452        )]));
1453
1454        assert_eq!(converted.arrow_type, expected_schema);
1455
1456        // Test conversion with inferred schema
1457        let converted_again =
1458            convert_schema(&schema, ProjectionMask::all(), Some(schema_fields))?.unwrap();
1459
1460        assert_eq!(converted_again.arrow_type, converted.arrow_type);
1461
1462        // Test conversion with modified schema (change lists to either LargeList or FixedSizeList)
1463        // as well as changing Binary to Utf8 or BinaryView
1464        let modified_schema_fields = Fields::from(vec![Arc::new(
1465            arrow_schema::Field::new(
1466                "my_col_1",
1467                DataType::LargeList(Arc::new(arrow_schema::Field::new(
1468                    "my_col_1",
1469                    DataType::Struct(Fields::from(vec![
1470                        Arc::new(
1471                            arrow_schema::Field::new("my_col_2", DataType::LargeBinary, true)
1472                                .with_field_id(2),
1473                        ),
1474                        Arc::new(
1475                            arrow_schema::Field::new(
1476                                "my_col_3",
1477                                DataType::LargeList(Arc::new(arrow_schema::Field::new(
1478                                    "my_col_3",
1479                                    DataType::Utf8,
1480                                    false,
1481                                ))),
1482                                false,
1483                            )
1484                            // add the field id to the outer list
1485                            .with_field_id(3),
1486                        ),
1487                        Arc::new(
1488                            arrow_schema::Field::new(
1489                                "my_col_4",
1490                                DataType::FixedSizeList(
1491                                    Arc::new(arrow_schema::Field::new(
1492                                        "my_col_4",
1493                                        DataType::Struct(Fields::from(vec![
1494                                            Arc::new(
1495                                                arrow_schema::Field::new(
1496                                                    "my_col_5",
1497                                                    DataType::Int64,
1498                                                    true,
1499                                                )
1500                                                .with_field_id(5),
1501                                            ),
1502                                            Arc::new(
1503                                                arrow_schema::Field::new(
1504                                                    "my_col_6",
1505                                                    DataType::LargeList(Arc::new(
1506                                                        arrow_schema::Field::new(
1507                                                            "my_col_6",
1508                                                            DataType::BinaryView,
1509                                                            false,
1510                                                        ),
1511                                                    )),
1512                                                    false,
1513                                                )
1514                                                // add the field id to the outer list
1515                                                .with_field_id(6),
1516                                            ),
1517                                        ])),
1518                                        false,
1519                                    )),
1520                                    3,
1521                                ),
1522                                false,
1523                            )
1524                            // add the field id to the outer list
1525                            .with_field_id(4),
1526                        ),
1527                    ])),
1528                    false,
1529                ))),
1530                false,
1531            )
1532            // add the field id to the outer list
1533            .with_field_id(1),
1534        )]);
1535
1536        let converted_with_modified = convert_schema(
1537            &schema,
1538            ProjectionMask::all(),
1539            Some(&modified_schema_fields),
1540        )?
1541        .unwrap();
1542
1543        assert_eq!(
1544            converted_with_modified.arrow_type,
1545            DataType::Struct(modified_schema_fields)
1546        );
1547
1548        Ok(())
1549    }
1550
1551    /// Backwards-compatibility: LIST with nullable element type - 1 - standard
1552    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L452-L466)
1553    #[test]
1554    fn list_nullable_element_standard() -> crate::errors::Result<()> {
1555        test_expected_type(
1556            "
1557            message root {
1558              optional group f1 (LIST) {
1559                repeated group list {
1560                  optional int32 element;
1561                }
1562              }
1563            }",
1564            Fields::from(vec![Field::new(
1565                "f1",
1566                DataType::List(Arc::new(Field::new("element", DataType::Int32, true))),
1567                true,
1568            )]),
1569        )
1570    }
1571
1572    /// Backwards-compatibility: LIST with nullable element type - 2
1573    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L468-L482)
1574    #[test]
1575    fn list_nullable_element_nested() -> crate::errors::Result<()> {
1576        test_expected_type(
1577            "
1578            message root {
1579              optional group f1 (LIST) {
1580                repeated group element {
1581                  optional int32 num;
1582                }
1583              }
1584            }",
1585            Fields::from(vec![Field::new(
1586                "f1",
1587                DataType::List(Arc::new(Field::new("num", DataType::Int32, true))),
1588                true,
1589            )]),
1590        )
1591    }
1592
1593    /// Backwards-compatibility: LIST with non-nullable element type - 1 - standard
1594    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L484-L495)
1595    #[test]
1596    fn list_required_element_standard() -> crate::errors::Result<()> {
1597        test_expected_type(
1598            "
1599            message root {
1600              optional group f1 (LIST) {
1601                repeated group list {
1602                  required int32 element;
1603                }
1604              }
1605            }",
1606            Fields::from(vec![Field::new(
1607                "f1",
1608                DataType::List(Arc::new(Field::new("element", DataType::Int32, false))),
1609                true,
1610            )]),
1611        )
1612    }
1613
1614    /// Backwards-compatibility: LIST with non-nullable element type - 2
1615    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L497-L508)
1616    #[test]
1617    fn list_required_element_nested() -> crate::errors::Result<()> {
1618        test_expected_type(
1619            "
1620            message root {
1621              optional group f1 (LIST) {
1622                repeated group element {
1623                  required int32 num;
1624                }
1625              }
1626            }",
1627            Fields::from(vec![Field::new(
1628                "f1",
1629                DataType::List(Arc::new(Field::new("num", DataType::Int32, false))),
1630                true,
1631            )]),
1632        )
1633    }
1634
1635    /// Backwards-compatibility: LIST with non-nullable element type - 3
1636    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L510-L519)
1637    #[test]
1638    fn list_required_element_primitive() -> crate::errors::Result<()> {
1639        test_expected_type(
1640            "
1641            message root {
1642              optional group f1 (LIST) {
1643                repeated int32 element;
1644              }
1645            }",
1646            Fields::from(vec![Field::new(
1647                "f1",
1648                DataType::List(Arc::new(Field::new("element", DataType::Int32, false))),
1649                true,
1650            )]),
1651        )
1652    }
1653
1654    /// Backwards-compatibility: LIST with non-nullable element type - 4
1655    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L521-L540)
1656    #[test]
1657    fn list_required_element_struct() -> crate::errors::Result<()> {
1658        test_expected_type(
1659            "
1660            message root {
1661              optional group f1 (LIST) {
1662                repeated group element {
1663                  required binary str (UTF8);
1664                  required int32 num;
1665                }
1666              }
1667            }",
1668            Fields::from(vec![Field::new(
1669                "f1",
1670                DataType::List(Arc::new(Field::new(
1671                    "element",
1672                    DataType::Struct(Fields::from(vec![
1673                        Field::new("str", DataType::Utf8, false),
1674                        Field::new("num", DataType::Int32, false),
1675                    ])),
1676                    false,
1677                ))),
1678                true,
1679            )]),
1680        )
1681    }
1682
1683    /// Backwards-compatibility: LIST with non-nullable element type - 5 - parquet-avro style
1684    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L542-L559)
1685    #[test]
1686    fn list_required_element_avro_style() -> crate::errors::Result<()> {
1687        test_expected_type(
1688            "
1689            message root {
1690              optional group f1 (LIST) {
1691                repeated group array {
1692                  required binary str (UTF8);
1693                }
1694              }
1695            }",
1696            Fields::from(vec![Field::new(
1697                "f1",
1698                DataType::List(Arc::new(Field::new(
1699                    "array",
1700                    DataType::Struct(Fields::from(vec![Field::new("str", DataType::Utf8, false)])),
1701                    false,
1702                ))),
1703                true,
1704            )]),
1705        )
1706    }
1707
1708    /// Backwards-compatibility: LIST with non-nullable element type - 6 - parquet-thrift style
1709    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L561-L578)
1710    #[test]
1711    fn list_required_element_thrift_style() -> crate::errors::Result<()> {
1712        test_expected_type(
1713            "
1714            message root {
1715              optional group f1 (LIST) {
1716                repeated group f1_tuple {
1717                  required binary str (UTF8);
1718                }
1719              }
1720            }",
1721            Fields::from(vec![Field::new(
1722                "f1",
1723                DataType::List(Arc::new(Field::new(
1724                    "f1_tuple",
1725                    DataType::Struct(Fields::from(vec![Field::new("str", DataType::Utf8, false)])),
1726                    false,
1727                ))),
1728                true,
1729            )]),
1730        )
1731    }
1732
1733    /// Backwards-compatibility: MAP with non-nullable value type - 1 - standard
1734    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L652-L667)
1735    #[test]
1736    fn map_required_value_standard() -> crate::errors::Result<()> {
1737        test_expected_type(
1738            "
1739            message root {
1740              optional group f1 (MAP) {
1741                repeated group key_value {
1742                  required int32 key;
1743                  required binary value (UTF8);
1744                }
1745              }
1746            }",
1747            Fields::from(vec![Field::new_map(
1748                "f1",
1749                "key_value",
1750                Field::new("key", DataType::Int32, false),
1751                Field::new("value", DataType::Utf8, false),
1752                false,
1753                true,
1754            )]),
1755        )
1756    }
1757
1758    /// Backwards-compatibility: MAP with non-nullable value type - 2
1759    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L669-L684)
1760    #[test]
1761    fn map_required_value_map_key_value() -> crate::errors::Result<()> {
1762        test_expected_type(
1763            "
1764            message root {
1765              optional group f1 (MAP_KEY_VALUE) {
1766                repeated group map {
1767                  required int32 num;
1768                  required binary str (UTF8);
1769                }
1770              }
1771            }",
1772            Fields::from(vec![Field::new_map(
1773                "f1",
1774                "map",
1775                Field::new("num", DataType::Int32, false),
1776                Field::new("str", DataType::Utf8, false),
1777                false,
1778                true,
1779            )]),
1780        )
1781    }
1782
1783    /// Backwards-compatibility: MAP with non-nullable value type - 3 - prior to 1.4.x
1784    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L686-L701)
1785    #[test]
1786    fn map_required_value_legacy() -> crate::errors::Result<()> {
1787        test_expected_type(
1788            "
1789            message root {
1790              optional group f1 (MAP) {
1791                repeated group map (MAP_KEY_VALUE) {
1792                  required int32 key;
1793                  required binary value (UTF8);
1794                }
1795              }
1796            }",
1797            Fields::from(vec![Field::new_map(
1798                "f1",
1799                "map",
1800                Field::new("key", DataType::Int32, false),
1801                Field::new("value", DataType::Utf8, false),
1802                false,
1803                true,
1804            )]),
1805        )
1806    }
1807
1808    /// Backwards-compatibility: MAP with nullable value type - 1 - standard
1809    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L703-L718)
1810    #[test]
1811    fn map_optional_value_standard() -> crate::errors::Result<()> {
1812        test_expected_type(
1813            "
1814            message root {
1815              optional group f1 (MAP) {
1816                repeated group key_value {
1817                  required int32 key;
1818                  optional binary value (UTF8);
1819                }
1820              }
1821            }",
1822            Fields::from(vec![Field::new_map(
1823                "f1",
1824                "key_value",
1825                Field::new("key", DataType::Int32, false),
1826                Field::new("value", DataType::Utf8, true),
1827                false,
1828                true,
1829            )]),
1830        )
1831    }
1832
1833    /// Backwards-compatibility: MAP with nullable value type - 2
1834    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L720-L735)
1835    #[test]
1836    fn map_optional_value_map_key_value() -> crate::errors::Result<()> {
1837        test_expected_type(
1838            "
1839            message root {
1840              optional group f1 (MAP_KEY_VALUE) {
1841                repeated group map {
1842                  required int32 num;
1843                  optional binary str (UTF8);
1844                }
1845              }
1846            }",
1847            Fields::from(vec![Field::new_map(
1848                "f1",
1849                "map",
1850                Field::new("num", DataType::Int32, false),
1851                Field::new("str", DataType::Utf8, true),
1852                false,
1853                true,
1854            )]),
1855        )
1856    }
1857
1858    /// Backwards-compatibility: MAP with nullable value type - 3 - parquet-avro style
1859    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L737-L752)
1860    #[test]
1861    fn map_optional_value_avro_style() -> crate::errors::Result<()> {
1862        test_expected_type(
1863            "
1864            message root {
1865              optional group f1 (MAP) {
1866                repeated group map (MAP_KEY_VALUE) {
1867                  required int32 key;
1868                  optional binary value (UTF8);
1869                }
1870              }
1871            }",
1872            Fields::from(vec![Field::new_map(
1873                "f1",
1874                "map",
1875                Field::new("key", DataType::Int32, false),
1876                Field::new("value", DataType::Utf8, true),
1877                false,
1878                true,
1879            )]),
1880        )
1881    }
1882}
parquet/arrow/schema/complex.rs

parquet/arrow/schema/
complex.rs
