parquet/arrow/schema/
complex.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use std::collections::HashMap;
6use std::sync::Arc;
7
8use crate::arrow::schema::extension::try_add_extension_type;
9use crate::arrow::schema::primitive::convert_primitive;
10use crate::arrow::schema::virtual_type::{RowGroupIndex, RowNumber};
11use crate::arrow::{PARQUET_FIELD_ID_META_KEY, ProjectionMask};
12use crate::basic::{ConvertedType, Repetition};
13use crate::errors::ParquetError;
14use crate::errors::Result;
15use crate::schema::types::{SchemaDescriptor, Type, TypePtr};
16use arrow_schema::{DataType, Field, Fields, SchemaBuilder, extension::ExtensionType};
17
18fn get_repetition(t: &Type) -> Repetition {
19    let info = t.get_basic_info();
20    match info.has_repetition() {
21        true => info.repetition(),
22        false => Repetition::REQUIRED,
23    }
24}
25
26/// Representation of a parquet schema element, in terms of arrow schema elements
27#[derive(Debug, Clone)]
28pub struct ParquetField {
29    /// The level which represents an insertion into the current list
30    /// i.e. guaranteed to be > 0 for an element of list type
31    pub rep_level: i16,
32    /// The level at which this field is fully defined,
33    /// i.e. guaranteed to be > 0 for a nullable type or child of a
34    /// nullable type
35    pub def_level: i16,
36    /// Whether this field is nullable
37    pub nullable: bool,
38    /// The arrow type of the column data
39    ///
40    /// Note: In certain cases the data stored in parquet may have been coerced
41    /// to a different type and will require conversion on read (e.g. Date64 and Interval)
42    pub arrow_type: DataType,
43    /// The type of this field
44    pub field_type: ParquetFieldType,
45}
46
47impl ParquetField {
48    /// Converts `self` into an arrow list, with its current type as the field type
49    ///
50    /// This is used to convert repeated columns, into their arrow representation
51    fn into_list(self, name: &str) -> Self {
52        ParquetField {
53            rep_level: self.rep_level,
54            def_level: self.def_level,
55            nullable: false,
56            arrow_type: DataType::List(Arc::new(Field::new(name, self.arrow_type.clone(), false))),
57            field_type: ParquetFieldType::Group {
58                children: vec![self],
59            },
60        }
61    }
62
63    /// Converts `self` into an arrow list, with its current type as the field type
64    /// accept an optional `list_data_type` to specify the type of list to create
65    ///
66    /// This is used to convert [deprecated repeated columns] (not in a list), into their arrow representation
67    ///
68    /// [deprecated repeated columns]: https://github.com/apache/parquet-format/blob/9fd57b59e0ce1a82a69237dcf8977d3e72a2965d/LogicalTypes.md?plain=1#L649-L650
69    fn into_list_with_arrow_list_hint(
70        self,
71        parquet_field_type: &Type,
72        list_data_type: Option<DataType>,
73    ) -> Result<Self, ParquetError> {
74        let arrow_field = match &list_data_type {
75            Some(
76                DataType::List(field_hint)
77                | DataType::LargeList(field_hint)
78                | DataType::FixedSizeList(field_hint, _),
79            ) => Some(field_hint.as_ref()),
80            Some(_) => {
81                return Err(general_err!(
82                    "Internal error: should be validated earlier that list_data_type is only a type of list"
83                ));
84            }
85            None => None,
86        };
87
88        let arrow_field = convert_field(
89            parquet_field_type,
90            &self,
91            arrow_field,
92            // Only add the field id to the list and not to the element
93            false,
94        )?
95        .with_nullable(false);
96
97        Ok(ParquetField {
98            rep_level: self.rep_level,
99            def_level: self.def_level,
100            nullable: false,
101            arrow_type: match list_data_type {
102                Some(DataType::List(_)) => DataType::List(Arc::new(arrow_field)),
103                Some(DataType::LargeList(_)) => DataType::LargeList(Arc::new(arrow_field)),
104                Some(DataType::FixedSizeList(_, len)) => {
105                    DataType::FixedSizeList(Arc::new(arrow_field), len)
106                }
107                _ => DataType::List(Arc::new(arrow_field)),
108            },
109            field_type: ParquetFieldType::Group {
110                children: vec![self],
111            },
112        })
113    }
114
115    /// Returns a list of [`ParquetField`] children if this is a group type
116    pub fn children(&self) -> Option<&[Self]> {
117        match &self.field_type {
118            ParquetFieldType::Primitive { .. } => None,
119            ParquetFieldType::Group { children } => Some(children),
120            ParquetFieldType::Virtual(_) => None,
121        }
122    }
123}
124
125/// Types of virtual columns that can be computed at read time
126#[derive(Debug, Clone, Copy, PartialEq)]
127pub enum VirtualColumnType {
128    /// Row number within the file
129    RowNumber,
130    /// Row group index
131    RowGroupIndex,
132}
133
134#[derive(Debug, Clone)]
135pub enum ParquetFieldType {
136    Primitive {
137        /// The index of the column in parquet
138        col_idx: usize,
139        /// The type of the column in parquet
140        primitive_type: TypePtr,
141    },
142    Group {
143        children: Vec<ParquetField>,
144    },
145    /// Virtual column that doesn't exist in the parquet file
146    /// but is computed at read time (e.g., row_number)
147    Virtual(VirtualColumnType),
148}
149
150/// Encodes the context of the parent of the field currently under consideration
151struct VisitorContext {
152    rep_level: i16,
153    def_level: i16,
154    /// An optional [`DataType`] sourced from the embedded arrow schema
155    data_type: Option<DataType>,
156
157    /// Whether to treat repeated types as list from arrow types
158    /// when true, if data_type provided it should be DataType::List() (or other list type)
159    /// and the list field data type would be treated as the hint for the parquet type
160    ///
161    /// when false, if data_type provided it will be treated as the hint without unwrapping
162    ///
163    /// This is for supporting [deprecated parquet list representation][1]
164    ///
165    /// [1]: https://github.com/apache/parquet-format/blob/38818fa0e7efd54b535001a4448030a40619c2a3/LogicalTypes.md?plain=1#L718-L806
166    treat_repeated_as_list_arrow_hint: bool,
167}
168
169impl VisitorContext {
170    /// Compute the resulting definition level, repetition level and nullability
171    /// for a child field with the given [`Repetition`]
172    fn levels(&self, repetition: Repetition) -> (i16, i16, bool) {
173        match repetition {
174            Repetition::OPTIONAL => (self.def_level + 1, self.rep_level, true),
175            Repetition::REQUIRED => (self.def_level, self.rep_level, false),
176            Repetition::REPEATED => (self.def_level + 1, self.rep_level + 1, false),
177        }
178    }
179}
180
181/// Walks the parquet schema in a depth-first fashion in order to map it to arrow data structures
182///
183/// See [Logical Types] for more information on the conversion algorithm
184///
185/// [Logical Types]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md
186struct Visitor {
187    /// The column index of the next leaf column
188    next_col_idx: usize,
189
190    /// Mask of columns to include
191    mask: ProjectionMask,
192}
193
194impl Visitor {
195    fn visit_primitive(
196        &mut self,
197        primitive_type: &TypePtr,
198        context: VisitorContext,
199    ) -> Result<Option<ParquetField>> {
200        let col_idx = self.next_col_idx;
201        self.next_col_idx += 1;
202
203        if !self.mask.leaf_included(col_idx) {
204            return Ok(None);
205        }
206
207        let repetition = get_repetition(primitive_type);
208        let (def_level, rep_level, nullable) = context.levels(repetition);
209
210        let primitive_arrow_data_type = match repetition {
211            Repetition::REPEATED if context.treat_repeated_as_list_arrow_hint => {
212                let arrow_field = match &context.data_type {
213                    Some(DataType::List(f)) => Some(f.as_ref()),
214                    Some(DataType::LargeList(f)) => Some(f.as_ref()),
215                    Some(DataType::FixedSizeList(f, _)) => Some(f.as_ref()),
216                    Some(d) => {
217                        return Err(arrow_err!(
218                            "incompatible arrow schema, expected list got {} for repeated primitive field",
219                            d
220                        ));
221                    }
222                    None => None,
223                };
224
225                arrow_field.map(|f| f.data_type().clone())
226            }
227            _ => context.data_type.clone(),
228        };
229
230        let arrow_type = convert_primitive(primitive_type, primitive_arrow_data_type)?;
231
232        let primitive_field = ParquetField {
233            rep_level,
234            def_level,
235            nullable,
236            arrow_type,
237            field_type: ParquetFieldType::Primitive {
238                primitive_type: primitive_type.clone(),
239                col_idx,
240            },
241        };
242
243        Ok(Some(match repetition {
244            Repetition::REPEATED if context.treat_repeated_as_list_arrow_hint => {
245                primitive_field.into_list_with_arrow_list_hint(primitive_type, context.data_type)?
246            }
247            Repetition::REPEATED => primitive_field.into_list(primitive_type.name()),
248            _ => primitive_field,
249        }))
250    }
251
252    fn visit_struct(
253        &mut self,
254        struct_type: &TypePtr,
255        context: VisitorContext,
256    ) -> Result<Option<ParquetField>> {
257        // The root type will not have a repetition level
258        let repetition = get_repetition(struct_type);
259        let (def_level, rep_level, nullable) = context.levels(repetition);
260
261        let parquet_fields = struct_type.get_fields();
262
263        // Extract any arrow fields from the hints
264        let arrow_struct = match repetition {
265            Repetition::REPEATED if context.treat_repeated_as_list_arrow_hint => {
266                let arrow_field = match &context.data_type {
267                    Some(DataType::List(f)) => Some(f.as_ref()),
268                    Some(DataType::LargeList(f)) => Some(f.as_ref()),
269                    Some(DataType::FixedSizeList(f, _)) => Some(f.as_ref()),
270                    Some(d) => {
271                        return Err(arrow_err!(
272                            "incompatible arrow schema, expected list got {} for repeated struct field",
273                            d
274                        ));
275                    }
276                    None => None,
277                };
278
279                arrow_field.map(|f| f.data_type())
280            }
281            _ => context.data_type.as_ref(),
282        };
283
284        let arrow_fields = match &arrow_struct {
285            Some(DataType::Struct(fields)) => {
286                if fields.len() != parquet_fields.len() {
287                    return Err(arrow_err!(
288                        "incompatible arrow schema, expected {} struct fields got {}",
289                        parquet_fields.len(),
290                        fields.len()
291                    ));
292                }
293                Some(fields)
294            }
295            Some(d) => {
296                return Err(arrow_err!(
297                    "incompatible arrow schema, expected struct got {}",
298                    d
299                ));
300            }
301            None => None,
302        };
303
304        let mut child_fields = SchemaBuilder::with_capacity(parquet_fields.len());
305        let mut children = Vec::with_capacity(parquet_fields.len());
306
307        // Perform a DFS of children
308        for (idx, parquet_field) in parquet_fields.iter().enumerate() {
309            let data_type = match arrow_fields {
310                Some(fields) => {
311                    let field = &fields[idx];
312                    if field.name() != parquet_field.name() {
313                        return Err(arrow_err!(
314                            "incompatible arrow schema, expected field named {} got {}",
315                            parquet_field.name(),
316                            field.name()
317                        ));
318                    }
319                    Some(field.data_type().clone())
320                }
321                None => None,
322            };
323
324            let arrow_field = arrow_fields.map(|x| &*x[idx]);
325            let child_ctx = VisitorContext {
326                rep_level,
327                def_level,
328                data_type,
329
330                // Always true: each child is independently responsible for its own
331                // repeated-to-list conversion. The parent's flag may be false when
332                // this struct's own repetition is consumed by an outer visit_list
333                // backward-compat path, but that only applies to the struct itself,
334                // not its children. A repeated child's arrow hint will be List<...>
335                // and needs to be unwrapped accordingly.
336                treat_repeated_as_list_arrow_hint: true,
337            };
338
339            if let Some(child) = self.dispatch(parquet_field, child_ctx)? {
340                // The child type returned may be different from what is encoded in the arrow
341                // schema in the event of a mismatch or a projection
342                child_fields.push(convert_field(parquet_field, &child, arrow_field, true)?);
343                children.push(child);
344            }
345        }
346
347        if children.is_empty() {
348            return Ok(None);
349        }
350
351        let struct_field = ParquetField {
352            rep_level,
353            def_level,
354            nullable,
355            arrow_type: DataType::Struct(child_fields.finish().fields),
356            field_type: ParquetFieldType::Group { children },
357        };
358
359        Ok(Some(match repetition {
360            Repetition::REPEATED if context.treat_repeated_as_list_arrow_hint => {
361                struct_field.into_list_with_arrow_list_hint(struct_type, context.data_type)?
362            }
363            Repetition::REPEATED => struct_field.into_list(struct_type.name()),
364            _ => struct_field,
365        }))
366    }
367
368    fn visit_map(
369        &mut self,
370        map_type: &TypePtr,
371        context: VisitorContext,
372    ) -> Result<Option<ParquetField>> {
373        let rep_level = context.rep_level + 1;
374        let (def_level, nullable) = match get_repetition(map_type) {
375            Repetition::REQUIRED => (context.def_level + 1, false),
376            Repetition::OPTIONAL => (context.def_level + 2, true),
377            Repetition::REPEATED => return Err(arrow_err!("Map cannot be repeated")),
378        };
379
380        if map_type.get_fields().len() != 1 {
381            return Err(arrow_err!(
382                "Map field must have exactly one key_value child, found {}",
383                map_type.get_fields().len()
384            ));
385        }
386
387        // Add map entry (key_value) to context
388        let map_key_value = &map_type.get_fields()[0];
389        if map_key_value.get_basic_info().repetition() != Repetition::REPEATED {
390            return Err(arrow_err!("Child of map field must be repeated"));
391        }
392
393        // According to the specification the values are optional (#1642).
394        // In this case, return the keys as a list.
395        if map_key_value.get_fields().len() == 1 {
396            return self.visit_list(map_type, context);
397        }
398
399        if map_key_value.get_fields().len() != 2 {
400            return Err(arrow_err!(
401                "Child of map field must have two children, found {}",
402                map_key_value.get_fields().len()
403            ));
404        }
405
406        // Get key and value, and create context for each
407        let map_key = &map_key_value.get_fields()[0];
408        let map_value = &map_key_value.get_fields()[1];
409
410        match map_key.get_basic_info().repetition() {
411            Repetition::REPEATED => {
412                return Err(arrow_err!("Map keys cannot be repeated"));
413            }
414            Repetition::REQUIRED | Repetition::OPTIONAL => {
415                // Relaxed check for having repetition REQUIRED as there exists
416                // parquet writers and files that do not conform to this standard.
417                // This allows us to consume a broader range of existing files even
418                // if they are out of spec.
419            }
420        }
421
422        if map_value.get_basic_info().repetition() == Repetition::REPEATED {
423            return Err(arrow_err!("Map values cannot be repeated"));
424        }
425
426        // Extract the arrow fields
427        let (arrow_map, arrow_key, arrow_value, sorted) = match &context.data_type {
428            Some(DataType::Map(field, sorted)) => match field.data_type() {
429                DataType::Struct(fields) => {
430                    if fields.len() != 2 {
431                        return Err(arrow_err!(
432                            "Map data type should contain struct with two children, got {}",
433                            fields.len()
434                        ));
435                    }
436
437                    (Some(field), Some(&*fields[0]), Some(&*fields[1]), *sorted)
438                }
439                d => {
440                    return Err(arrow_err!("Map data type should contain struct got {}", d));
441                }
442            },
443            Some(d) => {
444                return Err(arrow_err!(
445                    "incompatible arrow schema, expected map got {}",
446                    d
447                ));
448            }
449            None => (None, None, None, false),
450        };
451
452        let maybe_key = {
453            let context = VisitorContext {
454                rep_level,
455                def_level,
456                data_type: arrow_key.map(|x| x.data_type().clone()),
457                // Key is not repeated
458                treat_repeated_as_list_arrow_hint: false,
459            };
460
461            self.dispatch(map_key, context)?
462        };
463
464        let maybe_value = {
465            let context = VisitorContext {
466                rep_level,
467                def_level,
468                data_type: arrow_value.map(|x| x.data_type().clone()),
469                // Value type can be repeated
470                treat_repeated_as_list_arrow_hint: true,
471            };
472
473            self.dispatch(map_value, context)?
474        };
475
476        // Need both columns to be projected
477        match (maybe_key, maybe_value) {
478            (Some(key), Some(value)) => {
479                let key_field = Arc::new(
480                    convert_field(map_key, &key, arrow_key, true)?
481                        // The key is always non-nullable (#5630)
482                        .with_nullable(false),
483                );
484                let value_field = Arc::new(convert_field(map_value, &value, arrow_value, true)?);
485                let field_metadata = arrow_map
486                    .map(|field| field.metadata().clone())
487                    .unwrap_or_default();
488
489                let map_field = Field::new_struct(
490                    map_key_value.name(),
491                    [key_field, value_field],
492                    false, // The inner map field is always non-nullable (#1697)
493                )
494                .with_metadata(field_metadata);
495
496                Ok(Some(ParquetField {
497                    rep_level,
498                    def_level,
499                    nullable,
500                    arrow_type: DataType::Map(Arc::new(map_field), sorted),
501                    field_type: ParquetFieldType::Group {
502                        children: vec![key, value],
503                    },
504                }))
505            }
506            _ => Ok(None),
507        }
508    }
509
510    fn visit_list(
511        &mut self,
512        list_type: &TypePtr,
513        context: VisitorContext,
514    ) -> Result<Option<ParquetField>> {
515        if list_type.is_primitive() {
516            return Err(arrow_err!(
517                "{:?} is a list type and can't be processed as primitive.",
518                list_type
519            ));
520        }
521
522        let fields = list_type.get_fields();
523        if fields.len() != 1 {
524            return Err(arrow_err!(
525                "list type must have a single child, found {}",
526                fields.len()
527            ));
528        }
529
530        let repeated_field = &fields[0];
531        if get_repetition(repeated_field) != Repetition::REPEATED {
532            return Err(arrow_err!("List child must be repeated"));
533        }
534
535        // If the list is nullable
536        let (def_level, nullable) = match list_type.get_basic_info().repetition() {
537            Repetition::REQUIRED => (context.def_level, false),
538            Repetition::OPTIONAL => (context.def_level + 1, true),
539            Repetition::REPEATED => return Err(arrow_err!("List type cannot be repeated")),
540        };
541
542        let arrow_field = match &context.data_type {
543            Some(DataType::List(f)) => Some(f.as_ref()),
544            Some(DataType::LargeList(f)) => Some(f.as_ref()),
545            Some(DataType::FixedSizeList(f, _)) => Some(f.as_ref()),
546            Some(DataType::ListView(f)) => Some(f.as_ref()),
547            Some(DataType::LargeListView(f)) => Some(f.as_ref()),
548            Some(d) => {
549                return Err(arrow_err!(
550                    "incompatible arrow schema, expected list got {}",
551                    d
552                ));
553            }
554            None => None,
555        };
556
557        if repeated_field.is_primitive() {
558            // If the repeated field is not a group, then its type is the element type and elements are required.
559            //
560            // required/optional group my_list (LIST) {
561            //   repeated int32 element;
562            // }
563            //
564            let context = VisitorContext {
565                rep_level: context.rep_level,
566                def_level,
567                data_type: arrow_field.map(|f| f.data_type().clone()),
568                treat_repeated_as_list_arrow_hint: false,
569            };
570
571            return match self.visit_primitive(repeated_field, context) {
572                Ok(Some(mut field)) => {
573                    // visit_primitive will infer a non-nullable list, update if necessary
574                    field.nullable = nullable;
575                    Ok(Some(field))
576                }
577                r => r,
578            };
579        }
580
581        // test to see if the repeated field is a struct or one-tuple
582        let items = repeated_field.get_fields();
583        if items.len() != 1
584            || (!repeated_field.is_list()
585                && !repeated_field.has_single_repeated_child()
586                && (repeated_field.name() == "array"
587                    || repeated_field.name() == format!("{}_tuple", list_type.name())))
588        {
589            // If the repeated field is a group with multiple fields, then its type is the element
590            // type and elements are required.
591            //
592            // If the repeated field is a group with one field and is named either array or uses
593            // the LIST-annotated group's name with _tuple appended then the repeated type is the
594            // element type and elements are required. But this rule only applies if the
595            // repeated field is not annotated, and the single child field is not `repeated`.
596            let context = VisitorContext {
597                rep_level: context.rep_level,
598                def_level,
599                data_type: arrow_field.map(|f| f.data_type().clone()),
600                treat_repeated_as_list_arrow_hint: false,
601            };
602
603            return match self.visit_struct(repeated_field, context) {
604                Ok(Some(mut field)) => {
605                    field.nullable = nullable;
606                    Ok(Some(field))
607                }
608                r => r,
609            };
610        }
611
612        // Regular list handling logic
613        let item_type = &items[0];
614        let rep_level = context.rep_level + 1;
615        let def_level = def_level + 1;
616
617        let new_context = VisitorContext {
618            def_level,
619            rep_level,
620            data_type: arrow_field.map(|f| f.data_type().clone()),
621            treat_repeated_as_list_arrow_hint: true,
622        };
623
624        match self.dispatch(item_type, new_context) {
625            Ok(Some(item)) => {
626                let item_field = Arc::new(convert_field(item_type, &item, arrow_field, true)?);
627
628                // Use arrow type as hint for index size
629                let arrow_type = match context.data_type {
630                    Some(DataType::LargeList(_)) => DataType::LargeList(item_field),
631                    Some(DataType::FixedSizeList(_, len)) => {
632                        DataType::FixedSizeList(item_field, len)
633                    }
634                    Some(DataType::ListView(_)) => DataType::ListView(item_field),
635                    Some(DataType::LargeListView(_)) => DataType::LargeListView(item_field),
636                    _ => DataType::List(item_field),
637                };
638
639                Ok(Some(ParquetField {
640                    rep_level,
641                    def_level,
642                    nullable,
643                    arrow_type,
644                    field_type: ParquetFieldType::Group {
645                        children: vec![item],
646                    },
647                }))
648            }
649            r => r,
650        }
651    }
652
653    fn dispatch(
654        &mut self,
655        cur_type: &TypePtr,
656        context: VisitorContext,
657    ) -> Result<Option<ParquetField>> {
658        if cur_type.is_primitive() {
659            self.visit_primitive(cur_type, context)
660        } else {
661            match cur_type.get_basic_info().converted_type() {
662                ConvertedType::LIST => self.visit_list(cur_type, context),
663                ConvertedType::MAP | ConvertedType::MAP_KEY_VALUE => {
664                    self.visit_map(cur_type, context)
665                }
666                _ => self.visit_struct(cur_type, context),
667            }
668        }
669    }
670}
671
672/// Converts a virtual Arrow [`Field`] to a [`ParquetField`]
673///
674/// Virtual fields don't correspond to any data in the parquet file,
675/// but are computed at read time (e.g., row_number)
676///
677/// The levels are computed based on the parent context:
678/// - If nullable: def_level = parent_def_level + 1
679/// - If required: def_level = parent_def_level
680/// - rep_level = parent_rep_level (virtual fields are not repeated)
681pub(super) fn convert_virtual_field(
682    arrow_field: &Field,
683    parent_rep_level: i16,
684    parent_def_level: i16,
685) -> Result<ParquetField> {
686    let nullable = arrow_field.is_nullable();
687    let def_level = if nullable {
688        parent_def_level + 1
689    } else {
690        parent_def_level
691    };
692
693    // Determine the virtual column type based on the extension type name
694    let extension_name = arrow_field.extension_type_name().ok_or_else(|| {
695        ParquetError::ArrowError(format!(
696            "virtual column field '{}' must have an extension type",
697            arrow_field.name()
698        ))
699    })?;
700
701    let virtual_type = match extension_name {
702        RowNumber::NAME => VirtualColumnType::RowNumber,
703        RowGroupIndex::NAME => VirtualColumnType::RowGroupIndex,
704        _ => {
705            return Err(ParquetError::ArrowError(format!(
706                "unsupported virtual column type '{}' for field '{}'",
707                extension_name,
708                arrow_field.name()
709            )));
710        }
711    };
712
713    Ok(ParquetField {
714        rep_level: parent_rep_level,
715        def_level,
716        nullable,
717        arrow_type: arrow_field.data_type().clone(),
718        field_type: ParquetFieldType::Virtual(virtual_type),
719    })
720}
721
722/// Computes the Arrow [`Field`] for a child column
723///
724/// The resulting Arrow [`Field`] will have the type dictated by the Parquet `field`, a name
725/// dictated by the `parquet_type`, and any metadata from `arrow_hint`
726fn convert_field(
727    parquet_type: &Type,
728    field: &ParquetField,
729    arrow_hint: Option<&Field>,
730    add_field_id: bool,
731) -> Result<Field, ParquetError> {
732    let name = parquet_type.name();
733    let data_type = field.arrow_type.clone();
734    let nullable = field.nullable;
735
736    match arrow_hint {
737        Some(hint) => {
738            // If the inferred type is a dictionary, preserve dictionary metadata
739            #[expect(deprecated)]
740            let field = match (&data_type, hint.dict_id(), hint.dict_is_ordered()) {
741                (DataType::Dictionary(_, _), Some(id), Some(ordered)) =>
742                {
743                    #[expect(deprecated)]
744                    Field::new_dict(name, data_type, nullable, id, ordered)
745                }
746                _ => Field::new(name, data_type, nullable),
747            };
748
749            Ok(field.with_metadata(hint.metadata().clone()))
750        }
751        None => {
752            let mut ret = Field::new(name, data_type, nullable);
753            let basic_info = parquet_type.get_basic_info();
754            if add_field_id && basic_info.has_id() {
755                let mut meta = HashMap::with_capacity(1);
756                meta.insert(
757                    PARQUET_FIELD_ID_META_KEY.to_string(),
758                    basic_info.id().to_string(),
759                );
760                ret.set_metadata(meta);
761            }
762            try_add_extension_type(ret, parquet_type)
763        }
764    }
765}
766
767/// Computes the [`ParquetField`] for the provided [`SchemaDescriptor`] with `leaf_columns` listing
768/// the indexes of leaf columns to project, and `embedded_arrow_schema` the optional
769/// [`Fields`] embedded in the parquet metadata
770///
771/// Note: This does not support out of order column projection
772pub fn convert_schema(
773    schema: &SchemaDescriptor,
774    mask: ProjectionMask,
775    embedded_arrow_schema: Option<&Fields>,
776) -> Result<Option<ParquetField>> {
777    let mut visitor = Visitor {
778        next_col_idx: 0,
779        mask,
780    };
781
782    let context = VisitorContext {
783        rep_level: 0,
784        def_level: 0,
785        data_type: embedded_arrow_schema.map(|fields| DataType::Struct(fields.clone())),
786        treat_repeated_as_list_arrow_hint: true,
787    };
788
789    visitor.dispatch(&schema.root_schema_ptr(), context)
790}
791
792/// Computes the [`ParquetField`] for the provided `parquet_type`
793pub fn convert_type(parquet_type: &TypePtr) -> Result<ParquetField> {
794    let mut visitor = Visitor {
795        next_col_idx: 0,
796        mask: ProjectionMask::all(),
797    };
798
799    let context = VisitorContext {
800        rep_level: 0,
801        def_level: 0,
802        data_type: None,
803        // We might be inside list
804        treat_repeated_as_list_arrow_hint: false,
805    };
806
807    Ok(visitor.dispatch(parquet_type, context)?.unwrap())
808}
809
810#[cfg(test)]
811mod tests {
812    use crate::arrow::schema::complex::convert_schema;
813    use crate::arrow::{PARQUET_FIELD_ID_META_KEY, ProjectionMask};
814    use crate::schema::parser::parse_message_type;
815    use crate::schema::types::SchemaDescriptor;
816    use arrow_schema::{DataType, Field, Fields};
817    use std::sync::Arc;
818
819    trait WithFieldId {
820        fn with_field_id(self, id: i32) -> Self;
821    }
822    impl WithFieldId for arrow_schema::Field {
823        fn with_field_id(self, id: i32) -> Self {
824            let mut metadata = self.metadata().clone();
825            metadata.insert(PARQUET_FIELD_ID_META_KEY.to_string(), id.to_string());
826            self.with_metadata(metadata)
827        }
828    }
829
830    fn test_roundtrip(message_type: &str) -> crate::errors::Result<()> {
831        let parsed_input_schema = Arc::new(parse_message_type(message_type)?);
832        let schema = SchemaDescriptor::new(parsed_input_schema);
833
834        let converted = convert_schema(&schema, ProjectionMask::all(), None)?.unwrap();
835
836        let DataType::Struct(schema_fields) = &converted.arrow_type else {
837            panic!("Expected struct from convert_schema");
838        };
839
840        // Should be able to convert the same thing
841        let converted_again =
842            convert_schema(&schema, ProjectionMask::all(), Some(schema_fields))?.unwrap();
843
844        // Assert that we changed to Utf8
845        assert_eq!(converted_again.arrow_type, converted.arrow_type);
846
847        Ok(())
848    }
849
850    fn test_expected_type(
851        message_type: &str,
852        expected_fields: Fields,
853    ) -> crate::errors::Result<()> {
854        test_roundtrip(message_type)?;
855
856        let parsed_input_schema = Arc::new(parse_message_type(message_type)?);
857        let schema = SchemaDescriptor::new(parsed_input_schema);
858
859        let converted = convert_schema(&schema, ProjectionMask::all(), None)?.unwrap();
860
861        let DataType::Struct(schema_fields) = &converted.arrow_type else {
862            panic!("Expected struct from convert_schema");
863        };
864
865        assert_eq!(schema_fields, &expected_fields);
866
867        Ok(())
868    }
869
870    /// Taken from the example in [Parquet Format - Nested Types - Lists - Backward-compatibility rules](https://github.com/apache/parquet-format/blob/9fd57b59e0ce1a82a69237dcf8977d3e72a2965d/LogicalTypes.md?plain=1#L766-L769)
871    #[test]
872    fn basic_backward_compatible_list_1() -> crate::errors::Result<()> {
873        test_expected_type(
874            "
875            message schema {
876                optional group my_list (LIST) {
877                  repeated int32 element;
878                }
879            }
880        ",
881            Fields::from(vec![
882                // Rule 1: List<Integer> (nullable list, non-null elements)
883                Field::new(
884                    "my_list",
885                    DataType::List(Arc::new(Field::new("element", DataType::Int32, false))),
886                    true,
887                ),
888            ]),
889        )
890    }
891
892    /// Taken from the example in [Parquet Format - Nested Types - Lists - Backward-compatibility rules](https://github.com/apache/parquet-format/blob/9fd57b59e0ce1a82a69237dcf8977d3e72a2965d/LogicalTypes.md?plain=1#L771-L777)
893    #[test]
894    fn basic_backward_compatible_list_2() -> crate::errors::Result<()> {
895        test_expected_type(
896            "
897            message schema {
898              optional group my_list (LIST) {
899                  repeated group element {
900                    required binary str (STRING);
901                    required int32 num;
902                  }
903              }
904            }
905        ",
906            Fields::from(vec![
907                // Rule 2: List<Tuple<String, Integer>> (nullable list, non-null elements)
908                Field::new(
909                    "my_list",
910                    DataType::List(Arc::new(Field::new(
911                        "element",
912                        DataType::Struct(Fields::from(vec![
913                            Field::new("str", DataType::Utf8, false),
914                            Field::new("num", DataType::Int32, false),
915                        ])),
916                        false,
917                    ))),
918                    true,
919                ),
920            ]),
921        )
922    }
923
924    /// Taken from the example in [Parquet Format - Nested Types - Lists - Backward-compatibility rules](https://github.com/apache/parquet-format/blob/9fd57b59e0ce1a82a69237dcf8977d3e72a2965d/LogicalTypes.md?plain=1#L779-L784)
925    #[test]
926    fn basic_backward_compatible_list_3() -> crate::errors::Result<()> {
927        test_expected_type(
928            "
929            message schema {
930              optional group my_list (LIST) {
931                  repeated group array (LIST) {
932                    repeated int32 array;
933                  }
934              }
935            }
936        ",
937            Fields::from(vec![
938                // Rule 3: List<List<Integer>> (nullable outer list, non-null elements)
939                Field::new(
940                    "my_list",
941                    DataType::List(Arc::new(Field::new(
942                        "array",
943                        DataType::List(Arc::new(Field::new("array", DataType::Int32, false))),
944                        false,
945                    ))),
946                    true,
947                ),
948            ]),
949        )
950    }
951
952    /// Taken from the example in [Parquet Format - Nested Types - Lists - Backward-compatibility rules](https://github.com/apache/parquet-format/blob/9fd57b59e0ce1a82a69237dcf8977d3e72a2965d/LogicalTypes.md?plain=1#L786-L791)
953    #[test]
954    fn basic_backward_compatible_list_4_1() -> crate::errors::Result<()> {
955        test_expected_type(
956            "
957            message schema {
958              optional group my_list (LIST) {
959                  repeated group array {
960                    required binary str (STRING);
961                  }
962              }
963            }
964        ",
965            Fields::from(vec![
966                // Rule 4: List<OneTuple<String>> (nullable list, non-null elements)
967                Field::new(
968                    "my_list",
969                    DataType::List(Arc::new(Field::new(
970                        "array",
971                        DataType::Struct(Fields::from(vec![Field::new(
972                            "str",
973                            DataType::Utf8,
974                            false,
975                        )])),
976                        false,
977                    ))),
978                    true,
979                ),
980            ]),
981        )
982    }
983
984    /// Taken from the example in [Parquet Format - Nested Types - Lists - Backward-compatibility rules](https://github.com/apache/parquet-format/blob/9fd57b59e0ce1a82a69237dcf8977d3e72a2965d/LogicalTypes.md?plain=1#L793-L798)
985    #[test]
986    fn basic_backward_compatible_list_4_2() -> crate::errors::Result<()> {
987        test_expected_type(
988            "
989            message schema {
990                optional group my_list (LIST) {
991                    repeated group my_list_tuple {
992                        required binary str (STRING);
993                    }
994                }
995            }
996        ",
997            Fields::from(vec![
998                // Rule 4: List<OneTuple<String>> (nullable list, non-null elements)
999                Field::new(
1000                    "my_list",
1001                    DataType::List(Arc::new(Field::new(
1002                        "my_list_tuple",
1003                        DataType::Struct(Fields::from(vec![Field::new(
1004                            "str",
1005                            DataType::Utf8,
1006                            false,
1007                        )])),
1008                        false,
1009                    ))),
1010                    true,
1011                ),
1012            ]),
1013        )
1014    }
1015
1016    /// Taken from the example in [Parquet Format - Nested Types - Lists - Backward-compatibility rules](https://github.com/apache/parquet-format/blob/9fd57b59e0ce1a82a69237dcf8977d3e72a2965d/LogicalTypes.md?plain=1#L800-L805)
1017    #[test]
1018    fn basic_backward_compatible_list_5() -> crate::errors::Result<()> {
1019        test_expected_type(
1020            "
1021            message schema {
1022                optional group my_list (LIST) {
1023                    repeated group element {
1024                        optional binary str (STRING);
1025                    }
1026                }
1027            }
1028        ",
1029            Fields::from(vec![
1030                // Rule 5: List<String>  (nullable list, nullable elements)
1031                Field::new(
1032                    "my_list",
1033                    DataType::List(Arc::new(Field::new("str", DataType::Utf8, true))),
1034                    true,
1035                ),
1036            ]),
1037        )
1038    }
1039
1040    #[test]
1041    fn basic_backward_compatible_map_1() -> crate::errors::Result<()> {
1042        test_expected_type(
1043            "
1044            message schema {
1045                optional group my_map (MAP) {
1046                  repeated group map {
1047                    required binary str (STRING);
1048                    required int32 num;
1049                  }
1050                }
1051            }
1052        ",
1053            Fields::from(vec![
1054                // Map<String, Integer> (nullable map, non-null values)
1055                Field::new(
1056                    "my_map",
1057                    DataType::Map(
1058                        Arc::new(Field::new(
1059                            "map",
1060                            DataType::Struct(Fields::from(vec![
1061                                Field::new("str", DataType::Utf8, false),
1062                                Field::new("num", DataType::Int32, false),
1063                            ])),
1064                            false,
1065                        )),
1066                        false,
1067                    ),
1068                    true,
1069                ),
1070            ]),
1071        )
1072    }
1073
1074    #[test]
1075    fn basic_backward_compatible_map_2() -> crate::errors::Result<()> {
1076        test_expected_type(
1077            "
1078            message schema {
1079                optional group my_map (MAP_KEY_VALUE) {
1080                  repeated group map {
1081                    required binary key (STRING);
1082                    optional int32 value;
1083                  }
1084                }
1085            }
1086        ",
1087            Fields::from(vec![
1088                // Map<String, Integer> (nullable map, nullable values)
1089                Field::new(
1090                    "my_map",
1091                    DataType::Map(
1092                        Arc::new(Field::new(
1093                            "map",
1094                            DataType::Struct(Fields::from(vec![
1095                                Field::new("key", DataType::Utf8, false),
1096                                Field::new("value", DataType::Int32, true),
1097                            ])),
1098                            false,
1099                        )),
1100                        false,
1101                    ),
1102                    true,
1103                ),
1104            ]),
1105        )
1106    }
1107
1108    #[test]
1109    fn convert_schema_with_nested_list_repeated_primitive() -> crate::errors::Result<()> {
1110        test_roundtrip(
1111            "
1112            message schema {
1113                optional group f1 (LIST) {
1114                    repeated group element {
1115                        repeated int32 element;
1116                    }
1117                }
1118            }
1119        ",
1120        )
1121    }
1122
1123    #[test]
1124    fn convert_schema_with_repeated_primitive_keep_field_id() -> crate::errors::Result<()> {
1125        let message_type = "
1126    message schema {
1127      repeated BYTE_ARRAY col_1 = 1;
1128    }
1129    ";
1130
1131        let parsed_input_schema = Arc::new(parse_message_type(message_type)?);
1132        let schema = SchemaDescriptor::new(parsed_input_schema);
1133
1134        let converted = convert_schema(&schema, ProjectionMask::all(), None)?.unwrap();
1135
1136        let DataType::Struct(schema_fields) = &converted.arrow_type else {
1137            panic!("Expected struct from convert_schema");
1138        };
1139
1140        assert_eq!(schema_fields.len(), 1);
1141
1142        let expected_schema = DataType::Struct(Fields::from(vec![Arc::new(
1143            arrow_schema::Field::new(
1144                "col_1",
1145                DataType::List(Arc::new(
1146                    // No metadata on inner field
1147                    arrow_schema::Field::new("col_1", DataType::Binary, false),
1148                )),
1149                false,
1150            )
1151            // add the field id to the outer list
1152            .with_field_id(1),
1153        )]));
1154
1155        assert_eq!(converted.arrow_type, expected_schema);
1156
1157        Ok(())
1158    }
1159
1160    #[test]
1161    fn convert_schema_with_repeated_primitive_should_use_inferred_schema()
1162    -> crate::errors::Result<()> {
1163        let message_type = "
1164    message schema {
1165      repeated BYTE_ARRAY col_1 = 1;
1166    }
1167    ";
1168
1169        let parsed_input_schema = Arc::new(parse_message_type(message_type)?);
1170        let schema = SchemaDescriptor::new(parsed_input_schema);
1171
1172        let converted = convert_schema(&schema, ProjectionMask::all(), None)?.unwrap();
1173
1174        let DataType::Struct(schema_fields) = &converted.arrow_type else {
1175            panic!("Expected struct from convert_schema");
1176        };
1177
1178        assert_eq!(schema_fields.len(), 1);
1179
1180        let expected_schema = DataType::Struct(Fields::from(vec![Arc::new(
1181            arrow_schema::Field::new(
1182                "col_1",
1183                DataType::List(Arc::new(arrow_schema::Field::new(
1184                    "col_1",
1185                    DataType::Binary,
1186                    false,
1187                ))),
1188                false,
1189            )
1190            .with_metadata(schema_fields[0].metadata().clone()),
1191        )]));
1192
1193        assert_eq!(converted.arrow_type, expected_schema);
1194
1195        let utf8_instead_of_binary = Fields::from(vec![Arc::new(
1196            arrow_schema::Field::new(
1197                "col_1",
1198                DataType::List(Arc::new(arrow_schema::Field::new(
1199                    "col_1",
1200                    DataType::Utf8,
1201                    false,
1202                ))),
1203                false,
1204            )
1205            .with_metadata(schema_fields[0].metadata().clone()),
1206        )]);
1207
1208        // Should be able to convert the same thing
1209        let converted_again = convert_schema(
1210            &schema,
1211            ProjectionMask::all(),
1212            Some(&utf8_instead_of_binary),
1213        )?
1214        .unwrap();
1215
1216        // Assert that we changed to Utf8
1217        assert_eq!(
1218            converted_again.arrow_type,
1219            DataType::Struct(utf8_instead_of_binary)
1220        );
1221
1222        Ok(())
1223    }
1224
1225    #[test]
1226    fn convert_schema_with_repeated_primitive_should_use_inferred_schema_for_list_as_well()
1227    -> crate::errors::Result<()> {
1228        let message_type = "
1229    message schema {
1230      repeated BYTE_ARRAY col_1 = 1;
1231    }
1232    ";
1233
1234        let parsed_input_schema = Arc::new(parse_message_type(message_type)?);
1235        let schema = SchemaDescriptor::new(parsed_input_schema);
1236
1237        let converted = convert_schema(&schema, ProjectionMask::all(), None)?.unwrap();
1238
1239        let DataType::Struct(schema_fields) = &converted.arrow_type else {
1240            panic!("Expected struct from convert_schema");
1241        };
1242
1243        assert_eq!(schema_fields.len(), 1);
1244
1245        let expected_schema = DataType::Struct(Fields::from(vec![Arc::new(
1246            arrow_schema::Field::new(
1247                "col_1",
1248                DataType::List(Arc::new(arrow_schema::Field::new(
1249                    "col_1",
1250                    DataType::Binary,
1251                    false,
1252                ))),
1253                false,
1254            )
1255            .with_metadata(schema_fields[0].metadata().clone()),
1256        )]));
1257
1258        assert_eq!(converted.arrow_type, expected_schema);
1259
1260        let utf8_instead_of_binary = Fields::from(vec![Arc::new(
1261            arrow_schema::Field::new(
1262                "col_1",
1263                // Inferring as LargeList instead of List
1264                DataType::LargeList(Arc::new(arrow_schema::Field::new(
1265                    "col_1",
1266                    DataType::Utf8,
1267                    false,
1268                ))),
1269                false,
1270            )
1271            .with_metadata(schema_fields[0].metadata().clone()),
1272        )]);
1273
1274        // Should be able to convert the same thing
1275        let converted_again = convert_schema(
1276            &schema,
1277            ProjectionMask::all(),
1278            Some(&utf8_instead_of_binary),
1279        )?
1280        .unwrap();
1281
1282        // Assert that we changed to Utf8
1283        assert_eq!(
1284            converted_again.arrow_type,
1285            DataType::Struct(utf8_instead_of_binary)
1286        );
1287
1288        Ok(())
1289    }
1290
1291    #[test]
1292    fn convert_schema_with_repeated_struct_and_inferred_schema() -> crate::errors::Result<()> {
1293        test_roundtrip(
1294            "
1295    message schema {
1296        repeated group my_col_1 = 1 {
1297          optional binary my_col_2 = 2;
1298          optional binary my_col_3 = 3;
1299          optional group my_col_4 = 4 {
1300            optional int64 my_col_5 = 5;
1301            optional int32 my_col_6 = 6;
1302          }
1303        }
1304    }
1305    ",
1306        )
1307    }
1308
1309    #[test]
1310    fn convert_schema_with_repeated_struct_and_inferred_schema_and_field_id()
1311    -> crate::errors::Result<()> {
1312        let message_type = "
1313    message schema {
1314        repeated group my_col_1 = 1 {
1315          optional binary my_col_2 = 2;
1316          optional binary my_col_3 = 3;
1317          optional group my_col_4 = 4 {
1318            optional int64 my_col_5 = 5;
1319            optional int32 my_col_6 = 6;
1320          }
1321        }
1322    }
1323    ";
1324
1325        let parsed_input_schema = Arc::new(parse_message_type(message_type)?);
1326        let schema = SchemaDescriptor::new(parsed_input_schema);
1327
1328        let converted = convert_schema(&schema, ProjectionMask::all(), None)?.unwrap();
1329
1330        let DataType::Struct(schema_fields) = &converted.arrow_type else {
1331            panic!("Expected struct from convert_schema");
1332        };
1333
1334        assert_eq!(schema_fields.len(), 1);
1335
1336        // Should be able to convert the same thing
1337        let converted_again =
1338            convert_schema(&schema, ProjectionMask::all(), Some(schema_fields))?.unwrap();
1339
1340        // Assert that we changed to Utf8
1341        assert_eq!(converted_again.arrow_type, converted.arrow_type);
1342
1343        Ok(())
1344    }
1345
1346    #[test]
1347    fn convert_schema_with_nested_repeated_struct_and_primitives() -> crate::errors::Result<()> {
1348        let message_type = "
1349message schema {
1350    repeated group my_col_1 = 1 {
1351        optional binary my_col_2 = 2;
1352        repeated BYTE_ARRAY my_col_3 = 3;
1353        repeated group my_col_4 = 4 {
1354            optional int64 my_col_5 = 5;
1355            repeated binary my_col_6 = 6;
1356        }
1357    }
1358}
1359";
1360
1361        let parsed_input_schema = Arc::new(parse_message_type(message_type)?);
1362        let schema = SchemaDescriptor::new(parsed_input_schema);
1363
1364        let converted = convert_schema(&schema, ProjectionMask::all(), None)?.unwrap();
1365
1366        let DataType::Struct(schema_fields) = &converted.arrow_type else {
1367            panic!("Expected struct from convert_schema");
1368        };
1369
1370        assert_eq!(schema_fields.len(), 1);
1371
1372        // Build expected schema
1373        let expected_schema = DataType::Struct(Fields::from(vec![Arc::new(
1374            arrow_schema::Field::new(
1375                "my_col_1",
1376                DataType::List(Arc::new(arrow_schema::Field::new(
1377                    "my_col_1",
1378                    DataType::Struct(Fields::from(vec![
1379                        Arc::new(
1380                            arrow_schema::Field::new("my_col_2", DataType::Binary, true)
1381                                .with_field_id(2),
1382                        ),
1383                        Arc::new(
1384                            arrow_schema::Field::new(
1385                                "my_col_3",
1386                                DataType::List(Arc::new(arrow_schema::Field::new(
1387                                    "my_col_3",
1388                                    DataType::Binary,
1389                                    false,
1390                                ))),
1391                                false,
1392                            )
1393                            // add the field id to the outer list
1394                            .with_field_id(3),
1395                        ),
1396                        Arc::new(
1397                            arrow_schema::Field::new(
1398                                "my_col_4",
1399                                DataType::List(Arc::new(arrow_schema::Field::new(
1400                                    "my_col_4",
1401                                    DataType::Struct(Fields::from(vec![
1402                                        Arc::new(
1403                                            arrow_schema::Field::new(
1404                                                "my_col_5",
1405                                                DataType::Int64,
1406                                                true,
1407                                            )
1408                                            // add the field id to the outer list
1409                                            .with_field_id(5),
1410                                        ),
1411                                        Arc::new(
1412                                            arrow_schema::Field::new(
1413                                                "my_col_6",
1414                                                DataType::List(Arc::new(arrow_schema::Field::new(
1415                                                    "my_col_6",
1416                                                    DataType::Binary,
1417                                                    false,
1418                                                ))),
1419                                                false,
1420                                            )
1421                                            // add the field id to the outer list
1422                                            .with_field_id(6),
1423                                        ),
1424                                    ])),
1425                                    false,
1426                                ))),
1427                                false,
1428                            )
1429                            // add the field id to the outer list
1430                            .with_field_id(4),
1431                        ),
1432                    ])),
1433                    false,
1434                ))),
1435                false,
1436            )
1437            // add the field id to the outer list
1438            .with_field_id(1),
1439        )]));
1440
1441        assert_eq!(converted.arrow_type, expected_schema);
1442
1443        // Test conversion with inferred schema
1444        let converted_again =
1445            convert_schema(&schema, ProjectionMask::all(), Some(schema_fields))?.unwrap();
1446
1447        assert_eq!(converted_again.arrow_type, converted.arrow_type);
1448
1449        // Test conversion with modified schema (change lists to either LargeList or FixedSizeList)
1450        // as well as changing Binary to Utf8 or BinaryView
1451        let modified_schema_fields = Fields::from(vec![Arc::new(
1452            arrow_schema::Field::new(
1453                "my_col_1",
1454                DataType::LargeList(Arc::new(arrow_schema::Field::new(
1455                    "my_col_1",
1456                    DataType::Struct(Fields::from(vec![
1457                        Arc::new(
1458                            arrow_schema::Field::new("my_col_2", DataType::LargeBinary, true)
1459                                .with_field_id(2),
1460                        ),
1461                        Arc::new(
1462                            arrow_schema::Field::new(
1463                                "my_col_3",
1464                                DataType::LargeList(Arc::new(arrow_schema::Field::new(
1465                                    "my_col_3",
1466                                    DataType::Utf8,
1467                                    false,
1468                                ))),
1469                                false,
1470                            )
1471                            // add the field id to the outer list
1472                            .with_field_id(3),
1473                        ),
1474                        Arc::new(
1475                            arrow_schema::Field::new(
1476                                "my_col_4",
1477                                DataType::FixedSizeList(
1478                                    Arc::new(arrow_schema::Field::new(
1479                                        "my_col_4",
1480                                        DataType::Struct(Fields::from(vec![
1481                                            Arc::new(
1482                                                arrow_schema::Field::new(
1483                                                    "my_col_5",
1484                                                    DataType::Int64,
1485                                                    true,
1486                                                )
1487                                                .with_field_id(5),
1488                                            ),
1489                                            Arc::new(
1490                                                arrow_schema::Field::new(
1491                                                    "my_col_6",
1492                                                    DataType::LargeList(Arc::new(
1493                                                        arrow_schema::Field::new(
1494                                                            "my_col_6",
1495                                                            DataType::BinaryView,
1496                                                            false,
1497                                                        ),
1498                                                    )),
1499                                                    false,
1500                                                )
1501                                                // add the field id to the outer list
1502                                                .with_field_id(6),
1503                                            ),
1504                                        ])),
1505                                        false,
1506                                    )),
1507                                    3,
1508                                ),
1509                                false,
1510                            )
1511                            // add the field id to the outer list
1512                            .with_field_id(4),
1513                        ),
1514                    ])),
1515                    false,
1516                ))),
1517                false,
1518            )
1519            // add the field id to the outer list
1520            .with_field_id(1),
1521        )]);
1522
1523        let converted_with_modified = convert_schema(
1524            &schema,
1525            ProjectionMask::all(),
1526            Some(&modified_schema_fields),
1527        )?
1528        .unwrap();
1529
1530        assert_eq!(
1531            converted_with_modified.arrow_type,
1532            DataType::Struct(modified_schema_fields)
1533        );
1534
1535        Ok(())
1536    }
1537
1538    /// Backwards-compatibility: LIST with nullable element type - 1 - standard
1539    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L452-L466)
1540    #[test]
1541    fn list_nullable_element_standard() -> crate::errors::Result<()> {
1542        test_expected_type(
1543            "
1544            message root {
1545              optional group f1 (LIST) {
1546                repeated group list {
1547                  optional int32 element;
1548                }
1549              }
1550            }",
1551            Fields::from(vec![Field::new(
1552                "f1",
1553                DataType::List(Arc::new(Field::new("element", DataType::Int32, true))),
1554                true,
1555            )]),
1556        )
1557    }
1558
1559    /// Backwards-compatibility: LIST with nullable element type - 2
1560    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L468-L482)
1561    #[test]
1562    fn list_nullable_element_nested() -> crate::errors::Result<()> {
1563        test_expected_type(
1564            "
1565            message root {
1566              optional group f1 (LIST) {
1567                repeated group element {
1568                  optional int32 num;
1569                }
1570              }
1571            }",
1572            Fields::from(vec![Field::new(
1573                "f1",
1574                DataType::List(Arc::new(Field::new("num", DataType::Int32, true))),
1575                true,
1576            )]),
1577        )
1578    }
1579
1580    /// Backwards-compatibility: LIST with non-nullable element type - 1 - standard
1581    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L484-L495)
1582    #[test]
1583    fn list_required_element_standard() -> crate::errors::Result<()> {
1584        test_expected_type(
1585            "
1586            message root {
1587              optional group f1 (LIST) {
1588                repeated group list {
1589                  required int32 element;
1590                }
1591              }
1592            }",
1593            Fields::from(vec![Field::new(
1594                "f1",
1595                DataType::List(Arc::new(Field::new("element", DataType::Int32, false))),
1596                true,
1597            )]),
1598        )
1599    }
1600
1601    /// Backwards-compatibility: LIST with non-nullable element type - 2
1602    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L497-L508)
1603    #[test]
1604    fn list_required_element_nested() -> crate::errors::Result<()> {
1605        test_expected_type(
1606            "
1607            message root {
1608              optional group f1 (LIST) {
1609                repeated group element {
1610                  required int32 num;
1611                }
1612              }
1613            }",
1614            Fields::from(vec![Field::new(
1615                "f1",
1616                DataType::List(Arc::new(Field::new("num", DataType::Int32, false))),
1617                true,
1618            )]),
1619        )
1620    }
1621
1622    /// Backwards-compatibility: LIST with non-nullable element type - 3
1623    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L510-L519)
1624    #[test]
1625    fn list_required_element_primitive() -> crate::errors::Result<()> {
1626        test_expected_type(
1627            "
1628            message root {
1629              optional group f1 (LIST) {
1630                repeated int32 element;
1631              }
1632            }",
1633            Fields::from(vec![Field::new(
1634                "f1",
1635                DataType::List(Arc::new(Field::new("element", DataType::Int32, false))),
1636                true,
1637            )]),
1638        )
1639    }
1640
1641    /// Backwards-compatibility: LIST with non-nullable element type - 4
1642    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L521-L540)
1643    #[test]
1644    fn list_required_element_struct() -> crate::errors::Result<()> {
1645        test_expected_type(
1646            "
1647            message root {
1648              optional group f1 (LIST) {
1649                repeated group element {
1650                  required binary str (UTF8);
1651                  required int32 num;
1652                }
1653              }
1654            }",
1655            Fields::from(vec![Field::new(
1656                "f1",
1657                DataType::List(Arc::new(Field::new(
1658                    "element",
1659                    DataType::Struct(Fields::from(vec![
1660                        Field::new("str", DataType::Utf8, false),
1661                        Field::new("num", DataType::Int32, false),
1662                    ])),
1663                    false,
1664                ))),
1665                true,
1666            )]),
1667        )
1668    }
1669
1670    /// Backwards-compatibility: LIST with non-nullable element type - 5 - parquet-avro style
1671    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L542-L559)
1672    #[test]
1673    fn list_required_element_avro_style() -> crate::errors::Result<()> {
1674        test_expected_type(
1675            "
1676            message root {
1677              optional group f1 (LIST) {
1678                repeated group array {
1679                  required binary str (UTF8);
1680                }
1681              }
1682            }",
1683            Fields::from(vec![Field::new(
1684                "f1",
1685                DataType::List(Arc::new(Field::new(
1686                    "array",
1687                    DataType::Struct(Fields::from(vec![Field::new("str", DataType::Utf8, false)])),
1688                    false,
1689                ))),
1690                true,
1691            )]),
1692        )
1693    }
1694
1695    /// Backwards-compatibility: LIST with non-nullable element type - 6 - parquet-thrift style
1696    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L561-L578)
1697    #[test]
1698    fn list_required_element_thrift_style() -> crate::errors::Result<()> {
1699        test_expected_type(
1700            "
1701            message root {
1702              optional group f1 (LIST) {
1703                repeated group f1_tuple {
1704                  required binary str (UTF8);
1705                }
1706              }
1707            }",
1708            Fields::from(vec![Field::new(
1709                "f1",
1710                DataType::List(Arc::new(Field::new(
1711                    "f1_tuple",
1712                    DataType::Struct(Fields::from(vec![Field::new("str", DataType::Utf8, false)])),
1713                    false,
1714                ))),
1715                true,
1716            )]),
1717        )
1718    }
1719
1720    /// Backwards-compatibility: MAP with non-nullable value type - 1 - standard
1721    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L652-L667)
1722    #[test]
1723    fn map_required_value_standard() -> crate::errors::Result<()> {
1724        test_expected_type(
1725            "
1726            message root {
1727              optional group f1 (MAP) {
1728                repeated group key_value {
1729                  required int32 key;
1730                  required binary value (UTF8);
1731                }
1732              }
1733            }",
1734            Fields::from(vec![Field::new_map(
1735                "f1",
1736                "key_value",
1737                Field::new("key", DataType::Int32, false),
1738                Field::new("value", DataType::Utf8, false),
1739                false,
1740                true,
1741            )]),
1742        )
1743    }
1744
1745    /// Backwards-compatibility: MAP with non-nullable value type - 2
1746    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L669-L684)
1747    #[test]
1748    fn map_required_value_map_key_value() -> crate::errors::Result<()> {
1749        test_expected_type(
1750            "
1751            message root {
1752              optional group f1 (MAP_KEY_VALUE) {
1753                repeated group map {
1754                  required int32 num;
1755                  required binary str (UTF8);
1756                }
1757              }
1758            }",
1759            Fields::from(vec![Field::new_map(
1760                "f1",
1761                "map",
1762                Field::new("num", DataType::Int32, false),
1763                Field::new("str", DataType::Utf8, false),
1764                false,
1765                true,
1766            )]),
1767        )
1768    }
1769
1770    /// Backwards-compatibility: MAP with non-nullable value type - 3 - prior to 1.4.x
1771    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L686-L701)
1772    #[test]
1773    fn map_required_value_legacy() -> crate::errors::Result<()> {
1774        test_expected_type(
1775            "
1776            message root {
1777              optional group f1 (MAP) {
1778                repeated group map (MAP_KEY_VALUE) {
1779                  required int32 key;
1780                  required binary value (UTF8);
1781                }
1782              }
1783            }",
1784            Fields::from(vec![Field::new_map(
1785                "f1",
1786                "map",
1787                Field::new("key", DataType::Int32, false),
1788                Field::new("value", DataType::Utf8, false),
1789                false,
1790                true,
1791            )]),
1792        )
1793    }
1794
1795    /// Backwards-compatibility: MAP with nullable value type - 1 - standard
1796    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L703-L718)
1797    #[test]
1798    fn map_optional_value_standard() -> crate::errors::Result<()> {
1799        test_expected_type(
1800            "
1801            message root {
1802              optional group f1 (MAP) {
1803                repeated group key_value {
1804                  required int32 key;
1805                  optional binary value (UTF8);
1806                }
1807              }
1808            }",
1809            Fields::from(vec![Field::new_map(
1810                "f1",
1811                "key_value",
1812                Field::new("key", DataType::Int32, false),
1813                Field::new("value", DataType::Utf8, true),
1814                false,
1815                true,
1816            )]),
1817        )
1818    }
1819
1820    /// Backwards-compatibility: MAP with nullable value type - 2
1821    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L720-L735)
1822    #[test]
1823    fn map_optional_value_map_key_value() -> crate::errors::Result<()> {
1824        test_expected_type(
1825            "
1826            message root {
1827              optional group f1 (MAP_KEY_VALUE) {
1828                repeated group map {
1829                  required int32 num;
1830                  optional binary str (UTF8);
1831                }
1832              }
1833            }",
1834            Fields::from(vec![Field::new_map(
1835                "f1",
1836                "map",
1837                Field::new("num", DataType::Int32, false),
1838                Field::new("str", DataType::Utf8, true),
1839                false,
1840                true,
1841            )]),
1842        )
1843    }
1844
1845    /// Backwards-compatibility: MAP with nullable value type - 3 - parquet-avro style
1846    /// Taken from [Spark](https://github.com/apache/spark/blob/8ab50765cd793169091d983b50d87a391f6ac1f4/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala#L737-L752)
1847    #[test]
1848    fn map_optional_value_avro_style() -> crate::errors::Result<()> {
1849        test_expected_type(
1850            "
1851            message root {
1852              optional group f1 (MAP) {
1853                repeated group map (MAP_KEY_VALUE) {
1854                  required int32 key;
1855                  optional binary value (UTF8);
1856                }
1857              }
1858            }",
1859            Fields::from(vec![Field::new_map(
1860                "f1",
1861                "map",
1862                Field::new("key", DataType::Int32, false),
1863                Field::new("value", DataType::Utf8, true),
1864                false,
1865                true,
1866            )]),
1867        )
1868    }
1869}
parquet/arrow/schema/complex.rs

parquet/arrow/schema/
complex.rs
