Skip to main content

parquet/schema/
types.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Contains structs and methods to build Parquet schema and schema descriptors.
6
7use std::vec::IntoIter;
8use std::{collections::HashMap, fmt, sync::Arc};
9
10use crate::file::metadata::HeapSize;
11use crate::file::metadata::thrift::SchemaElement;
12
13use crate::basic::{
14    ColumnOrder, ConvertedType, IntType, LogicalType, Repetition, SortOrder, TimeType, TimeUnit,
15    Type as PhysicalType,
16};
17use crate::errors::{ParquetError, Result};
18
19// ----------------------------------------------------------------------
20// Parquet Type definitions
21
22/// Type alias for `Arc<Type>`.
23pub type TypePtr = Arc<Type>;
24/// Type alias for `Arc<SchemaDescriptor>`.
25pub type SchemaDescPtr = Arc<SchemaDescriptor>;
26/// Type alias for `Arc<ColumnDescriptor>`.
27pub type ColumnDescPtr = Arc<ColumnDescriptor>;
28
29/// Representation of a Parquet type.
30///
31/// Used to describe primitive leaf fields and structs, including top-level schema.
32///
33/// Note that the top-level schema is represented using [`Type::GroupType`] whose
34/// repetition is `None`.
35#[derive(Clone, Debug, PartialEq)]
36pub enum Type {
37    /// Represents a primitive leaf field.
38    PrimitiveType {
39        /// Basic information about the type.
40        basic_info: BasicTypeInfo,
41        /// Physical type of this primitive type.
42        physical_type: PhysicalType,
43        /// Length of this type.
44        type_length: i32,
45        /// Scale of this type.
46        scale: i32,
47        /// Precision of this type.
48        precision: i32,
49    },
50    /// Represents a group of fields (similar to struct).
51    GroupType {
52        /// Basic information about the type.
53        basic_info: BasicTypeInfo,
54        /// Fields of this group type.
55        fields: Vec<TypePtr>,
56    },
57}
58
59impl HeapSize for Type {
60    fn heap_size(&self) -> usize {
61        match self {
62            Type::PrimitiveType { basic_info, .. } => basic_info.heap_size(),
63            Type::GroupType { basic_info, fields } => basic_info.heap_size() + fields.heap_size(),
64        }
65    }
66}
67
68impl Type {
69    /// Creates primitive type builder with provided field name and physical type.
70    pub fn primitive_type_builder(
71        name: &str,
72        physical_type: PhysicalType,
73    ) -> PrimitiveTypeBuilder<'_> {
74        PrimitiveTypeBuilder::new(name, physical_type)
75    }
76
77    /// Creates group type builder with provided column name.
78    pub fn group_type_builder(name: &str) -> GroupTypeBuilder<'_> {
79        GroupTypeBuilder::new(name)
80    }
81
82    /// Returns [`BasicTypeInfo`] information about the type.
83    pub fn get_basic_info(&self) -> &BasicTypeInfo {
84        match *self {
85            Type::PrimitiveType { ref basic_info, .. } => basic_info,
86            Type::GroupType { ref basic_info, .. } => basic_info,
87        }
88    }
89
90    /// Returns this type's field name.
91    pub fn name(&self) -> &str {
92        self.get_basic_info().name()
93    }
94
95    /// Gets the fields from this group type.
96    ///
97    /// # Panics
98    ///
99    /// Panics if called on a non-group type
100    // TODO: should we return `&[&Type]` here?
101    pub fn get_fields(&self) -> &[TypePtr] {
102        match *self {
103            Type::GroupType { ref fields, .. } => &fields[..],
104            Type::PrimitiveType { .. } => panic!("Cannot call get_fields() on a non-group type"),
105        }
106    }
107
108    /// Gets physical type of this primitive type.
109    ///
110    /// # Panics
111    ///
112    /// Panics if called on a non-primitive type
113    pub fn get_physical_type(&self) -> PhysicalType {
114        match *self {
115            Type::PrimitiveType {
116                basic_info: _,
117                physical_type,
118                ..
119            } => physical_type,
120            Type::GroupType { .. } => {
121                panic!("Cannot call get_physical_type() on a non-primitive type")
122            }
123        }
124    }
125
126    /// Gets precision of this primitive type.
127    ///
128    /// # Panics
129    ///
130    /// Panics if called on a non-primitive type
131    pub fn get_precision(&self) -> i32 {
132        match *self {
133            Type::PrimitiveType { precision, .. } => precision,
134            Type::GroupType { .. } => panic!("Cannot call get_precision() on non-primitive type"),
135        }
136    }
137
138    /// Gets scale of this primitive type.
139    ///
140    /// # Panics
141    ///
142    /// Panics if called on a non-primitive type
143    pub fn get_scale(&self) -> i32 {
144        match *self {
145            Type::PrimitiveType { scale, .. } => scale,
146            Type::GroupType { .. } => panic!("Cannot call get_scale() on non-primitive type"),
147        }
148    }
149
150    /// Checks if `sub_type` schema is part of current schema.
151    /// This method can be used to check if projected columns are part of the root schema.
152    pub fn check_contains(&self, sub_type: &Type) -> bool {
153        // Names match, and repetitions match or not set for both
154        let basic_match = self.get_basic_info().name() == sub_type.get_basic_info().name()
155            && (self.is_schema() && sub_type.is_schema()
156                || !self.is_schema()
157                    && !sub_type.is_schema()
158                    && self.get_basic_info().repetition()
159                        == sub_type.get_basic_info().repetition());
160
161        match *self {
162            Type::PrimitiveType { .. } if basic_match && sub_type.is_primitive() => {
163                self.get_physical_type() == sub_type.get_physical_type()
164            }
165            Type::GroupType { .. } if basic_match && sub_type.is_group() => {
166                // build hashmap of name -> TypePtr
167                let mut field_map = HashMap::new();
168                for field in self.get_fields() {
169                    field_map.insert(field.name(), field);
170                }
171
172                for field in sub_type.get_fields() {
173                    if !field_map
174                        .get(field.name())
175                        .map(|type_| type_.check_contains(field))
176                        .unwrap_or(false)
177                    {
178                        return false;
179                    }
180                }
181                true
182            }
183            _ => false,
184        }
185    }
186
187    /// Returns `true` if this type is a primitive type, `false` otherwise.
188    pub fn is_primitive(&self) -> bool {
189        matches!(*self, Type::PrimitiveType { .. })
190    }
191
192    /// Returns `true` if this type is a group type, `false` otherwise.
193    pub fn is_group(&self) -> bool {
194        matches!(*self, Type::GroupType { .. })
195    }
196
197    /// Returns `true` if this type is the top-level schema type (message type).
198    pub fn is_schema(&self) -> bool {
199        match *self {
200            Type::GroupType { ref basic_info, .. } => !basic_info.has_repetition(),
201            Type::PrimitiveType { .. } => false,
202        }
203    }
204
205    /// Returns `true` if this type is repeated or optional.
206    /// If this type doesn't have repetition defined, we treat it as required.
207    pub fn is_optional(&self) -> bool {
208        self.get_basic_info().has_repetition()
209            && self.get_basic_info().repetition() != Repetition::REQUIRED
210    }
211
212    /// Returns `true` if this type is annotated as a list.
213    pub(crate) fn is_list(&self) -> bool {
214        if self.is_group() {
215            let basic_info = self.get_basic_info();
216            if let Some(logical_type) = basic_info.logical_type_ref() {
217                return logical_type == &LogicalType::List;
218            }
219            return basic_info.converted_type() == ConvertedType::LIST;
220        }
221        false
222    }
223
224    /// Returns `true` if this type is a group with a single child field that is `repeated`.
225    pub(crate) fn has_single_repeated_child(&self) -> bool {
226        if self.is_group() {
227            let children = self.get_fields();
228            return children.len() == 1
229                && children[0].get_basic_info().has_repetition()
230                && children[0].get_basic_info().repetition() == Repetition::REPEATED;
231        }
232        false
233    }
234}
235
236/// A builder for primitive types. All attributes are optional
237/// except the name and physical type.
238/// Note that if not specified explicitly, `Repetition::OPTIONAL` is used.
239pub struct PrimitiveTypeBuilder<'a> {
240    name: &'a str,
241    repetition: Repetition,
242    physical_type: PhysicalType,
243    converted_type: ConvertedType,
244    logical_type: Option<LogicalType>,
245    length: i32,
246    precision: i32,
247    scale: i32,
248    id: Option<i32>,
249}
250
251impl<'a> PrimitiveTypeBuilder<'a> {
252    /// Creates new primitive type builder with provided field name and physical type.
253    pub fn new(name: &'a str, physical_type: PhysicalType) -> Self {
254        Self {
255            name,
256            repetition: Repetition::OPTIONAL,
257            physical_type,
258            converted_type: ConvertedType::NONE,
259            logical_type: None,
260            length: -1,
261            precision: -1,
262            scale: -1,
263            id: None,
264        }
265    }
266
267    /// Sets [`Repetition`] for this field and returns itself.
268    pub fn with_repetition(self, repetition: Repetition) -> Self {
269        Self { repetition, ..self }
270    }
271
272    /// Sets [`ConvertedType`] for this field and returns itself.
273    pub fn with_converted_type(self, converted_type: ConvertedType) -> Self {
274        Self {
275            converted_type,
276            ..self
277        }
278    }
279
280    /// Sets [`LogicalType`] for this field and returns itself.
281    /// If only the logical type is populated for a primitive type, the converted type
282    /// will be automatically populated, and can thus be omitted.
283    pub fn with_logical_type(self, logical_type: Option<LogicalType>) -> Self {
284        Self {
285            logical_type,
286            ..self
287        }
288    }
289
290    /// Sets type length and returns itself.
291    /// This is only applied to FIXED_LEN_BYTE_ARRAY and INT96 (INTERVAL) types, because
292    /// they maintain fixed size underlying byte array.
293    /// By default, value is `0`.
294    pub fn with_length(self, length: i32) -> Self {
295        Self { length, ..self }
296    }
297
298    /// Sets precision for Parquet DECIMAL physical type and returns itself.
299    /// By default, it equals to `0` and used only for decimal context.
300    pub fn with_precision(self, precision: i32) -> Self {
301        Self { precision, ..self }
302    }
303
304    /// Sets scale for Parquet DECIMAL physical type and returns itself.
305    /// By default, it equals to `0` and used only for decimal context.
306    pub fn with_scale(self, scale: i32) -> Self {
307        Self { scale, ..self }
308    }
309
310    /// Sets optional field id and returns itself.
311    pub fn with_id(self, id: Option<i32>) -> Self {
312        Self { id, ..self }
313    }
314
315    /// Creates a new `PrimitiveType` instance from the collected attributes.
316    /// Returns `Err` in case of any building conditions are not met.
317    pub fn build(self) -> Result<Type> {
318        let sort_order = ColumnOrder::column_order_for_type(
319            self.logical_type.as_ref(),
320            self.converted_type,
321            self.physical_type,
322        )
323        .sort_order();
324        let mut basic_info = BasicTypeInfo {
325            name: String::from(self.name),
326            repetition: Some(self.repetition),
327            converted_type: self.converted_type,
328            logical_type: self.logical_type.clone(),
329            id: self.id,
330            sort_order,
331        };
332
333        // Check length before logical type, since it is used for logical type validation.
334        if self.physical_type == PhysicalType::FIXED_LEN_BYTE_ARRAY && self.length < 0 {
335            return Err(general_err!(
336                "Invalid FIXED_LEN_BYTE_ARRAY length: {} for field '{}'",
337                self.length,
338                self.name
339            ));
340        }
341
342        if let Some(logical_type) = &self.logical_type {
343            // If a converted type is populated, check that it is consistent with
344            // its logical type
345            if self.converted_type != ConvertedType::NONE {
346                if ConvertedType::from(self.logical_type.clone()) != self.converted_type {
347                    return Err(general_err!(
348                        "Logical type {:?} is incompatible with converted type {} for field '{}'",
349                        logical_type,
350                        self.converted_type,
351                        self.name
352                    ));
353                }
354            } else {
355                // Populate the converted type for backwards compatibility
356                basic_info.converted_type = self.logical_type.clone().into();
357            }
358            // Check that logical type and physical type are compatible
359            match (logical_type, self.physical_type) {
360                (LogicalType::Map | LogicalType::List | LogicalType::File, _) => {
361                    return Err(general_err!(
362                        "{:?} cannot be applied to a primitive type for field '{}'",
363                        logical_type,
364                        self.name
365                    ));
366                }
367                (LogicalType::Enum, PhysicalType::BYTE_ARRAY) => {}
368                (LogicalType::Decimal(decimal), _) => {
369                    // Check that scale and precision are consistent with legacy values
370                    if decimal.scale != self.scale {
371                        return Err(general_err!(
372                            "DECIMAL logical type scale {} must match self.scale {} for field '{}'",
373                            decimal.scale,
374                            self.scale,
375                            self.name
376                        ));
377                    }
378                    if decimal.precision != self.precision {
379                        return Err(general_err!(
380                            "DECIMAL logical type precision {} must match self.precision {} for field '{}'",
381                            decimal.precision,
382                            self.precision,
383                            self.name
384                        ));
385                    }
386                    self.check_decimal_precision_scale()?;
387                }
388                (LogicalType::Date, PhysicalType::INT32) => {}
389                (
390                    LogicalType::Time(TimeType {
391                        unit: TimeUnit::MILLIS,
392                        ..
393                    }),
394                    PhysicalType::INT32,
395                ) => {}
396                (LogicalType::Time(time), PhysicalType::INT64) => {
397                    if time.unit == TimeUnit::MILLIS {
398                        return Err(general_err!(
399                            "Cannot use millisecond unit on INT64 type for field '{}'",
400                            self.name
401                        ));
402                    }
403                }
404                (LogicalType::Timestamp(_), PhysicalType::INT64) => {}
405                (LogicalType::Integer(int), PhysicalType::INT32) if int.bit_width <= 32 => {}
406                (LogicalType::Integer(int), PhysicalType::INT64) if int.bit_width == 64 => {}
407                // Null type
408                (LogicalType::Unknown, _) => {}
409                (LogicalType::String, PhysicalType::BYTE_ARRAY) => {}
410                (LogicalType::Json, PhysicalType::BYTE_ARRAY) => {}
411                (LogicalType::Bson, PhysicalType::BYTE_ARRAY) => {}
412                (LogicalType::Geometry(_), PhysicalType::BYTE_ARRAY) => {}
413                (LogicalType::Geography(_), PhysicalType::BYTE_ARRAY) => {}
414                (LogicalType::Uuid, PhysicalType::FIXED_LEN_BYTE_ARRAY) if self.length == 16 => {}
415                (LogicalType::Uuid, PhysicalType::FIXED_LEN_BYTE_ARRAY) => {
416                    return Err(general_err!(
417                        "UUID cannot annotate field '{}' because it is not a FIXED_LEN_BYTE_ARRAY(16) field",
418                        self.name
419                    ));
420                }
421                (LogicalType::Float16, PhysicalType::FIXED_LEN_BYTE_ARRAY) if self.length == 2 => {}
422                (LogicalType::Float16, PhysicalType::FIXED_LEN_BYTE_ARRAY) => {
423                    return Err(general_err!(
424                        "FLOAT16 cannot annotate field '{}' because it is not a FIXED_LEN_BYTE_ARRAY(2) field",
425                        self.name
426                    ));
427                }
428                // unknown logical type means just use physical type
429                (LogicalType::_Unknown { .. }, _) => {}
430                (a, b) => {
431                    return Err(general_err!(
432                        "Cannot annotate {:?} from {} for field '{}'",
433                        a,
434                        b,
435                        self.name
436                    ));
437                }
438            }
439        }
440
441        match self.converted_type {
442            ConvertedType::NONE => {}
443            ConvertedType::UTF8 | ConvertedType::BSON | ConvertedType::JSON => {
444                if self.physical_type != PhysicalType::BYTE_ARRAY {
445                    return Err(general_err!(
446                        "{} cannot annotate field '{}' because it is not a BYTE_ARRAY field",
447                        self.converted_type,
448                        self.name
449                    ));
450                }
451            }
452            ConvertedType::DECIMAL => {
453                self.check_decimal_precision_scale()?;
454            }
455            ConvertedType::DATE
456            | ConvertedType::TIME_MILLIS
457            | ConvertedType::UINT_8
458            | ConvertedType::UINT_16
459            | ConvertedType::UINT_32
460            | ConvertedType::INT_8
461            | ConvertedType::INT_16
462            | ConvertedType::INT_32 => {
463                if self.physical_type != PhysicalType::INT32 {
464                    return Err(general_err!(
465                        "{} cannot annotate field '{}' because it is not a INT32 field",
466                        self.converted_type,
467                        self.name
468                    ));
469                }
470            }
471            ConvertedType::TIME_MICROS
472            | ConvertedType::TIMESTAMP_MILLIS
473            | ConvertedType::TIMESTAMP_MICROS
474            | ConvertedType::UINT_64
475            | ConvertedType::INT_64 => {
476                if self.physical_type != PhysicalType::INT64 {
477                    return Err(general_err!(
478                        "{} cannot annotate field '{}' because it is not a INT64 field",
479                        self.converted_type,
480                        self.name
481                    ));
482                }
483            }
484            ConvertedType::INTERVAL => {
485                if self.physical_type != PhysicalType::FIXED_LEN_BYTE_ARRAY || self.length != 12 {
486                    return Err(general_err!(
487                        "INTERVAL cannot annotate field '{}' because it is not a FIXED_LEN_BYTE_ARRAY(12) field",
488                        self.name
489                    ));
490                }
491            }
492            ConvertedType::ENUM => {
493                if self.physical_type != PhysicalType::BYTE_ARRAY {
494                    return Err(general_err!(
495                        "ENUM cannot annotate field '{}' because it is not a BYTE_ARRAY field",
496                        self.name
497                    ));
498                }
499            }
500            _ => {
501                return Err(general_err!(
502                    "{} cannot be applied to primitive field '{}'",
503                    self.converted_type,
504                    self.name
505                ));
506            }
507        }
508
509        Ok(Type::PrimitiveType {
510            basic_info,
511            physical_type: self.physical_type,
512            type_length: self.length,
513            scale: self.scale,
514            precision: self.precision,
515        })
516    }
517
518    #[inline]
519    fn check_decimal_precision_scale(&self) -> Result<()> {
520        match self.physical_type {
521            PhysicalType::INT32
522            | PhysicalType::INT64
523            | PhysicalType::BYTE_ARRAY
524            | PhysicalType::FIXED_LEN_BYTE_ARRAY => (),
525            _ => {
526                return Err(general_err!(
527                    "DECIMAL can only annotate INT32, INT64, BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY"
528                ));
529            }
530        }
531
532        // Precision is required and must be a non-zero positive integer.
533        if self.precision < 1 {
534            return Err(general_err!(
535                "Invalid DECIMAL precision: {}",
536                self.precision
537            ));
538        }
539
540        // Scale must be zero or a positive integer less than the precision.
541        if self.scale < 0 {
542            return Err(general_err!("Invalid DECIMAL scale: {}", self.scale));
543        }
544
545        if self.scale > self.precision {
546            return Err(general_err!(
547                "Invalid DECIMAL: scale ({}) cannot be greater than precision \
548             ({})",
549                self.scale,
550                self.precision
551            ));
552        }
553
554        // Check precision and scale based on physical type limitations.
555        match self.physical_type {
556            PhysicalType::INT32 => {
557                if self.precision > 9 {
558                    return Err(general_err!(
559                        "Cannot represent INT32 as DECIMAL with precision {}",
560                        self.precision
561                    ));
562                }
563            }
564            PhysicalType::INT64 => {
565                if self.precision > 18 {
566                    return Err(general_err!(
567                        "Cannot represent INT64 as DECIMAL with precision {}",
568                        self.precision
569                    ));
570                }
571            }
572            PhysicalType::FIXED_LEN_BYTE_ARRAY => {
573                let length = self
574                    .length
575                    .checked_mul(8)
576                    .ok_or(general_err!("Invalid length {} for Decimal", self.length))?;
577                let max_precision = (2f64.powi(length - 1) - 1f64).log10().floor() as i32;
578
579                if self.precision > max_precision {
580                    return Err(general_err!(
581                        "Cannot represent FIXED_LEN_BYTE_ARRAY as DECIMAL with length {} and \
582                        precision {}. The max precision can only be {}",
583                        self.length,
584                        self.precision,
585                        max_precision
586                    ));
587                }
588            }
589            _ => (), // For BYTE_ARRAY precision is not limited
590        }
591
592        Ok(())
593    }
594}
595
596/// A builder for group types. All attributes are optional except the name.
597/// Note that if not specified explicitly, `None` is used as the repetition of the group,
598/// which means it is a root (message) type.
599pub struct GroupTypeBuilder<'a> {
600    name: &'a str,
601    repetition: Option<Repetition>,
602    converted_type: ConvertedType,
603    logical_type: Option<LogicalType>,
604    fields: Vec<TypePtr>,
605    id: Option<i32>,
606}
607
608impl<'a> GroupTypeBuilder<'a> {
609    /// Creates new group type builder with provided field name.
610    pub fn new(name: &'a str) -> Self {
611        Self {
612            name,
613            repetition: None,
614            converted_type: ConvertedType::NONE,
615            logical_type: None,
616            fields: Vec::new(),
617            id: None,
618        }
619    }
620
621    /// Sets [`Repetition`] for this field and returns itself.
622    pub fn with_repetition(mut self, repetition: Repetition) -> Self {
623        self.repetition = Some(repetition);
624        self
625    }
626
627    /// Sets [`ConvertedType`] for this field and returns itself.
628    pub fn with_converted_type(self, converted_type: ConvertedType) -> Self {
629        Self {
630            converted_type,
631            ..self
632        }
633    }
634
635    /// Sets [`LogicalType`] for this field and returns itself.
636    pub fn with_logical_type(self, logical_type: Option<LogicalType>) -> Self {
637        Self {
638            logical_type,
639            ..self
640        }
641    }
642
643    /// Sets a list of fields that should be child nodes of this field.
644    /// Returns updated self.
645    pub fn with_fields(self, fields: Vec<TypePtr>) -> Self {
646        Self { fields, ..self }
647    }
648
649    /// Sets optional field id and returns itself.
650    pub fn with_id(self, id: Option<i32>) -> Self {
651        Self { id, ..self }
652    }
653
654    /// Creates a new `GroupType` instance from the gathered attributes.
655    pub fn build(self) -> Result<Type> {
656        if matches!(&self.logical_type, Some(LogicalType::File)) {
657            validate_file_type_fields(self.name, &self.fields)?;
658        }
659        let mut basic_info = BasicTypeInfo {
660            name: String::from(self.name),
661            repetition: self.repetition,
662            converted_type: self.converted_type,
663            logical_type: self.logical_type.clone(),
664            id: self.id,
665            sort_order: SortOrder::UNDEFINED,
666        };
667        // Populate the converted type if only the logical type is populated
668        if self.logical_type.is_some() && self.converted_type == ConvertedType::NONE {
669            basic_info.converted_type = self.logical_type.into();
670        }
671        Ok(Type::GroupType {
672            basic_info,
673            fields: self.fields,
674        })
675    }
676}
677
678/// Validates the fields of a `FILE`-annotated group against the Parquet
679/// [specification].
680///
681/// A `FILE` group annotates a reference to a range of bytes. Every field is
682/// optional both in the schema and in the data: a writer may omit any field
683/// from the group definition, and any field that is present must have a field
684/// repetition type of `OPTIONAL`. The recognized fields (identified by name)
685/// and their expected physical/logical types are:
686///
687/// | Field          | Physical type | Logical type |
688/// |----------------|---------------|--------------|
689/// | `uri`          | `BYTE_ARRAY`  | `STRING`     |
690/// | `offset`       | `INT64`       | —            |
691/// | `size`         | `INT64`       | —            |
692/// | `content_type` | `BYTE_ARRAY`  | `STRING`     |
693/// | `checksum`     | `BYTE_ARRAY`  | `STRING`     |
694/// | `inline`       | `BYTE_ARRAY`  | —            |
695///
696/// [specification]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#file
697fn validate_file_type_fields(name: &str, fields: &[TypePtr]) -> Result<()> {
698    // (name, expected physical type, expected logical type)
699    const VALID_FIELDS: &[(&str, PhysicalType, Option<LogicalType>)] = &[
700        ("uri", PhysicalType::BYTE_ARRAY, Some(LogicalType::String)),
701        ("offset", PhysicalType::INT64, None),
702        ("size", PhysicalType::INT64, None),
703        (
704            "content_type",
705            PhysicalType::BYTE_ARRAY,
706            Some(LogicalType::String),
707        ),
708        (
709            "checksum",
710            PhysicalType::BYTE_ARRAY,
711            Some(LogicalType::String),
712        ),
713        ("inline", PhysicalType::BYTE_ARRAY, None),
714    ];
715
716    for field in fields {
717        let field_name = field.get_basic_info().name();
718        let Some((_, expected_physical, expected_logical)) =
719            VALID_FIELDS.iter().find(|(n, _, _)| *n == field_name)
720        else {
721            return Err(general_err!(
722                "FILE type group '{}' contains unrecognized field '{}'. \
723                 Valid fields are: uri, offset, size, content_type, checksum, inline",
724                name,
725                field_name
726            ));
727        };
728
729        // Every field present in a FILE group must be OPTIONAL.
730        let is_optional = field.get_basic_info().has_repetition()
731            && field.get_basic_info().repetition() == Repetition::OPTIONAL;
732        if !is_optional {
733            return Err(general_err!(
734                "FILE type field '{}' must be OPTIONAL in group '{}'",
735                field_name,
736                name
737            ));
738        }
739
740        // FILE fields are always primitives with a fixed physical type.
741        if field.is_group() {
742            return Err(general_err!(
743                "FILE type field '{}' in group '{}' must be a primitive type",
744                field_name,
745                name
746            ));
747        }
748        if field.get_physical_type() != *expected_physical {
749            return Err(general_err!(
750                "FILE type field '{}' in group '{}' must have physical type {:?}",
751                field_name,
752                name,
753                expected_physical
754            ));
755        }
756        if field.get_basic_info().logical_type_ref() != expected_logical.as_ref() {
757            return Err(general_err!(
758                "FILE type field '{}' in group '{}' must have logical type {:?}",
759                field_name,
760                name,
761                expected_logical
762            ));
763        }
764    }
765    Ok(())
766}
767
768/// Basic type info. This contains information such as the name of the type,
769/// the repetition level, the logical type and the kind of the type (group, primitive).
770#[derive(Clone, Debug, PartialEq, Eq)]
771pub struct BasicTypeInfo {
772    name: String,
773    repetition: Option<Repetition>,
774    converted_type: ConvertedType,
775    logical_type: Option<LogicalType>,
776    id: Option<i32>,
777    sort_order: SortOrder,
778}
779
780impl HeapSize for BasicTypeInfo {
781    fn heap_size(&self) -> usize {
782        // no heap allocations in any other subfield
783        self.name.heap_size()
784    }
785}
786
787impl BasicTypeInfo {
788    /// Returns field name.
789    pub fn name(&self) -> &str {
790        &self.name
791    }
792
793    /// Returns `true` if type has repetition field set, `false` otherwise.
794    /// This is mostly applied to group type, because primitive type always has
795    /// repetition set.
796    pub fn has_repetition(&self) -> bool {
797        self.repetition.is_some()
798    }
799
800    /// Returns [`Repetition`] value for the type.
801    ///
802    /// # Panics
803    ///
804    /// Panics if the repetition is not set, see [`Self::has_repetition`]
805    pub fn repetition(&self) -> Repetition {
806        assert!(self.repetition.is_some());
807        self.repetition.unwrap()
808    }
809
810    /// Returns [`ConvertedType`] value for the type.
811    pub fn converted_type(&self) -> ConvertedType {
812        self.converted_type
813    }
814
815    /// Return a reference to the [`LogicalType`] value for the type.
816    pub fn logical_type_ref(&self) -> Option<&LogicalType> {
817        self.logical_type.as_ref()
818    }
819
820    /// Returns `true` if id is set, `false` otherwise.
821    pub fn has_id(&self) -> bool {
822        self.id.is_some()
823    }
824
825    /// Returns id value for the type.
826    ///
827    /// # Panics
828    ///
829    /// Panics if the id is not set, see [`Self::has_id`]
830    pub fn id(&self) -> i32 {
831        assert!(self.id.is_some());
832        self.id.unwrap()
833    }
834
835    /// Returns [`SortOrder`] for the type.
836    pub fn sort_order(&self) -> SortOrder {
837        self.sort_order
838    }
839}
840
841// ----------------------------------------------------------------------
842// Parquet descriptor definitions
843
844/// Represents the location of a column in a Parquet schema
845///
846/// # Example: refer to column named `'my_column'`
847/// ```
848/// # use parquet::schema::types::ColumnPath;
849/// let column_path = ColumnPath::from("my_column");
850/// ```
851///
852/// # Example: refer to column named `c` in a nested struct `{a: {b: {c: ...}}}`
853/// ```
854/// # use parquet::schema::types::ColumnPath;
855/// // form path 'a.b.c'
856/// let column_path = ColumnPath::from(vec![
857///   String::from("a"),
858///   String::from("b"),
859///   String::from("c")
860/// ]);
861/// ```
862#[derive(Clone, PartialEq, Debug, Eq, Hash)]
863pub struct ColumnPath {
864    parts: Vec<String>,
865}
866
867impl HeapSize for ColumnPath {
868    fn heap_size(&self) -> usize {
869        self.parts.heap_size()
870    }
871}
872
873impl ColumnPath {
874    /// Creates new column path from vector of field names.
875    pub fn new(parts: Vec<String>) -> Self {
876        ColumnPath { parts }
877    }
878
879    /// Returns string representation of this column path.
880    /// ```rust
881    /// use parquet::schema::types::ColumnPath;
882    ///
883    /// let path = ColumnPath::new(vec!["a".to_string(), "b".to_string(), "c".to_string()]);
884    /// assert_eq!(&path.string(), "a.b.c");
885    /// ```
886    pub fn string(&self) -> String {
887        self.parts.join(".")
888    }
889
890    /// Appends more components to end of column path.
891    /// ```rust
892    /// use parquet::schema::types::ColumnPath;
893    ///
894    /// let mut path = ColumnPath::new(vec!["a".to_string(), "b".to_string(), "c"
895    /// .to_string()]);
896    /// assert_eq!(&path.string(), "a.b.c");
897    ///
898    /// path.append(vec!["d".to_string(), "e".to_string()]);
899    /// assert_eq!(&path.string(), "a.b.c.d.e");
900    /// ```
901    pub fn append(&mut self, mut tail: Vec<String>) {
902        self.parts.append(&mut tail);
903    }
904
905    /// Returns a slice of path components.
906    pub fn parts(&self) -> &[String] {
907        &self.parts
908    }
909}
910
911impl fmt::Display for ColumnPath {
912    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
913        write!(f, "{:?}", self.string())
914    }
915}
916
917impl From<Vec<String>> for ColumnPath {
918    fn from(parts: Vec<String>) -> Self {
919        ColumnPath { parts }
920    }
921}
922
923impl From<&str> for ColumnPath {
924    fn from(single_path: &str) -> Self {
925        let s = String::from(single_path);
926        ColumnPath::from(s)
927    }
928}
929
930impl From<String> for ColumnPath {
931    fn from(single_path: String) -> Self {
932        let v = vec![single_path];
933        ColumnPath { parts: v }
934    }
935}
936
937impl AsRef<[String]> for ColumnPath {
938    fn as_ref(&self) -> &[String] {
939        &self.parts
940    }
941}
942
943/// Physical type for leaf-level primitive columns.
944///
945/// Also includes the maximum definition and repetition levels required to
946/// re-assemble nested data.
947#[derive(Debug, PartialEq)]
948pub struct ColumnDescriptor {
949    /// The "leaf" primitive type of this column
950    primitive_type: TypePtr,
951
952    /// The maximum definition level for this column
953    max_def_level: i16,
954
955    /// The maximum repetition level for this column
956    max_rep_level: i16,
957
958    /// The definition level at the nearest REPEATED ancestor, or 0 if none.
959    repeated_ancestor_def_level: i16,
960
961    /// The path of this column. For instance, "a.b.c.d".
962    path: ColumnPath,
963}
964
965impl HeapSize for ColumnDescriptor {
966    fn heap_size(&self) -> usize {
967        // Don't include the heap size of primitive_type, this is already
968        // accounted for via SchemaDescriptor::schema
969        self.path.heap_size()
970    }
971}
972
973impl ColumnDescriptor {
974    /// Creates new descriptor for leaf-level column.
975    pub fn new(
976        primitive_type: TypePtr,
977        max_def_level: i16,
978        max_rep_level: i16,
979        path: ColumnPath,
980    ) -> Self {
981        Self::new_with_repeated_ancestor(primitive_type, max_def_level, max_rep_level, path, 0)
982    }
983
984    pub(crate) fn new_with_repeated_ancestor(
985        primitive_type: TypePtr,
986        max_def_level: i16,
987        max_rep_level: i16,
988        path: ColumnPath,
989        repeated_ancestor_def_level: i16,
990    ) -> Self {
991        Self {
992            primitive_type,
993            max_def_level,
994            max_rep_level,
995            repeated_ancestor_def_level,
996            path,
997        }
998    }
999
1000    /// Returns maximum definition level for this column.
1001    #[inline]
1002    pub fn max_def_level(&self) -> i16 {
1003        self.max_def_level
1004    }
1005
1006    /// Returns maximum repetition level for this column.
1007    #[inline]
1008    pub fn max_rep_level(&self) -> i16 {
1009        self.max_rep_level
1010    }
1011
1012    /// Returns the definition level at the nearest REPEATED ancestor, or 0 if none.
1013    #[inline]
1014    pub fn repeated_ancestor_def_level(&self) -> i16 {
1015        self.repeated_ancestor_def_level
1016    }
1017
1018    /// Returns [`ColumnPath`] for this column.
1019    pub fn path(&self) -> &ColumnPath {
1020        &self.path
1021    }
1022
1023    /// Returns self type [`Type`] for this leaf column.
1024    pub fn self_type(&self) -> &Type {
1025        self.primitive_type.as_ref()
1026    }
1027
1028    /// Returns self type [`TypePtr`]  for this leaf
1029    /// column.
1030    pub fn self_type_ptr(&self) -> TypePtr {
1031        self.primitive_type.clone()
1032    }
1033
1034    /// Returns [`BasicTypeInfo`] information for this leaf column.
1035    pub fn get_basic_info(&self) -> &BasicTypeInfo {
1036        self.primitive_type.get_basic_info()
1037    }
1038
1039    /// Returns column name.
1040    pub fn name(&self) -> &str {
1041        self.primitive_type.name()
1042    }
1043
1044    /// Returns [`ConvertedType`] for this column.
1045    pub fn converted_type(&self) -> ConvertedType {
1046        self.primitive_type.get_basic_info().converted_type()
1047    }
1048
1049    /// Returns a reference to the [`LogicalType`] for this column.
1050    pub fn logical_type_ref(&self) -> Option<&LogicalType> {
1051        self.primitive_type.get_basic_info().logical_type_ref()
1052    }
1053
1054    /// Returns physical type for this column.
1055    ///
1056    /// # Panics
1057    ///
1058    /// Panics if called on a non-primitive type
1059    pub fn physical_type(&self) -> PhysicalType {
1060        match self.primitive_type.as_ref() {
1061            Type::PrimitiveType { physical_type, .. } => *physical_type,
1062            Type::GroupType { .. } => panic!("Expected primitive type!"),
1063        }
1064    }
1065
1066    /// Returns type length for this column.
1067    ///
1068    /// # Panics
1069    ///
1070    /// Panics if called on a non-primitive type
1071    pub fn type_length(&self) -> i32 {
1072        match self.primitive_type.as_ref() {
1073            Type::PrimitiveType { type_length, .. } => *type_length,
1074            Type::GroupType { .. } => panic!("Expected primitive type!"),
1075        }
1076    }
1077
1078    /// Returns type precision for this column.
1079    ///
1080    /// # Panics
1081    ///
1082    /// Panics if called on a non-primitive type
1083    pub fn type_precision(&self) -> i32 {
1084        match self.primitive_type.as_ref() {
1085            Type::PrimitiveType { precision, .. } => *precision,
1086            Type::GroupType { .. } => panic!("Expected primitive type!"),
1087        }
1088    }
1089
1090    /// Returns type scale for this column.
1091    ///
1092    /// # Panics
1093    ///
1094    /// Panics if called on a non-primitive type
1095    pub fn type_scale(&self) -> i32 {
1096        match self.primitive_type.as_ref() {
1097            Type::PrimitiveType { scale, .. } => *scale,
1098            Type::GroupType { .. } => panic!("Expected primitive type!"),
1099        }
1100    }
1101
1102    /// Returns the sort order for this column as currently defined for the logical or
1103    /// physical type.
1104    ///
1105    /// Returns `SortOrder::UNDEFINED` for non-primitive types.
1106    pub fn sort_order(&self) -> SortOrder {
1107        self.primitive_type.get_basic_info().sort_order()
1108    }
1109}
1110
1111/// Schema of a Parquet file.
1112///
1113/// Encapsulates the file's schema ([`Type`]) and [`ColumnDescriptor`]s for
1114/// each primitive (leaf) column.
1115///
1116/// # Example
1117/// ```
1118/// # use std::sync::Arc;
1119/// use parquet::schema::types::{SchemaDescriptor, Type};
1120/// use parquet::basic; // note there are two `Type`s that are different
1121/// // Schema for a table with two columns: "a" (int64) and "b" (int32, stored as a date)
1122/// let descriptor = SchemaDescriptor::new(
1123///   Arc::new(
1124///     Type::group_type_builder("my_schema")
1125///       .with_fields(vec![
1126///         Arc::new(
1127///          Type::primitive_type_builder("a", basic::Type::INT64)
1128///           .build().unwrap()
1129///         ),
1130///         Arc::new(
1131///          Type::primitive_type_builder("b", basic::Type::INT32)
1132///           .with_converted_type(basic::ConvertedType::DATE)
1133///           .with_logical_type(Some(basic::LogicalType::Date))
1134///           .build().unwrap()
1135///         ),
1136///      ])
1137///      .build().unwrap()
1138///   )
1139/// );
1140/// ```
1141#[derive(PartialEq, Clone)]
1142pub struct SchemaDescriptor {
1143    /// The top-level logical schema (the "message" type).
1144    ///
1145    /// This must be a [`Type::GroupType`] where each field is a root
1146    /// column type in the schema.
1147    schema: TypePtr,
1148
1149    /// The descriptors for the physical type of each leaf column in this schema
1150    ///
1151    /// Constructed from `schema` in DFS order.
1152    leaves: Vec<ColumnDescPtr>,
1153
1154    /// Mapping from a leaf column's index to the root column index that it
1155    /// comes from.
1156    ///
1157    /// For instance: the leaf `a.b.c.d` would have a link back to `a`:
1158    /// ```text
1159    /// -- a  <-----+
1160    /// -- -- b     |
1161    /// -- -- -- c  |
1162    /// -- -- -- -- d
1163    /// ```
1164    leaf_to_base: Vec<usize>,
1165}
1166
1167impl fmt::Debug for SchemaDescriptor {
1168    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1169        // Skip leaves and leaf_to_base as they only a cache information already found in `schema`
1170        f.debug_struct("SchemaDescriptor")
1171            .field("schema", &self.schema)
1172            .finish()
1173    }
1174}
1175
1176// Need to implement HeapSize in this module as the fields are private
1177impl HeapSize for SchemaDescriptor {
1178    fn heap_size(&self) -> usize {
1179        self.schema.heap_size() + self.leaves.heap_size() + self.leaf_to_base.heap_size()
1180    }
1181}
1182
1183impl SchemaDescriptor {
1184    /// Creates new schema descriptor from Parquet schema.
1185    pub fn new(tp: TypePtr) -> Self {
1186        const INIT_SCHEMA_DEPTH: usize = 16;
1187        assert!(tp.is_group(), "SchemaDescriptor should take a GroupType");
1188        // unwrap should be safe since we just asserted tp is a group
1189        let n_leaves = num_leaves(&tp).unwrap();
1190        let mut leaves = Vec::with_capacity(n_leaves);
1191        let mut leaf_to_base = Vec::with_capacity(n_leaves);
1192        let mut path = Vec::with_capacity(INIT_SCHEMA_DEPTH);
1193        for (root_idx, f) in tp.get_fields().iter().enumerate() {
1194            path.clear();
1195            build_tree(
1196                f,
1197                root_idx,
1198                0,
1199                0,
1200                0,
1201                &mut leaves,
1202                &mut leaf_to_base,
1203                &mut path,
1204            );
1205        }
1206
1207        Self {
1208            schema: tp,
1209            leaves,
1210            leaf_to_base,
1211        }
1212    }
1213
1214    /// Returns [`ColumnDescriptor`] for a field position.
1215    ///
1216    /// # Panics
1217    ///
1218    /// Panics if `i >= self.num_columns()`
1219    pub fn column(&self, i: usize) -> ColumnDescPtr {
1220        assert!(
1221            i < self.leaves.len(),
1222            "Index out of bound: {} not in [0, {})",
1223            i,
1224            self.leaves.len()
1225        );
1226        self.leaves[i].clone()
1227    }
1228
1229    /// Returns slice of [`ColumnDescriptor`].
1230    pub fn columns(&self) -> &[ColumnDescPtr] {
1231        &self.leaves
1232    }
1233
1234    /// Returns number of leaf-level columns.
1235    pub fn num_columns(&self) -> usize {
1236        self.leaves.len()
1237    }
1238
1239    /// Returns column root [`Type`] for a leaf position.
1240    pub fn get_column_root(&self, i: usize) -> &Type {
1241        let result = self.column_root_of(i);
1242        result.as_ref()
1243    }
1244
1245    /// Returns column root [`Type`] pointer for a leaf position.
1246    ///
1247    /// # Panics
1248    ///
1249    /// Panics if `i >= self.num_columns()`
1250    pub fn get_column_root_ptr(&self, i: usize) -> TypePtr {
1251        let result = self.column_root_of(i);
1252        result.clone()
1253    }
1254
1255    /// Returns the index of the root column for a field position
1256    ///
1257    /// # Panics
1258    ///
1259    /// Panics if `leaf` is out of bounds
1260    pub fn get_column_root_idx(&self, leaf: usize) -> usize {
1261        assert!(
1262            leaf < self.leaves.len(),
1263            "Index out of bound: {} not in [0, {})",
1264            leaf,
1265            self.leaves.len()
1266        );
1267
1268        *self
1269            .leaf_to_base
1270            .get(leaf)
1271            .unwrap_or_else(|| panic!("Expected a value for index {leaf} but found None"))
1272    }
1273
1274    fn column_root_of(&self, i: usize) -> &TypePtr {
1275        &self.schema.get_fields()[self.get_column_root_idx(i)]
1276    }
1277
1278    /// Returns schema as [`Type`].
1279    pub fn root_schema(&self) -> &Type {
1280        self.schema.as_ref()
1281    }
1282
1283    /// Returns schema as [`TypePtr`] for cheap cloning.
1284    pub fn root_schema_ptr(&self) -> TypePtr {
1285        self.schema.clone()
1286    }
1287
1288    /// Returns schema name.
1289    pub fn name(&self) -> &str {
1290        self.schema.name()
1291    }
1292}
1293
1294// walk tree and count nodes
1295pub(crate) fn num_nodes(tp: &TypePtr) -> Result<usize> {
1296    if !tp.is_group() {
1297        return Err(general_err!("Root schema must be Group type"));
1298    }
1299    let mut n_nodes = 1usize; // count root
1300    for f in tp.get_fields() {
1301        count_nodes(f, &mut n_nodes);
1302    }
1303    Ok(n_nodes)
1304}
1305
1306pub(crate) fn count_nodes(tp: &TypePtr, n_nodes: &mut usize) {
1307    *n_nodes += 1;
1308    if let Type::GroupType { fields, .. } = tp.as_ref() {
1309        for f in fields {
1310            count_nodes(f, n_nodes);
1311        }
1312    }
1313}
1314
1315// do a quick walk of the tree to get proper sizing for SchemaDescriptor arrays
1316fn num_leaves(tp: &TypePtr) -> Result<usize> {
1317    if !tp.is_group() {
1318        return Err(general_err!("Root schema must be Group type"));
1319    }
1320    let mut n_leaves = 0usize;
1321    for f in tp.get_fields() {
1322        count_leaves(f, &mut n_leaves);
1323    }
1324    Ok(n_leaves)
1325}
1326
1327fn count_leaves(tp: &TypePtr, n_leaves: &mut usize) {
1328    match tp.as_ref() {
1329        Type::PrimitiveType { .. } => *n_leaves += 1,
1330        Type::GroupType { fields, .. } => {
1331            for f in fields {
1332                count_leaves(f, n_leaves);
1333            }
1334        }
1335    }
1336}
1337
1338#[expect(clippy::too_many_arguments)]
1339fn build_tree<'a>(
1340    tp: &'a TypePtr,
1341    root_idx: usize,
1342    mut max_rep_level: i16,
1343    mut max_def_level: i16,
1344    mut repeated_ancestor_def_level: i16,
1345    leaves: &mut Vec<ColumnDescPtr>,
1346    leaf_to_base: &mut Vec<usize>,
1347    path_so_far: &mut Vec<&'a str>,
1348) {
1349    assert!(tp.get_basic_info().has_repetition());
1350
1351    path_so_far.push(tp.name());
1352    match tp.get_basic_info().repetition() {
1353        Repetition::OPTIONAL => {
1354            max_def_level += 1;
1355        }
1356        Repetition::REPEATED => {
1357            max_def_level += 1;
1358            max_rep_level += 1;
1359            repeated_ancestor_def_level = max_def_level;
1360        }
1361        Repetition::REQUIRED => {}
1362    }
1363
1364    match tp.as_ref() {
1365        Type::PrimitiveType { .. } => {
1366            let mut path: Vec<String> = vec![];
1367            path.extend(path_so_far.iter().copied().map(String::from));
1368            let desc = ColumnDescriptor::new_with_repeated_ancestor(
1369                tp.clone(),
1370                max_def_level,
1371                max_rep_level,
1372                ColumnPath::new(path),
1373                repeated_ancestor_def_level,
1374            );
1375            leaves.push(Arc::new(desc));
1376            leaf_to_base.push(root_idx);
1377        }
1378        Type::GroupType { fields, .. } => {
1379            for f in fields {
1380                build_tree(
1381                    f,
1382                    root_idx,
1383                    max_rep_level,
1384                    max_def_level,
1385                    repeated_ancestor_def_level,
1386                    leaves,
1387                    leaf_to_base,
1388                    path_so_far,
1389                );
1390                path_so_far.pop();
1391            }
1392        }
1393    }
1394}
1395
1396/// Checks if the logical type is valid.
1397fn check_logical_type(logical_type: Option<&LogicalType>) -> Result<()> {
1398    if let Some(LogicalType::Integer(IntType { bit_width, .. })) = logical_type
1399        && *bit_width != 8
1400        && *bit_width != 16
1401        && *bit_width != 32
1402        && *bit_width != 64
1403    {
1404        return Err(general_err!(
1405            "Bit width must be 8, 16, 32, or 64 for Integer logical type"
1406        ));
1407    }
1408    Ok(())
1409}
1410
1411// convert thrift decoded array of `SchemaElement` into this crate's representation of
1412// parquet types. this function consumes `elements`.
1413pub(crate) fn parquet_schema_from_array(elements: Vec<SchemaElement<'_>>) -> Result<TypePtr> {
1414    let mut index = 0;
1415    let num_elements = elements.len();
1416    let mut schema_nodes = Vec::with_capacity(1); // there should only be one element when done
1417
1418    // turn into iterator so we can take ownership of elements of the vector
1419    let mut elements = elements.into_iter();
1420
1421    while index < num_elements {
1422        let t = schema_from_array_helper(&mut elements, num_elements, index)?;
1423        index = t.0;
1424        schema_nodes.push(t.1);
1425    }
1426    if schema_nodes.len() != 1 {
1427        return Err(general_err!(
1428            "Expected exactly one root node, but found {}",
1429            schema_nodes.len()
1430        ));
1431    }
1432
1433    if !schema_nodes[0].is_group() {
1434        return Err(general_err!("Expected root node to be a group type"));
1435    }
1436
1437    Ok(schema_nodes.remove(0))
1438}
1439
1440// recursive helper function for schema conversion
1441fn schema_from_array_helper(
1442    elements: &mut IntoIter<SchemaElement<'_>>,
1443    num_elements: usize,
1444    index: usize,
1445) -> Result<(usize, TypePtr)> {
1446    // Whether or not the current node is root (message type).
1447    // There is only one message type node in the schema tree.
1448    let is_root_node = index == 0;
1449
1450    if index >= num_elements {
1451        return Err(general_err!(
1452            "Index out of bound, index = {}, len = {}",
1453            index,
1454            num_elements
1455        ));
1456    }
1457    let element = elements.next().expect("schema vector should not be empty");
1458
1459    // Check for empty schema
1460    if let (true, None | Some(0)) = (is_root_node, element.num_children) {
1461        let builder = Type::group_type_builder(element.name);
1462        return Ok((index + 1, Arc::new(builder.build().unwrap())));
1463    }
1464
1465    let converted_type = element.converted_type.unwrap_or(ConvertedType::NONE);
1466
1467    // LogicalType is preferred to ConvertedType, but both may be present.
1468    let logical_type = element.logical_type;
1469
1470    check_logical_type(logical_type.as_ref())?;
1471
1472    let field_id = element.field_id;
1473    match element.num_children {
1474        // From parquet-format:
1475        //   The children count is used to construct the nested relationship.
1476        //   This field is not set when the element is a primitive type
1477        // Sometimes parquet-cpp sets num_children field to 0 for primitive types, so we
1478        // have to handle this case too.
1479        None | Some(0) => {
1480            // primitive type
1481            if element.repetition_type.is_none() {
1482                return Err(general_err!(
1483                    "Repetition level must be defined for a primitive type"
1484                ));
1485            }
1486            let repetition = element.repetition_type.unwrap();
1487            if let Some(physical_type) = element.r#type {
1488                let length = element.type_length.unwrap_or(-1);
1489                let scale = element.scale.unwrap_or(-1);
1490                let precision = element.precision.unwrap_or(-1);
1491                let name = element.name;
1492                let builder = Type::primitive_type_builder(name, physical_type)
1493                    .with_repetition(repetition)
1494                    .with_converted_type(converted_type)
1495                    .with_logical_type(logical_type)
1496                    .with_length(length)
1497                    .with_precision(precision)
1498                    .with_scale(scale)
1499                    .with_id(field_id);
1500                Ok((index + 1, Arc::new(builder.build()?)))
1501            } else {
1502                let mut builder = Type::group_type_builder(element.name)
1503                    .with_converted_type(converted_type)
1504                    .with_logical_type(logical_type)
1505                    .with_id(field_id);
1506                if !is_root_node {
1507                    // Sometimes parquet-cpp and parquet-mr set repetition level REQUIRED or
1508                    // REPEATED for root node.
1509                    //
1510                    // We only set repetition for group types that are not top-level message
1511                    // type. According to parquet-format:
1512                    //   Root of the schema does not have a repetition_type.
1513                    //   All other types must have one.
1514                    builder = builder.with_repetition(repetition);
1515                }
1516                Ok((index + 1, Arc::new(builder.build().unwrap())))
1517            }
1518        }
1519        Some(n) => {
1520            let repetition = element.repetition_type;
1521
1522            let num_children = usize::try_from(n)?;
1523            if num_children > elements.len() {
1524                return Err(general_err!(
1525                    "Schema children count {} exceeds remaining schema elements {}",
1526                    num_children,
1527                    elements.len()
1528                ));
1529            }
1530            let mut fields = Vec::with_capacity(num_children);
1531            let mut next_index = index + 1;
1532            for _ in 0..num_children {
1533                let child_result = schema_from_array_helper(elements, num_elements, next_index)?;
1534                next_index = child_result.0;
1535                fields.push(child_result.1);
1536            }
1537
1538            let mut builder = Type::group_type_builder(element.name)
1539                .with_converted_type(converted_type)
1540                .with_logical_type(logical_type)
1541                .with_fields(fields)
1542                .with_id(field_id);
1543
1544            // Sometimes parquet-cpp and parquet-mr set repetition level REQUIRED or
1545            // REPEATED for root node.
1546            //
1547            // We only set repetition for group types that are not top-level message
1548            // type. According to parquet-format:
1549            //   Root of the schema does not have a repetition_type.
1550            //   All other types must have one.
1551            if !is_root_node {
1552                let Some(rep) = repetition else {
1553                    return Err(general_err!(
1554                        "Repetition level must be defined for non-root types"
1555                    ));
1556                };
1557                builder = builder.with_repetition(rep);
1558            }
1559            Ok((next_index, Arc::new(builder.build()?)))
1560        }
1561    }
1562}
1563
1564#[cfg(test)]
1565mod tests {
1566    use super::*;
1567
1568    use crate::{
1569        file::metadata::thrift::tests::{buf_to_schema_list, roundtrip_schema, schema_to_buf},
1570        schema::parser::parse_message_type,
1571    };
1572
1573    // TODO: add tests for v2 types
1574
1575    #[test]
1576    fn test_primitive_type() {
1577        let mut result = Type::primitive_type_builder("foo", PhysicalType::INT32)
1578            .with_logical_type(Some(LogicalType::integer(32, true)))
1579            .with_id(Some(0))
1580            .build();
1581        assert!(result.is_ok());
1582
1583        if let Ok(tp) = result {
1584            assert!(tp.is_primitive());
1585            assert!(!tp.is_group());
1586            let basic_info = tp.get_basic_info();
1587            assert_eq!(basic_info.repetition(), Repetition::OPTIONAL);
1588            assert_eq!(
1589                basic_info.logical_type_ref(),
1590                Some(&LogicalType::integer(32, true))
1591            );
1592            assert_eq!(basic_info.converted_type(), ConvertedType::INT_32);
1593            assert_eq!(basic_info.id(), 0);
1594            match tp {
1595                Type::PrimitiveType { physical_type, .. } => {
1596                    assert_eq!(physical_type, PhysicalType::INT32);
1597                }
1598                Type::GroupType { .. } => panic!(),
1599            }
1600        }
1601
1602        // Test illegal inputs with logical type
1603        result = Type::primitive_type_builder("foo", PhysicalType::INT64)
1604            .with_repetition(Repetition::REPEATED)
1605            .with_logical_type(Some(LogicalType::integer(8, true)))
1606            .build();
1607        assert!(result.is_err());
1608        if let Err(e) = result {
1609            assert_eq!(
1610                format!("{e}"),
1611                "Parquet error: Cannot annotate Integer(IntType { bit_width: 8, is_signed: true }) from INT64 for field 'foo'"
1612            );
1613        }
1614
1615        // Test illegal inputs with converted type
1616        result = Type::primitive_type_builder("foo", PhysicalType::INT64)
1617            .with_repetition(Repetition::REPEATED)
1618            .with_converted_type(ConvertedType::BSON)
1619            .build();
1620        assert!(result.is_err());
1621        if let Err(e) = result {
1622            assert_eq!(
1623                format!("{e}"),
1624                "Parquet error: BSON cannot annotate field 'foo' because it is not a BYTE_ARRAY field"
1625            );
1626        }
1627
1628        result = Type::primitive_type_builder("foo", PhysicalType::INT96)
1629            .with_repetition(Repetition::REQUIRED)
1630            .with_converted_type(ConvertedType::DECIMAL)
1631            .with_precision(-1)
1632            .with_scale(-1)
1633            .build();
1634        assert!(result.is_err());
1635        if let Err(e) = result {
1636            assert_eq!(
1637                format!("{e}"),
1638                "Parquet error: DECIMAL can only annotate INT32, INT64, BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY"
1639            );
1640        }
1641
1642        result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
1643            .with_repetition(Repetition::REQUIRED)
1644            .with_logical_type(Some(LogicalType::decimal(32, 12)))
1645            .with_precision(-1)
1646            .with_scale(-1)
1647            .build();
1648        assert!(result.is_err());
1649        if let Err(e) = result {
1650            assert_eq!(
1651                format!("{e}"),
1652                "Parquet error: DECIMAL logical type scale 32 must match self.scale -1 for field 'foo'"
1653            );
1654        }
1655
1656        result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
1657            .with_repetition(Repetition::REQUIRED)
1658            .with_converted_type(ConvertedType::DECIMAL)
1659            .with_precision(-1)
1660            .with_scale(-1)
1661            .build();
1662        assert!(result.is_err());
1663        if let Err(e) = result {
1664            assert_eq!(
1665                format!("{e}"),
1666                "Parquet error: Invalid DECIMAL precision: -1"
1667            );
1668        }
1669
1670        result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
1671            .with_repetition(Repetition::REQUIRED)
1672            .with_converted_type(ConvertedType::DECIMAL)
1673            .with_precision(0)
1674            .with_scale(-1)
1675            .build();
1676        assert!(result.is_err());
1677        if let Err(e) = result {
1678            assert_eq!(
1679                format!("{e}"),
1680                "Parquet error: Invalid DECIMAL precision: 0"
1681            );
1682        }
1683
1684        result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
1685            .with_repetition(Repetition::REQUIRED)
1686            .with_converted_type(ConvertedType::DECIMAL)
1687            .with_precision(1)
1688            .with_scale(-1)
1689            .build();
1690        assert!(result.is_err());
1691        if let Err(e) = result {
1692            assert_eq!(format!("{e}"), "Parquet error: Invalid DECIMAL scale: -1");
1693        }
1694
1695        result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
1696            .with_repetition(Repetition::REQUIRED)
1697            .with_converted_type(ConvertedType::DECIMAL)
1698            .with_precision(1)
1699            .with_scale(2)
1700            .build();
1701        assert!(result.is_err());
1702        if let Err(e) = result {
1703            assert_eq!(
1704                format!("{e}"),
1705                "Parquet error: Invalid DECIMAL: scale (2) cannot be greater than precision (1)"
1706            );
1707        }
1708
1709        // It is OK if precision == scale
1710        result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
1711            .with_repetition(Repetition::REQUIRED)
1712            .with_converted_type(ConvertedType::DECIMAL)
1713            .with_precision(1)
1714            .with_scale(1)
1715            .build();
1716        assert!(result.is_ok());
1717
1718        result = Type::primitive_type_builder("foo", PhysicalType::INT32)
1719            .with_repetition(Repetition::REQUIRED)
1720            .with_converted_type(ConvertedType::DECIMAL)
1721            .with_precision(18)
1722            .with_scale(2)
1723            .build();
1724        assert!(result.is_err());
1725        if let Err(e) = result {
1726            assert_eq!(
1727                format!("{e}"),
1728                "Parquet error: Cannot represent INT32 as DECIMAL with precision 18"
1729            );
1730        }
1731
1732        result = Type::primitive_type_builder("foo", PhysicalType::INT64)
1733            .with_repetition(Repetition::REQUIRED)
1734            .with_converted_type(ConvertedType::DECIMAL)
1735            .with_precision(32)
1736            .with_scale(2)
1737            .build();
1738        assert!(result.is_err());
1739        if let Err(e) = result {
1740            assert_eq!(
1741                format!("{e}"),
1742                "Parquet error: Cannot represent INT64 as DECIMAL with precision 32"
1743            );
1744        }
1745
1746        result = Type::primitive_type_builder("foo", PhysicalType::FIXED_LEN_BYTE_ARRAY)
1747            .with_repetition(Repetition::REQUIRED)
1748            .with_converted_type(ConvertedType::DECIMAL)
1749            .with_length(5)
1750            .with_precision(12)
1751            .with_scale(2)
1752            .build();
1753        assert!(result.is_err());
1754        if let Err(e) = result {
1755            assert_eq!(
1756                format!("{e}"),
1757                "Parquet error: Cannot represent FIXED_LEN_BYTE_ARRAY as DECIMAL with length 5 and precision 12. The max precision can only be 11"
1758            );
1759        }
1760
1761        result = Type::primitive_type_builder("foo", PhysicalType::INT64)
1762            .with_repetition(Repetition::REQUIRED)
1763            .with_converted_type(ConvertedType::UINT_8)
1764            .build();
1765        assert!(result.is_err());
1766        if let Err(e) = result {
1767            assert_eq!(
1768                format!("{e}"),
1769                "Parquet error: UINT_8 cannot annotate field 'foo' because it is not a INT32 field"
1770            );
1771        }
1772
1773        result = Type::primitive_type_builder("foo", PhysicalType::INT32)
1774            .with_repetition(Repetition::REQUIRED)
1775            .with_converted_type(ConvertedType::TIME_MICROS)
1776            .build();
1777        assert!(result.is_err());
1778        if let Err(e) = result {
1779            assert_eq!(
1780                format!("{e}"),
1781                "Parquet error: TIME_MICROS cannot annotate field 'foo' because it is not a INT64 field"
1782            );
1783        }
1784
1785        result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
1786            .with_repetition(Repetition::REQUIRED)
1787            .with_converted_type(ConvertedType::INTERVAL)
1788            .build();
1789        assert!(result.is_err());
1790        if let Err(e) = result {
1791            assert_eq!(
1792                format!("{e}"),
1793                "Parquet error: INTERVAL cannot annotate field 'foo' because it is not a FIXED_LEN_BYTE_ARRAY(12) field"
1794            );
1795        }
1796
1797        result = Type::primitive_type_builder("foo", PhysicalType::FIXED_LEN_BYTE_ARRAY)
1798            .with_repetition(Repetition::REQUIRED)
1799            .with_converted_type(ConvertedType::INTERVAL)
1800            .with_length(1)
1801            .build();
1802        assert!(result.is_err());
1803        if let Err(e) = result {
1804            assert_eq!(
1805                format!("{e}"),
1806                "Parquet error: INTERVAL cannot annotate field 'foo' because it is not a FIXED_LEN_BYTE_ARRAY(12) field"
1807            );
1808        }
1809
1810        result = Type::primitive_type_builder("foo", PhysicalType::INT32)
1811            .with_repetition(Repetition::REQUIRED)
1812            .with_converted_type(ConvertedType::ENUM)
1813            .build();
1814        assert!(result.is_err());
1815        if let Err(e) = result {
1816            assert_eq!(
1817                format!("{e}"),
1818                "Parquet error: ENUM cannot annotate field 'foo' because it is not a BYTE_ARRAY field"
1819            );
1820        }
1821
1822        result = Type::primitive_type_builder("foo", PhysicalType::INT32)
1823            .with_repetition(Repetition::REQUIRED)
1824            .with_converted_type(ConvertedType::MAP)
1825            .build();
1826        assert!(result.is_err());
1827        if let Err(e) = result {
1828            assert_eq!(
1829                format!("{e}"),
1830                "Parquet error: MAP cannot be applied to primitive field 'foo'"
1831            );
1832        }
1833
1834        result = Type::primitive_type_builder("foo", PhysicalType::FIXED_LEN_BYTE_ARRAY)
1835            .with_repetition(Repetition::REQUIRED)
1836            .with_converted_type(ConvertedType::DECIMAL)
1837            .with_length(-1)
1838            .build();
1839        assert!(result.is_err());
1840        if let Err(e) = result {
1841            assert_eq!(
1842                format!("{e}"),
1843                "Parquet error: Invalid FIXED_LEN_BYTE_ARRAY length: -1 for field 'foo'"
1844            );
1845        }
1846
1847        result = Type::primitive_type_builder("foo", PhysicalType::FIXED_LEN_BYTE_ARRAY)
1848            .with_repetition(Repetition::REQUIRED)
1849            .with_logical_type(Some(LogicalType::Float16))
1850            .with_length(2)
1851            .build();
1852        assert!(result.is_ok());
1853
1854        // Can't be other than FIXED_LEN_BYTE_ARRAY for physical type
1855        result = Type::primitive_type_builder("foo", PhysicalType::FLOAT)
1856            .with_repetition(Repetition::REQUIRED)
1857            .with_logical_type(Some(LogicalType::Float16))
1858            .with_length(2)
1859            .build();
1860        assert!(result.is_err());
1861        if let Err(e) = result {
1862            assert_eq!(
1863                format!("{e}"),
1864                "Parquet error: Cannot annotate Float16 from FLOAT for field 'foo'"
1865            );
1866        }
1867
1868        // Must have length 2
1869        result = Type::primitive_type_builder("foo", PhysicalType::FIXED_LEN_BYTE_ARRAY)
1870            .with_repetition(Repetition::REQUIRED)
1871            .with_logical_type(Some(LogicalType::Float16))
1872            .with_length(4)
1873            .build();
1874        assert!(result.is_err());
1875        if let Err(e) = result {
1876            assert_eq!(
1877                format!("{e}"),
1878                "Parquet error: FLOAT16 cannot annotate field 'foo' because it is not a FIXED_LEN_BYTE_ARRAY(2) field"
1879            );
1880        }
1881
1882        // Must have length 16
1883        result = Type::primitive_type_builder("foo", PhysicalType::FIXED_LEN_BYTE_ARRAY)
1884            .with_repetition(Repetition::REQUIRED)
1885            .with_logical_type(Some(LogicalType::Uuid))
1886            .with_length(15)
1887            .build();
1888        assert!(result.is_err());
1889        if let Err(e) = result {
1890            assert_eq!(
1891                format!("{e}"),
1892                "Parquet error: UUID cannot annotate field 'foo' because it is not a FIXED_LEN_BYTE_ARRAY(16) field"
1893            );
1894        }
1895
1896        // test unknown logical types are ok
1897        result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
1898            .with_logical_type(Some(LogicalType::_Unknown { field_id: 100 }))
1899            .build();
1900        assert!(result.is_ok());
1901    }
1902
1903    #[test]
1904    fn test_group_type() {
1905        let f1 = Type::primitive_type_builder("f1", PhysicalType::INT32)
1906            .with_converted_type(ConvertedType::INT_32)
1907            .with_id(Some(0))
1908            .build();
1909        assert!(f1.is_ok());
1910        let f2 = Type::primitive_type_builder("f2", PhysicalType::BYTE_ARRAY)
1911            .with_converted_type(ConvertedType::UTF8)
1912            .with_id(Some(1))
1913            .build();
1914        assert!(f2.is_ok());
1915
1916        let fields = vec![Arc::new(f1.unwrap()), Arc::new(f2.unwrap())];
1917
1918        let result = Type::group_type_builder("foo")
1919            .with_repetition(Repetition::REPEATED)
1920            .with_logical_type(Some(LogicalType::List))
1921            .with_fields(fields)
1922            .with_id(Some(1))
1923            .build();
1924        assert!(result.is_ok());
1925
1926        let tp = result.unwrap();
1927        let basic_info = tp.get_basic_info();
1928        assert!(tp.is_group());
1929        assert!(!tp.is_primitive());
1930        assert_eq!(basic_info.repetition(), Repetition::REPEATED);
1931        assert_eq!(basic_info.logical_type_ref(), Some(&LogicalType::List));
1932        assert_eq!(basic_info.converted_type(), ConvertedType::LIST);
1933        assert_eq!(basic_info.id(), 1);
1934        assert_eq!(tp.get_fields().len(), 2);
1935        assert_eq!(tp.get_fields()[0].name(), "f1");
1936        assert_eq!(tp.get_fields()[1].name(), "f2");
1937    }
1938
1939    #[test]
1940    fn test_column_descriptor() {
1941        let result = test_column_descriptor_helper();
1942        assert!(
1943            result.is_ok(),
1944            "Expected result to be OK but got err:\n {}",
1945            result.unwrap_err()
1946        );
1947    }
1948
1949    fn test_column_descriptor_helper() -> Result<()> {
1950        let tp = Type::primitive_type_builder("name", PhysicalType::BYTE_ARRAY)
1951            .with_converted_type(ConvertedType::UTF8)
1952            .build()?;
1953
1954        let descr = ColumnDescriptor::new(Arc::new(tp), 4, 1, ColumnPath::from("name"));
1955
1956        assert_eq!(descr.path(), &ColumnPath::from("name"));
1957        assert_eq!(descr.converted_type(), ConvertedType::UTF8);
1958        assert_eq!(descr.physical_type(), PhysicalType::BYTE_ARRAY);
1959        assert_eq!(descr.max_def_level(), 4);
1960        assert_eq!(descr.max_rep_level(), 1);
1961        assert_eq!(descr.name(), "name");
1962        assert_eq!(descr.type_length(), -1);
1963        assert_eq!(descr.type_precision(), -1);
1964        assert_eq!(descr.type_scale(), -1);
1965
1966        Ok(())
1967    }
1968
1969    #[test]
1970    fn test_schema_descriptor() {
1971        let result = test_schema_descriptor_helper();
1972        assert!(
1973            result.is_ok(),
1974            "Expected result to be OK but got err:\n {}",
1975            result.unwrap_err()
1976        );
1977    }
1978
1979    // A helper fn to avoid handling the results from type creation
1980    fn test_schema_descriptor_helper() -> Result<()> {
1981        let mut fields = vec![];
1982
1983        let inta = Type::primitive_type_builder("a", PhysicalType::INT32)
1984            .with_repetition(Repetition::REQUIRED)
1985            .with_converted_type(ConvertedType::INT_32)
1986            .build()?;
1987        fields.push(Arc::new(inta));
1988        let intb = Type::primitive_type_builder("b", PhysicalType::INT64)
1989            .with_converted_type(ConvertedType::INT_64)
1990            .build()?;
1991        fields.push(Arc::new(intb));
1992        let intc = Type::primitive_type_builder("c", PhysicalType::BYTE_ARRAY)
1993            .with_repetition(Repetition::REPEATED)
1994            .with_converted_type(ConvertedType::UTF8)
1995            .build()?;
1996        fields.push(Arc::new(intc));
1997
1998        // 3-level list encoding
1999        let item1 = Type::primitive_type_builder("item1", PhysicalType::INT64)
2000            .with_repetition(Repetition::REQUIRED)
2001            .with_converted_type(ConvertedType::INT_64)
2002            .build()?;
2003        let item2 = Type::primitive_type_builder("item2", PhysicalType::BOOLEAN).build()?;
2004        let item3 = Type::primitive_type_builder("item3", PhysicalType::INT32)
2005            .with_repetition(Repetition::REPEATED)
2006            .with_converted_type(ConvertedType::INT_32)
2007            .build()?;
2008        let list = Type::group_type_builder("records")
2009            .with_repetition(Repetition::REPEATED)
2010            .with_converted_type(ConvertedType::LIST)
2011            .with_fields(vec![Arc::new(item1), Arc::new(item2), Arc::new(item3)])
2012            .build()?;
2013        let bag = Type::group_type_builder("bag")
2014            .with_repetition(Repetition::OPTIONAL)
2015            .with_fields(vec![Arc::new(list)])
2016            .build()?;
2017        fields.push(Arc::new(bag));
2018
2019        let schema = Type::group_type_builder("schema")
2020            .with_repetition(Repetition::REPEATED)
2021            .with_fields(fields)
2022            .build()?;
2023        let descr = SchemaDescriptor::new(Arc::new(schema));
2024
2025        let nleaves = 6;
2026        assert_eq!(descr.num_columns(), nleaves);
2027
2028        //                             mdef mrep
2029        // required int32 a            0    0
2030        // optional int64 b            1    0
2031        // repeated byte_array c       1    1
2032        // optional group bag          1    0
2033        //   repeated group records    2    1
2034        //     required int64 item1    2    1
2035        //     optional boolean item2  3    1
2036        //     repeated int32 item3    3    2
2037        let ex_max_def_levels = [0, 1, 1, 2, 3, 3];
2038        let ex_max_rep_levels = [0, 0, 1, 1, 1, 2];
2039
2040        for i in 0..nleaves {
2041            let col = descr.column(i);
2042            assert_eq!(col.max_def_level(), ex_max_def_levels[i], "{i}");
2043            assert_eq!(col.max_rep_level(), ex_max_rep_levels[i], "{i}");
2044        }
2045
2046        assert_eq!(descr.column(0).path().string(), "a");
2047        assert_eq!(descr.column(1).path().string(), "b");
2048        assert_eq!(descr.column(2).path().string(), "c");
2049        assert_eq!(descr.column(3).path().string(), "bag.records.item1");
2050        assert_eq!(descr.column(4).path().string(), "bag.records.item2");
2051        assert_eq!(descr.column(5).path().string(), "bag.records.item3");
2052
2053        assert_eq!(descr.get_column_root(0).name(), "a");
2054        assert_eq!(descr.get_column_root(3).name(), "bag");
2055        assert_eq!(descr.get_column_root(4).name(), "bag");
2056        assert_eq!(descr.get_column_root(5).name(), "bag");
2057
2058        Ok(())
2059    }
2060
2061    #[test]
2062    fn test_schema_build_tree_def_rep_levels() {
2063        let message_type = "
2064    message spark_schema {
2065      REQUIRED INT32 a;
2066      OPTIONAL group b {
2067        OPTIONAL INT32 _1;
2068        OPTIONAL INT32 _2;
2069      }
2070      OPTIONAL group c (LIST) {
2071        REPEATED group list {
2072          OPTIONAL INT32 element;
2073        }
2074      }
2075    }
2076    ";
2077        let schema = parse_message_type(message_type).expect("should parse schema");
2078        let descr = SchemaDescriptor::new(Arc::new(schema));
2079        // required int32 a
2080        assert_eq!(descr.column(0).max_def_level(), 0);
2081        assert_eq!(descr.column(0).max_rep_level(), 0);
2082        // optional int32 b._1
2083        assert_eq!(descr.column(1).max_def_level(), 2);
2084        assert_eq!(descr.column(1).max_rep_level(), 0);
2085        // optional int32 b._2
2086        assert_eq!(descr.column(2).max_def_level(), 2);
2087        assert_eq!(descr.column(2).max_rep_level(), 0);
2088        // repeated optional int32 c.list.element
2089        assert_eq!(descr.column(3).max_def_level(), 3);
2090        assert_eq!(descr.column(3).max_rep_level(), 1);
2091    }
2092
2093    #[test]
2094    fn test_schema_build_tree_repeated_ancestor_def_level() {
2095        // Flat columns: no REPEATED ancestor → repeated_ancestor_def_level = 0
2096        let message_type = "
2097    message m {
2098      REQUIRED INT32 a;
2099      OPTIONAL INT32 b;
2100      OPTIONAL group s {
2101        OPTIONAL INT32 x;
2102      }
2103    }
2104    ";
2105        let schema = parse_message_type(message_type).expect("should parse schema");
2106        let descr = SchemaDescriptor::new(Arc::new(schema));
2107        assert_eq!(descr.column(0).repeated_ancestor_def_level(), 0); // a
2108        assert_eq!(descr.column(1).repeated_ancestor_def_level(), 0); // b
2109        assert_eq!(descr.column(2).repeated_ancestor_def_level(), 0); // s.x
2110
2111        // Standard list: OPTIONAL outer, REPEATED group, OPTIONAL element
2112        // repeated_ancestor_def_level is the def_level at the REPEATED group (= 2)
2113        let message_type = "
2114    message m {
2115      OPTIONAL group c (LIST) {
2116        REPEATED group list {
2117          OPTIONAL INT32 element;
2118        }
2119      }
2120    }
2121    ";
2122        let schema = parse_message_type(message_type).expect("should parse schema");
2123        let descr = SchemaDescriptor::new(Arc::new(schema));
2124        // c(optional)=1, list(repeated)=2, element(optional)=3
2125        assert_eq!(descr.column(0).max_def_level(), 3);
2126        assert_eq!(descr.column(0).max_rep_level(), 1);
2127        assert_eq!(descr.column(0).repeated_ancestor_def_level(), 2);
2128
2129        // Required list: REQUIRED outer, REPEATED group, REQUIRED element
2130        // No OPTIONAL nodes between REPEATED and leaf, so repeated_ancestor_def_level == max_def_level
2131        let message_type = "
2132    message m {
2133      REQUIRED group c (LIST) {
2134        REPEATED group list {
2135          REQUIRED INT32 element;
2136        }
2137      }
2138    }
2139    ";
2140        let schema = parse_message_type(message_type).expect("should parse schema");
2141        let descr = SchemaDescriptor::new(Arc::new(schema));
2142        // list(repeated)=1, element(required)=1
2143        assert_eq!(descr.column(0).max_def_level(), 1);
2144        assert_eq!(descr.column(0).max_rep_level(), 1);
2145        assert_eq!(descr.column(0).repeated_ancestor_def_level(), 1);
2146
2147        // Nested lists: innermost REPEATED wins
2148        let message_type = "
2149    message m {
2150      OPTIONAL group outer (LIST) {
2151        REPEATED group list {
2152          OPTIONAL group inner (LIST) {
2153            REPEATED group list2 {
2154              OPTIONAL INT32 element;
2155            }
2156          }
2157        }
2158      }
2159    }
2160    ";
2161        let schema = parse_message_type(message_type).expect("should parse schema");
2162        let descr = SchemaDescriptor::new(Arc::new(schema));
2163        // outer(opt)=1, list(rep)=2, inner(opt)=3, list2(rep)=4, element(opt)=5
2164        assert_eq!(descr.column(0).max_def_level(), 5);
2165        assert_eq!(descr.column(0).max_rep_level(), 2);
2166        assert_eq!(descr.column(0).repeated_ancestor_def_level(), 4);
2167
2168        // Struct inside list: all sibling leaves share the same repeated_ancestor_def_level
2169        let message_type = "
2170    message m {
2171      OPTIONAL group bag (LIST) {
2172        REPEATED group list {
2173          REQUIRED group item {
2174            OPTIONAL INT32 x;
2175            REQUIRED INT32 y;
2176          }
2177        }
2178      }
2179    }
2180    ";
2181        let schema = parse_message_type(message_type).expect("should parse schema");
2182        let descr = SchemaDescriptor::new(Arc::new(schema));
2183        // bag(opt)=1, list(rep)=2, item(req)=2, x(opt)=3
2184        assert_eq!(descr.column(0).repeated_ancestor_def_level(), 2); // bag.list.item.x
2185        // bag(opt)=1, list(rep)=2, item(req)=2, y(req)=2
2186        assert_eq!(descr.column(1).repeated_ancestor_def_level(), 2); // bag.list.item.y
2187
2188        // Map type: key (required) and value (optional) under the same REPEATED group
2189        let message_type = "
2190    message m {
2191      OPTIONAL group my_map (MAP) {
2192        REPEATED group key_value {
2193          REQUIRED BYTE_ARRAY key (UTF8);
2194          OPTIONAL INT32 value;
2195        }
2196      }
2197    }
2198    ";
2199        let schema = parse_message_type(message_type).expect("should parse schema");
2200        let descr = SchemaDescriptor::new(Arc::new(schema));
2201        // my_map(opt)=1, key_value(rep)=2, key(req)=2
2202        assert_eq!(descr.column(0).max_def_level(), 2);
2203        assert_eq!(descr.column(0).repeated_ancestor_def_level(), 2); // key: max_def == repeated_ancestor
2204        // my_map(opt)=1, key_value(rep)=2, value(opt)=3
2205        assert_eq!(descr.column(1).max_def_level(), 3);
2206        assert_eq!(descr.column(1).repeated_ancestor_def_level(), 2); // value: max_def > repeated_ancestor
2207    }
2208
2209    #[test]
2210    #[should_panic(expected = "Cannot call get_physical_type() on a non-primitive type")]
2211    fn test_get_physical_type_panic() {
2212        let list = Type::group_type_builder("records")
2213            .with_repetition(Repetition::REPEATED)
2214            .build()
2215            .unwrap();
2216        list.get_physical_type();
2217    }
2218
2219    #[test]
2220    fn test_get_physical_type_primitive() {
2221        let f = Type::primitive_type_builder("f", PhysicalType::INT64)
2222            .build()
2223            .unwrap();
2224        assert_eq!(f.get_physical_type(), PhysicalType::INT64);
2225
2226        let f = Type::primitive_type_builder("f", PhysicalType::BYTE_ARRAY)
2227            .build()
2228            .unwrap();
2229        assert_eq!(f.get_physical_type(), PhysicalType::BYTE_ARRAY);
2230    }
2231
2232    #[test]
2233    fn test_check_contains_primitive_primitive() {
2234        // OK
2235        let f1 = Type::primitive_type_builder("f", PhysicalType::INT32)
2236            .build()
2237            .unwrap();
2238        let f2 = Type::primitive_type_builder("f", PhysicalType::INT32)
2239            .build()
2240            .unwrap();
2241        assert!(f1.check_contains(&f2));
2242
2243        // OK: different logical type does not affect check_contains
2244        let f1 = Type::primitive_type_builder("f", PhysicalType::INT32)
2245            .with_converted_type(ConvertedType::UINT_8)
2246            .build()
2247            .unwrap();
2248        let f2 = Type::primitive_type_builder("f", PhysicalType::INT32)
2249            .with_converted_type(ConvertedType::UINT_16)
2250            .build()
2251            .unwrap();
2252        assert!(f1.check_contains(&f2));
2253
2254        // KO: different name
2255        let f1 = Type::primitive_type_builder("f1", PhysicalType::INT32)
2256            .build()
2257            .unwrap();
2258        let f2 = Type::primitive_type_builder("f2", PhysicalType::INT32)
2259            .build()
2260            .unwrap();
2261        assert!(!f1.check_contains(&f2));
2262
2263        // KO: different type
2264        let f1 = Type::primitive_type_builder("f", PhysicalType::INT32)
2265            .build()
2266            .unwrap();
2267        let f2 = Type::primitive_type_builder("f", PhysicalType::INT64)
2268            .build()
2269            .unwrap();
2270        assert!(!f1.check_contains(&f2));
2271
2272        // KO: different repetition
2273        let f1 = Type::primitive_type_builder("f", PhysicalType::INT32)
2274            .with_repetition(Repetition::REQUIRED)
2275            .build()
2276            .unwrap();
2277        let f2 = Type::primitive_type_builder("f", PhysicalType::INT32)
2278            .with_repetition(Repetition::OPTIONAL)
2279            .build()
2280            .unwrap();
2281        assert!(!f1.check_contains(&f2));
2282    }
2283
2284    // function to create a new group type for testing
2285    fn test_new_group_type(name: &str, repetition: Repetition, types: Vec<Type>) -> Type {
2286        Type::group_type_builder(name)
2287            .with_repetition(repetition)
2288            .with_fields(types.into_iter().map(Arc::new).collect())
2289            .build()
2290            .unwrap()
2291    }
2292
2293    #[test]
2294    fn test_check_contains_group_group() {
2295        // OK: should match okay with empty fields
2296        let f1 = Type::group_type_builder("f").build().unwrap();
2297        let f2 = Type::group_type_builder("f").build().unwrap();
2298        assert!(f1.check_contains(&f2));
2299        assert!(!f1.is_optional());
2300
2301        // OK: fields match
2302        let f1 = test_new_group_type(
2303            "f",
2304            Repetition::REPEATED,
2305            vec![
2306                Type::primitive_type_builder("f1", PhysicalType::INT32)
2307                    .build()
2308                    .unwrap(),
2309                Type::primitive_type_builder("f2", PhysicalType::INT64)
2310                    .build()
2311                    .unwrap(),
2312            ],
2313        );
2314        let f2 = test_new_group_type(
2315            "f",
2316            Repetition::REPEATED,
2317            vec![
2318                Type::primitive_type_builder("f1", PhysicalType::INT32)
2319                    .build()
2320                    .unwrap(),
2321                Type::primitive_type_builder("f2", PhysicalType::INT64)
2322                    .build()
2323                    .unwrap(),
2324            ],
2325        );
2326        assert!(f1.check_contains(&f2));
2327
2328        // OK: subset of fields
2329        let f1 = test_new_group_type(
2330            "f",
2331            Repetition::REPEATED,
2332            vec![
2333                Type::primitive_type_builder("f1", PhysicalType::INT32)
2334                    .build()
2335                    .unwrap(),
2336                Type::primitive_type_builder("f2", PhysicalType::INT64)
2337                    .build()
2338                    .unwrap(),
2339            ],
2340        );
2341        let f2 = test_new_group_type(
2342            "f",
2343            Repetition::REPEATED,
2344            vec![
2345                Type::primitive_type_builder("f2", PhysicalType::INT64)
2346                    .build()
2347                    .unwrap(),
2348            ],
2349        );
2350        assert!(f1.check_contains(&f2));
2351
2352        // KO: different name
2353        let f1 = Type::group_type_builder("f1").build().unwrap();
2354        let f2 = Type::group_type_builder("f2").build().unwrap();
2355        assert!(!f1.check_contains(&f2));
2356
2357        // KO: different repetition
2358        let f1 = Type::group_type_builder("f")
2359            .with_repetition(Repetition::OPTIONAL)
2360            .build()
2361            .unwrap();
2362        let f2 = Type::group_type_builder("f")
2363            .with_repetition(Repetition::REPEATED)
2364            .build()
2365            .unwrap();
2366        assert!(!f1.check_contains(&f2));
2367
2368        // KO: different fields
2369        let f1 = test_new_group_type(
2370            "f",
2371            Repetition::REPEATED,
2372            vec![
2373                Type::primitive_type_builder("f1", PhysicalType::INT32)
2374                    .build()
2375                    .unwrap(),
2376                Type::primitive_type_builder("f2", PhysicalType::INT64)
2377                    .build()
2378                    .unwrap(),
2379            ],
2380        );
2381        let f2 = test_new_group_type(
2382            "f",
2383            Repetition::REPEATED,
2384            vec![
2385                Type::primitive_type_builder("f1", PhysicalType::INT32)
2386                    .build()
2387                    .unwrap(),
2388                Type::primitive_type_builder("f2", PhysicalType::BOOLEAN)
2389                    .build()
2390                    .unwrap(),
2391            ],
2392        );
2393        assert!(!f1.check_contains(&f2));
2394
2395        // KO: different fields
2396        let f1 = test_new_group_type(
2397            "f",
2398            Repetition::REPEATED,
2399            vec![
2400                Type::primitive_type_builder("f1", PhysicalType::INT32)
2401                    .build()
2402                    .unwrap(),
2403                Type::primitive_type_builder("f2", PhysicalType::INT64)
2404                    .build()
2405                    .unwrap(),
2406            ],
2407        );
2408        let f2 = test_new_group_type(
2409            "f",
2410            Repetition::REPEATED,
2411            vec![
2412                Type::primitive_type_builder("f3", PhysicalType::INT32)
2413                    .build()
2414                    .unwrap(),
2415            ],
2416        );
2417        assert!(!f1.check_contains(&f2));
2418    }
2419
2420    #[test]
2421    fn test_check_contains_group_primitive() {
2422        // KO: should not match
2423        let f1 = Type::group_type_builder("f").build().unwrap();
2424        let f2 = Type::primitive_type_builder("f", PhysicalType::INT64)
2425            .build()
2426            .unwrap();
2427        assert!(!f1.check_contains(&f2));
2428        assert!(!f2.check_contains(&f1));
2429
2430        // KO: should not match when primitive field is part of group type
2431        let f1 = test_new_group_type(
2432            "f",
2433            Repetition::REPEATED,
2434            vec![
2435                Type::primitive_type_builder("f1", PhysicalType::INT32)
2436                    .build()
2437                    .unwrap(),
2438            ],
2439        );
2440        let f2 = Type::primitive_type_builder("f1", PhysicalType::INT32)
2441            .build()
2442            .unwrap();
2443        assert!(!f1.check_contains(&f2));
2444        assert!(!f2.check_contains(&f1));
2445
2446        // OK: match nested types
2447        let f1 = test_new_group_type(
2448            "a",
2449            Repetition::REPEATED,
2450            vec![
2451                test_new_group_type(
2452                    "b",
2453                    Repetition::REPEATED,
2454                    vec![
2455                        Type::primitive_type_builder("c", PhysicalType::INT32)
2456                            .build()
2457                            .unwrap(),
2458                    ],
2459                ),
2460                Type::primitive_type_builder("d", PhysicalType::INT64)
2461                    .build()
2462                    .unwrap(),
2463                Type::primitive_type_builder("e", PhysicalType::BOOLEAN)
2464                    .build()
2465                    .unwrap(),
2466            ],
2467        );
2468        let f2 = test_new_group_type(
2469            "a",
2470            Repetition::REPEATED,
2471            vec![test_new_group_type(
2472                "b",
2473                Repetition::REPEATED,
2474                vec![
2475                    Type::primitive_type_builder("c", PhysicalType::INT32)
2476                        .build()
2477                        .unwrap(),
2478                ],
2479            )],
2480        );
2481        assert!(f1.check_contains(&f2)); // should match
2482        assert!(!f2.check_contains(&f1)); // should fail
2483    }
2484
2485    #[test]
2486    fn test_schema_type_thrift_conversion_err() {
2487        let schema = Type::primitive_type_builder("col", PhysicalType::INT32)
2488            .build()
2489            .unwrap();
2490        let schema = Arc::new(schema);
2491        let thrift_schema = schema_to_buf(&schema);
2492        assert!(thrift_schema.is_err());
2493        if let Err(e) = thrift_schema {
2494            assert_eq!(
2495                format!("{e}"),
2496                "Parquet error: Root schema must be Group type"
2497            );
2498        }
2499    }
2500
2501    #[test]
2502    fn test_schema_type_thrift_conversion() {
2503        let message_type = "
2504    message conversions {
2505      REQUIRED INT64 id;
2506      OPTIONAL FIXED_LEN_BYTE_ARRAY (2) f16 (FLOAT16);
2507      OPTIONAL group int_array_Array (LIST) {
2508        REPEATED group list {
2509          OPTIONAL group element (LIST) {
2510            REPEATED group list {
2511              OPTIONAL INT32 element;
2512            }
2513          }
2514        }
2515      }
2516      OPTIONAL group int_map (MAP) {
2517        REPEATED group map (MAP_KEY_VALUE) {
2518          REQUIRED BYTE_ARRAY key (UTF8);
2519          OPTIONAL INT32 value;
2520        }
2521      }
2522      OPTIONAL group int_Map_Array (LIST) {
2523        REPEATED group list {
2524          OPTIONAL group g (MAP) {
2525            REPEATED group map (MAP_KEY_VALUE) {
2526              REQUIRED BYTE_ARRAY key (UTF8);
2527              OPTIONAL group value {
2528                OPTIONAL group H {
2529                  OPTIONAL group i (LIST) {
2530                    REPEATED group list {
2531                      OPTIONAL DOUBLE element;
2532                    }
2533                  }
2534                }
2535              }
2536            }
2537          }
2538        }
2539      }
2540      OPTIONAL group nested_struct {
2541        OPTIONAL INT32 A;
2542        OPTIONAL group b (LIST) {
2543          REPEATED group list {
2544            REQUIRED FIXED_LEN_BYTE_ARRAY (16) element;
2545          }
2546        }
2547      }
2548    }
2549    ";
2550        let expected_schema = parse_message_type(message_type).unwrap();
2551        let result_schema = roundtrip_schema(Arc::new(expected_schema.clone())).unwrap();
2552        assert_eq!(result_schema, Arc::new(expected_schema));
2553    }
2554
2555    #[test]
2556    fn test_schema_type_thrift_conversion_decimal() {
2557        let message_type = "
2558    message decimals {
2559      OPTIONAL INT32 field0;
2560      OPTIONAL INT64 field1 (DECIMAL (18, 2));
2561      OPTIONAL FIXED_LEN_BYTE_ARRAY (16) field2 (DECIMAL (38, 18));
2562      OPTIONAL BYTE_ARRAY field3 (DECIMAL (9));
2563    }
2564    ";
2565        let expected_schema = parse_message_type(message_type).unwrap();
2566        let result_schema = roundtrip_schema(Arc::new(expected_schema.clone())).unwrap();
2567        assert_eq!(result_schema, Arc::new(expected_schema));
2568    }
2569
2570    // Tests schema conversion from thrift, when num_children is set to Some(0) for a
2571    // primitive type.
2572    #[test]
2573    fn test_schema_from_thrift_with_num_children_set() {
2574        // schema definition written by parquet-cpp version 1.3.2-SNAPSHOT
2575        let message_type = "
2576    message schema {
2577      OPTIONAL BYTE_ARRAY id (UTF8);
2578      OPTIONAL BYTE_ARRAY name (UTF8);
2579      OPTIONAL BYTE_ARRAY message (UTF8);
2580      OPTIONAL INT32 type (UINT_8);
2581      OPTIONAL INT64 author_time (TIMESTAMP_MILLIS);
2582      OPTIONAL INT64 __index_level_0__;
2583    }
2584    ";
2585
2586        let expected_schema = Arc::new(parse_message_type(message_type).unwrap());
2587        let mut buf = schema_to_buf(&expected_schema).unwrap();
2588        let mut thrift_schema = buf_to_schema_list(&mut buf).unwrap();
2589
2590        // Change all of None to Some(0)
2591        for elem in &mut thrift_schema[..] {
2592            if elem.num_children.is_none() {
2593                elem.num_children = Some(0);
2594            }
2595        }
2596
2597        let result_schema = parquet_schema_from_array(thrift_schema).unwrap();
2598        assert_eq!(result_schema, expected_schema);
2599    }
2600
2601    // Sometimes parquet-cpp sets repetition level for the root node, which is against
2602    // the format definition, but we need to handle it by setting it back to None.
2603    #[test]
2604    fn test_schema_from_thrift_root_has_repetition() {
2605        // schema definition written by parquet-cpp version 1.3.2-SNAPSHOT
2606        let message_type = "
2607    message schema {
2608      OPTIONAL BYTE_ARRAY a (UTF8);
2609      OPTIONAL INT32 b (UINT_8);
2610    }
2611    ";
2612
2613        let expected_schema = Arc::new(parse_message_type(message_type).unwrap());
2614        let mut buf = schema_to_buf(&expected_schema).unwrap();
2615        let mut thrift_schema = buf_to_schema_list(&mut buf).unwrap();
2616        thrift_schema[0].repetition_type = Some(Repetition::REQUIRED);
2617
2618        let result_schema = parquet_schema_from_array(thrift_schema).unwrap();
2619        assert_eq!(result_schema, expected_schema);
2620    }
2621
2622    #[test]
2623    fn test_schema_from_thrift_group_has_no_child() {
2624        let message_type = "message schema {}";
2625
2626        let expected_schema = Arc::new(parse_message_type(message_type).unwrap());
2627        let mut buf = schema_to_buf(&expected_schema).unwrap();
2628        let mut thrift_schema = buf_to_schema_list(&mut buf).unwrap();
2629        thrift_schema[0].repetition_type = Some(Repetition::REQUIRED);
2630
2631        let result_schema = parquet_schema_from_array(thrift_schema).unwrap();
2632        assert_eq!(result_schema, expected_schema);
2633    }
2634
2635    /// Builds an `OPTIONAL` primitive field for use inside a `FILE` group.
2636    fn file_field(name: &str, physical: PhysicalType, logical: Option<LogicalType>) -> TypePtr {
2637        Arc::new(
2638            Type::primitive_type_builder(name, physical)
2639                .with_repetition(Repetition::OPTIONAL)
2640                .with_logical_type(logical)
2641                .build()
2642                .unwrap(),
2643        )
2644    }
2645
2646    /// The full set of recognized `FILE` fields, all `OPTIONAL`, per the spec.
2647    fn all_file_fields() -> Vec<TypePtr> {
2648        vec![
2649            file_field("uri", PhysicalType::BYTE_ARRAY, Some(LogicalType::String)),
2650            file_field("offset", PhysicalType::INT64, None),
2651            file_field("size", PhysicalType::INT64, None),
2652            file_field(
2653                "content_type",
2654                PhysicalType::BYTE_ARRAY,
2655                Some(LogicalType::String),
2656            ),
2657            file_field(
2658                "checksum",
2659                PhysicalType::BYTE_ARRAY,
2660                Some(LogicalType::String),
2661            ),
2662            file_field("inline", PhysicalType::BYTE_ARRAY, None),
2663        ]
2664    }
2665
2666    #[test]
2667    fn test_file_logical_type_roundtrip() {
2668        let file_group = Arc::new(
2669            Type::group_type_builder("f")
2670                .with_repetition(Repetition::REQUIRED)
2671                .with_logical_type(Some(LogicalType::File))
2672                .with_fields(all_file_fields())
2673                .build()
2674                .unwrap(),
2675        );
2676        let schema = Arc::new(
2677            Type::group_type_builder("example")
2678                .with_fields(vec![file_group])
2679                .build()
2680                .unwrap(),
2681        );
2682        let result = roundtrip_schema(schema.clone()).unwrap();
2683        assert_eq!(result, schema);
2684        assert_eq!(
2685            result.get_fields()[0].get_basic_info().logical_type_ref(),
2686            Some(&LogicalType::File)
2687        );
2688    }
2689
2690    #[test]
2691    fn test_file_logical_type_all_fields() {
2692        let result = Type::group_type_builder("file_field")
2693            .with_repetition(Repetition::REQUIRED)
2694            .with_logical_type(Some(LogicalType::File))
2695            .with_fields(all_file_fields())
2696            .build();
2697        assert!(result.is_ok());
2698        assert_eq!(result.unwrap().get_fields().len(), 6);
2699    }
2700
2701    #[test]
2702    fn test_file_logical_type_uri_only() {
2703        // Every field is optional, so a group may define just `uri`.
2704        let result = Type::group_type_builder("file_field")
2705            .with_repetition(Repetition::REQUIRED)
2706            .with_logical_type(Some(LogicalType::File))
2707            .with_fields(vec![file_field(
2708                "uri",
2709                PhysicalType::BYTE_ARRAY,
2710                Some(LogicalType::String),
2711            )])
2712            .build();
2713        assert!(result.is_ok());
2714        assert_eq!(
2715            result.unwrap().get_basic_info().logical_type_ref(),
2716            Some(&LogicalType::File)
2717        );
2718    }
2719
2720    #[test]
2721    fn test_file_logical_type_inline_only() {
2722        // An inline-only group need only define `inline`.
2723        let result = Type::group_type_builder("inline_file")
2724            .with_repetition(Repetition::REQUIRED)
2725            .with_logical_type(Some(LogicalType::File))
2726            .with_fields(vec![file_field("inline", PhysicalType::BYTE_ARRAY, None)])
2727            .build();
2728        assert!(result.is_ok());
2729    }
2730
2731    #[test]
2732    fn test_file_logical_type_empty_group_is_allowed() {
2733        // No field is mandatory in the schema, so an empty group is valid.
2734        let result = Type::group_type_builder("empty_file")
2735            .with_repetition(Repetition::REQUIRED)
2736            .with_logical_type(Some(LogicalType::File))
2737            .with_fields(vec![])
2738            .build();
2739        assert!(result.is_ok());
2740    }
2741
2742    #[test]
2743    fn test_file_logical_type_rejects_unrecognized_field() {
2744        let unknown_field = file_field("unknown_field", PhysicalType::BYTE_ARRAY, None);
2745        let result = Type::group_type_builder("bad_file")
2746            .with_repetition(Repetition::REQUIRED)
2747            .with_logical_type(Some(LogicalType::File))
2748            .with_fields(vec![
2749                file_field("uri", PhysicalType::BYTE_ARRAY, Some(LogicalType::String)),
2750                unknown_field,
2751            ])
2752            .build();
2753        assert_eq!(
2754            result.unwrap_err().to_string(),
2755            "Parquet error: FILE type group 'bad_file' contains unrecognized field \
2756             'unknown_field'. Valid fields are: uri, offset, size, content_type, \
2757             checksum, inline"
2758        );
2759    }
2760
2761    #[test]
2762    fn test_file_logical_type_requires_optional_fields() {
2763        // A REQUIRED field is no longer valid: every field must be OPTIONAL.
2764        let uri_field = Arc::new(
2765            Type::primitive_type_builder("uri", PhysicalType::BYTE_ARRAY)
2766                .with_repetition(Repetition::REQUIRED)
2767                .with_logical_type(Some(LogicalType::String))
2768                .build()
2769                .unwrap(),
2770        );
2771        let result = Type::group_type_builder("required_uri")
2772            .with_repetition(Repetition::REQUIRED)
2773            .with_logical_type(Some(LogicalType::File))
2774            .with_fields(vec![uri_field])
2775            .build();
2776        assert_eq!(
2777            result.unwrap_err().to_string(),
2778            "Parquet error: FILE type field 'uri' must be OPTIONAL in group 'required_uri'"
2779        );
2780    }
2781
2782    #[test]
2783    fn test_file_logical_type_rejects_wrong_physical_type() {
2784        // `size` must be an INT64, not a BYTE_ARRAY.
2785        let bad_size = file_field("size", PhysicalType::BYTE_ARRAY, None);
2786        let result = Type::group_type_builder("bad_size")
2787            .with_repetition(Repetition::REQUIRED)
2788            .with_logical_type(Some(LogicalType::File))
2789            .with_fields(vec![bad_size])
2790            .build();
2791        assert_eq!(
2792            result.unwrap_err().to_string(),
2793            "Parquet error: FILE type field 'size' in group 'bad_size' must have physical type INT64"
2794        );
2795    }
2796
2797    #[test]
2798    fn test_file_logical_type_rejects_wrong_logical_type() {
2799        // `uri` must carry the STRING logical type.
2800        let bad_uri = file_field("uri", PhysicalType::BYTE_ARRAY, None);
2801        let result = Type::group_type_builder("bad_uri")
2802            .with_repetition(Repetition::REQUIRED)
2803            .with_logical_type(Some(LogicalType::File))
2804            .with_fields(vec![bad_uri])
2805            .build();
2806        assert_eq!(
2807            result.unwrap_err().to_string(),
2808            "Parquet error: FILE type field 'uri' in group 'bad_uri' must have logical type \
2809             Some(String)"
2810        );
2811    }
2812
2813    #[test]
2814    fn test_file_logical_type_not_allowed_on_primitive() {
2815        let result = Type::primitive_type_builder("bad", PhysicalType::BYTE_ARRAY)
2816            .with_repetition(Repetition::REQUIRED)
2817            .with_logical_type(Some(LogicalType::File))
2818            .build();
2819        assert!(result.is_err());
2820    }
2821
2822    #[test]
2823    fn test_parquet_schema_from_array_rejects_huge_num_children() {
2824        let schema = Arc::new(parse_message_type("message schema {}").unwrap());
2825        let mut buf = schema_to_buf(&schema).unwrap();
2826        let mut elements = buf_to_schema_list(&mut buf).unwrap();
2827        elements[0].num_children = Some(i32::MAX);
2828        let err = parquet_schema_from_array(elements).unwrap_err().to_string();
2829        assert!(
2830            err.contains("Schema children count 2147483647 exceeds remaining schema elements 0"),
2831            "{err}"
2832        );
2833    }
2834
2835    #[test]
2836    fn test_parquet_schema_from_array_rejects_children_exceeding_remaining_elements() {
2837        let schema = Arc::new(
2838            parse_message_type("message schema { REQUIRED group nested { REQUIRED INT32 leaf; } }")
2839                .unwrap(),
2840        );
2841        let mut buf = schema_to_buf(&schema).unwrap();
2842        let mut elements = buf_to_schema_list(&mut buf).unwrap();
2843        // Two children fit in the original three-element array, but only one
2844        // element remains after consuming the root and nested group.
2845        elements[1].num_children = Some(2);
2846        let err = parquet_schema_from_array(elements).unwrap_err().to_string();
2847        assert!(
2848            err.contains("Schema children count 2 exceeds remaining schema elements 1"),
2849            "{err}"
2850        );
2851    }
2852
2853    #[test]
2854    fn test_parquet_schema_from_array_rejects_negative_num_children() {
2855        let elements = vec![SchemaElement {
2856            r#type: None,
2857            type_length: None,
2858            repetition_type: Some(Repetition::REQUIRED),
2859            name: "schema",
2860            num_children: Some(-1),
2861            converted_type: None,
2862            scale: None,
2863            precision: None,
2864            field_id: None,
2865            logical_type: None,
2866        }];
2867        let result = parquet_schema_from_array(elements);
2868        assert!(result.unwrap_err().to_string().contains("Integer overflow"));
2869    }
2870}