Skip to main content

parquet/arrow/schema/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Converting Parquet schema <--> Arrow schema: [`ArrowSchemaConverter`] and [parquet_to_arrow_schema]
6
7use base64::Engine;
8use base64::prelude::BASE64_STANDARD;
9use std::collections::HashMap;
10use std::sync::Arc;
11
12use arrow_ipc::writer;
13use arrow_schema::{DataType, Field, FieldRef, Fields, Schema, TimeUnit};
14
15use crate::basic::{
16    ConvertedType, LogicalType, Repetition, TimeUnit as ParquetTimeUnit, Type as PhysicalType,
17};
18use crate::errors::{ParquetError, Result};
19use crate::file::{metadata::KeyValue, properties::WriterProperties};
20use crate::schema::types::{ColumnDescriptor, SchemaDescriptor, Type};
21
22mod complex;
23mod extension;
24mod primitive;
25pub mod virtual_type;
26
27use super::PARQUET_FIELD_ID_META_KEY;
28use crate::arrow::ProjectionMask;
29use crate::arrow::schema::extension::{
30    has_extension_type, logical_type_for_binary, logical_type_for_binary_view,
31    logical_type_for_fixed_size_binary, logical_type_for_string, logical_type_for_struct,
32    try_add_extension_type,
33};
34pub(crate) use complex::{ParquetField, ParquetFieldType, VirtualColumnType};
35
36/// Convert Parquet schema to Arrow schema including optional metadata
37///
38/// Attempts to decode any existing Arrow schema metadata, falling back
39/// to converting the Parquet schema column-wise
40pub fn parquet_to_arrow_schema(
41    parquet_schema: &SchemaDescriptor,
42    key_value_metadata: Option<&Vec<KeyValue>>,
43) -> Result<Schema> {
44    parquet_to_arrow_schema_by_columns(parquet_schema, ProjectionMask::all(), key_value_metadata)
45}
46
47/// Convert parquet schema to arrow schema including optional metadata,
48/// only preserving some leaf columns.
49pub fn parquet_to_arrow_schema_by_columns(
50    parquet_schema: &SchemaDescriptor,
51    mask: ProjectionMask,
52    key_value_metadata: Option<&Vec<KeyValue>>,
53) -> Result<Schema> {
54    Ok(parquet_to_arrow_schema_and_fields(parquet_schema, mask, key_value_metadata, &[])?.0)
55}
56
57/// Determines the Arrow Schema from a Parquet schema
58///
59/// Looks for an Arrow schema metadata "hint" (see
60/// [`parquet_to_arrow_field_levels`]), and uses it if present to ensure
61/// lossless round trips.
62pub(crate) fn parquet_to_arrow_schema_and_fields(
63    parquet_schema: &SchemaDescriptor,
64    mask: ProjectionMask,
65    key_value_metadata: Option<&Vec<KeyValue>>,
66    virtual_columns: &[FieldRef],
67) -> Result<(Schema, Option<ParquetField>)> {
68    let mut metadata = parse_key_value_metadata(key_value_metadata).unwrap_or_default();
69    let maybe_schema = metadata
70        .remove(super::ARROW_SCHEMA_META_KEY)
71        .map(|value| get_arrow_schema_from_metadata(&value))
72        .transpose()?;
73
74    // Add the Arrow metadata to the Parquet metadata skipping keys that collide
75    if let Some(arrow_schema) = &maybe_schema {
76        arrow_schema.metadata().iter().for_each(|(k, v)| {
77            metadata.entry(k.clone()).or_insert_with(|| v.clone());
78        });
79    }
80
81    let hint = maybe_schema.as_ref().map(|s| s.fields());
82    let field_levels =
83        parquet_to_arrow_field_levels_with_virtual(parquet_schema, mask, hint, virtual_columns)?;
84    let schema = Schema::new_with_metadata(field_levels.fields, metadata);
85    Ok((schema, field_levels.levels))
86}
87
88/// Schema information necessary to decode a parquet file as arrow [`Fields`]
89///
90/// In particular this stores the dremel-level information necessary to correctly
91/// interpret the encoded definition and repetition levels
92///
93/// Note: this is an opaque container intended to be used with lower-level APIs
94/// within this crate
95#[derive(Debug, Clone)]
96pub struct FieldLevels {
97    pub(crate) fields: Fields,
98    pub(crate) levels: Option<ParquetField>,
99}
100
101/// Convert a parquet [`SchemaDescriptor`] to [`FieldLevels`]
102///
103/// Columns not included within [`ProjectionMask`] will be ignored.
104///
105/// The optional `hint` parameter is the desired Arrow schema. See the
106/// [`arrow`] module documentation for more information.
107///
108/// [`arrow`]: crate::arrow
109///
110/// # Notes:
111/// Where a field type in `hint` is compatible with the corresponding parquet type in `schema`, it
112/// will be used, otherwise the default arrow type for the given parquet column type will be used.
113///
114/// This is to accommodate arrow types that cannot be round-tripped through parquet natively.
115/// Depending on the parquet writer, this can lead to a mismatch between a file's parquet schema
116/// and its embedded arrow schema. The parquet `schema` must be treated as authoritative in such
117/// an event. See [#1663](https://github.com/apache/arrow-rs/issues/1663) for more information
118///
119/// Note: this is a low-level API, most users will want to make use of the higher-level
120/// [`parquet_to_arrow_schema`] for decoding metadata from a parquet file.
121pub fn parquet_to_arrow_field_levels(
122    schema: &SchemaDescriptor,
123    mask: ProjectionMask,
124    hint: Option<&Fields>,
125) -> Result<FieldLevels> {
126    parquet_to_arrow_field_levels_with_virtual(schema, mask, hint, &[])
127}
128
129/// Convert a parquet [`SchemaDescriptor`] to [`FieldLevels`] with support for virtual columns
130///
131/// Columns not included within [`ProjectionMask`] will be ignored.
132///
133/// The optional `hint` parameter is the desired Arrow schema. See the
134/// [`arrow`] module documentation for more information.
135///
136/// [`arrow`]: crate::arrow
137///
138/// # Arguments
139/// * `schema` - The Parquet schema descriptor
140/// * `mask` - Projection mask to select which columns to include
141/// * `hint` - Optional hint for Arrow field types to use instead of defaults
142/// * `virtual_columns` - Virtual columns to append to the schema (e.g., row numbers)
143///
144/// # Notes:
145/// Where a field type in `hint` is compatible with the corresponding parquet type in `schema`, it
146/// will be used, otherwise the default arrow type for the given parquet column type will be used.
147///
148/// Virtual columns are columns that don't exist in the Parquet file but are generated during reading.
149/// They must have extension type names starting with "arrow.virtual.".
150///
151/// This is to accommodate arrow types that cannot be round-tripped through parquet natively.
152/// Depending on the parquet writer, this can lead to a mismatch between a file's parquet schema
153/// and its embedded arrow schema. The parquet `schema` must be treated as authoritative in such
154/// an event. See [#1663](https://github.com/apache/arrow-rs/issues/1663) for more information
155///
156/// Note: this is a low-level API, most users will want to make use of the higher-level
157/// [`parquet_to_arrow_schema`] for decoding metadata from a parquet file.
158pub fn parquet_to_arrow_field_levels_with_virtual(
159    schema: &SchemaDescriptor,
160    mask: ProjectionMask,
161    hint: Option<&Fields>,
162    virtual_columns: &[FieldRef],
163) -> Result<FieldLevels> {
164    // Validate that all fields are virtual columns
165    for field in virtual_columns {
166        if !virtual_type::is_virtual_column(field) {
167            return Err(ParquetError::General(format!(
168                "Field '{}' is not a virtual column. Virtual columns must have extension type names starting with 'arrow.virtual.'",
169                field.name()
170            )));
171        }
172    }
173
174    // Convert the regular schema first
175    let mut parquet_field = match complex::convert_schema(schema, mask, hint)? {
176        Some(field) => field,
177        None if virtual_columns.is_empty() => {
178            return Ok(FieldLevels {
179                fields: Fields::empty(),
180                levels: None,
181            });
182        }
183        None => {
184            // No regular fields, but we have virtual columns - create empty root struct
185            ParquetField {
186                rep_level: 0,
187                def_level: 0,
188                nullable: false,
189                arrow_type: DataType::Struct(Fields::empty()),
190                field_type: ParquetFieldType::Group {
191                    children: Vec::new(),
192                },
193            }
194        }
195    };
196
197    // Append virtual columns if any
198    if !virtual_columns.is_empty() {
199        match &mut parquet_field.field_type {
200            ParquetFieldType::Group { children } => {
201                // Get the mutable fields from the struct type
202                let DataType::Struct(ref mut fields) = parquet_field.arrow_type else {
203                    unreachable!("Root field must be a struct");
204                };
205
206                // Convert to mutable Vec to append
207                let mut fields_vec: Vec<FieldRef> = fields.iter().cloned().collect();
208
209                // Append each virtual column
210                for virtual_column in virtual_columns {
211                    // Virtual columns can only be added at the root level
212                    assert_eq!(
213                        parquet_field.rep_level, 0,
214                        "Virtual columns can only be added at rep level 0"
215                    );
216                    assert_eq!(
217                        parquet_field.def_level, 0,
218                        "Virtual columns can only be added at def level 0"
219                    );
220
221                    fields_vec.push(virtual_column.clone());
222                    let virtual_parquet_field = complex::convert_virtual_field(
223                        virtual_column,
224                        parquet_field.rep_level,
225                        parquet_field.def_level,
226                    )?;
227                    children.push(virtual_parquet_field);
228                }
229
230                // Update the fields
231                parquet_field.arrow_type = DataType::Struct(Fields::from(fields_vec));
232            }
233            _ => unreachable!("Root field must be a group"),
234        }
235    }
236
237    match &parquet_field.arrow_type {
238        DataType::Struct(fields) => Ok(FieldLevels {
239            fields: fields.clone(),
240            levels: Some(parquet_field),
241        }),
242        _ => unreachable!(),
243    }
244}
245
246/// Try to convert Arrow schema metadata into a schema
247fn get_arrow_schema_from_metadata(encoded_meta: &str) -> Result<Schema> {
248    let decoded = BASE64_STANDARD.decode(encoded_meta);
249    match decoded {
250        Ok(bytes) => {
251            let slice = if bytes.len() > 8 && bytes[0..4] == [255u8; 4] {
252                &bytes[8..]
253            } else {
254                bytes.as_slice()
255            };
256            match arrow_ipc::root_as_message(slice) {
257                Ok(message) => {
258                    let schema = message
259                        .header_as_schema()
260                        .ok_or_else(|| arrow_err!("the message is not Arrow Schema"))?;
261                    arrow_ipc::convert::try_fb_to_schema(schema).map_err(Into::into)
262                }
263                Err(err) => {
264                    // The flatbuffers implementation returns an error on verification error.
265                    Err(arrow_err!(
266                        "Unable to get root as message stored in {}: {:?}",
267                        super::ARROW_SCHEMA_META_KEY,
268                        err
269                    ))
270                }
271            }
272        }
273        Err(err) => {
274            // The C++ implementation returns an error if the schema can't be parsed.
275            Err(arrow_err!(
276                "Unable to decode the encoded schema stored in {}, {:?}",
277                super::ARROW_SCHEMA_META_KEY,
278                err
279            ))
280        }
281    }
282}
283
284/// Encodes the Arrow schema into the IPC format, and base64 encodes it
285pub fn encode_arrow_schema(schema: &Schema) -> String {
286    let options = writer::IpcWriteOptions::default();
287    let mut dictionary_tracker = writer::DictionaryTracker::new(true);
288    let data_gen = writer::IpcDataGenerator::default();
289    let mut serialized_schema =
290        data_gen.schema_to_bytes_with_dictionary_tracker(schema, &mut dictionary_tracker, &options);
291
292    // manually prepending the length to the schema as arrow uses the legacy IPC format
293    // TODO: change after addressing ARROW-9777
294    let schema_len = serialized_schema.ipc_message.len();
295    let mut len_prefix_schema = Vec::with_capacity(schema_len + 8);
296    len_prefix_schema.append(&mut vec![255u8, 255, 255, 255]);
297    len_prefix_schema.append((schema_len as u32).to_le_bytes().to_vec().as_mut());
298    len_prefix_schema.append(&mut serialized_schema.ipc_message);
299
300    BASE64_STANDARD.encode(&len_prefix_schema)
301}
302
303/// Reject map keys that are nullable after resolving their encoded layout.
304/// This validates the writer's logical contract, not a batch's physical layout.
305pub(crate) fn validate_map_key_type(data_type: &DataType) -> Result<()> {
306    match data_type {
307        DataType::Map(entries, _) => {
308            let DataType::Struct(fields) = entries.data_type() else {
309                return Err(arrow_err!(
310                    "DataType::Map should contain a struct field child"
311                ));
312            };
313            if fields.len() != 2 {
314                return Err(arrow_err!("DataType::Map entries must contain two fields"));
315            }
316            let key = &fields[0];
317            let mut key_type = key.data_type();
318            let mut nullable = key.is_nullable();
319            loop {
320                match key_type {
321                    DataType::Dictionary(_, value) => key_type = value,
322                    DataType::RunEndEncoded(_, value) => {
323                        nullable |= value.is_nullable();
324                        key_type = value.data_type();
325                    }
326                    _ => break,
327                }
328            }
329            if nullable {
330                return Err(arrow_err!(
331                    "Map key field '{}' must be non-nullable, including its encoded values",
332                    key.name()
333                ));
334            }
335            validate_map_key_type(entries.data_type())?;
336        }
337        DataType::Struct(fields) => {
338            for field in fields {
339                validate_map_key_type(field.data_type())?;
340            }
341        }
342        DataType::List(field)
343        | DataType::LargeList(field)
344        | DataType::FixedSizeList(field, _)
345        | DataType::ListView(field)
346        | DataType::LargeListView(field)
347        | DataType::RunEndEncoded(_, field) => validate_map_key_type(field.data_type())?,
348        DataType::Dictionary(_, value) => validate_map_key_type(value)?,
349        _ => {}
350    }
351    Ok(())
352}
353
354/// Normalize `field` for Arrow schema metadata.
355///
356/// This removes wrappers the Parquet reader does not reconstruct: REE at any
357/// nesting depth and dictionaries around nested values. Scalar dictionaries,
358/// including `Utf8View` and `BinaryView`, are retained in the metadata.
359fn normalize_metadata_field(field: &Field) -> Field {
360    match field.data_type() {
361        DataType::RunEndEncoded(_, value_field) => {
362            let value = normalize_metadata_field(value_field);
363            field
364                .clone()
365                .with_data_type(value.data_type().clone())
366                .with_nullable(field.is_nullable() || value.is_nullable())
367        }
368        DataType::Dictionary(key, value_type) => {
369            // Dictionary values have no Field, so use a synthetic one to retain
370            // the nullability of any REE value while normalizing recursively.
371            let value_field = Field::new("", value_type.as_ref().clone(), false);
372            let value = normalize_metadata_field(&value_field);
373            let value_type = value.data_type();
374            // Parquet expands nested dictionary values into leaf columns.
375            // Scalar view dictionaries are supported by the reader and must remain.
376            let data_type = if value_type.is_nested() {
377                value_type.clone()
378            } else {
379                DataType::Dictionary(key.clone(), Box::new(value_type.clone()))
380            };
381            field
382                .clone()
383                .with_data_type(data_type)
384                .with_nullable(field.is_nullable() || value.is_nullable())
385        }
386        _ => field
387            .clone()
388            .with_data_type(normalize_metadata_children(field.data_type())),
389    }
390}
391
392fn normalize_metadata_children(dt: &DataType) -> DataType {
393    match dt {
394        DataType::Struct(fields) => DataType::Struct(
395            fields
396                .iter()
397                .map(|f| Arc::new(normalize_metadata_field(f)))
398                .collect(),
399        ),
400        DataType::List(f) => DataType::List(Arc::new(normalize_metadata_field(f))),
401        DataType::LargeList(f) => DataType::LargeList(Arc::new(normalize_metadata_field(f))),
402        DataType::FixedSizeList(f, n) => {
403            DataType::FixedSizeList(Arc::new(normalize_metadata_field(f)), *n)
404        }
405        DataType::ListView(f) => DataType::ListView(Arc::new(normalize_metadata_field(f))),
406        DataType::LargeListView(f) => {
407            DataType::LargeListView(Arc::new(normalize_metadata_field(f)))
408        }
409        DataType::Map(f, sorted) => DataType::Map(Arc::new(normalize_metadata_field(f)), *sorted),
410        _ => dt.clone(),
411    }
412}
413
414/// Whether `dt` needs normalization before being written as Arrow schema metadata.
415fn needs_metadata_normalization(dt: &DataType) -> bool {
416    match dt {
417        DataType::RunEndEncoded(_, _) => true,
418        DataType::Struct(fields) => fields
419            .iter()
420            .any(|f| needs_metadata_normalization(f.data_type())),
421        DataType::List(f)
422        | DataType::LargeList(f)
423        | DataType::FixedSizeList(f, _)
424        | DataType::ListView(f)
425        | DataType::LargeListView(f)
426        | DataType::Map(f, _) => needs_metadata_normalization(f.data_type()),
427        DataType::Dictionary(_, value) => value.is_nested() || needs_metadata_normalization(value),
428        _ => false,
429    }
430}
431
432/// Mutates writer metadata by storing the encoded Arrow schema hint in
433/// [`ARROW_SCHEMA_META_KEY`].
434///
435/// If there is an existing Arrow schema metadata, it is replaced.
436///
437/// [`ARROW_SCHEMA_META_KEY`]: crate::arrow::ARROW_SCHEMA_META_KEY
438pub fn add_encoded_arrow_schema_to_metadata(schema: &Schema, props: &mut WriterProperties) {
439    let needs_normalization = schema
440        .fields()
441        .iter()
442        .any(|f| needs_metadata_normalization(f.data_type()));
443    let metadata_schema;
444    let schema = if needs_normalization {
445        let fields: Vec<Field> = schema
446            .fields()
447            .iter()
448            .map(|f| normalize_metadata_field(f))
449            .collect();
450        metadata_schema = Schema::new_with_metadata(fields, schema.metadata().clone());
451        &metadata_schema
452    } else {
453        schema
454    };
455    let encoded = encode_arrow_schema(schema);
456
457    let schema_kv = KeyValue {
458        key: super::ARROW_SCHEMA_META_KEY.to_string(),
459        value: Some(encoded),
460    };
461
462    let meta = props
463        .key_value_metadata
464        .get_or_insert_with(Default::default);
465
466    // check if ARROW:schema exists, and overwrite it
467    let schema_meta = meta
468        .iter()
469        .enumerate()
470        .find(|(_, kv)| kv.key.as_str() == super::ARROW_SCHEMA_META_KEY);
471    match schema_meta {
472        Some((i, _)) => {
473            meta.remove(i);
474            meta.push(schema_kv);
475        }
476        None => {
477            meta.push(schema_kv);
478        }
479    }
480}
481
482/// Converter for Arrow schema to Parquet schema
483///
484/// See the documentation on the [`arrow`] module for background
485/// information on how Arrow schema is represented in Parquet.
486///
487/// [`arrow`]: crate::arrow
488///
489/// # Example:
490/// ```
491/// # use std::sync::Arc;
492/// # use arrow_schema::{Field, Schema, DataType};
493/// # use parquet::arrow::ArrowSchemaConverter;
494/// use parquet::schema::types::{SchemaDescriptor, Type};
495/// use parquet::basic; // note there are two `Type`s in the following example
496/// // create an Arrow Schema
497/// let arrow_schema = Schema::new(vec![
498///   Field::new("a", DataType::Int64, true),
499///   Field::new("b", DataType::Date32, true),
500/// ]);
501/// // convert the Arrow schema to a Parquet schema
502/// let parquet_schema = ArrowSchemaConverter::new()
503///   .convert(&arrow_schema)
504///   .unwrap();
505///
506/// let expected_parquet_schema = SchemaDescriptor::new(
507///   Arc::new(
508///     Type::group_type_builder("arrow_schema")
509///       .with_fields(vec![
510///         Arc::new(
511///          Type::primitive_type_builder("a", basic::Type::INT64)
512///           .build().unwrap()
513///         ),
514///         Arc::new(
515///          Type::primitive_type_builder("b", basic::Type::INT32)
516///           .with_converted_type(basic::ConvertedType::DATE)
517///           .with_logical_type(Some(basic::LogicalType::Date))
518///           .build().unwrap()
519///         ),
520///      ])
521///      .build().unwrap()
522///   )
523/// );
524/// assert_eq!(parquet_schema, expected_parquet_schema);
525/// ```
526#[derive(Debug)]
527pub struct ArrowSchemaConverter<'a> {
528    /// Name of the root schema in Parquet
529    schema_root: &'a str,
530    /// Should we coerce Arrow types to compatible Parquet types?
531    ///
532    /// See docs on [`Self::with_coerce_types`]
533    coerce_types: bool,
534}
535
536impl Default for ArrowSchemaConverter<'_> {
537    fn default() -> Self {
538        Self::new()
539    }
540}
541
542impl<'a> ArrowSchemaConverter<'a> {
543    /// Create a new converter
544    pub fn new() -> Self {
545        Self {
546            schema_root: "arrow_schema",
547            coerce_types: false,
548        }
549    }
550
551    /// Should Arrow types be coerced into Parquet native types (default `false`).
552    ///
553    /// Setting this option to `true` will result in Parquet files that can be
554    /// read by more readers, but may lose precision for Arrow types such as
555    /// [`DataType::Date64`] which have no direct [corresponding Parquet type].
556    ///
557    /// By default, this converter does not coerce to native Parquet types. Enabling type
558    /// coercion allows for meaningful representations that do not require
559    /// downstream readers to consider the embedded Arrow schema, and can allow
560    /// for greater compatibility with other Parquet implementations. However,
561    /// type coercion also prevents data from being losslessly round-tripped.
562    ///
563    /// # Discussion
564    ///
565    /// Some Arrow types such as `Date64`, `Timestamp` and `Interval` have no
566    /// corresponding Parquet logical type. Thus, they can not be losslessly
567    /// round-tripped when stored using the appropriate Parquet logical type.
568    /// For example, some Date64 values may be truncated when stored with
569    /// parquet's native 32 bit date type.
570    ///
571    /// For [`List`] and [`Map`] types, some Parquet readers expect certain
572    /// schema elements to have specific names (earlier versions of the spec
573    /// were somewhat ambiguous on this point). Type coercion will use the names
574    /// prescribed by the Parquet specification, potentially losing naming
575    /// metadata from the Arrow schema.
576    ///
577    /// [`List`]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists
578    /// [`Map`]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#maps
579    /// [corresponding Parquet type]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#date
580    ///
581    pub fn with_coerce_types(mut self, coerce_types: bool) -> Self {
582        self.coerce_types = coerce_types;
583        self
584    }
585
586    /// Set the root schema element name (defaults to `"arrow_schema"`).
587    pub fn schema_root(mut self, schema_root: &'a str) -> Self {
588        self.schema_root = schema_root;
589        self
590    }
591
592    /// Convert the specified Arrow [`Schema`] to the desired Parquet [`SchemaDescriptor`]
593    ///
594    /// See example in [`ArrowSchemaConverter`]
595    pub fn convert(&self, schema: &Schema) -> Result<SchemaDescriptor> {
596        for field in schema.fields() {
597            validate_map_key_type(field.data_type())?;
598        }
599        let fields = schema
600            .fields()
601            .iter()
602            .map(|field| arrow_to_parquet_type(field, self.coerce_types).map(Arc::new))
603            .collect::<Result<_>>()?;
604        let group = Type::group_type_builder(self.schema_root)
605            .with_fields(fields)
606            .build()?;
607        Ok(SchemaDescriptor::new(Arc::new(group)))
608    }
609}
610
611fn parse_key_value_metadata(
612    key_value_metadata: Option<&Vec<KeyValue>>,
613) -> Option<HashMap<String, String>> {
614    match key_value_metadata {
615        Some(key_values) => {
616            let map: HashMap<String, String> = key_values
617                .iter()
618                .filter_map(|kv| {
619                    kv.value
620                        .as_ref()
621                        .map(|value| (kv.key.clone(), value.clone()))
622                })
623                .collect();
624
625            if map.is_empty() { None } else { Some(map) }
626        }
627        None => None,
628    }
629}
630
631/// Convert parquet column schema to arrow field.
632pub fn parquet_to_arrow_field(parquet_column: &ColumnDescriptor) -> Result<Field> {
633    let field = complex::convert_type(&parquet_column.self_type_ptr())?;
634    let mut ret = Field::new(parquet_column.name(), field.arrow_type, field.nullable);
635
636    let parquet_type = parquet_column.self_type();
637    let basic_info = parquet_type.get_basic_info();
638
639    let mut hash_map_size = 0;
640    if basic_info.has_id() {
641        hash_map_size += 1;
642    }
643    if has_extension_type(parquet_type) {
644        hash_map_size += 1;
645    }
646    if hash_map_size == 0 {
647        return Ok(ret);
648    }
649    ret.set_metadata(HashMap::with_capacity(hash_map_size));
650    if basic_info.has_id() {
651        ret.metadata_mut().insert(
652            PARQUET_FIELD_ID_META_KEY.to_string(),
653            basic_info.id().to_string(),
654        );
655    }
656    try_add_extension_type(ret, parquet_column.self_type())
657}
658
659pub fn decimal_length_from_precision(precision: u8) -> usize {
660    // digits = floor(log_10(2^(8*n - 1) - 1))  // definition in parquet's logical types
661    // ceil(digits) = log10(2^(8*n - 1) - 1)
662    // 10^ceil(digits) = 2^(8*n - 1) - 1
663    // 10^ceil(digits) + 1 = 2^(8*n - 1)
664    // log2(10^ceil(digits) + 1) = (8*n - 1)
665    // log2(10^ceil(digits) + 1) + 1 = 8*n
666    // (log2(10^ceil(a) + 1) + 1) / 8 = n
667    (((10.0_f64.powi(precision as i32) + 1.0).log2() + 1.0) / 8.0).ceil() as usize
668}
669
670/// Convert an arrow field to a parquet `Type`
671pub(crate) fn arrow_to_parquet_type(field: &Field, coerce_types: bool) -> Result<Type> {
672    const PARQUET_LIST_ELEMENT_NAME: &str = "element";
673    const PARQUET_MAP_STRUCT_NAME: &str = "key_value";
674    const PARQUET_KEY_FIELD_NAME: &str = "key";
675    const PARQUET_VALUE_FIELD_NAME: &str = "value";
676
677    let name = field.name().as_str();
678    let repetition = if field.is_nullable() {
679        Repetition::OPTIONAL
680    } else {
681        Repetition::REQUIRED
682    };
683    let id = field_id(field);
684    // create type from field
685    match field.data_type() {
686        DataType::Null => Type::primitive_type_builder(name, PhysicalType::INT32)
687            .with_logical_type(Some(LogicalType::Unknown))
688            .with_repetition(repetition)
689            .with_id(id)
690            .build(),
691        DataType::Boolean => Type::primitive_type_builder(name, PhysicalType::BOOLEAN)
692            .with_repetition(repetition)
693            .with_id(id)
694            .build(),
695        DataType::Int8 => Type::primitive_type_builder(name, PhysicalType::INT32)
696            .with_logical_type(Some(LogicalType::integer(8, true)))
697            .with_repetition(repetition)
698            .with_id(id)
699            .build(),
700        DataType::Int16 => Type::primitive_type_builder(name, PhysicalType::INT32)
701            .with_logical_type(Some(LogicalType::integer(16, true)))
702            .with_repetition(repetition)
703            .with_id(id)
704            .build(),
705        DataType::Int32 => Type::primitive_type_builder(name, PhysicalType::INT32)
706            .with_repetition(repetition)
707            .with_id(id)
708            .build(),
709        DataType::Int64 => Type::primitive_type_builder(name, PhysicalType::INT64)
710            .with_repetition(repetition)
711            .with_id(id)
712            .build(),
713        DataType::UInt8 => Type::primitive_type_builder(name, PhysicalType::INT32)
714            .with_logical_type(Some(LogicalType::integer(8, false)))
715            .with_repetition(repetition)
716            .with_id(id)
717            .build(),
718        DataType::UInt16 => Type::primitive_type_builder(name, PhysicalType::INT32)
719            .with_logical_type(Some(LogicalType::integer(16, false)))
720            .with_repetition(repetition)
721            .with_id(id)
722            .build(),
723        DataType::UInt32 => Type::primitive_type_builder(name, PhysicalType::INT32)
724            .with_logical_type(Some(LogicalType::integer(32, false)))
725            .with_repetition(repetition)
726            .with_id(id)
727            .build(),
728        DataType::UInt64 => Type::primitive_type_builder(name, PhysicalType::INT64)
729            .with_logical_type(Some(LogicalType::integer(64, false)))
730            .with_repetition(repetition)
731            .with_id(id)
732            .build(),
733        DataType::Float16 => Type::primitive_type_builder(name, PhysicalType::FIXED_LEN_BYTE_ARRAY)
734            .with_repetition(repetition)
735            .with_id(id)
736            .with_logical_type(Some(LogicalType::Float16))
737            .with_length(2)
738            .build(),
739        DataType::Float32 => Type::primitive_type_builder(name, PhysicalType::FLOAT)
740            .with_repetition(repetition)
741            .with_id(id)
742            .build(),
743        DataType::Float64 => Type::primitive_type_builder(name, PhysicalType::DOUBLE)
744            .with_repetition(repetition)
745            .with_id(id)
746            .build(),
747        DataType::Timestamp(TimeUnit::Second, _) => {
748            // Cannot represent seconds in LogicalType
749            Type::primitive_type_builder(name, PhysicalType::INT64)
750                .with_repetition(repetition)
751                .with_id(id)
752                .build()
753        }
754        DataType::Timestamp(time_unit, tz) => {
755            Type::primitive_type_builder(name, PhysicalType::INT64)
756                .with_logical_type(Some(LogicalType::timestamp(
757                    // If timezone set, values are normalized to UTC timezone
758                    matches!(tz, Some(z) if !z.as_ref().is_empty()),
759                    match time_unit {
760                        TimeUnit::Second => unreachable!(),
761                        TimeUnit::Millisecond => ParquetTimeUnit::MILLIS,
762                        TimeUnit::Microsecond => ParquetTimeUnit::MICROS,
763                        TimeUnit::Nanosecond => ParquetTimeUnit::NANOS,
764                    },
765                )))
766                .with_repetition(repetition)
767                .with_id(id)
768                .build()
769        }
770        DataType::Date32 => Type::primitive_type_builder(name, PhysicalType::INT32)
771            .with_logical_type(Some(LogicalType::Date))
772            .with_repetition(repetition)
773            .with_id(id)
774            .build(),
775        DataType::Date64 => {
776            if coerce_types {
777                Type::primitive_type_builder(name, PhysicalType::INT32)
778                    .with_logical_type(Some(LogicalType::Date))
779                    .with_repetition(repetition)
780                    .with_id(id)
781                    .build()
782            } else {
783                Type::primitive_type_builder(name, PhysicalType::INT64)
784                    .with_repetition(repetition)
785                    .with_id(id)
786                    .build()
787            }
788        }
789        DataType::Time32(TimeUnit::Second) => {
790            // Cannot represent seconds in LogicalType
791            Type::primitive_type_builder(name, PhysicalType::INT32)
792                .with_repetition(repetition)
793                .with_id(id)
794                .build()
795        }
796        DataType::Time32(unit) => Type::primitive_type_builder(name, PhysicalType::INT32)
797            .with_logical_type(Some(LogicalType::time(
798                field.metadata().contains_key("adjusted_to_utc"),
799                match unit {
800                    TimeUnit::Millisecond => ParquetTimeUnit::MILLIS,
801                    u => unreachable!("Invalid unit for Time32: {:?}", u),
802                },
803            )))
804            .with_repetition(repetition)
805            .with_id(id)
806            .build(),
807        DataType::Time64(unit) => Type::primitive_type_builder(name, PhysicalType::INT64)
808            .with_logical_type(Some(LogicalType::time(
809                field.metadata().contains_key("adjusted_to_utc"),
810                match unit {
811                    TimeUnit::Microsecond => ParquetTimeUnit::MICROS,
812                    TimeUnit::Nanosecond => ParquetTimeUnit::NANOS,
813                    u => unreachable!("Invalid unit for Time64: {:?}", u),
814                },
815            )))
816            .with_repetition(repetition)
817            .with_id(id)
818            .build(),
819        DataType::Duration(_) => Type::primitive_type_builder(name, PhysicalType::INT64)
820            .with_repetition(repetition)
821            .with_id(id)
822            .build(),
823        DataType::Interval(_) => {
824            Type::primitive_type_builder(name, PhysicalType::FIXED_LEN_BYTE_ARRAY)
825                .with_converted_type(ConvertedType::INTERVAL)
826                .with_repetition(repetition)
827                .with_id(id)
828                .with_length(12)
829                .build()
830        }
831        DataType::Binary | DataType::LargeBinary => {
832            Type::primitive_type_builder(name, PhysicalType::BYTE_ARRAY)
833                .with_repetition(repetition)
834                .with_id(id)
835                .with_logical_type(logical_type_for_binary(field))
836                .build()
837        }
838        DataType::FixedSizeBinary(length) => {
839            Type::primitive_type_builder(name, PhysicalType::FIXED_LEN_BYTE_ARRAY)
840                .with_repetition(repetition)
841                .with_id(id)
842                .with_length(*length)
843                .with_logical_type(logical_type_for_fixed_size_binary(field))
844                .build()
845        }
846        DataType::BinaryView => Type::primitive_type_builder(name, PhysicalType::BYTE_ARRAY)
847            .with_repetition(repetition)
848            .with_id(id)
849            .with_logical_type(logical_type_for_binary_view(field))
850            .build(),
851        DataType::Decimal32(precision, scale)
852        | DataType::Decimal64(precision, scale)
853        | DataType::Decimal128(precision, scale)
854        | DataType::Decimal256(precision, scale) => {
855            // Decimal precision determines the Parquet physical type to use.
856            // Following the: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal
857            let (physical_type, length) = if *precision >= 1 && *precision <= 9 {
858                (PhysicalType::INT32, -1)
859            } else if *precision <= 18 {
860                (PhysicalType::INT64, -1)
861            } else {
862                (
863                    PhysicalType::FIXED_LEN_BYTE_ARRAY,
864                    decimal_length_from_precision(*precision) as i32,
865                )
866            };
867            Type::primitive_type_builder(name, physical_type)
868                .with_repetition(repetition)
869                .with_id(id)
870                .with_length(length)
871                .with_logical_type(Some(LogicalType::decimal(*scale as i32, *precision as i32)))
872                .with_precision(*precision as i32)
873                .with_scale(*scale as i32)
874                .build()
875        }
876        DataType::Utf8 | DataType::LargeUtf8 => {
877            Type::primitive_type_builder(name, PhysicalType::BYTE_ARRAY)
878                .with_logical_type(logical_type_for_string(field))
879                .with_repetition(repetition)
880                .with_id(id)
881                .build()
882        }
883        DataType::Utf8View => Type::primitive_type_builder(name, PhysicalType::BYTE_ARRAY)
884            .with_logical_type(logical_type_for_string(field))
885            .with_repetition(repetition)
886            .with_id(id)
887            .build(),
888        DataType::List(f)
889        | DataType::FixedSizeList(f, _)
890        | DataType::LargeList(f)
891        | DataType::ListView(f)
892        | DataType::LargeListView(f) => {
893            let field_ref = if coerce_types && f.name() != PARQUET_LIST_ELEMENT_NAME {
894                // Ensure proper naming per the Parquet specification
895                let ff = f.as_ref().clone().with_name(PARQUET_LIST_ELEMENT_NAME);
896                Arc::new(arrow_to_parquet_type(&ff, coerce_types)?)
897            } else {
898                Arc::new(arrow_to_parquet_type(f, coerce_types)?)
899            };
900
901            Type::group_type_builder(name)
902                .with_fields(vec![Arc::new(
903                    Type::group_type_builder("list")
904                        .with_fields(vec![field_ref])
905                        .with_repetition(Repetition::REPEATED)
906                        .build()?,
907                )])
908                .with_logical_type(Some(LogicalType::List))
909                .with_repetition(repetition)
910                .with_id(id)
911                .build()
912        }
913        DataType::Struct(fields) => {
914            if fields.is_empty() {
915                return Err(arrow_err!("Parquet does not support writing empty structs",));
916            }
917            // recursively convert children to types/nodes
918            let fields = fields
919                .iter()
920                .map(|f| arrow_to_parquet_type(f, coerce_types).map(Arc::new))
921                .collect::<Result<_>>()?;
922            Type::group_type_builder(name)
923                .with_fields(fields)
924                .with_repetition(repetition)
925                .with_id(id)
926                .with_logical_type(logical_type_for_struct(field))
927                .build()
928        }
929        DataType::Map(field, _) => {
930            if let DataType::Struct(struct_fields) = field.data_type() {
931                // If coercing then set inner struct name to "key_value"
932                let map_struct_name = if coerce_types {
933                    PARQUET_MAP_STRUCT_NAME
934                } else {
935                    field.name()
936                };
937
938                // If coercing then ensure struct fields are named "key" and "value"
939                let fix_map_field = |name: &str, fld: &Arc<Field>| -> Result<Arc<Type>> {
940                    if coerce_types && fld.name() != name {
941                        let f = fld.as_ref().clone().with_name(name);
942                        Ok(Arc::new(arrow_to_parquet_type(&f, coerce_types)?))
943                    } else {
944                        Ok(Arc::new(arrow_to_parquet_type(fld, coerce_types)?))
945                    }
946                };
947                let key_field = fix_map_field(PARQUET_KEY_FIELD_NAME, &struct_fields[0])?;
948                let val_field = fix_map_field(PARQUET_VALUE_FIELD_NAME, &struct_fields[1])?;
949
950                Type::group_type_builder(name)
951                    .with_fields(vec![Arc::new(
952                        Type::group_type_builder(map_struct_name)
953                            .with_fields(vec![key_field, val_field])
954                            .with_repetition(Repetition::REPEATED)
955                            .build()?,
956                    )])
957                    .with_logical_type(Some(LogicalType::Map))
958                    .with_repetition(repetition)
959                    .with_id(id)
960                    .build()
961            } else {
962                Err(arrow_err!(
963                    "DataType::Map should contain a struct field child",
964                ))
965            }
966        }
967        DataType::Union(_, _) => unimplemented!("See ARROW-8817."),
968        DataType::Dictionary(_, value) => {
969            // Dictionary encoding not handled at the schema level
970            let dict_field = field.clone().with_data_type(value.as_ref().clone());
971            arrow_to_parquet_type(&dict_field, coerce_types)
972        }
973        DataType::RunEndEncoded(_, value) => {
974            // Run-end encoding is a physical layout. The Parquet column uses the
975            // run value type and is optional when either the run array or its
976            // values are nullable, so schema conversion recurses into the value
977            // field here.
978            let value_field = field
979                .clone()
980                .with_data_type(value.data_type().clone())
981                .with_nullable(field.is_nullable() || value.is_nullable());
982            arrow_to_parquet_type(&value_field, coerce_types)
983        }
984    }
985}
986
987fn field_id(field: &Field) -> Option<i32> {
988    let value = field.metadata().get(super::PARQUET_FIELD_ID_META_KEY)?;
989    value.parse().ok() // Fail quietly if not a valid integer
990}
991
992#[cfg(test)]
993mod tests {
994    use super::*;
995
996    use std::{collections::HashMap, sync::Arc};
997
998    use crate::arrow::PARQUET_FIELD_ID_META_KEY;
999    use crate::file::metadata::KeyValue;
1000    use crate::file::reader::FileReader;
1001    use crate::{
1002        arrow::{ArrowWriter, arrow_reader::ParquetRecordBatchReaderBuilder},
1003        schema::{parser::parse_message_type, types::SchemaDescriptor},
1004    };
1005    use arrow::datatypes::{DataType, Field, IntervalUnit, TimeUnit};
1006
1007    #[test]
1008    fn test_flat_primitives() {
1009        let message_type = "
1010        message test_schema {
1011            REQUIRED BOOLEAN boolean;
1012            REQUIRED INT32   int8  (INT_8);
1013            REQUIRED INT32   int16 (INT_16);
1014            REQUIRED INT32   uint8 (INTEGER(8,false));
1015            REQUIRED INT32   uint16 (INTEGER(16,false));
1016            REQUIRED INT32   int32;
1017            REQUIRED INT64   int64;
1018            OPTIONAL DOUBLE  double;
1019            OPTIONAL FLOAT   float;
1020            OPTIONAL FIXED_LEN_BYTE_ARRAY (2) float16 (FLOAT16);
1021            OPTIONAL BINARY  string (UTF8);
1022            OPTIONAL BINARY  string_2 (STRING);
1023            OPTIONAL BINARY  json (JSON);
1024        }
1025        ";
1026        let parquet_group_type = parse_message_type(message_type).unwrap();
1027
1028        let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1029        let converted_arrow_schema = parquet_to_arrow_schema(&parquet_schema, None).unwrap();
1030
1031        let arrow_fields = Fields::from(vec![
1032            Field::new("boolean", DataType::Boolean, false),
1033            Field::new("int8", DataType::Int8, false),
1034            Field::new("int16", DataType::Int16, false),
1035            Field::new("uint8", DataType::UInt8, false),
1036            Field::new("uint16", DataType::UInt16, false),
1037            Field::new("int32", DataType::Int32, false),
1038            Field::new("int64", DataType::Int64, false),
1039            Field::new("double", DataType::Float64, true),
1040            Field::new("float", DataType::Float32, true),
1041            Field::new("float16", DataType::Float16, true),
1042            Field::new("string", DataType::Utf8, true),
1043            Field::new("string_2", DataType::Utf8, true),
1044            json_field(),
1045        ]);
1046
1047        assert_eq!(&arrow_fields, converted_arrow_schema.fields());
1048    }
1049
1050    /// Return the expected Field for a Parquet column annotated with
1051    /// the JSON logical type.
1052    fn json_field() -> Field {
1053        #[cfg(feature = "arrow_canonical_extension_types")]
1054        {
1055            Field::new("json", DataType::Utf8, true)
1056                .with_extension_type(arrow_schema::extension::Json::default())
1057        }
1058        #[cfg(not(feature = "arrow_canonical_extension_types"))]
1059        {
1060            Field::new("json", DataType::Utf8, true)
1061        }
1062    }
1063
1064    #[test]
1065    fn test_decimal_fields() {
1066        let message_type = "
1067        message test_schema {
1068                    REQUIRED INT32 decimal1 (DECIMAL(4,2));
1069                    REQUIRED INT64 decimal2 (DECIMAL(12,2));
1070                    REQUIRED FIXED_LEN_BYTE_ARRAY (16) decimal3 (DECIMAL(30,2));
1071                    REQUIRED BYTE_ARRAY decimal4 (DECIMAL(33,2));
1072                    REQUIRED BYTE_ARRAY decimal5 (DECIMAL(38,2));
1073                    REQUIRED FIXED_LEN_BYTE_ARRAY (17) decimal6 (DECIMAL(39,2));
1074                    REQUIRED BYTE_ARRAY decimal7 (DECIMAL(39,2));
1075        }
1076        ";
1077
1078        let parquet_group_type = parse_message_type(message_type).unwrap();
1079
1080        let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1081        let converted_arrow_schema = parquet_to_arrow_schema(&parquet_schema, None).unwrap();
1082
1083        let arrow_fields = Fields::from(vec![
1084            Field::new("decimal1", DataType::Decimal128(4, 2), false),
1085            Field::new("decimal2", DataType::Decimal128(12, 2), false),
1086            Field::new("decimal3", DataType::Decimal128(30, 2), false),
1087            Field::new("decimal4", DataType::Decimal128(33, 2), false),
1088            Field::new("decimal5", DataType::Decimal128(38, 2), false),
1089            Field::new("decimal6", DataType::Decimal256(39, 2), false),
1090            Field::new("decimal7", DataType::Decimal256(39, 2), false),
1091        ]);
1092        assert_eq!(&arrow_fields, converted_arrow_schema.fields());
1093    }
1094
1095    #[test]
1096    fn test_decimal_precision_physical_types() {
1097        let cases = [
1098            (1, PhysicalType::INT32, None),
1099            (9, PhysicalType::INT32, None),
1100            (10, PhysicalType::INT64, None),
1101            (18, PhysicalType::INT64, None),
1102            (19, PhysicalType::FIXED_LEN_BYTE_ARRAY, Some(9)),
1103            (38, PhysicalType::FIXED_LEN_BYTE_ARRAY, Some(16)),
1104        ];
1105
1106        for (precision, expected_type, expected_length) in cases {
1107            let arrow_schema = Schema::new(vec![Field::new(
1108                "decimal",
1109                DataType::Decimal128(precision, 0),
1110                false,
1111            )]);
1112            let parquet_schema = ArrowSchemaConverter::new().convert(&arrow_schema).unwrap();
1113            let column = parquet_schema.column(0);
1114
1115            assert_eq!(
1116                column.physical_type(),
1117                expected_type,
1118                "precision {precision}"
1119            );
1120            if let Some(expected_length) = expected_length {
1121                assert_eq!(
1122                    column.type_length(),
1123                    expected_length,
1124                    "precision {precision}"
1125                );
1126            }
1127        }
1128    }
1129
1130    #[test]
1131    fn test_byte_array_fields() {
1132        let message_type = "
1133        message test_schema {
1134            REQUIRED BYTE_ARRAY binary;
1135            REQUIRED FIXED_LEN_BYTE_ARRAY (20) fixed_binary;
1136        }
1137        ";
1138
1139        let parquet_group_type = parse_message_type(message_type).unwrap();
1140
1141        let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1142        let converted_arrow_schema = parquet_to_arrow_schema(&parquet_schema, None).unwrap();
1143
1144        let arrow_fields = Fields::from(vec![
1145            Field::new("binary", DataType::Binary, false),
1146            Field::new("fixed_binary", DataType::FixedSizeBinary(20), false),
1147        ]);
1148        assert_eq!(&arrow_fields, converted_arrow_schema.fields());
1149    }
1150
1151    #[test]
1152    fn test_duplicate_fields() {
1153        let message_type = "
1154        message test_schema {
1155            REQUIRED BOOLEAN boolean;
1156            REQUIRED INT32 int8 (INT_8);
1157        }
1158        ";
1159
1160        let parquet_group_type = parse_message_type(message_type).unwrap();
1161
1162        let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1163        let converted_arrow_schema = parquet_to_arrow_schema(&parquet_schema, None).unwrap();
1164
1165        let arrow_fields = Fields::from(vec![
1166            Field::new("boolean", DataType::Boolean, false),
1167            Field::new("int8", DataType::Int8, false),
1168        ]);
1169        assert_eq!(&arrow_fields, converted_arrow_schema.fields());
1170
1171        let converted_arrow_schema =
1172            parquet_to_arrow_schema_by_columns(&parquet_schema, ProjectionMask::all(), None)
1173                .unwrap();
1174        assert_eq!(&arrow_fields, converted_arrow_schema.fields());
1175    }
1176
1177    #[test]
1178    fn test_parquet_lists() {
1179        let mut arrow_fields = Vec::new();
1180
1181        // LIST encoding example taken from parquet-format/LogicalTypes.md
1182        let message_type = "
1183        message test_schema {
1184          REQUIRED GROUP my_list (LIST) {
1185            REPEATED GROUP list {
1186              OPTIONAL BINARY element (UTF8);
1187            }
1188          }
1189          OPTIONAL GROUP my_list (LIST) {
1190            REPEATED GROUP list {
1191              REQUIRED BINARY element (UTF8);
1192            }
1193          }
1194          OPTIONAL GROUP array_of_arrays (LIST) {
1195            REPEATED GROUP list {
1196              REQUIRED GROUP element (LIST) {
1197                REPEATED GROUP list {
1198                  REQUIRED INT32 element;
1199                }
1200              }
1201            }
1202          }
1203          OPTIONAL GROUP my_list (LIST) {
1204            REPEATED GROUP element {
1205              REQUIRED BINARY str (UTF8);
1206            }
1207          }
1208          OPTIONAL GROUP my_list (LIST) {
1209            REPEATED INT32 element;
1210          }
1211          OPTIONAL GROUP my_list (LIST) {
1212            REPEATED GROUP element {
1213              REQUIRED BINARY str (UTF8);
1214              REQUIRED INT32 num;
1215            }
1216          }
1217          OPTIONAL GROUP my_list (LIST) {
1218            REPEATED GROUP array {
1219              REQUIRED BINARY str (UTF8);
1220            }
1221
1222          }
1223          OPTIONAL GROUP my_list (LIST) {
1224            REPEATED GROUP my_list_tuple {
1225              REQUIRED BINARY str (UTF8);
1226            }
1227          }
1228          REPEATED INT32 name;
1229        }
1230        ";
1231
1232        // // List<String> (list non-null, elements nullable)
1233        // required group my_list (LIST) {
1234        //   repeated group list {
1235        //     optional binary element (UTF8);
1236        //   }
1237        // }
1238        {
1239            arrow_fields.push(Field::new_list(
1240                "my_list",
1241                Field::new("element", DataType::Utf8, true),
1242                false,
1243            ));
1244        }
1245
1246        // // List<String> (list nullable, elements non-null)
1247        // optional group my_list (LIST) {
1248        //   repeated group list {
1249        //     required binary element (UTF8);
1250        //   }
1251        // }
1252        {
1253            arrow_fields.push(Field::new_list(
1254                "my_list",
1255                Field::new("element", DataType::Utf8, false),
1256                true,
1257            ));
1258        }
1259
1260        // Element types can be nested structures. For example, a list of lists:
1261        //
1262        // // List<List<Integer>>
1263        // optional group array_of_arrays (LIST) {
1264        //   repeated group list {
1265        //     required group element (LIST) {
1266        //       repeated group list {
1267        //         required int32 element;
1268        //       }
1269        //     }
1270        //   }
1271        // }
1272        {
1273            let arrow_inner_list = Field::new("element", DataType::Int32, false);
1274            arrow_fields.push(Field::new_list(
1275                "array_of_arrays",
1276                Field::new_list("element", arrow_inner_list, false),
1277                true,
1278            ));
1279        }
1280
1281        // // List<String> (list nullable, elements non-null)
1282        // optional group my_list (LIST) {
1283        //   repeated group element {
1284        //     required binary str (UTF8);
1285        //   };
1286        // }
1287        {
1288            arrow_fields.push(Field::new_list(
1289                "my_list",
1290                Field::new("str", DataType::Utf8, false),
1291                true,
1292            ));
1293        }
1294
1295        // // List<Integer> (nullable list, non-null elements)
1296        // optional group my_list (LIST) {
1297        //   repeated int32 element;
1298        // }
1299        {
1300            arrow_fields.push(Field::new_list(
1301                "my_list",
1302                Field::new("element", DataType::Int32, false),
1303                true,
1304            ));
1305        }
1306
1307        // // List<Tuple<String, Integer>> (nullable list, non-null elements)
1308        // optional group my_list (LIST) {
1309        //   repeated group element {
1310        //     required binary str (UTF8);
1311        //     required int32 num;
1312        //   };
1313        // }
1314        {
1315            let fields = vec![
1316                Field::new("str", DataType::Utf8, false),
1317                Field::new("num", DataType::Int32, false),
1318            ];
1319            arrow_fields.push(Field::new_list(
1320                "my_list",
1321                Field::new_struct("element", fields, false),
1322                true,
1323            ));
1324        }
1325
1326        // // List<OneTuple<String>> (nullable list, non-null elements)
1327        // optional group my_list (LIST) {
1328        //   repeated group array {
1329        //     required binary str (UTF8);
1330        //   };
1331        // }
1332        // Special case: group is named array
1333        {
1334            let fields = vec![Field::new("str", DataType::Utf8, false)];
1335            arrow_fields.push(Field::new_list(
1336                "my_list",
1337                Field::new_struct("array", fields, false),
1338                true,
1339            ));
1340        }
1341
1342        // // List<OneTuple<String>> (nullable list, non-null elements)
1343        // optional group my_list (LIST) {
1344        //   repeated group my_list_tuple {
1345        //     required binary str (UTF8);
1346        //   };
1347        // }
1348        // Special case: group named ends in _tuple
1349        {
1350            let fields = vec![Field::new("str", DataType::Utf8, false)];
1351            arrow_fields.push(Field::new_list(
1352                "my_list",
1353                Field::new_struct("my_list_tuple", fields, false),
1354                true,
1355            ));
1356        }
1357
1358        // One-level encoding: Only allows required lists with required cells
1359        //   repeated value_type name
1360        {
1361            arrow_fields.push(Field::new_list(
1362                "name",
1363                Field::new("name", DataType::Int32, false),
1364                false,
1365            ));
1366        }
1367
1368        let parquet_group_type = parse_message_type(message_type).unwrap();
1369
1370        let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1371        let converted_arrow_schema = parquet_to_arrow_schema(&parquet_schema, None).unwrap();
1372        let converted_fields = converted_arrow_schema.fields();
1373
1374        assert_eq!(arrow_fields.len(), converted_fields.len());
1375        for i in 0..arrow_fields.len() {
1376            assert_eq!(&arrow_fields[i], converted_fields[i].as_ref(), "{i}");
1377        }
1378    }
1379
1380    #[test]
1381    fn test_parquet_list_nullable() {
1382        let mut arrow_fields = Vec::new();
1383
1384        let message_type = "
1385        message test_schema {
1386          REQUIRED GROUP my_list1 (LIST) {
1387            REPEATED GROUP list {
1388              OPTIONAL BINARY element (UTF8);
1389            }
1390          }
1391          OPTIONAL GROUP my_list2 (LIST) {
1392            REPEATED GROUP list {
1393              REQUIRED BINARY element (UTF8);
1394            }
1395          }
1396          REQUIRED GROUP my_list3 (LIST) {
1397            REPEATED GROUP list {
1398              REQUIRED BINARY element (UTF8);
1399            }
1400          }
1401        }
1402        ";
1403
1404        // // List<String> (list non-null, elements nullable)
1405        // required group my_list1 (LIST) {
1406        //   repeated group list {
1407        //     optional binary element (UTF8);
1408        //   }
1409        // }
1410        {
1411            arrow_fields.push(Field::new_list(
1412                "my_list1",
1413                Field::new("element", DataType::Utf8, true),
1414                false,
1415            ));
1416        }
1417
1418        // // List<String> (list nullable, elements non-null)
1419        // optional group my_list2 (LIST) {
1420        //   repeated group list {
1421        //     required binary element (UTF8);
1422        //   }
1423        // }
1424        {
1425            arrow_fields.push(Field::new_list(
1426                "my_list2",
1427                Field::new("element", DataType::Utf8, false),
1428                true,
1429            ));
1430        }
1431
1432        // // List<String> (list non-null, elements non-null)
1433        // repeated group my_list3 (LIST) {
1434        //   repeated group list {
1435        //     required binary element (UTF8);
1436        //   }
1437        // }
1438        {
1439            arrow_fields.push(Field::new_list(
1440                "my_list3",
1441                Field::new("element", DataType::Utf8, false),
1442                false,
1443            ));
1444        }
1445
1446        let parquet_group_type = parse_message_type(message_type).unwrap();
1447
1448        let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1449        let converted_arrow_schema = parquet_to_arrow_schema(&parquet_schema, None).unwrap();
1450        let converted_fields = converted_arrow_schema.fields();
1451
1452        assert_eq!(arrow_fields.len(), converted_fields.len());
1453        for i in 0..arrow_fields.len() {
1454            assert_eq!(&arrow_fields[i], converted_fields[i].as_ref());
1455        }
1456    }
1457
1458    #[test]
1459    fn test_parquet_maps() {
1460        let mut arrow_fields = Vec::new();
1461
1462        // LIST encoding example taken from parquet-format/LogicalTypes.md
1463        let message_type = "
1464        message test_schema {
1465          REQUIRED group my_map1 (MAP) {
1466            REPEATED group key_value {
1467              REQUIRED binary key (UTF8);
1468              OPTIONAL int32 value;
1469            }
1470          }
1471          OPTIONAL group my_map2 (MAP) {
1472            REPEATED group map {
1473              REQUIRED binary str (UTF8);
1474              REQUIRED int32 num;
1475            }
1476          }
1477          OPTIONAL group my_map3 (MAP_KEY_VALUE) {
1478            REPEATED group map {
1479              REQUIRED binary key (UTF8);
1480              OPTIONAL int32 value;
1481            }
1482          }
1483          REQUIRED group my_map4 (MAP) {
1484            REPEATED group map {
1485              OPTIONAL binary key (UTF8);
1486              REQUIRED int32 value;
1487            }
1488          }
1489        }
1490        ";
1491
1492        // // Map<String, Integer>
1493        // required group my_map (MAP) {
1494        //   repeated group key_value {
1495        //     required binary key (UTF8);
1496        //     optional int32 value;
1497        //   }
1498        // }
1499        {
1500            arrow_fields.push(Field::new_map(
1501                "my_map1",
1502                "key_value",
1503                Field::new("key", DataType::Utf8, false),
1504                Field::new("value", DataType::Int32, true),
1505                false,
1506                false,
1507            ));
1508        }
1509
1510        // // Map<String, Integer> (nullable map, non-null values)
1511        // optional group my_map (MAP) {
1512        //   repeated group map {
1513        //     required binary str (UTF8);
1514        //     required int32 num;
1515        //   }
1516        // }
1517        {
1518            arrow_fields.push(Field::new_map(
1519                "my_map2",
1520                "map",
1521                Field::new("str", DataType::Utf8, false),
1522                Field::new("num", DataType::Int32, false),
1523                false,
1524                true,
1525            ));
1526        }
1527
1528        // // Map<String, Integer> (nullable map, nullable values)
1529        // optional group my_map (MAP_KEY_VALUE) {
1530        //   repeated group map {
1531        //     required binary key (UTF8);
1532        //     optional int32 value;
1533        //   }
1534        // }
1535        {
1536            arrow_fields.push(Field::new_map(
1537                "my_map3",
1538                "map",
1539                Field::new("key", DataType::Utf8, false),
1540                Field::new("value", DataType::Int32, true),
1541                false,
1542                true,
1543            ));
1544        }
1545
1546        // // Map<String, Integer> (non-compliant nullable key)
1547        // group my_map (MAP_KEY_VALUE) {
1548        //   repeated group map {
1549        //     optional binary key (UTF8);
1550        //     required int32 value;
1551        //   }
1552        // }
1553        {
1554            arrow_fields.push(Field::new_map(
1555                "my_map4",
1556                "map",
1557                Field::new("key", DataType::Utf8, false), // The key is always non-nullable (#5630)
1558                Field::new("value", DataType::Int32, false),
1559                false,
1560                false,
1561            ));
1562        }
1563
1564        let parquet_group_type = parse_message_type(message_type).unwrap();
1565
1566        let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1567        let converted_arrow_schema = parquet_to_arrow_schema(&parquet_schema, None).unwrap();
1568        let converted_fields = converted_arrow_schema.fields();
1569
1570        assert_eq!(arrow_fields.len(), converted_fields.len());
1571        for i in 0..arrow_fields.len() {
1572            assert_eq!(&arrow_fields[i], converted_fields[i].as_ref());
1573        }
1574    }
1575
1576    #[test]
1577    fn test_nested_schema() {
1578        let mut arrow_fields = Vec::new();
1579        {
1580            let group1_fields = Fields::from(vec![
1581                Field::new("leaf1", DataType::Boolean, false),
1582                Field::new("leaf2", DataType::Int32, false),
1583            ]);
1584            let group1_struct = Field::new("group1", DataType::Struct(group1_fields), false);
1585            arrow_fields.push(group1_struct);
1586
1587            let leaf3_field = Field::new("leaf3", DataType::Int64, false);
1588            arrow_fields.push(leaf3_field);
1589        }
1590
1591        let message_type = "
1592        message test_schema {
1593          REQUIRED GROUP group1 {
1594            REQUIRED BOOLEAN leaf1;
1595            REQUIRED INT32 leaf2;
1596          }
1597          REQUIRED INT64 leaf3;
1598        }
1599        ";
1600        let parquet_group_type = parse_message_type(message_type).unwrap();
1601
1602        let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1603        let converted_arrow_schema = parquet_to_arrow_schema(&parquet_schema, None).unwrap();
1604        let converted_fields = converted_arrow_schema.fields();
1605
1606        assert_eq!(arrow_fields.len(), converted_fields.len());
1607        for i in 0..arrow_fields.len() {
1608            assert_eq!(&arrow_fields[i], converted_fields[i].as_ref());
1609        }
1610    }
1611
1612    #[test]
1613    fn test_nested_schema_partial() {
1614        let mut arrow_fields = Vec::new();
1615        {
1616            let group1_fields = vec![Field::new("leaf1", DataType::Int64, false)].into();
1617            let group1 = Field::new("group1", DataType::Struct(group1_fields), false);
1618            arrow_fields.push(group1);
1619
1620            let group2_fields = vec![Field::new("leaf4", DataType::Int64, false)].into();
1621            let group2 = Field::new("group2", DataType::Struct(group2_fields), false);
1622            arrow_fields.push(group2);
1623
1624            arrow_fields.push(Field::new("leaf5", DataType::Int64, false));
1625        }
1626
1627        let message_type = "
1628        message test_schema {
1629          REQUIRED GROUP group1 {
1630            REQUIRED INT64 leaf1;
1631            REQUIRED INT64 leaf2;
1632          }
1633          REQUIRED  GROUP group2 {
1634            REQUIRED INT64 leaf3;
1635            REQUIRED INT64 leaf4;
1636          }
1637          REQUIRED INT64 leaf5;
1638        }
1639        ";
1640        let parquet_group_type = parse_message_type(message_type).unwrap();
1641
1642        // Expected partial arrow schema (columns 0, 3, 4):
1643        // required group group1 {
1644        //   required int64 leaf1;
1645        // }
1646        // required group group2 {
1647        //   required int64 leaf4;
1648        // }
1649        // required int64 leaf5;
1650
1651        let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1652        let mask = ProjectionMask::leaves(&parquet_schema, [3, 0, 4, 4]);
1653        let converted_arrow_schema =
1654            parquet_to_arrow_schema_by_columns(&parquet_schema, mask, None).unwrap();
1655        let converted_fields = converted_arrow_schema.fields();
1656
1657        assert_eq!(arrow_fields.len(), converted_fields.len());
1658        for i in 0..arrow_fields.len() {
1659            assert_eq!(&arrow_fields[i], converted_fields[i].as_ref());
1660        }
1661    }
1662
1663    #[test]
1664    fn test_nested_schema_partial_ordering() {
1665        let mut arrow_fields = Vec::new();
1666        {
1667            let group1_fields = vec![Field::new("leaf1", DataType::Int64, false)].into();
1668            let group1 = Field::new("group1", DataType::Struct(group1_fields), false);
1669            arrow_fields.push(group1);
1670
1671            let group2_fields = vec![Field::new("leaf4", DataType::Int64, false)].into();
1672            let group2 = Field::new("group2", DataType::Struct(group2_fields), false);
1673            arrow_fields.push(group2);
1674
1675            arrow_fields.push(Field::new("leaf5", DataType::Int64, false));
1676        }
1677
1678        let message_type = "
1679        message test_schema {
1680          REQUIRED GROUP group1 {
1681            REQUIRED INT64 leaf1;
1682            REQUIRED INT64 leaf2;
1683          }
1684          REQUIRED  GROUP group2 {
1685            REQUIRED INT64 leaf3;
1686            REQUIRED INT64 leaf4;
1687          }
1688          REQUIRED INT64 leaf5;
1689        }
1690        ";
1691        let parquet_group_type = parse_message_type(message_type).unwrap();
1692
1693        // Expected partial arrow schema (columns 3, 4, 0):
1694        // required group group1 {
1695        //   required int64 leaf1;
1696        // }
1697        // required group group2 {
1698        //   required int64 leaf4;
1699        // }
1700        // required int64 leaf5;
1701
1702        let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1703        let mask = ProjectionMask::leaves(&parquet_schema, [3, 0, 4]);
1704        let converted_arrow_schema =
1705            parquet_to_arrow_schema_by_columns(&parquet_schema, mask, None).unwrap();
1706        let converted_fields = converted_arrow_schema.fields();
1707
1708        assert_eq!(arrow_fields.len(), converted_fields.len());
1709        for i in 0..arrow_fields.len() {
1710            assert_eq!(&arrow_fields[i], converted_fields[i].as_ref());
1711        }
1712
1713        let mask =
1714            ProjectionMask::columns(&parquet_schema, ["group2.leaf4", "group1.leaf1", "leaf5"]);
1715        let converted_arrow_schema =
1716            parquet_to_arrow_schema_by_columns(&parquet_schema, mask, None).unwrap();
1717        let converted_fields = converted_arrow_schema.fields();
1718
1719        assert_eq!(arrow_fields.len(), converted_fields.len());
1720        for i in 0..arrow_fields.len() {
1721            assert_eq!(&arrow_fields[i], converted_fields[i].as_ref());
1722        }
1723    }
1724
1725    #[test]
1726    fn test_repeated_nested_schema() {
1727        let mut arrow_fields = Vec::new();
1728        {
1729            arrow_fields.push(Field::new("leaf1", DataType::Int32, true));
1730
1731            let inner_group_list = Field::new_list(
1732                "innerGroup",
1733                Field::new_struct(
1734                    "innerGroup",
1735                    vec![Field::new("leaf3", DataType::Int32, true)],
1736                    false,
1737                ),
1738                false,
1739            );
1740
1741            let outer_group_list = Field::new_list(
1742                "outerGroup",
1743                Field::new_struct(
1744                    "outerGroup",
1745                    vec![Field::new("leaf2", DataType::Int32, true), inner_group_list],
1746                    false,
1747                ),
1748                false,
1749            );
1750            arrow_fields.push(outer_group_list);
1751        }
1752
1753        let message_type = "
1754        message test_schema {
1755          OPTIONAL INT32 leaf1;
1756          REPEATED GROUP outerGroup {
1757            OPTIONAL INT32 leaf2;
1758            REPEATED GROUP innerGroup {
1759              OPTIONAL INT32 leaf3;
1760            }
1761          }
1762        }
1763        ";
1764        let parquet_group_type = parse_message_type(message_type).unwrap();
1765
1766        let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1767        let converted_arrow_schema = parquet_to_arrow_schema(&parquet_schema, None).unwrap();
1768        let converted_fields = converted_arrow_schema.fields();
1769
1770        assert_eq!(arrow_fields.len(), converted_fields.len());
1771        for i in 0..arrow_fields.len() {
1772            assert_eq!(&arrow_fields[i], converted_fields[i].as_ref());
1773        }
1774    }
1775
1776    #[test]
1777    fn test_column_desc_to_field() {
1778        let message_type = "
1779        message test_schema {
1780            REQUIRED BOOLEAN boolean;
1781            REQUIRED INT32   int8  (INT_8);
1782            REQUIRED INT32   uint8 (INTEGER(8,false));
1783            REQUIRED INT32   int16 (INT_16);
1784            REQUIRED INT32   uint16 (INTEGER(16,false));
1785            REQUIRED INT32   int32;
1786            REQUIRED INT64   int64;
1787            OPTIONAL DOUBLE  double;
1788            OPTIONAL FLOAT   float;
1789            OPTIONAL FIXED_LEN_BYTE_ARRAY (2) float16 (FLOAT16);
1790            OPTIONAL BINARY  string (UTF8);
1791            REPEATED BOOLEAN bools;
1792            OPTIONAL INT32   date       (DATE);
1793            OPTIONAL INT32   time_milli (TIME_MILLIS);
1794            OPTIONAL INT64   time_micro (TIME_MICROS);
1795            OPTIONAL INT64   time_nano (TIME(NANOS,false));
1796            OPTIONAL INT64   ts_milli (TIMESTAMP_MILLIS);
1797            REQUIRED INT64   ts_micro (TIMESTAMP_MICROS);
1798            REQUIRED INT64   ts_nano (TIMESTAMP(NANOS,true));
1799            REPEATED INT32   int_list;
1800            REPEATED BINARY  byte_list;
1801            REPEATED BINARY  string_list (UTF8);
1802            REQUIRED INT32 decimal_int32 (DECIMAL(8,2));
1803            REQUIRED INT64 decimal_int64 (DECIMAL(16,2));
1804            REQUIRED FIXED_LEN_BYTE_ARRAY (13) decimal_fix_length (DECIMAL(30,2));
1805        }
1806        ";
1807        let parquet_group_type = parse_message_type(message_type).unwrap();
1808
1809        let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1810        let converted_arrow_fields = parquet_schema
1811            .columns()
1812            .iter()
1813            .map(|c| parquet_to_arrow_field(c).unwrap())
1814            .collect::<Vec<Field>>();
1815
1816        let arrow_fields = vec![
1817            Field::new("boolean", DataType::Boolean, false),
1818            Field::new("int8", DataType::Int8, false),
1819            Field::new("uint8", DataType::UInt8, false),
1820            Field::new("int16", DataType::Int16, false),
1821            Field::new("uint16", DataType::UInt16, false),
1822            Field::new("int32", DataType::Int32, false),
1823            Field::new("int64", DataType::Int64, false),
1824            Field::new("double", DataType::Float64, true),
1825            Field::new("float", DataType::Float32, true),
1826            Field::new("float16", DataType::Float16, true),
1827            Field::new("string", DataType::Utf8, true),
1828            Field::new_list(
1829                "bools",
1830                Field::new("bools", DataType::Boolean, false),
1831                false,
1832            ),
1833            Field::new("date", DataType::Date32, true),
1834            Field::new("time_milli", DataType::Time32(TimeUnit::Millisecond), true),
1835            Field::new("time_micro", DataType::Time64(TimeUnit::Microsecond), true),
1836            Field::new("time_nano", DataType::Time64(TimeUnit::Nanosecond), true),
1837            Field::new(
1838                "ts_milli",
1839                DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())),
1840                true,
1841            ),
1842            Field::new(
1843                "ts_micro",
1844                DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
1845                false,
1846            ),
1847            Field::new(
1848                "ts_nano",
1849                DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),
1850                false,
1851            ),
1852            Field::new_list(
1853                "int_list",
1854                Field::new("int_list", DataType::Int32, false),
1855                false,
1856            ),
1857            Field::new_list(
1858                "byte_list",
1859                Field::new("byte_list", DataType::Binary, false),
1860                false,
1861            ),
1862            Field::new_list(
1863                "string_list",
1864                Field::new("string_list", DataType::Utf8, false),
1865                false,
1866            ),
1867            Field::new("decimal_int32", DataType::Decimal128(8, 2), false),
1868            Field::new("decimal_int64", DataType::Decimal128(16, 2), false),
1869            Field::new("decimal_fix_length", DataType::Decimal128(30, 2), false),
1870        ];
1871
1872        assert_eq!(arrow_fields, converted_arrow_fields);
1873    }
1874
1875    #[test]
1876    fn test_coerced_map_list() {
1877        // Create Arrow schema with non-Parquet naming
1878        let arrow_fields = vec![
1879            Field::new_list(
1880                "my_list",
1881                Field::new("item", DataType::Boolean, true),
1882                false,
1883            ),
1884            Field::new_map(
1885                "my_map",
1886                "my_entries",
1887                Field::new("my_keys", DataType::Utf8, false),
1888                Field::new("my_values", DataType::Int32, true),
1889                false,
1890                true,
1891            ),
1892        ];
1893        let arrow_schema = Schema::new(arrow_fields);
1894
1895        // Create Parquet schema with coerced names
1896        let message_type = "
1897        message parquet_schema {
1898            REQUIRED GROUP my_list (LIST) {
1899                REPEATED GROUP list {
1900                    OPTIONAL BOOLEAN element;
1901                }
1902            }
1903            OPTIONAL GROUP my_map (MAP) {
1904                REPEATED GROUP key_value {
1905                    REQUIRED BINARY key (STRING);
1906                    OPTIONAL INT32 value;
1907                }
1908            }
1909        }
1910        ";
1911        let parquet_group_type = parse_message_type(message_type).unwrap();
1912        let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1913        let converted_arrow_schema = ArrowSchemaConverter::new()
1914            .with_coerce_types(true)
1915            .convert(&arrow_schema)
1916            .unwrap();
1917        assert_eq!(
1918            parquet_schema.columns().len(),
1919            converted_arrow_schema.columns().len()
1920        );
1921
1922        // Create Parquet schema without coerced names
1923        let message_type = "
1924        message parquet_schema {
1925            REQUIRED GROUP my_list (LIST) {
1926                REPEATED GROUP list {
1927                    OPTIONAL BOOLEAN item;
1928                }
1929            }
1930            OPTIONAL GROUP my_map (MAP) {
1931                REPEATED GROUP my_entries {
1932                    REQUIRED BINARY my_keys (STRING);
1933                    OPTIONAL INT32 my_values;
1934                }
1935            }
1936        }
1937        ";
1938        let parquet_group_type = parse_message_type(message_type).unwrap();
1939        let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1940        let converted_arrow_schema = ArrowSchemaConverter::new()
1941            .with_coerce_types(false)
1942            .convert(&arrow_schema)
1943            .unwrap();
1944        assert_eq!(
1945            parquet_schema.columns().len(),
1946            converted_arrow_schema.columns().len()
1947        );
1948    }
1949
1950    #[test]
1951    fn test_field_to_column_desc() {
1952        let message_type = "
1953        message arrow_schema {
1954            REQUIRED BOOLEAN boolean;
1955            REQUIRED INT32   int8  (INT_8);
1956            REQUIRED INT32   int16 (INTEGER(16,true));
1957            REQUIRED INT32   int32;
1958            REQUIRED INT64   int64;
1959            OPTIONAL DOUBLE  double;
1960            OPTIONAL FLOAT   float;
1961            OPTIONAL FIXED_LEN_BYTE_ARRAY (2) float16 (FLOAT16);
1962            OPTIONAL BINARY  string (STRING);
1963            OPTIONAL GROUP   bools (LIST) {
1964                REPEATED GROUP list {
1965                    OPTIONAL BOOLEAN element;
1966                }
1967            }
1968            REQUIRED GROUP   bools_non_null (LIST) {
1969                REPEATED GROUP list {
1970                    REQUIRED BOOLEAN element;
1971                }
1972            }
1973            OPTIONAL INT32   date       (DATE);
1974            OPTIONAL INT32   time_milli (TIME(MILLIS,false));
1975            OPTIONAL INT32   time_milli_utc (TIME(MILLIS,true));
1976            OPTIONAL INT64   time_micro (TIME_MICROS);
1977            OPTIONAL INT64   time_micro_utc (TIME(MICROS, true));
1978            OPTIONAL INT64   ts_milli (TIMESTAMP_MILLIS);
1979            REQUIRED INT64   ts_micro (TIMESTAMP(MICROS,false));
1980            REQUIRED INT64   ts_seconds;
1981            REQUIRED INT64   ts_micro_utc (TIMESTAMP(MICROS, true));
1982            REQUIRED INT64   ts_millis_zero_offset (TIMESTAMP(MILLIS, true));
1983            REQUIRED INT64   ts_millis_zero_negative_offset (TIMESTAMP(MILLIS, true));
1984            REQUIRED INT64   ts_micro_non_utc (TIMESTAMP(MICROS, true));
1985            REQUIRED GROUP struct {
1986                REQUIRED BOOLEAN bools;
1987                REQUIRED INT32 uint32 (INTEGER(32,false));
1988                REQUIRED GROUP   int32 (LIST) {
1989                    REPEATED GROUP list {
1990                        OPTIONAL INT32 element;
1991                    }
1992                }
1993            }
1994            REQUIRED BINARY  dictionary_strings (STRING);
1995            REQUIRED INT32 decimal_int32 (DECIMAL(8,2));
1996            REQUIRED INT64 decimal_int64 (DECIMAL(16,2));
1997            REQUIRED FIXED_LEN_BYTE_ARRAY (13) decimal_fix_length (DECIMAL(30,2));
1998            REQUIRED FIXED_LEN_BYTE_ARRAY (16) decimal128 (DECIMAL(38,2));
1999            REQUIRED FIXED_LEN_BYTE_ARRAY (17) decimal256 (DECIMAL(39,2));
2000        }
2001        ";
2002        let parquet_group_type = parse_message_type(message_type).unwrap();
2003
2004        let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
2005
2006        let arrow_fields = vec![
2007            Field::new("boolean", DataType::Boolean, false),
2008            Field::new("int8", DataType::Int8, false),
2009            Field::new("int16", DataType::Int16, false),
2010            Field::new("int32", DataType::Int32, false),
2011            Field::new("int64", DataType::Int64, false),
2012            Field::new("double", DataType::Float64, true),
2013            Field::new("float", DataType::Float32, true),
2014            Field::new("float16", DataType::Float16, true),
2015            Field::new("string", DataType::Utf8, true),
2016            Field::new_list(
2017                "bools",
2018                Field::new("element", DataType::Boolean, true),
2019                true,
2020            ),
2021            Field::new_list(
2022                "bools_non_null",
2023                Field::new("element", DataType::Boolean, false),
2024                false,
2025            ),
2026            Field::new("date", DataType::Date32, true),
2027            Field::new("time_milli", DataType::Time32(TimeUnit::Millisecond), true),
2028            Field::new(
2029                "time_milli_utc",
2030                DataType::Time32(TimeUnit::Millisecond),
2031                true,
2032            )
2033            .with_metadata(HashMap::from_iter(vec![(
2034                "adjusted_to_utc".to_string(),
2035                String::new(),
2036            )])),
2037            Field::new("time_micro", DataType::Time64(TimeUnit::Microsecond), true),
2038            Field::new(
2039                "time_micro_utc",
2040                DataType::Time64(TimeUnit::Microsecond),
2041                true,
2042            )
2043            .with_metadata(HashMap::from_iter(vec![(
2044                "adjusted_to_utc".to_string(),
2045                String::new(),
2046            )])),
2047            Field::new(
2048                "ts_milli",
2049                DataType::Timestamp(TimeUnit::Millisecond, None),
2050                true,
2051            ),
2052            Field::new(
2053                "ts_micro",
2054                DataType::Timestamp(TimeUnit::Microsecond, None),
2055                false,
2056            ),
2057            Field::new(
2058                "ts_seconds",
2059                DataType::Timestamp(TimeUnit::Second, Some("UTC".into())),
2060                false,
2061            ),
2062            Field::new(
2063                "ts_micro_utc",
2064                DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
2065                false,
2066            ),
2067            Field::new(
2068                "ts_millis_zero_offset",
2069                DataType::Timestamp(TimeUnit::Millisecond, Some("+00:00".into())),
2070                false,
2071            ),
2072            Field::new(
2073                "ts_millis_zero_negative_offset",
2074                DataType::Timestamp(TimeUnit::Millisecond, Some("-00:00".into())),
2075                false,
2076            ),
2077            Field::new(
2078                "ts_micro_non_utc",
2079                DataType::Timestamp(TimeUnit::Microsecond, Some("+01:00".into())),
2080                false,
2081            ),
2082            Field::new_struct(
2083                "struct",
2084                vec![
2085                    Field::new("bools", DataType::Boolean, false),
2086                    Field::new("uint32", DataType::UInt32, false),
2087                    Field::new_list("int32", Field::new("element", DataType::Int32, true), false),
2088                ],
2089                false,
2090            ),
2091            Field::new_dictionary("dictionary_strings", DataType::Int32, DataType::Utf8, false),
2092            Field::new("decimal_int32", DataType::Decimal128(8, 2), false),
2093            Field::new("decimal_int64", DataType::Decimal128(16, 2), false),
2094            Field::new("decimal_fix_length", DataType::Decimal128(30, 2), false),
2095            Field::new("decimal128", DataType::Decimal128(38, 2), false),
2096            Field::new("decimal256", DataType::Decimal256(39, 2), false),
2097        ];
2098        let arrow_schema = Schema::new(arrow_fields);
2099        let converted_arrow_schema = ArrowSchemaConverter::new().convert(&arrow_schema).unwrap();
2100
2101        assert_eq!(
2102            parquet_schema.columns().len(),
2103            converted_arrow_schema.columns().len()
2104        );
2105        parquet_schema
2106            .columns()
2107            .iter()
2108            .zip(converted_arrow_schema.columns())
2109            .for_each(|(a, b)| {
2110                // Only check logical type if it's set on the Parquet side.
2111                // This is because the Arrow conversion always sets logical type,
2112                // even if there wasn't originally one.
2113                // This is not an issue, but is an inconvenience for this test.
2114                match a.logical_type_ref() {
2115                    Some(_) => {
2116                        assert_eq!(a, b)
2117                    }
2118                    None => {
2119                        assert_eq!(a.name(), b.name());
2120                        assert_eq!(a.physical_type(), b.physical_type());
2121                        assert_eq!(a.converted_type(), b.converted_type());
2122                    }
2123                }
2124            });
2125    }
2126
2127    #[test]
2128    #[should_panic(expected = "Parquet does not support writing empty structs")]
2129    fn test_empty_struct_field() {
2130        let arrow_fields = vec![Field::new(
2131            "struct",
2132            DataType::Struct(Fields::empty()),
2133            false,
2134        )];
2135        let arrow_schema = Schema::new(arrow_fields);
2136        let converted_arrow_schema = ArrowSchemaConverter::new()
2137            .with_coerce_types(true)
2138            .convert(&arrow_schema);
2139
2140        converted_arrow_schema.unwrap();
2141    }
2142
2143    #[test]
2144    fn test_run_end_schema_conversion_recurses_into_value_type() {
2145        fn run_end_type(value_type: DataType, nullable_values: bool) -> DataType {
2146            DataType::RunEndEncoded(
2147                Arc::new(Field::new("run_ends", DataType::Int32, false)),
2148                Arc::new(Field::new("values", value_type, nullable_values)),
2149            )
2150        }
2151
2152        // A leaf REE contributes the value's physical type and combined
2153        // nullability to the Parquet schema.
2154        let leaf_schema = Schema::new(vec![Field::new(
2155            "ree",
2156            run_end_type(DataType::Int32, true),
2157            false,
2158        )]);
2159        let converted = ArrowSchemaConverter::new().convert(&leaf_schema).unwrap();
2160        assert_eq!(converted.columns().len(), 1);
2161        assert_eq!(converted.column(0).name(), "ree");
2162        assert_eq!(converted.column(0).physical_type(), PhysicalType::INT32);
2163        assert_eq!(converted.column(0).max_def_level(), 1);
2164
2165        // Non-leaf value types recurse into the value type rather than
2166        // exposing Arrow's physical run-end wrapper in the Parquet schema.
2167        let list_schema = Schema::new(vec![Field::new(
2168            "ree_list",
2169            run_end_type(
2170                DataType::List(Arc::new(Field::new("element", DataType::Int32, true))),
2171                true,
2172            ),
2173            true,
2174        )]);
2175        let converted = ArrowSchemaConverter::new().convert(&list_schema).unwrap();
2176        assert_eq!(converted.columns().len(), 1);
2177        assert_eq!(converted.column(0).physical_type(), PhysicalType::INT32);
2178        assert!(
2179            converted.column(0).max_rep_level() >= 1,
2180            "list element should be repeated"
2181        );
2182
2183        let dictionary_schema = Schema::new(vec![Field::new(
2184            "ree_dictionary",
2185            run_end_type(
2186                DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)),
2187                true,
2188            ),
2189            true,
2190        )]);
2191        let converted = ArrowSchemaConverter::new()
2192            .convert(&dictionary_schema)
2193            .unwrap();
2194        assert_eq!(converted.columns().len(), 1);
2195        assert_eq!(
2196            converted.column(0).physical_type(),
2197            PhysicalType::BYTE_ARRAY
2198        );
2199    }
2200
2201    fn map_type_with_key(key: Field) -> DataType {
2202        DataType::Map(
2203            Arc::new(Field::new(
2204                "entries",
2205                DataType::Struct(vec![key, Field::new("value", DataType::Int32, true)].into()),
2206                false,
2207            )),
2208            false,
2209        )
2210    }
2211
2212    #[test]
2213    fn test_map_key_schema_rejects_logically_nullable_keys() {
2214        let ree = DataType::RunEndEncoded(
2215            Arc::new(Field::new("run_ends", DataType::Int32, false)),
2216            Arc::new(Field::new("values", DataType::Int32, true)),
2217        );
2218        let dictionary_ree = DataType::Dictionary(Box::new(DataType::Int8), Box::new(ree.clone()));
2219        let nested_ree = DataType::RunEndEncoded(
2220            Arc::new(Field::new("run_ends", DataType::Int32, false)),
2221            Arc::new(Field::new("values", dictionary_ree.clone(), false)),
2222        );
2223        for key in [
2224            Field::new("key", DataType::Int32, true),
2225            Field::new("key", ree, false),
2226            Field::new("key", dictionary_ree, false),
2227            Field::new("key", nested_ree, false),
2228        ] {
2229            let map = map_type_with_key(key);
2230            for data_type in [
2231                map.clone(),
2232                DataType::Struct(vec![Field::new("nested", map.clone(), true)].into()),
2233                DataType::List(Arc::new(Field::new("item", map.clone(), true))),
2234                DataType::Dictionary(Box::new(DataType::Int8), Box::new(map.clone())),
2235                DataType::RunEndEncoded(
2236                    Arc::new(Field::new("run_ends", DataType::Int32, false)),
2237                    Arc::new(Field::new("values", map, true)),
2238                ),
2239            ] {
2240                let schema = Schema::new(vec![Field::new("map", data_type, true)]);
2241                for coerce_types in [false, true] {
2242                    let err = ArrowSchemaConverter::new()
2243                        .with_coerce_types(coerce_types)
2244                        .convert(&schema)
2245                        .unwrap_err();
2246                    assert!(
2247                        err.to_string()
2248                            .contains("Map key field 'key' must be non-nullable"),
2249                        "{err}"
2250                    );
2251                }
2252            }
2253        }
2254    }
2255
2256    #[test]
2257    fn test_map_key_schema_preserves_required_keys_and_nullable_members() {
2258        for key_type in [
2259            DataType::Int32,
2260            DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Int32)),
2261            DataType::RunEndEncoded(
2262                Arc::new(Field::new("run_ends", DataType::Int32, false)),
2263                Arc::new(Field::new("values", DataType::Int32, false)),
2264            ),
2265            DataType::Struct(vec![Field::new("member", DataType::Int32, true)].into()),
2266        ] {
2267            let schema = Schema::new(vec![Field::new(
2268                "map",
2269                map_type_with_key(Field::new("key", key_type, false)),
2270                true,
2271            )]);
2272            let parquet = ArrowSchemaConverter::new().convert(&schema).unwrap();
2273            let key = &parquet.root_schema().get_fields()[0].get_fields()[0].get_fields()[0];
2274            assert_eq!(key.get_basic_info().repetition(), Repetition::REQUIRED);
2275
2276            let mut props = WriterProperties::builder().build();
2277            add_encoded_arrow_schema_to_metadata(&schema, &mut props);
2278            let encoded = props
2279                .key_value_metadata()
2280                .unwrap()
2281                .iter()
2282                .find(|kv| kv.key == crate::arrow::ARROW_SCHEMA_META_KEY)
2283                .unwrap()
2284                .value
2285                .as_deref()
2286                .unwrap();
2287            let decoded = get_arrow_schema_from_metadata(encoded).unwrap();
2288            let DataType::Map(entries, _) = decoded.field(0).data_type() else {
2289                unreachable!()
2290            };
2291            let DataType::Struct(fields) = entries.data_type() else {
2292                unreachable!()
2293            };
2294            assert!(!fields[0].is_nullable());
2295            assert!(fields[1].is_nullable());
2296            if let DataType::Struct(members) = fields[0].data_type() {
2297                assert!(members[0].is_nullable());
2298            }
2299        }
2300    }
2301
2302    #[test]
2303    fn test_arrow_metadata_normalization_merges_ree_value_nullability() {
2304        let run_ends = Arc::new(Field::new("run_ends", DataType::Int32, false));
2305        let inner = Arc::new(Field::new("inner", DataType::Int32, true));
2306        let nested = Arc::new(Field::new(
2307            "nested",
2308            DataType::RunEndEncoded(Arc::clone(&run_ends), inner),
2309            false,
2310        ));
2311        let field = Field::new("outer", DataType::RunEndEncoded(run_ends, nested), false);
2312
2313        let normalized = normalize_metadata_field(&field);
2314        assert_eq!(normalized.data_type(), &DataType::Int32);
2315        assert!(normalized.is_nullable());
2316    }
2317
2318    #[test]
2319    fn test_arrow_metadata_normalization_descends_dictionary_values() {
2320        let ree = DataType::RunEndEncoded(
2321            Arc::new(Field::new("run_ends", DataType::Int32, false)),
2322            Arc::new(Field::new("values", DataType::Int64, true)),
2323        );
2324        let field = Field::new(
2325            "dictionary_ree",
2326            DataType::Dictionary(Box::new(DataType::Int8), Box::new(ree)),
2327            false,
2328        );
2329
2330        assert!(needs_metadata_normalization(field.data_type()));
2331        let normalized = normalize_metadata_field(&field);
2332        assert_eq!(
2333            normalized.data_type(),
2334            &DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Int64))
2335        );
2336        assert!(normalized.is_nullable());
2337        assert!(!needs_metadata_normalization(normalized.data_type()));
2338
2339        let mut props = WriterProperties::builder().build();
2340        add_encoded_arrow_schema_to_metadata(&Schema::new(vec![field]), &mut props);
2341        let encoded = props
2342            .key_value_metadata()
2343            .unwrap()
2344            .iter()
2345            .find(|kv| kv.key == crate::arrow::ARROW_SCHEMA_META_KEY)
2346            .unwrap()
2347            .value
2348            .as_deref()
2349            .unwrap();
2350        let decoded = get_arrow_schema_from_metadata(encoded).unwrap();
2351        assert_eq!(decoded.field(0), &normalized);
2352    }
2353
2354    #[test]
2355    fn test_arrow_metadata_normalization_preserves_view_dictionary_siblings() {
2356        let dictionary = |dt| DataType::Dictionary(Box::new(DataType::Int8), Box::new(dt));
2357        let ree = DataType::RunEndEncoded(
2358            Arc::new(Field::new(
2359                Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
2360                DataType::Int32,
2361                false,
2362            )),
2363            Arc::new(Field::new(
2364                Field::REE_VALUES_FIELD_DEFAULT_NAME,
2365                DataType::Int32,
2366                true,
2367            )),
2368        );
2369        let fields = vec![
2370            // This field forces normalization of the schema's metadata.
2371            Field::new(
2372                "nested",
2373                dictionary(DataType::Struct(
2374                    vec![Field::new("child", ree, false)].into(),
2375                )),
2376                false,
2377            ),
2378            // Scalar view dictionaries are reader-supported, just like Utf8
2379            // dictionaries. Normalizing their sibling must preserve these hints.
2380            Field::new("view", dictionary(DataType::Utf8View), true),
2381            Field::new("ordinary", dictionary(DataType::Utf8), true),
2382            Field::new("binary_view", dictionary(DataType::BinaryView), true),
2383        ];
2384        let schema = Schema::new(fields);
2385        assert!(needs_metadata_normalization(schema.field(0).data_type()));
2386        let mut props = WriterProperties::builder().build();
2387        add_encoded_arrow_schema_to_metadata(&schema, &mut props);
2388        let encoded = props
2389            .key_value_metadata()
2390            .unwrap()
2391            .iter()
2392            .find(|kv| kv.key == crate::arrow::ARROW_SCHEMA_META_KEY)
2393            .unwrap()
2394            .value
2395            .as_deref()
2396            .unwrap();
2397        let decoded = get_arrow_schema_from_metadata(encoded).unwrap();
2398        let DataType::Struct(children) = decoded.field(0).data_type() else {
2399            panic!("nested dictionary not erased")
2400        };
2401        assert_eq!(children[0].name(), "child");
2402        assert_eq!(children[0].data_type(), &DataType::Int32);
2403        assert!(children[0].is_nullable());
2404        assert!(!decoded.field(0).is_nullable());
2405        assert_eq!(decoded.field(1), schema.field(1));
2406        assert_eq!(decoded.field(2), schema.field(2));
2407        assert_eq!(decoded.field(3), schema.field(3));
2408        assert!(!needs_metadata_normalization(schema.field(1).data_type()));
2409        assert!(!needs_metadata_normalization(schema.field(3).data_type()));
2410        let parquet = ArrowSchemaConverter::new().convert(&schema).unwrap();
2411        assert_eq!(parquet.column(0).path().parts(), &["nested", "child"]);
2412        assert_eq!(parquet.column(0).max_def_level(), 1);
2413    }
2414
2415    #[test]
2416    fn test_metadata() {
2417        let message_type = "
2418        message test_schema {
2419            OPTIONAL BINARY  string (STRING);
2420        }
2421        ";
2422        let parquet_group_type = parse_message_type(message_type).unwrap();
2423
2424        let key_value_metadata = vec![
2425            KeyValue::new("foo".to_owned(), Some("bar".to_owned())),
2426            KeyValue::new("baz".to_owned(), None),
2427        ];
2428
2429        let mut expected_metadata: HashMap<String, String> = HashMap::new();
2430        expected_metadata.insert("foo".to_owned(), "bar".to_owned());
2431
2432        let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
2433        let converted_arrow_schema =
2434            parquet_to_arrow_schema(&parquet_schema, Some(&key_value_metadata)).unwrap();
2435
2436        assert_eq!(converted_arrow_schema.metadata(), &expected_metadata);
2437    }
2438
2439    #[test]
2440    fn test_arrow_schema_roundtrip() -> Result<()> {
2441        let meta = |a: &[(&str, &str)]| -> HashMap<String, String> {
2442            a.iter()
2443                .map(|(a, b)| (a.to_string(), b.to_string()))
2444                .collect()
2445        };
2446
2447        let schema = Schema::new_with_metadata(
2448            vec![
2449                Field::new("c1", DataType::Utf8, false)
2450                    .with_metadata(meta(&[("Key", "Foo"), (PARQUET_FIELD_ID_META_KEY, "2")])),
2451                Field::new("c2", DataType::Binary, false),
2452                Field::new("c3", DataType::FixedSizeBinary(3), false),
2453                Field::new("c4", DataType::Boolean, false),
2454                Field::new("c5", DataType::Date32, false),
2455                Field::new("c6", DataType::Date64, false),
2456                Field::new("c7", DataType::Time32(TimeUnit::Second), false),
2457                Field::new("c8", DataType::Time32(TimeUnit::Millisecond), false),
2458                Field::new("c13", DataType::Time64(TimeUnit::Microsecond), false),
2459                Field::new("c14", DataType::Time64(TimeUnit::Nanosecond), false),
2460                Field::new("c15", DataType::Timestamp(TimeUnit::Second, None), false),
2461                Field::new(
2462                    "c16",
2463                    DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())),
2464                    false,
2465                ),
2466                Field::new(
2467                    "c17",
2468                    DataType::Timestamp(TimeUnit::Microsecond, Some("Africa/Johannesburg".into())),
2469                    false,
2470                ),
2471                Field::new(
2472                    "c18",
2473                    DataType::Timestamp(TimeUnit::Nanosecond, None),
2474                    false,
2475                ),
2476                Field::new("c19", DataType::Interval(IntervalUnit::DayTime), false),
2477                Field::new("c20", DataType::Interval(IntervalUnit::YearMonth), false),
2478                Field::new_list(
2479                    "c21",
2480                    Field::new_list_field(DataType::Boolean, true)
2481                        .with_metadata(meta(&[("Key", "Bar"), (PARQUET_FIELD_ID_META_KEY, "5")])),
2482                    false,
2483                )
2484                .with_metadata(meta(&[(PARQUET_FIELD_ID_META_KEY, "4")])),
2485                Field::new(
2486                    "c22",
2487                    DataType::FixedSizeList(
2488                        Arc::new(Field::new_list_field(DataType::Boolean, true)),
2489                        5,
2490                    ),
2491                    false,
2492                ),
2493                Field::new_list(
2494                    "c23",
2495                    Field::new_large_list(
2496                        "inner",
2497                        Field::new_list_field(
2498                            DataType::Struct(
2499                                vec![
2500                                    Field::new("a", DataType::Int16, true),
2501                                    Field::new("b", DataType::Float64, false),
2502                                    Field::new("c", DataType::Float32, false),
2503                                    Field::new("d", DataType::Float16, false),
2504                                ]
2505                                .into(),
2506                            ),
2507                            false,
2508                        ),
2509                        true,
2510                    ),
2511                    false,
2512                ),
2513                Field::new(
2514                    "c24",
2515                    DataType::Struct(Fields::from(vec![
2516                        Field::new("a", DataType::Utf8, false),
2517                        Field::new("b", DataType::UInt16, false),
2518                    ])),
2519                    false,
2520                ),
2521                Field::new("c25", DataType::Interval(IntervalUnit::YearMonth), true),
2522                Field::new("c26", DataType::Interval(IntervalUnit::DayTime), true),
2523                // Duration types not supported
2524                // Field::new("c27", DataType::Duration(TimeUnit::Second), false),
2525                // Field::new("c28", DataType::Duration(TimeUnit::Millisecond), false),
2526                // Field::new("c29", DataType::Duration(TimeUnit::Microsecond), false),
2527                // Field::new("c30", DataType::Duration(TimeUnit::Nanosecond), false),
2528                #[expect(deprecated)]
2529                Field::new_dict(
2530                    "c31",
2531                    DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
2532                    true,
2533                    123,
2534                    true,
2535                )
2536                .with_metadata(meta(&[(PARQUET_FIELD_ID_META_KEY, "6")])),
2537                Field::new("c32", DataType::LargeBinary, true),
2538                Field::new("c33", DataType::LargeUtf8, true),
2539                Field::new_large_list(
2540                    "c34",
2541                    Field::new_list(
2542                        "inner",
2543                        Field::new_list_field(
2544                            DataType::Struct(
2545                                vec![
2546                                    Field::new("a", DataType::Int16, true),
2547                                    Field::new("b", DataType::Float64, true),
2548                                ]
2549                                .into(),
2550                            ),
2551                            true,
2552                        ),
2553                        true,
2554                    ),
2555                    true,
2556                ),
2557                Field::new("c35", DataType::Null, true),
2558                Field::new("c36", DataType::Decimal128(2, 1), false),
2559                Field::new("c37", DataType::Decimal256(50, 20), false),
2560                Field::new("c38", DataType::Decimal128(18, 12), true),
2561                Field::new_map(
2562                    "c39",
2563                    "key_value",
2564                    Field::new("key", DataType::Utf8, false),
2565                    Field::new_list("value", Field::new("element", DataType::Utf8, true), true),
2566                    false, // fails to roundtrip keys_sorted
2567                    true,
2568                ),
2569                Field::new_map(
2570                    "c40",
2571                    "my_entries",
2572                    Field::new("my_key", DataType::Utf8, false)
2573                        .with_metadata(meta(&[(PARQUET_FIELD_ID_META_KEY, "8")])),
2574                    Field::new_list(
2575                        "my_value",
2576                        Field::new_list_field(DataType::Utf8, true)
2577                            .with_metadata(meta(&[(PARQUET_FIELD_ID_META_KEY, "10")])),
2578                        true,
2579                    )
2580                    .with_metadata(meta(&[(PARQUET_FIELD_ID_META_KEY, "9")])),
2581                    false, // fails to roundtrip keys_sorted
2582                    true,
2583                )
2584                .with_metadata(meta(&[(PARQUET_FIELD_ID_META_KEY, "7")])),
2585                Field::new_map(
2586                    "c41",
2587                    "my_entries",
2588                    Field::new("my_key", DataType::Utf8, false),
2589                    Field::new_list(
2590                        "my_value",
2591                        Field::new_list_field(DataType::Utf8, true)
2592                            .with_metadata(meta(&[(PARQUET_FIELD_ID_META_KEY, "11")])),
2593                        true,
2594                    ),
2595                    false, // fails to roundtrip keys_sorted
2596                    false,
2597                ),
2598                Field::new("c42", DataType::Decimal32(5, 2), false),
2599                Field::new("c43", DataType::Decimal64(18, 12), true),
2600            ],
2601            meta(&[("Key", "Value")]),
2602        );
2603
2604        // write to an empty parquet file so that schema is serialized
2605        let file = tempfile::tempfile().unwrap();
2606        let writer =
2607            ArrowWriter::try_new(file.try_clone().unwrap(), Arc::new(schema.clone()), None)?;
2608        writer.close()?;
2609
2610        // read file back
2611        let arrow_reader = ParquetRecordBatchReaderBuilder::try_new(file).unwrap();
2612
2613        // Check arrow schema
2614        let read_schema = arrow_reader.schema();
2615        assert_eq!(&schema, read_schema.as_ref());
2616
2617        // Walk schema finding field IDs
2618        let mut stack = Vec::with_capacity(10);
2619        let mut out = Vec::with_capacity(10);
2620
2621        let root = arrow_reader.parquet_schema().root_schema_ptr();
2622        stack.push((root.name().to_string(), root));
2623
2624        while let Some((p, t)) = stack.pop() {
2625            if t.is_group() {
2626                for f in t.get_fields() {
2627                    stack.push((format!("{p}.{}", f.name()), f.clone()))
2628                }
2629            }
2630
2631            let info = t.get_basic_info();
2632            if info.has_id() {
2633                out.push(format!("{p} -> {}", info.id()))
2634            }
2635        }
2636        out.sort_unstable();
2637        let out: Vec<_> = out.iter().map(|x| x.as_str()).collect();
2638
2639        assert_eq!(
2640            &out,
2641            &[
2642                "arrow_schema.c1 -> 2",
2643                "arrow_schema.c21 -> 4",
2644                "arrow_schema.c21.list.item -> 5",
2645                "arrow_schema.c31 -> 6",
2646                "arrow_schema.c40 -> 7",
2647                "arrow_schema.c40.my_entries.my_key -> 8",
2648                "arrow_schema.c40.my_entries.my_value -> 9",
2649                "arrow_schema.c40.my_entries.my_value.list.item -> 10",
2650                "arrow_schema.c41.my_entries.my_value.list.item -> 11",
2651            ]
2652        );
2653
2654        Ok(())
2655    }
2656
2657    #[test]
2658    fn test_read_parquet_field_ids_raw() -> Result<()> {
2659        let meta = |a: &[(&str, &str)]| -> HashMap<String, String> {
2660            a.iter()
2661                .map(|(a, b)| (a.to_string(), b.to_string()))
2662                .collect()
2663        };
2664        let schema = Schema::new_with_metadata(
2665            vec![
2666                Field::new("c1", DataType::Utf8, true)
2667                    .with_metadata(meta(&[(PARQUET_FIELD_ID_META_KEY, "1")])),
2668                Field::new("c2", DataType::Utf8, true)
2669                    .with_metadata(meta(&[(PARQUET_FIELD_ID_META_KEY, "2")])),
2670            ],
2671            HashMap::new(),
2672        );
2673
2674        let writer = ArrowWriter::try_new(vec![], Arc::new(schema.clone()), None)?;
2675        let parquet_bytes = writer.into_inner()?;
2676
2677        let reader =
2678            crate::file::reader::SerializedFileReader::new(bytes::Bytes::from(parquet_bytes))?;
2679        let schema_descriptor = reader.metadata().file_metadata().schema_descr_ptr();
2680
2681        // don't pass metadata so field ids are read from Parquet and not from serialized Arrow schema
2682        let arrow_schema = crate::arrow::parquet_to_arrow_schema(&schema_descriptor, None)?;
2683
2684        let parq_schema_descr = ArrowSchemaConverter::new()
2685            .with_coerce_types(true)
2686            .convert(&arrow_schema)?;
2687        let parq_fields = parq_schema_descr.root_schema().get_fields();
2688        assert_eq!(parq_fields.len(), 2);
2689        assert_eq!(parq_fields[0].get_basic_info().id(), 1);
2690        assert_eq!(parq_fields[1].get_basic_info().id(), 2);
2691
2692        Ok(())
2693    }
2694
2695    #[test]
2696    fn test_arrow_schema_roundtrip_lists() -> Result<()> {
2697        let metadata = HashMap::from([("Key".to_string(), "Value".to_string())]);
2698
2699        let schema = Schema::new_with_metadata(
2700            vec![
2701                Field::new_list("c21", Field::new("array", DataType::Boolean, true), false),
2702                Field::new(
2703                    "c22",
2704                    DataType::FixedSizeList(
2705                        Arc::new(Field::new("items", DataType::Boolean, false)),
2706                        5,
2707                    ),
2708                    false,
2709                ),
2710                Field::new_list(
2711                    "c23",
2712                    Field::new_large_list(
2713                        "items",
2714                        Field::new_struct(
2715                            "items",
2716                            vec![
2717                                Field::new("a", DataType::Int16, true),
2718                                Field::new("b", DataType::Float64, false),
2719                            ],
2720                            true,
2721                        ),
2722                        true,
2723                    ),
2724                    true,
2725                ),
2726            ],
2727            metadata,
2728        );
2729
2730        // write to an empty parquet file so that schema is serialized
2731        let file = tempfile::tempfile().unwrap();
2732        let writer =
2733            ArrowWriter::try_new(file.try_clone().unwrap(), Arc::new(schema.clone()), None)?;
2734        writer.close()?;
2735
2736        // read file back
2737        let arrow_reader = ParquetRecordBatchReaderBuilder::try_new(file).unwrap();
2738        let read_schema = arrow_reader.schema();
2739        assert_eq!(&schema, read_schema.as_ref());
2740        Ok(())
2741    }
2742
2743    #[test]
2744    fn test_get_arrow_schema_from_metadata() {
2745        assert!(get_arrow_schema_from_metadata("").is_err());
2746    }
2747
2748    #[test]
2749    #[cfg(feature = "arrow_canonical_extension_types")]
2750    fn arrow_uuid_to_parquet_uuid() -> Result<()> {
2751        use arrow_schema::extension::Uuid;
2752        let arrow_schema = Schema::new(vec![
2753            Field::new("uuid", DataType::FixedSizeBinary(16), false).with_extension_type(Uuid),
2754        ]);
2755
2756        let parquet_schema = ArrowSchemaConverter::new().convert(&arrow_schema)?;
2757
2758        assert_eq!(
2759            parquet_schema.column(0).logical_type_ref(),
2760            Some(&LogicalType::Uuid)
2761        );
2762
2763        let arrow_schema = parquet_to_arrow_schema(&parquet_schema, None)?;
2764        assert_eq!(arrow_schema.field(0).try_extension_type::<Uuid>()?, Uuid);
2765
2766        Ok(())
2767    }
2768
2769    #[test]
2770    #[cfg(feature = "arrow_canonical_extension_types")]
2771    fn arrow_json_to_parquet_json() -> Result<()> {
2772        use arrow_schema::extension::Json;
2773        let arrow_schema = Schema::new(vec![
2774            Field::new("json", DataType::Utf8, false).with_extension_type(Json::default()),
2775        ]);
2776
2777        let parquet_schema = ArrowSchemaConverter::new().convert(&arrow_schema)?;
2778
2779        assert_eq!(
2780            parquet_schema.column(0).logical_type_ref(),
2781            Some(&LogicalType::Json)
2782        );
2783
2784        let arrow_schema = parquet_to_arrow_schema(&parquet_schema, None)?;
2785        assert_eq!(
2786            arrow_schema.field(0).try_extension_type::<Json>()?,
2787            Json::default()
2788        );
2789
2790        Ok(())
2791    }
2792
2793    #[test]
2794    #[cfg(feature = "variant_experimental")]
2795    fn arrow_variant_to_parquet_variant() -> Result<()> {
2796        use parquet_variant_compute::VariantType;
2797        let variant_fields = Fields::from(vec![
2798            Field::new("metadata", DataType::BinaryView, false),
2799            Field::new("value", DataType::BinaryView, true),
2800        ]);
2801        let arrow_schema = Schema::new(vec![
2802            Field::new("v", DataType::Struct(variant_fields), true)
2803                .with_extension_type(VariantType),
2804        ]);
2805
2806        let parquet_schema = ArrowSchemaConverter::new().convert(&arrow_schema)?;
2807
2808        // The Parquet spec requires the VARIANT annotation to carry the
2809        // specification version (`VARIANT(1)`); Spark rejects files without it
2810        let root = parquet_schema.root_schema();
2811        assert_eq!(
2812            root.get_fields()[0].get_basic_info().logical_type_ref(),
2813            Some(&LogicalType::variant(Some(1)))
2814        );
2815
2816        let arrow_schema = parquet_to_arrow_schema(&parquet_schema, None)?;
2817        assert!(
2818            arrow_schema
2819                .field(0)
2820                .has_valid_extension_type::<VariantType>()
2821        );
2822
2823        Ok(())
2824    }
2825
2826    #[test]
2827    fn test_parquet_to_arrow_field_levels_with_virtual_rejects_non_virtual() {
2828        let message_type = "
2829        message test_schema {
2830            REQUIRED INT32 id;
2831        }
2832        ";
2833        let parquet_schema = Arc::new(parse_message_type(message_type).unwrap());
2834        let descriptor = SchemaDescriptor::new(parquet_schema);
2835
2836        // Try to pass a regular field (not a virtual column)
2837        let regular_field = Arc::new(Field::new("regular_column", DataType::Int64, false));
2838        let result = parquet_to_arrow_field_levels_with_virtual(
2839            &descriptor,
2840            ProjectionMask::all(),
2841            None,
2842            &[regular_field],
2843        );
2844
2845        assert!(result.is_err());
2846        assert!(
2847            result
2848                .unwrap_err()
2849                .to_string()
2850                .contains("is not a virtual column")
2851        );
2852    }
2853}