1use base64::Engine;
8use base64::prelude::BASE64_STANDARD;
9use std::collections::HashMap;
10use std::sync::Arc;
11
12use arrow_ipc::writer;
13use arrow_schema::{DataType, Field, FieldRef, Fields, Schema, TimeUnit};
14
15use crate::basic::{
16 ConvertedType, LogicalType, Repetition, TimeUnit as ParquetTimeUnit, Type as PhysicalType,
17};
18use crate::errors::{ParquetError, Result};
19use crate::file::{metadata::KeyValue, properties::WriterProperties};
20use crate::schema::types::{ColumnDescriptor, SchemaDescriptor, Type};
21
22mod complex;
23mod extension;
24mod primitive;
25pub mod virtual_type;
26
27use super::PARQUET_FIELD_ID_META_KEY;
28use crate::arrow::ProjectionMask;
29use crate::arrow::schema::extension::{
30 has_extension_type, logical_type_for_binary, logical_type_for_binary_view,
31 logical_type_for_fixed_size_binary, logical_type_for_string, logical_type_for_struct,
32 try_add_extension_type,
33};
34pub(crate) use complex::{ParquetField, ParquetFieldType, VirtualColumnType};
35
36pub fn parquet_to_arrow_schema(
41 parquet_schema: &SchemaDescriptor,
42 key_value_metadata: Option<&Vec<KeyValue>>,
43) -> Result<Schema> {
44 parquet_to_arrow_schema_by_columns(parquet_schema, ProjectionMask::all(), key_value_metadata)
45}
46
47pub fn parquet_to_arrow_schema_by_columns(
50 parquet_schema: &SchemaDescriptor,
51 mask: ProjectionMask,
52 key_value_metadata: Option<&Vec<KeyValue>>,
53) -> Result<Schema> {
54 Ok(parquet_to_arrow_schema_and_fields(parquet_schema, mask, key_value_metadata, &[])?.0)
55}
56
57pub(crate) fn parquet_to_arrow_schema_and_fields(
63 parquet_schema: &SchemaDescriptor,
64 mask: ProjectionMask,
65 key_value_metadata: Option<&Vec<KeyValue>>,
66 virtual_columns: &[FieldRef],
67) -> Result<(Schema, Option<ParquetField>)> {
68 let mut metadata = parse_key_value_metadata(key_value_metadata).unwrap_or_default();
69 let maybe_schema = metadata
70 .remove(super::ARROW_SCHEMA_META_KEY)
71 .map(|value| get_arrow_schema_from_metadata(&value))
72 .transpose()?;
73
74 if let Some(arrow_schema) = &maybe_schema {
76 arrow_schema.metadata().iter().for_each(|(k, v)| {
77 metadata.entry(k.clone()).or_insert_with(|| v.clone());
78 });
79 }
80
81 let hint = maybe_schema.as_ref().map(|s| s.fields());
82 let field_levels =
83 parquet_to_arrow_field_levels_with_virtual(parquet_schema, mask, hint, virtual_columns)?;
84 let schema = Schema::new_with_metadata(field_levels.fields, metadata);
85 Ok((schema, field_levels.levels))
86}
87
88#[derive(Debug, Clone)]
96pub struct FieldLevels {
97 pub(crate) fields: Fields,
98 pub(crate) levels: Option<ParquetField>,
99}
100
101pub fn parquet_to_arrow_field_levels(
122 schema: &SchemaDescriptor,
123 mask: ProjectionMask,
124 hint: Option<&Fields>,
125) -> Result<FieldLevels> {
126 parquet_to_arrow_field_levels_with_virtual(schema, mask, hint, &[])
127}
128
129pub fn parquet_to_arrow_field_levels_with_virtual(
159 schema: &SchemaDescriptor,
160 mask: ProjectionMask,
161 hint: Option<&Fields>,
162 virtual_columns: &[FieldRef],
163) -> Result<FieldLevels> {
164 for field in virtual_columns {
166 if !virtual_type::is_virtual_column(field) {
167 return Err(ParquetError::General(format!(
168 "Field '{}' is not a virtual column. Virtual columns must have extension type names starting with 'arrow.virtual.'",
169 field.name()
170 )));
171 }
172 }
173
174 let mut parquet_field = match complex::convert_schema(schema, mask, hint)? {
176 Some(field) => field,
177 None if virtual_columns.is_empty() => {
178 return Ok(FieldLevels {
179 fields: Fields::empty(),
180 levels: None,
181 });
182 }
183 None => {
184 ParquetField {
186 rep_level: 0,
187 def_level: 0,
188 nullable: false,
189 arrow_type: DataType::Struct(Fields::empty()),
190 field_type: ParquetFieldType::Group {
191 children: Vec::new(),
192 },
193 }
194 }
195 };
196
197 if !virtual_columns.is_empty() {
199 match &mut parquet_field.field_type {
200 ParquetFieldType::Group { children } => {
201 let DataType::Struct(ref mut fields) = parquet_field.arrow_type else {
203 unreachable!("Root field must be a struct");
204 };
205
206 let mut fields_vec: Vec<FieldRef> = fields.iter().cloned().collect();
208
209 for virtual_column in virtual_columns {
211 assert_eq!(
213 parquet_field.rep_level, 0,
214 "Virtual columns can only be added at rep level 0"
215 );
216 assert_eq!(
217 parquet_field.def_level, 0,
218 "Virtual columns can only be added at def level 0"
219 );
220
221 fields_vec.push(virtual_column.clone());
222 let virtual_parquet_field = complex::convert_virtual_field(
223 virtual_column,
224 parquet_field.rep_level,
225 parquet_field.def_level,
226 )?;
227 children.push(virtual_parquet_field);
228 }
229
230 parquet_field.arrow_type = DataType::Struct(Fields::from(fields_vec));
232 }
233 _ => unreachable!("Root field must be a group"),
234 }
235 }
236
237 match &parquet_field.arrow_type {
238 DataType::Struct(fields) => Ok(FieldLevels {
239 fields: fields.clone(),
240 levels: Some(parquet_field),
241 }),
242 _ => unreachable!(),
243 }
244}
245
246fn get_arrow_schema_from_metadata(encoded_meta: &str) -> Result<Schema> {
248 let decoded = BASE64_STANDARD.decode(encoded_meta);
249 match decoded {
250 Ok(bytes) => {
251 let slice = if bytes.len() > 8 && bytes[0..4] == [255u8; 4] {
252 &bytes[8..]
253 } else {
254 bytes.as_slice()
255 };
256 match arrow_ipc::root_as_message(slice) {
257 Ok(message) => {
258 let schema = message
259 .header_as_schema()
260 .ok_or_else(|| arrow_err!("the message is not Arrow Schema"))?;
261 arrow_ipc::convert::try_fb_to_schema(schema).map_err(Into::into)
262 }
263 Err(err) => {
264 Err(arrow_err!(
266 "Unable to get root as message stored in {}: {:?}",
267 super::ARROW_SCHEMA_META_KEY,
268 err
269 ))
270 }
271 }
272 }
273 Err(err) => {
274 Err(arrow_err!(
276 "Unable to decode the encoded schema stored in {}, {:?}",
277 super::ARROW_SCHEMA_META_KEY,
278 err
279 ))
280 }
281 }
282}
283
284pub fn encode_arrow_schema(schema: &Schema) -> String {
286 let options = writer::IpcWriteOptions::default();
287 let mut dictionary_tracker = writer::DictionaryTracker::new(true);
288 let data_gen = writer::IpcDataGenerator::default();
289 let mut serialized_schema =
290 data_gen.schema_to_bytes_with_dictionary_tracker(schema, &mut dictionary_tracker, &options);
291
292 let schema_len = serialized_schema.ipc_message.len();
295 let mut len_prefix_schema = Vec::with_capacity(schema_len + 8);
296 len_prefix_schema.append(&mut vec![255u8, 255, 255, 255]);
297 len_prefix_schema.append((schema_len as u32).to_le_bytes().to_vec().as_mut());
298 len_prefix_schema.append(&mut serialized_schema.ipc_message);
299
300 BASE64_STANDARD.encode(&len_prefix_schema)
301}
302
303pub(crate) fn validate_map_key_type(data_type: &DataType) -> Result<()> {
306 match data_type {
307 DataType::Map(entries, _) => {
308 let DataType::Struct(fields) = entries.data_type() else {
309 return Err(arrow_err!(
310 "DataType::Map should contain a struct field child"
311 ));
312 };
313 if fields.len() != 2 {
314 return Err(arrow_err!("DataType::Map entries must contain two fields"));
315 }
316 let key = &fields[0];
317 let mut key_type = key.data_type();
318 let mut nullable = key.is_nullable();
319 loop {
320 match key_type {
321 DataType::Dictionary(_, value) => key_type = value,
322 DataType::RunEndEncoded(_, value) => {
323 nullable |= value.is_nullable();
324 key_type = value.data_type();
325 }
326 _ => break,
327 }
328 }
329 if nullable {
330 return Err(arrow_err!(
331 "Map key field '{}' must be non-nullable, including its encoded values",
332 key.name()
333 ));
334 }
335 validate_map_key_type(entries.data_type())?;
336 }
337 DataType::Struct(fields) => {
338 for field in fields {
339 validate_map_key_type(field.data_type())?;
340 }
341 }
342 DataType::List(field)
343 | DataType::LargeList(field)
344 | DataType::FixedSizeList(field, _)
345 | DataType::ListView(field)
346 | DataType::LargeListView(field)
347 | DataType::RunEndEncoded(_, field) => validate_map_key_type(field.data_type())?,
348 DataType::Dictionary(_, value) => validate_map_key_type(value)?,
349 _ => {}
350 }
351 Ok(())
352}
353
354fn normalize_metadata_field(field: &Field) -> Field {
360 match field.data_type() {
361 DataType::RunEndEncoded(_, value_field) => {
362 let value = normalize_metadata_field(value_field);
363 field
364 .clone()
365 .with_data_type(value.data_type().clone())
366 .with_nullable(field.is_nullable() || value.is_nullable())
367 }
368 DataType::Dictionary(key, value_type) => {
369 let value_field = Field::new("", value_type.as_ref().clone(), false);
372 let value = normalize_metadata_field(&value_field);
373 let value_type = value.data_type();
374 let data_type = if value_type.is_nested() {
377 value_type.clone()
378 } else {
379 DataType::Dictionary(key.clone(), Box::new(value_type.clone()))
380 };
381 field
382 .clone()
383 .with_data_type(data_type)
384 .with_nullable(field.is_nullable() || value.is_nullable())
385 }
386 _ => field
387 .clone()
388 .with_data_type(normalize_metadata_children(field.data_type())),
389 }
390}
391
392fn normalize_metadata_children(dt: &DataType) -> DataType {
393 match dt {
394 DataType::Struct(fields) => DataType::Struct(
395 fields
396 .iter()
397 .map(|f| Arc::new(normalize_metadata_field(f)))
398 .collect(),
399 ),
400 DataType::List(f) => DataType::List(Arc::new(normalize_metadata_field(f))),
401 DataType::LargeList(f) => DataType::LargeList(Arc::new(normalize_metadata_field(f))),
402 DataType::FixedSizeList(f, n) => {
403 DataType::FixedSizeList(Arc::new(normalize_metadata_field(f)), *n)
404 }
405 DataType::ListView(f) => DataType::ListView(Arc::new(normalize_metadata_field(f))),
406 DataType::LargeListView(f) => {
407 DataType::LargeListView(Arc::new(normalize_metadata_field(f)))
408 }
409 DataType::Map(f, sorted) => DataType::Map(Arc::new(normalize_metadata_field(f)), *sorted),
410 _ => dt.clone(),
411 }
412}
413
414fn needs_metadata_normalization(dt: &DataType) -> bool {
416 match dt {
417 DataType::RunEndEncoded(_, _) => true,
418 DataType::Struct(fields) => fields
419 .iter()
420 .any(|f| needs_metadata_normalization(f.data_type())),
421 DataType::List(f)
422 | DataType::LargeList(f)
423 | DataType::FixedSizeList(f, _)
424 | DataType::ListView(f)
425 | DataType::LargeListView(f)
426 | DataType::Map(f, _) => needs_metadata_normalization(f.data_type()),
427 DataType::Dictionary(_, value) => value.is_nested() || needs_metadata_normalization(value),
428 _ => false,
429 }
430}
431
432pub fn add_encoded_arrow_schema_to_metadata(schema: &Schema, props: &mut WriterProperties) {
439 let needs_normalization = schema
440 .fields()
441 .iter()
442 .any(|f| needs_metadata_normalization(f.data_type()));
443 let metadata_schema;
444 let schema = if needs_normalization {
445 let fields: Vec<Field> = schema
446 .fields()
447 .iter()
448 .map(|f| normalize_metadata_field(f))
449 .collect();
450 metadata_schema = Schema::new_with_metadata(fields, schema.metadata().clone());
451 &metadata_schema
452 } else {
453 schema
454 };
455 let encoded = encode_arrow_schema(schema);
456
457 let schema_kv = KeyValue {
458 key: super::ARROW_SCHEMA_META_KEY.to_string(),
459 value: Some(encoded),
460 };
461
462 let meta = props
463 .key_value_metadata
464 .get_or_insert_with(Default::default);
465
466 let schema_meta = meta
468 .iter()
469 .enumerate()
470 .find(|(_, kv)| kv.key.as_str() == super::ARROW_SCHEMA_META_KEY);
471 match schema_meta {
472 Some((i, _)) => {
473 meta.remove(i);
474 meta.push(schema_kv);
475 }
476 None => {
477 meta.push(schema_kv);
478 }
479 }
480}
481
482#[derive(Debug)]
527pub struct ArrowSchemaConverter<'a> {
528 schema_root: &'a str,
530 coerce_types: bool,
534}
535
536impl Default for ArrowSchemaConverter<'_> {
537 fn default() -> Self {
538 Self::new()
539 }
540}
541
542impl<'a> ArrowSchemaConverter<'a> {
543 pub fn new() -> Self {
545 Self {
546 schema_root: "arrow_schema",
547 coerce_types: false,
548 }
549 }
550
551 pub fn with_coerce_types(mut self, coerce_types: bool) -> Self {
582 self.coerce_types = coerce_types;
583 self
584 }
585
586 pub fn schema_root(mut self, schema_root: &'a str) -> Self {
588 self.schema_root = schema_root;
589 self
590 }
591
592 pub fn convert(&self, schema: &Schema) -> Result<SchemaDescriptor> {
596 for field in schema.fields() {
597 validate_map_key_type(field.data_type())?;
598 }
599 let fields = schema
600 .fields()
601 .iter()
602 .map(|field| arrow_to_parquet_type(field, self.coerce_types).map(Arc::new))
603 .collect::<Result<_>>()?;
604 let group = Type::group_type_builder(self.schema_root)
605 .with_fields(fields)
606 .build()?;
607 Ok(SchemaDescriptor::new(Arc::new(group)))
608 }
609}
610
611fn parse_key_value_metadata(
612 key_value_metadata: Option<&Vec<KeyValue>>,
613) -> Option<HashMap<String, String>> {
614 match key_value_metadata {
615 Some(key_values) => {
616 let map: HashMap<String, String> = key_values
617 .iter()
618 .filter_map(|kv| {
619 kv.value
620 .as_ref()
621 .map(|value| (kv.key.clone(), value.clone()))
622 })
623 .collect();
624
625 if map.is_empty() { None } else { Some(map) }
626 }
627 None => None,
628 }
629}
630
631pub fn parquet_to_arrow_field(parquet_column: &ColumnDescriptor) -> Result<Field> {
633 let field = complex::convert_type(&parquet_column.self_type_ptr())?;
634 let mut ret = Field::new(parquet_column.name(), field.arrow_type, field.nullable);
635
636 let parquet_type = parquet_column.self_type();
637 let basic_info = parquet_type.get_basic_info();
638
639 let mut hash_map_size = 0;
640 if basic_info.has_id() {
641 hash_map_size += 1;
642 }
643 if has_extension_type(parquet_type) {
644 hash_map_size += 1;
645 }
646 if hash_map_size == 0 {
647 return Ok(ret);
648 }
649 ret.set_metadata(HashMap::with_capacity(hash_map_size));
650 if basic_info.has_id() {
651 ret.metadata_mut().insert(
652 PARQUET_FIELD_ID_META_KEY.to_string(),
653 basic_info.id().to_string(),
654 );
655 }
656 try_add_extension_type(ret, parquet_column.self_type())
657}
658
659pub fn decimal_length_from_precision(precision: u8) -> usize {
660 (((10.0_f64.powi(precision as i32) + 1.0).log2() + 1.0) / 8.0).ceil() as usize
668}
669
670pub(crate) fn arrow_to_parquet_type(field: &Field, coerce_types: bool) -> Result<Type> {
672 const PARQUET_LIST_ELEMENT_NAME: &str = "element";
673 const PARQUET_MAP_STRUCT_NAME: &str = "key_value";
674 const PARQUET_KEY_FIELD_NAME: &str = "key";
675 const PARQUET_VALUE_FIELD_NAME: &str = "value";
676
677 let name = field.name().as_str();
678 let repetition = if field.is_nullable() {
679 Repetition::OPTIONAL
680 } else {
681 Repetition::REQUIRED
682 };
683 let id = field_id(field);
684 match field.data_type() {
686 DataType::Null => Type::primitive_type_builder(name, PhysicalType::INT32)
687 .with_logical_type(Some(LogicalType::Unknown))
688 .with_repetition(repetition)
689 .with_id(id)
690 .build(),
691 DataType::Boolean => Type::primitive_type_builder(name, PhysicalType::BOOLEAN)
692 .with_repetition(repetition)
693 .with_id(id)
694 .build(),
695 DataType::Int8 => Type::primitive_type_builder(name, PhysicalType::INT32)
696 .with_logical_type(Some(LogicalType::integer(8, true)))
697 .with_repetition(repetition)
698 .with_id(id)
699 .build(),
700 DataType::Int16 => Type::primitive_type_builder(name, PhysicalType::INT32)
701 .with_logical_type(Some(LogicalType::integer(16, true)))
702 .with_repetition(repetition)
703 .with_id(id)
704 .build(),
705 DataType::Int32 => Type::primitive_type_builder(name, PhysicalType::INT32)
706 .with_repetition(repetition)
707 .with_id(id)
708 .build(),
709 DataType::Int64 => Type::primitive_type_builder(name, PhysicalType::INT64)
710 .with_repetition(repetition)
711 .with_id(id)
712 .build(),
713 DataType::UInt8 => Type::primitive_type_builder(name, PhysicalType::INT32)
714 .with_logical_type(Some(LogicalType::integer(8, false)))
715 .with_repetition(repetition)
716 .with_id(id)
717 .build(),
718 DataType::UInt16 => Type::primitive_type_builder(name, PhysicalType::INT32)
719 .with_logical_type(Some(LogicalType::integer(16, false)))
720 .with_repetition(repetition)
721 .with_id(id)
722 .build(),
723 DataType::UInt32 => Type::primitive_type_builder(name, PhysicalType::INT32)
724 .with_logical_type(Some(LogicalType::integer(32, false)))
725 .with_repetition(repetition)
726 .with_id(id)
727 .build(),
728 DataType::UInt64 => Type::primitive_type_builder(name, PhysicalType::INT64)
729 .with_logical_type(Some(LogicalType::integer(64, false)))
730 .with_repetition(repetition)
731 .with_id(id)
732 .build(),
733 DataType::Float16 => Type::primitive_type_builder(name, PhysicalType::FIXED_LEN_BYTE_ARRAY)
734 .with_repetition(repetition)
735 .with_id(id)
736 .with_logical_type(Some(LogicalType::Float16))
737 .with_length(2)
738 .build(),
739 DataType::Float32 => Type::primitive_type_builder(name, PhysicalType::FLOAT)
740 .with_repetition(repetition)
741 .with_id(id)
742 .build(),
743 DataType::Float64 => Type::primitive_type_builder(name, PhysicalType::DOUBLE)
744 .with_repetition(repetition)
745 .with_id(id)
746 .build(),
747 DataType::Timestamp(TimeUnit::Second, _) => {
748 Type::primitive_type_builder(name, PhysicalType::INT64)
750 .with_repetition(repetition)
751 .with_id(id)
752 .build()
753 }
754 DataType::Timestamp(time_unit, tz) => {
755 Type::primitive_type_builder(name, PhysicalType::INT64)
756 .with_logical_type(Some(LogicalType::timestamp(
757 matches!(tz, Some(z) if !z.as_ref().is_empty()),
759 match time_unit {
760 TimeUnit::Second => unreachable!(),
761 TimeUnit::Millisecond => ParquetTimeUnit::MILLIS,
762 TimeUnit::Microsecond => ParquetTimeUnit::MICROS,
763 TimeUnit::Nanosecond => ParquetTimeUnit::NANOS,
764 },
765 )))
766 .with_repetition(repetition)
767 .with_id(id)
768 .build()
769 }
770 DataType::Date32 => Type::primitive_type_builder(name, PhysicalType::INT32)
771 .with_logical_type(Some(LogicalType::Date))
772 .with_repetition(repetition)
773 .with_id(id)
774 .build(),
775 DataType::Date64 => {
776 if coerce_types {
777 Type::primitive_type_builder(name, PhysicalType::INT32)
778 .with_logical_type(Some(LogicalType::Date))
779 .with_repetition(repetition)
780 .with_id(id)
781 .build()
782 } else {
783 Type::primitive_type_builder(name, PhysicalType::INT64)
784 .with_repetition(repetition)
785 .with_id(id)
786 .build()
787 }
788 }
789 DataType::Time32(TimeUnit::Second) => {
790 Type::primitive_type_builder(name, PhysicalType::INT32)
792 .with_repetition(repetition)
793 .with_id(id)
794 .build()
795 }
796 DataType::Time32(unit) => Type::primitive_type_builder(name, PhysicalType::INT32)
797 .with_logical_type(Some(LogicalType::time(
798 field.metadata().contains_key("adjusted_to_utc"),
799 match unit {
800 TimeUnit::Millisecond => ParquetTimeUnit::MILLIS,
801 u => unreachable!("Invalid unit for Time32: {:?}", u),
802 },
803 )))
804 .with_repetition(repetition)
805 .with_id(id)
806 .build(),
807 DataType::Time64(unit) => Type::primitive_type_builder(name, PhysicalType::INT64)
808 .with_logical_type(Some(LogicalType::time(
809 field.metadata().contains_key("adjusted_to_utc"),
810 match unit {
811 TimeUnit::Microsecond => ParquetTimeUnit::MICROS,
812 TimeUnit::Nanosecond => ParquetTimeUnit::NANOS,
813 u => unreachable!("Invalid unit for Time64: {:?}", u),
814 },
815 )))
816 .with_repetition(repetition)
817 .with_id(id)
818 .build(),
819 DataType::Duration(_) => Type::primitive_type_builder(name, PhysicalType::INT64)
820 .with_repetition(repetition)
821 .with_id(id)
822 .build(),
823 DataType::Interval(_) => {
824 Type::primitive_type_builder(name, PhysicalType::FIXED_LEN_BYTE_ARRAY)
825 .with_converted_type(ConvertedType::INTERVAL)
826 .with_repetition(repetition)
827 .with_id(id)
828 .with_length(12)
829 .build()
830 }
831 DataType::Binary | DataType::LargeBinary => {
832 Type::primitive_type_builder(name, PhysicalType::BYTE_ARRAY)
833 .with_repetition(repetition)
834 .with_id(id)
835 .with_logical_type(logical_type_for_binary(field))
836 .build()
837 }
838 DataType::FixedSizeBinary(length) => {
839 Type::primitive_type_builder(name, PhysicalType::FIXED_LEN_BYTE_ARRAY)
840 .with_repetition(repetition)
841 .with_id(id)
842 .with_length(*length)
843 .with_logical_type(logical_type_for_fixed_size_binary(field))
844 .build()
845 }
846 DataType::BinaryView => Type::primitive_type_builder(name, PhysicalType::BYTE_ARRAY)
847 .with_repetition(repetition)
848 .with_id(id)
849 .with_logical_type(logical_type_for_binary_view(field))
850 .build(),
851 DataType::Decimal32(precision, scale)
852 | DataType::Decimal64(precision, scale)
853 | DataType::Decimal128(precision, scale)
854 | DataType::Decimal256(precision, scale) => {
855 let (physical_type, length) = if *precision >= 1 && *precision <= 9 {
858 (PhysicalType::INT32, -1)
859 } else if *precision <= 18 {
860 (PhysicalType::INT64, -1)
861 } else {
862 (
863 PhysicalType::FIXED_LEN_BYTE_ARRAY,
864 decimal_length_from_precision(*precision) as i32,
865 )
866 };
867 Type::primitive_type_builder(name, physical_type)
868 .with_repetition(repetition)
869 .with_id(id)
870 .with_length(length)
871 .with_logical_type(Some(LogicalType::decimal(*scale as i32, *precision as i32)))
872 .with_precision(*precision as i32)
873 .with_scale(*scale as i32)
874 .build()
875 }
876 DataType::Utf8 | DataType::LargeUtf8 => {
877 Type::primitive_type_builder(name, PhysicalType::BYTE_ARRAY)
878 .with_logical_type(logical_type_for_string(field))
879 .with_repetition(repetition)
880 .with_id(id)
881 .build()
882 }
883 DataType::Utf8View => Type::primitive_type_builder(name, PhysicalType::BYTE_ARRAY)
884 .with_logical_type(logical_type_for_string(field))
885 .with_repetition(repetition)
886 .with_id(id)
887 .build(),
888 DataType::List(f)
889 | DataType::FixedSizeList(f, _)
890 | DataType::LargeList(f)
891 | DataType::ListView(f)
892 | DataType::LargeListView(f) => {
893 let field_ref = if coerce_types && f.name() != PARQUET_LIST_ELEMENT_NAME {
894 let ff = f.as_ref().clone().with_name(PARQUET_LIST_ELEMENT_NAME);
896 Arc::new(arrow_to_parquet_type(&ff, coerce_types)?)
897 } else {
898 Arc::new(arrow_to_parquet_type(f, coerce_types)?)
899 };
900
901 Type::group_type_builder(name)
902 .with_fields(vec![Arc::new(
903 Type::group_type_builder("list")
904 .with_fields(vec![field_ref])
905 .with_repetition(Repetition::REPEATED)
906 .build()?,
907 )])
908 .with_logical_type(Some(LogicalType::List))
909 .with_repetition(repetition)
910 .with_id(id)
911 .build()
912 }
913 DataType::Struct(fields) => {
914 if fields.is_empty() {
915 return Err(arrow_err!("Parquet does not support writing empty structs",));
916 }
917 let fields = fields
919 .iter()
920 .map(|f| arrow_to_parquet_type(f, coerce_types).map(Arc::new))
921 .collect::<Result<_>>()?;
922 Type::group_type_builder(name)
923 .with_fields(fields)
924 .with_repetition(repetition)
925 .with_id(id)
926 .with_logical_type(logical_type_for_struct(field))
927 .build()
928 }
929 DataType::Map(field, _) => {
930 if let DataType::Struct(struct_fields) = field.data_type() {
931 let map_struct_name = if coerce_types {
933 PARQUET_MAP_STRUCT_NAME
934 } else {
935 field.name()
936 };
937
938 let fix_map_field = |name: &str, fld: &Arc<Field>| -> Result<Arc<Type>> {
940 if coerce_types && fld.name() != name {
941 let f = fld.as_ref().clone().with_name(name);
942 Ok(Arc::new(arrow_to_parquet_type(&f, coerce_types)?))
943 } else {
944 Ok(Arc::new(arrow_to_parquet_type(fld, coerce_types)?))
945 }
946 };
947 let key_field = fix_map_field(PARQUET_KEY_FIELD_NAME, &struct_fields[0])?;
948 let val_field = fix_map_field(PARQUET_VALUE_FIELD_NAME, &struct_fields[1])?;
949
950 Type::group_type_builder(name)
951 .with_fields(vec![Arc::new(
952 Type::group_type_builder(map_struct_name)
953 .with_fields(vec![key_field, val_field])
954 .with_repetition(Repetition::REPEATED)
955 .build()?,
956 )])
957 .with_logical_type(Some(LogicalType::Map))
958 .with_repetition(repetition)
959 .with_id(id)
960 .build()
961 } else {
962 Err(arrow_err!(
963 "DataType::Map should contain a struct field child",
964 ))
965 }
966 }
967 DataType::Union(_, _) => unimplemented!("See ARROW-8817."),
968 DataType::Dictionary(_, value) => {
969 let dict_field = field.clone().with_data_type(value.as_ref().clone());
971 arrow_to_parquet_type(&dict_field, coerce_types)
972 }
973 DataType::RunEndEncoded(_, value) => {
974 let value_field = field
979 .clone()
980 .with_data_type(value.data_type().clone())
981 .with_nullable(field.is_nullable() || value.is_nullable());
982 arrow_to_parquet_type(&value_field, coerce_types)
983 }
984 }
985}
986
987fn field_id(field: &Field) -> Option<i32> {
988 let value = field.metadata().get(super::PARQUET_FIELD_ID_META_KEY)?;
989 value.parse().ok() }
991
992#[cfg(test)]
993mod tests {
994 use super::*;
995
996 use std::{collections::HashMap, sync::Arc};
997
998 use crate::arrow::PARQUET_FIELD_ID_META_KEY;
999 use crate::file::metadata::KeyValue;
1000 use crate::file::reader::FileReader;
1001 use crate::{
1002 arrow::{ArrowWriter, arrow_reader::ParquetRecordBatchReaderBuilder},
1003 schema::{parser::parse_message_type, types::SchemaDescriptor},
1004 };
1005 use arrow::datatypes::{DataType, Field, IntervalUnit, TimeUnit};
1006
1007 #[test]
1008 fn test_flat_primitives() {
1009 let message_type = "
1010 message test_schema {
1011 REQUIRED BOOLEAN boolean;
1012 REQUIRED INT32 int8 (INT_8);
1013 REQUIRED INT32 int16 (INT_16);
1014 REQUIRED INT32 uint8 (INTEGER(8,false));
1015 REQUIRED INT32 uint16 (INTEGER(16,false));
1016 REQUIRED INT32 int32;
1017 REQUIRED INT64 int64;
1018 OPTIONAL DOUBLE double;
1019 OPTIONAL FLOAT float;
1020 OPTIONAL FIXED_LEN_BYTE_ARRAY (2) float16 (FLOAT16);
1021 OPTIONAL BINARY string (UTF8);
1022 OPTIONAL BINARY string_2 (STRING);
1023 OPTIONAL BINARY json (JSON);
1024 }
1025 ";
1026 let parquet_group_type = parse_message_type(message_type).unwrap();
1027
1028 let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1029 let converted_arrow_schema = parquet_to_arrow_schema(&parquet_schema, None).unwrap();
1030
1031 let arrow_fields = Fields::from(vec![
1032 Field::new("boolean", DataType::Boolean, false),
1033 Field::new("int8", DataType::Int8, false),
1034 Field::new("int16", DataType::Int16, false),
1035 Field::new("uint8", DataType::UInt8, false),
1036 Field::new("uint16", DataType::UInt16, false),
1037 Field::new("int32", DataType::Int32, false),
1038 Field::new("int64", DataType::Int64, false),
1039 Field::new("double", DataType::Float64, true),
1040 Field::new("float", DataType::Float32, true),
1041 Field::new("float16", DataType::Float16, true),
1042 Field::new("string", DataType::Utf8, true),
1043 Field::new("string_2", DataType::Utf8, true),
1044 json_field(),
1045 ]);
1046
1047 assert_eq!(&arrow_fields, converted_arrow_schema.fields());
1048 }
1049
1050 fn json_field() -> Field {
1053 #[cfg(feature = "arrow_canonical_extension_types")]
1054 {
1055 Field::new("json", DataType::Utf8, true)
1056 .with_extension_type(arrow_schema::extension::Json::default())
1057 }
1058 #[cfg(not(feature = "arrow_canonical_extension_types"))]
1059 {
1060 Field::new("json", DataType::Utf8, true)
1061 }
1062 }
1063
1064 #[test]
1065 fn test_decimal_fields() {
1066 let message_type = "
1067 message test_schema {
1068 REQUIRED INT32 decimal1 (DECIMAL(4,2));
1069 REQUIRED INT64 decimal2 (DECIMAL(12,2));
1070 REQUIRED FIXED_LEN_BYTE_ARRAY (16) decimal3 (DECIMAL(30,2));
1071 REQUIRED BYTE_ARRAY decimal4 (DECIMAL(33,2));
1072 REQUIRED BYTE_ARRAY decimal5 (DECIMAL(38,2));
1073 REQUIRED FIXED_LEN_BYTE_ARRAY (17) decimal6 (DECIMAL(39,2));
1074 REQUIRED BYTE_ARRAY decimal7 (DECIMAL(39,2));
1075 }
1076 ";
1077
1078 let parquet_group_type = parse_message_type(message_type).unwrap();
1079
1080 let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1081 let converted_arrow_schema = parquet_to_arrow_schema(&parquet_schema, None).unwrap();
1082
1083 let arrow_fields = Fields::from(vec![
1084 Field::new("decimal1", DataType::Decimal128(4, 2), false),
1085 Field::new("decimal2", DataType::Decimal128(12, 2), false),
1086 Field::new("decimal3", DataType::Decimal128(30, 2), false),
1087 Field::new("decimal4", DataType::Decimal128(33, 2), false),
1088 Field::new("decimal5", DataType::Decimal128(38, 2), false),
1089 Field::new("decimal6", DataType::Decimal256(39, 2), false),
1090 Field::new("decimal7", DataType::Decimal256(39, 2), false),
1091 ]);
1092 assert_eq!(&arrow_fields, converted_arrow_schema.fields());
1093 }
1094
1095 #[test]
1096 fn test_decimal_precision_physical_types() {
1097 let cases = [
1098 (1, PhysicalType::INT32, None),
1099 (9, PhysicalType::INT32, None),
1100 (10, PhysicalType::INT64, None),
1101 (18, PhysicalType::INT64, None),
1102 (19, PhysicalType::FIXED_LEN_BYTE_ARRAY, Some(9)),
1103 (38, PhysicalType::FIXED_LEN_BYTE_ARRAY, Some(16)),
1104 ];
1105
1106 for (precision, expected_type, expected_length) in cases {
1107 let arrow_schema = Schema::new(vec![Field::new(
1108 "decimal",
1109 DataType::Decimal128(precision, 0),
1110 false,
1111 )]);
1112 let parquet_schema = ArrowSchemaConverter::new().convert(&arrow_schema).unwrap();
1113 let column = parquet_schema.column(0);
1114
1115 assert_eq!(
1116 column.physical_type(),
1117 expected_type,
1118 "precision {precision}"
1119 );
1120 if let Some(expected_length) = expected_length {
1121 assert_eq!(
1122 column.type_length(),
1123 expected_length,
1124 "precision {precision}"
1125 );
1126 }
1127 }
1128 }
1129
1130 #[test]
1131 fn test_byte_array_fields() {
1132 let message_type = "
1133 message test_schema {
1134 REQUIRED BYTE_ARRAY binary;
1135 REQUIRED FIXED_LEN_BYTE_ARRAY (20) fixed_binary;
1136 }
1137 ";
1138
1139 let parquet_group_type = parse_message_type(message_type).unwrap();
1140
1141 let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1142 let converted_arrow_schema = parquet_to_arrow_schema(&parquet_schema, None).unwrap();
1143
1144 let arrow_fields = Fields::from(vec![
1145 Field::new("binary", DataType::Binary, false),
1146 Field::new("fixed_binary", DataType::FixedSizeBinary(20), false),
1147 ]);
1148 assert_eq!(&arrow_fields, converted_arrow_schema.fields());
1149 }
1150
1151 #[test]
1152 fn test_duplicate_fields() {
1153 let message_type = "
1154 message test_schema {
1155 REQUIRED BOOLEAN boolean;
1156 REQUIRED INT32 int8 (INT_8);
1157 }
1158 ";
1159
1160 let parquet_group_type = parse_message_type(message_type).unwrap();
1161
1162 let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1163 let converted_arrow_schema = parquet_to_arrow_schema(&parquet_schema, None).unwrap();
1164
1165 let arrow_fields = Fields::from(vec![
1166 Field::new("boolean", DataType::Boolean, false),
1167 Field::new("int8", DataType::Int8, false),
1168 ]);
1169 assert_eq!(&arrow_fields, converted_arrow_schema.fields());
1170
1171 let converted_arrow_schema =
1172 parquet_to_arrow_schema_by_columns(&parquet_schema, ProjectionMask::all(), None)
1173 .unwrap();
1174 assert_eq!(&arrow_fields, converted_arrow_schema.fields());
1175 }
1176
1177 #[test]
1178 fn test_parquet_lists() {
1179 let mut arrow_fields = Vec::new();
1180
1181 let message_type = "
1183 message test_schema {
1184 REQUIRED GROUP my_list (LIST) {
1185 REPEATED GROUP list {
1186 OPTIONAL BINARY element (UTF8);
1187 }
1188 }
1189 OPTIONAL GROUP my_list (LIST) {
1190 REPEATED GROUP list {
1191 REQUIRED BINARY element (UTF8);
1192 }
1193 }
1194 OPTIONAL GROUP array_of_arrays (LIST) {
1195 REPEATED GROUP list {
1196 REQUIRED GROUP element (LIST) {
1197 REPEATED GROUP list {
1198 REQUIRED INT32 element;
1199 }
1200 }
1201 }
1202 }
1203 OPTIONAL GROUP my_list (LIST) {
1204 REPEATED GROUP element {
1205 REQUIRED BINARY str (UTF8);
1206 }
1207 }
1208 OPTIONAL GROUP my_list (LIST) {
1209 REPEATED INT32 element;
1210 }
1211 OPTIONAL GROUP my_list (LIST) {
1212 REPEATED GROUP element {
1213 REQUIRED BINARY str (UTF8);
1214 REQUIRED INT32 num;
1215 }
1216 }
1217 OPTIONAL GROUP my_list (LIST) {
1218 REPEATED GROUP array {
1219 REQUIRED BINARY str (UTF8);
1220 }
1221
1222 }
1223 OPTIONAL GROUP my_list (LIST) {
1224 REPEATED GROUP my_list_tuple {
1225 REQUIRED BINARY str (UTF8);
1226 }
1227 }
1228 REPEATED INT32 name;
1229 }
1230 ";
1231
1232 {
1239 arrow_fields.push(Field::new_list(
1240 "my_list",
1241 Field::new("element", DataType::Utf8, true),
1242 false,
1243 ));
1244 }
1245
1246 {
1253 arrow_fields.push(Field::new_list(
1254 "my_list",
1255 Field::new("element", DataType::Utf8, false),
1256 true,
1257 ));
1258 }
1259
1260 {
1273 let arrow_inner_list = Field::new("element", DataType::Int32, false);
1274 arrow_fields.push(Field::new_list(
1275 "array_of_arrays",
1276 Field::new_list("element", arrow_inner_list, false),
1277 true,
1278 ));
1279 }
1280
1281 {
1288 arrow_fields.push(Field::new_list(
1289 "my_list",
1290 Field::new("str", DataType::Utf8, false),
1291 true,
1292 ));
1293 }
1294
1295 {
1300 arrow_fields.push(Field::new_list(
1301 "my_list",
1302 Field::new("element", DataType::Int32, false),
1303 true,
1304 ));
1305 }
1306
1307 {
1315 let fields = vec![
1316 Field::new("str", DataType::Utf8, false),
1317 Field::new("num", DataType::Int32, false),
1318 ];
1319 arrow_fields.push(Field::new_list(
1320 "my_list",
1321 Field::new_struct("element", fields, false),
1322 true,
1323 ));
1324 }
1325
1326 {
1334 let fields = vec![Field::new("str", DataType::Utf8, false)];
1335 arrow_fields.push(Field::new_list(
1336 "my_list",
1337 Field::new_struct("array", fields, false),
1338 true,
1339 ));
1340 }
1341
1342 {
1350 let fields = vec![Field::new("str", DataType::Utf8, false)];
1351 arrow_fields.push(Field::new_list(
1352 "my_list",
1353 Field::new_struct("my_list_tuple", fields, false),
1354 true,
1355 ));
1356 }
1357
1358 {
1361 arrow_fields.push(Field::new_list(
1362 "name",
1363 Field::new("name", DataType::Int32, false),
1364 false,
1365 ));
1366 }
1367
1368 let parquet_group_type = parse_message_type(message_type).unwrap();
1369
1370 let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1371 let converted_arrow_schema = parquet_to_arrow_schema(&parquet_schema, None).unwrap();
1372 let converted_fields = converted_arrow_schema.fields();
1373
1374 assert_eq!(arrow_fields.len(), converted_fields.len());
1375 for i in 0..arrow_fields.len() {
1376 assert_eq!(&arrow_fields[i], converted_fields[i].as_ref(), "{i}");
1377 }
1378 }
1379
1380 #[test]
1381 fn test_parquet_list_nullable() {
1382 let mut arrow_fields = Vec::new();
1383
1384 let message_type = "
1385 message test_schema {
1386 REQUIRED GROUP my_list1 (LIST) {
1387 REPEATED GROUP list {
1388 OPTIONAL BINARY element (UTF8);
1389 }
1390 }
1391 OPTIONAL GROUP my_list2 (LIST) {
1392 REPEATED GROUP list {
1393 REQUIRED BINARY element (UTF8);
1394 }
1395 }
1396 REQUIRED GROUP my_list3 (LIST) {
1397 REPEATED GROUP list {
1398 REQUIRED BINARY element (UTF8);
1399 }
1400 }
1401 }
1402 ";
1403
1404 {
1411 arrow_fields.push(Field::new_list(
1412 "my_list1",
1413 Field::new("element", DataType::Utf8, true),
1414 false,
1415 ));
1416 }
1417
1418 {
1425 arrow_fields.push(Field::new_list(
1426 "my_list2",
1427 Field::new("element", DataType::Utf8, false),
1428 true,
1429 ));
1430 }
1431
1432 {
1439 arrow_fields.push(Field::new_list(
1440 "my_list3",
1441 Field::new("element", DataType::Utf8, false),
1442 false,
1443 ));
1444 }
1445
1446 let parquet_group_type = parse_message_type(message_type).unwrap();
1447
1448 let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1449 let converted_arrow_schema = parquet_to_arrow_schema(&parquet_schema, None).unwrap();
1450 let converted_fields = converted_arrow_schema.fields();
1451
1452 assert_eq!(arrow_fields.len(), converted_fields.len());
1453 for i in 0..arrow_fields.len() {
1454 assert_eq!(&arrow_fields[i], converted_fields[i].as_ref());
1455 }
1456 }
1457
1458 #[test]
1459 fn test_parquet_maps() {
1460 let mut arrow_fields = Vec::new();
1461
1462 let message_type = "
1464 message test_schema {
1465 REQUIRED group my_map1 (MAP) {
1466 REPEATED group key_value {
1467 REQUIRED binary key (UTF8);
1468 OPTIONAL int32 value;
1469 }
1470 }
1471 OPTIONAL group my_map2 (MAP) {
1472 REPEATED group map {
1473 REQUIRED binary str (UTF8);
1474 REQUIRED int32 num;
1475 }
1476 }
1477 OPTIONAL group my_map3 (MAP_KEY_VALUE) {
1478 REPEATED group map {
1479 REQUIRED binary key (UTF8);
1480 OPTIONAL int32 value;
1481 }
1482 }
1483 REQUIRED group my_map4 (MAP) {
1484 REPEATED group map {
1485 OPTIONAL binary key (UTF8);
1486 REQUIRED int32 value;
1487 }
1488 }
1489 }
1490 ";
1491
1492 {
1500 arrow_fields.push(Field::new_map(
1501 "my_map1",
1502 "key_value",
1503 Field::new("key", DataType::Utf8, false),
1504 Field::new("value", DataType::Int32, true),
1505 false,
1506 false,
1507 ));
1508 }
1509
1510 {
1518 arrow_fields.push(Field::new_map(
1519 "my_map2",
1520 "map",
1521 Field::new("str", DataType::Utf8, false),
1522 Field::new("num", DataType::Int32, false),
1523 false,
1524 true,
1525 ));
1526 }
1527
1528 {
1536 arrow_fields.push(Field::new_map(
1537 "my_map3",
1538 "map",
1539 Field::new("key", DataType::Utf8, false),
1540 Field::new("value", DataType::Int32, true),
1541 false,
1542 true,
1543 ));
1544 }
1545
1546 {
1554 arrow_fields.push(Field::new_map(
1555 "my_map4",
1556 "map",
1557 Field::new("key", DataType::Utf8, false), Field::new("value", DataType::Int32, false),
1559 false,
1560 false,
1561 ));
1562 }
1563
1564 let parquet_group_type = parse_message_type(message_type).unwrap();
1565
1566 let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1567 let converted_arrow_schema = parquet_to_arrow_schema(&parquet_schema, None).unwrap();
1568 let converted_fields = converted_arrow_schema.fields();
1569
1570 assert_eq!(arrow_fields.len(), converted_fields.len());
1571 for i in 0..arrow_fields.len() {
1572 assert_eq!(&arrow_fields[i], converted_fields[i].as_ref());
1573 }
1574 }
1575
1576 #[test]
1577 fn test_nested_schema() {
1578 let mut arrow_fields = Vec::new();
1579 {
1580 let group1_fields = Fields::from(vec![
1581 Field::new("leaf1", DataType::Boolean, false),
1582 Field::new("leaf2", DataType::Int32, false),
1583 ]);
1584 let group1_struct = Field::new("group1", DataType::Struct(group1_fields), false);
1585 arrow_fields.push(group1_struct);
1586
1587 let leaf3_field = Field::new("leaf3", DataType::Int64, false);
1588 arrow_fields.push(leaf3_field);
1589 }
1590
1591 let message_type = "
1592 message test_schema {
1593 REQUIRED GROUP group1 {
1594 REQUIRED BOOLEAN leaf1;
1595 REQUIRED INT32 leaf2;
1596 }
1597 REQUIRED INT64 leaf3;
1598 }
1599 ";
1600 let parquet_group_type = parse_message_type(message_type).unwrap();
1601
1602 let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1603 let converted_arrow_schema = parquet_to_arrow_schema(&parquet_schema, None).unwrap();
1604 let converted_fields = converted_arrow_schema.fields();
1605
1606 assert_eq!(arrow_fields.len(), converted_fields.len());
1607 for i in 0..arrow_fields.len() {
1608 assert_eq!(&arrow_fields[i], converted_fields[i].as_ref());
1609 }
1610 }
1611
1612 #[test]
1613 fn test_nested_schema_partial() {
1614 let mut arrow_fields = Vec::new();
1615 {
1616 let group1_fields = vec![Field::new("leaf1", DataType::Int64, false)].into();
1617 let group1 = Field::new("group1", DataType::Struct(group1_fields), false);
1618 arrow_fields.push(group1);
1619
1620 let group2_fields = vec![Field::new("leaf4", DataType::Int64, false)].into();
1621 let group2 = Field::new("group2", DataType::Struct(group2_fields), false);
1622 arrow_fields.push(group2);
1623
1624 arrow_fields.push(Field::new("leaf5", DataType::Int64, false));
1625 }
1626
1627 let message_type = "
1628 message test_schema {
1629 REQUIRED GROUP group1 {
1630 REQUIRED INT64 leaf1;
1631 REQUIRED INT64 leaf2;
1632 }
1633 REQUIRED GROUP group2 {
1634 REQUIRED INT64 leaf3;
1635 REQUIRED INT64 leaf4;
1636 }
1637 REQUIRED INT64 leaf5;
1638 }
1639 ";
1640 let parquet_group_type = parse_message_type(message_type).unwrap();
1641
1642 let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1652 let mask = ProjectionMask::leaves(&parquet_schema, [3, 0, 4, 4]);
1653 let converted_arrow_schema =
1654 parquet_to_arrow_schema_by_columns(&parquet_schema, mask, None).unwrap();
1655 let converted_fields = converted_arrow_schema.fields();
1656
1657 assert_eq!(arrow_fields.len(), converted_fields.len());
1658 for i in 0..arrow_fields.len() {
1659 assert_eq!(&arrow_fields[i], converted_fields[i].as_ref());
1660 }
1661 }
1662
1663 #[test]
1664 fn test_nested_schema_partial_ordering() {
1665 let mut arrow_fields = Vec::new();
1666 {
1667 let group1_fields = vec![Field::new("leaf1", DataType::Int64, false)].into();
1668 let group1 = Field::new("group1", DataType::Struct(group1_fields), false);
1669 arrow_fields.push(group1);
1670
1671 let group2_fields = vec![Field::new("leaf4", DataType::Int64, false)].into();
1672 let group2 = Field::new("group2", DataType::Struct(group2_fields), false);
1673 arrow_fields.push(group2);
1674
1675 arrow_fields.push(Field::new("leaf5", DataType::Int64, false));
1676 }
1677
1678 let message_type = "
1679 message test_schema {
1680 REQUIRED GROUP group1 {
1681 REQUIRED INT64 leaf1;
1682 REQUIRED INT64 leaf2;
1683 }
1684 REQUIRED GROUP group2 {
1685 REQUIRED INT64 leaf3;
1686 REQUIRED INT64 leaf4;
1687 }
1688 REQUIRED INT64 leaf5;
1689 }
1690 ";
1691 let parquet_group_type = parse_message_type(message_type).unwrap();
1692
1693 let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1703 let mask = ProjectionMask::leaves(&parquet_schema, [3, 0, 4]);
1704 let converted_arrow_schema =
1705 parquet_to_arrow_schema_by_columns(&parquet_schema, mask, None).unwrap();
1706 let converted_fields = converted_arrow_schema.fields();
1707
1708 assert_eq!(arrow_fields.len(), converted_fields.len());
1709 for i in 0..arrow_fields.len() {
1710 assert_eq!(&arrow_fields[i], converted_fields[i].as_ref());
1711 }
1712
1713 let mask =
1714 ProjectionMask::columns(&parquet_schema, ["group2.leaf4", "group1.leaf1", "leaf5"]);
1715 let converted_arrow_schema =
1716 parquet_to_arrow_schema_by_columns(&parquet_schema, mask, None).unwrap();
1717 let converted_fields = converted_arrow_schema.fields();
1718
1719 assert_eq!(arrow_fields.len(), converted_fields.len());
1720 for i in 0..arrow_fields.len() {
1721 assert_eq!(&arrow_fields[i], converted_fields[i].as_ref());
1722 }
1723 }
1724
1725 #[test]
1726 fn test_repeated_nested_schema() {
1727 let mut arrow_fields = Vec::new();
1728 {
1729 arrow_fields.push(Field::new("leaf1", DataType::Int32, true));
1730
1731 let inner_group_list = Field::new_list(
1732 "innerGroup",
1733 Field::new_struct(
1734 "innerGroup",
1735 vec![Field::new("leaf3", DataType::Int32, true)],
1736 false,
1737 ),
1738 false,
1739 );
1740
1741 let outer_group_list = Field::new_list(
1742 "outerGroup",
1743 Field::new_struct(
1744 "outerGroup",
1745 vec![Field::new("leaf2", DataType::Int32, true), inner_group_list],
1746 false,
1747 ),
1748 false,
1749 );
1750 arrow_fields.push(outer_group_list);
1751 }
1752
1753 let message_type = "
1754 message test_schema {
1755 OPTIONAL INT32 leaf1;
1756 REPEATED GROUP outerGroup {
1757 OPTIONAL INT32 leaf2;
1758 REPEATED GROUP innerGroup {
1759 OPTIONAL INT32 leaf3;
1760 }
1761 }
1762 }
1763 ";
1764 let parquet_group_type = parse_message_type(message_type).unwrap();
1765
1766 let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1767 let converted_arrow_schema = parquet_to_arrow_schema(&parquet_schema, None).unwrap();
1768 let converted_fields = converted_arrow_schema.fields();
1769
1770 assert_eq!(arrow_fields.len(), converted_fields.len());
1771 for i in 0..arrow_fields.len() {
1772 assert_eq!(&arrow_fields[i], converted_fields[i].as_ref());
1773 }
1774 }
1775
1776 #[test]
1777 fn test_column_desc_to_field() {
1778 let message_type = "
1779 message test_schema {
1780 REQUIRED BOOLEAN boolean;
1781 REQUIRED INT32 int8 (INT_8);
1782 REQUIRED INT32 uint8 (INTEGER(8,false));
1783 REQUIRED INT32 int16 (INT_16);
1784 REQUIRED INT32 uint16 (INTEGER(16,false));
1785 REQUIRED INT32 int32;
1786 REQUIRED INT64 int64;
1787 OPTIONAL DOUBLE double;
1788 OPTIONAL FLOAT float;
1789 OPTIONAL FIXED_LEN_BYTE_ARRAY (2) float16 (FLOAT16);
1790 OPTIONAL BINARY string (UTF8);
1791 REPEATED BOOLEAN bools;
1792 OPTIONAL INT32 date (DATE);
1793 OPTIONAL INT32 time_milli (TIME_MILLIS);
1794 OPTIONAL INT64 time_micro (TIME_MICROS);
1795 OPTIONAL INT64 time_nano (TIME(NANOS,false));
1796 OPTIONAL INT64 ts_milli (TIMESTAMP_MILLIS);
1797 REQUIRED INT64 ts_micro (TIMESTAMP_MICROS);
1798 REQUIRED INT64 ts_nano (TIMESTAMP(NANOS,true));
1799 REPEATED INT32 int_list;
1800 REPEATED BINARY byte_list;
1801 REPEATED BINARY string_list (UTF8);
1802 REQUIRED INT32 decimal_int32 (DECIMAL(8,2));
1803 REQUIRED INT64 decimal_int64 (DECIMAL(16,2));
1804 REQUIRED FIXED_LEN_BYTE_ARRAY (13) decimal_fix_length (DECIMAL(30,2));
1805 }
1806 ";
1807 let parquet_group_type = parse_message_type(message_type).unwrap();
1808
1809 let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1810 let converted_arrow_fields = parquet_schema
1811 .columns()
1812 .iter()
1813 .map(|c| parquet_to_arrow_field(c).unwrap())
1814 .collect::<Vec<Field>>();
1815
1816 let arrow_fields = vec![
1817 Field::new("boolean", DataType::Boolean, false),
1818 Field::new("int8", DataType::Int8, false),
1819 Field::new("uint8", DataType::UInt8, false),
1820 Field::new("int16", DataType::Int16, false),
1821 Field::new("uint16", DataType::UInt16, false),
1822 Field::new("int32", DataType::Int32, false),
1823 Field::new("int64", DataType::Int64, false),
1824 Field::new("double", DataType::Float64, true),
1825 Field::new("float", DataType::Float32, true),
1826 Field::new("float16", DataType::Float16, true),
1827 Field::new("string", DataType::Utf8, true),
1828 Field::new_list(
1829 "bools",
1830 Field::new("bools", DataType::Boolean, false),
1831 false,
1832 ),
1833 Field::new("date", DataType::Date32, true),
1834 Field::new("time_milli", DataType::Time32(TimeUnit::Millisecond), true),
1835 Field::new("time_micro", DataType::Time64(TimeUnit::Microsecond), true),
1836 Field::new("time_nano", DataType::Time64(TimeUnit::Nanosecond), true),
1837 Field::new(
1838 "ts_milli",
1839 DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())),
1840 true,
1841 ),
1842 Field::new(
1843 "ts_micro",
1844 DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
1845 false,
1846 ),
1847 Field::new(
1848 "ts_nano",
1849 DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),
1850 false,
1851 ),
1852 Field::new_list(
1853 "int_list",
1854 Field::new("int_list", DataType::Int32, false),
1855 false,
1856 ),
1857 Field::new_list(
1858 "byte_list",
1859 Field::new("byte_list", DataType::Binary, false),
1860 false,
1861 ),
1862 Field::new_list(
1863 "string_list",
1864 Field::new("string_list", DataType::Utf8, false),
1865 false,
1866 ),
1867 Field::new("decimal_int32", DataType::Decimal128(8, 2), false),
1868 Field::new("decimal_int64", DataType::Decimal128(16, 2), false),
1869 Field::new("decimal_fix_length", DataType::Decimal128(30, 2), false),
1870 ];
1871
1872 assert_eq!(arrow_fields, converted_arrow_fields);
1873 }
1874
1875 #[test]
1876 fn test_coerced_map_list() {
1877 let arrow_fields = vec![
1879 Field::new_list(
1880 "my_list",
1881 Field::new("item", DataType::Boolean, true),
1882 false,
1883 ),
1884 Field::new_map(
1885 "my_map",
1886 "my_entries",
1887 Field::new("my_keys", DataType::Utf8, false),
1888 Field::new("my_values", DataType::Int32, true),
1889 false,
1890 true,
1891 ),
1892 ];
1893 let arrow_schema = Schema::new(arrow_fields);
1894
1895 let message_type = "
1897 message parquet_schema {
1898 REQUIRED GROUP my_list (LIST) {
1899 REPEATED GROUP list {
1900 OPTIONAL BOOLEAN element;
1901 }
1902 }
1903 OPTIONAL GROUP my_map (MAP) {
1904 REPEATED GROUP key_value {
1905 REQUIRED BINARY key (STRING);
1906 OPTIONAL INT32 value;
1907 }
1908 }
1909 }
1910 ";
1911 let parquet_group_type = parse_message_type(message_type).unwrap();
1912 let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1913 let converted_arrow_schema = ArrowSchemaConverter::new()
1914 .with_coerce_types(true)
1915 .convert(&arrow_schema)
1916 .unwrap();
1917 assert_eq!(
1918 parquet_schema.columns().len(),
1919 converted_arrow_schema.columns().len()
1920 );
1921
1922 let message_type = "
1924 message parquet_schema {
1925 REQUIRED GROUP my_list (LIST) {
1926 REPEATED GROUP list {
1927 OPTIONAL BOOLEAN item;
1928 }
1929 }
1930 OPTIONAL GROUP my_map (MAP) {
1931 REPEATED GROUP my_entries {
1932 REQUIRED BINARY my_keys (STRING);
1933 OPTIONAL INT32 my_values;
1934 }
1935 }
1936 }
1937 ";
1938 let parquet_group_type = parse_message_type(message_type).unwrap();
1939 let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
1940 let converted_arrow_schema = ArrowSchemaConverter::new()
1941 .with_coerce_types(false)
1942 .convert(&arrow_schema)
1943 .unwrap();
1944 assert_eq!(
1945 parquet_schema.columns().len(),
1946 converted_arrow_schema.columns().len()
1947 );
1948 }
1949
1950 #[test]
1951 fn test_field_to_column_desc() {
1952 let message_type = "
1953 message arrow_schema {
1954 REQUIRED BOOLEAN boolean;
1955 REQUIRED INT32 int8 (INT_8);
1956 REQUIRED INT32 int16 (INTEGER(16,true));
1957 REQUIRED INT32 int32;
1958 REQUIRED INT64 int64;
1959 OPTIONAL DOUBLE double;
1960 OPTIONAL FLOAT float;
1961 OPTIONAL FIXED_LEN_BYTE_ARRAY (2) float16 (FLOAT16);
1962 OPTIONAL BINARY string (STRING);
1963 OPTIONAL GROUP bools (LIST) {
1964 REPEATED GROUP list {
1965 OPTIONAL BOOLEAN element;
1966 }
1967 }
1968 REQUIRED GROUP bools_non_null (LIST) {
1969 REPEATED GROUP list {
1970 REQUIRED BOOLEAN element;
1971 }
1972 }
1973 OPTIONAL INT32 date (DATE);
1974 OPTIONAL INT32 time_milli (TIME(MILLIS,false));
1975 OPTIONAL INT32 time_milli_utc (TIME(MILLIS,true));
1976 OPTIONAL INT64 time_micro (TIME_MICROS);
1977 OPTIONAL INT64 time_micro_utc (TIME(MICROS, true));
1978 OPTIONAL INT64 ts_milli (TIMESTAMP_MILLIS);
1979 REQUIRED INT64 ts_micro (TIMESTAMP(MICROS,false));
1980 REQUIRED INT64 ts_seconds;
1981 REQUIRED INT64 ts_micro_utc (TIMESTAMP(MICROS, true));
1982 REQUIRED INT64 ts_millis_zero_offset (TIMESTAMP(MILLIS, true));
1983 REQUIRED INT64 ts_millis_zero_negative_offset (TIMESTAMP(MILLIS, true));
1984 REQUIRED INT64 ts_micro_non_utc (TIMESTAMP(MICROS, true));
1985 REQUIRED GROUP struct {
1986 REQUIRED BOOLEAN bools;
1987 REQUIRED INT32 uint32 (INTEGER(32,false));
1988 REQUIRED GROUP int32 (LIST) {
1989 REPEATED GROUP list {
1990 OPTIONAL INT32 element;
1991 }
1992 }
1993 }
1994 REQUIRED BINARY dictionary_strings (STRING);
1995 REQUIRED INT32 decimal_int32 (DECIMAL(8,2));
1996 REQUIRED INT64 decimal_int64 (DECIMAL(16,2));
1997 REQUIRED FIXED_LEN_BYTE_ARRAY (13) decimal_fix_length (DECIMAL(30,2));
1998 REQUIRED FIXED_LEN_BYTE_ARRAY (16) decimal128 (DECIMAL(38,2));
1999 REQUIRED FIXED_LEN_BYTE_ARRAY (17) decimal256 (DECIMAL(39,2));
2000 }
2001 ";
2002 let parquet_group_type = parse_message_type(message_type).unwrap();
2003
2004 let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
2005
2006 let arrow_fields = vec![
2007 Field::new("boolean", DataType::Boolean, false),
2008 Field::new("int8", DataType::Int8, false),
2009 Field::new("int16", DataType::Int16, false),
2010 Field::new("int32", DataType::Int32, false),
2011 Field::new("int64", DataType::Int64, false),
2012 Field::new("double", DataType::Float64, true),
2013 Field::new("float", DataType::Float32, true),
2014 Field::new("float16", DataType::Float16, true),
2015 Field::new("string", DataType::Utf8, true),
2016 Field::new_list(
2017 "bools",
2018 Field::new("element", DataType::Boolean, true),
2019 true,
2020 ),
2021 Field::new_list(
2022 "bools_non_null",
2023 Field::new("element", DataType::Boolean, false),
2024 false,
2025 ),
2026 Field::new("date", DataType::Date32, true),
2027 Field::new("time_milli", DataType::Time32(TimeUnit::Millisecond), true),
2028 Field::new(
2029 "time_milli_utc",
2030 DataType::Time32(TimeUnit::Millisecond),
2031 true,
2032 )
2033 .with_metadata(HashMap::from_iter(vec![(
2034 "adjusted_to_utc".to_string(),
2035 String::new(),
2036 )])),
2037 Field::new("time_micro", DataType::Time64(TimeUnit::Microsecond), true),
2038 Field::new(
2039 "time_micro_utc",
2040 DataType::Time64(TimeUnit::Microsecond),
2041 true,
2042 )
2043 .with_metadata(HashMap::from_iter(vec![(
2044 "adjusted_to_utc".to_string(),
2045 String::new(),
2046 )])),
2047 Field::new(
2048 "ts_milli",
2049 DataType::Timestamp(TimeUnit::Millisecond, None),
2050 true,
2051 ),
2052 Field::new(
2053 "ts_micro",
2054 DataType::Timestamp(TimeUnit::Microsecond, None),
2055 false,
2056 ),
2057 Field::new(
2058 "ts_seconds",
2059 DataType::Timestamp(TimeUnit::Second, Some("UTC".into())),
2060 false,
2061 ),
2062 Field::new(
2063 "ts_micro_utc",
2064 DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
2065 false,
2066 ),
2067 Field::new(
2068 "ts_millis_zero_offset",
2069 DataType::Timestamp(TimeUnit::Millisecond, Some("+00:00".into())),
2070 false,
2071 ),
2072 Field::new(
2073 "ts_millis_zero_negative_offset",
2074 DataType::Timestamp(TimeUnit::Millisecond, Some("-00:00".into())),
2075 false,
2076 ),
2077 Field::new(
2078 "ts_micro_non_utc",
2079 DataType::Timestamp(TimeUnit::Microsecond, Some("+01:00".into())),
2080 false,
2081 ),
2082 Field::new_struct(
2083 "struct",
2084 vec![
2085 Field::new("bools", DataType::Boolean, false),
2086 Field::new("uint32", DataType::UInt32, false),
2087 Field::new_list("int32", Field::new("element", DataType::Int32, true), false),
2088 ],
2089 false,
2090 ),
2091 Field::new_dictionary("dictionary_strings", DataType::Int32, DataType::Utf8, false),
2092 Field::new("decimal_int32", DataType::Decimal128(8, 2), false),
2093 Field::new("decimal_int64", DataType::Decimal128(16, 2), false),
2094 Field::new("decimal_fix_length", DataType::Decimal128(30, 2), false),
2095 Field::new("decimal128", DataType::Decimal128(38, 2), false),
2096 Field::new("decimal256", DataType::Decimal256(39, 2), false),
2097 ];
2098 let arrow_schema = Schema::new(arrow_fields);
2099 let converted_arrow_schema = ArrowSchemaConverter::new().convert(&arrow_schema).unwrap();
2100
2101 assert_eq!(
2102 parquet_schema.columns().len(),
2103 converted_arrow_schema.columns().len()
2104 );
2105 parquet_schema
2106 .columns()
2107 .iter()
2108 .zip(converted_arrow_schema.columns())
2109 .for_each(|(a, b)| {
2110 match a.logical_type_ref() {
2115 Some(_) => {
2116 assert_eq!(a, b)
2117 }
2118 None => {
2119 assert_eq!(a.name(), b.name());
2120 assert_eq!(a.physical_type(), b.physical_type());
2121 assert_eq!(a.converted_type(), b.converted_type());
2122 }
2123 }
2124 });
2125 }
2126
2127 #[test]
2128 #[should_panic(expected = "Parquet does not support writing empty structs")]
2129 fn test_empty_struct_field() {
2130 let arrow_fields = vec![Field::new(
2131 "struct",
2132 DataType::Struct(Fields::empty()),
2133 false,
2134 )];
2135 let arrow_schema = Schema::new(arrow_fields);
2136 let converted_arrow_schema = ArrowSchemaConverter::new()
2137 .with_coerce_types(true)
2138 .convert(&arrow_schema);
2139
2140 converted_arrow_schema.unwrap();
2141 }
2142
2143 #[test]
2144 fn test_run_end_schema_conversion_recurses_into_value_type() {
2145 fn run_end_type(value_type: DataType, nullable_values: bool) -> DataType {
2146 DataType::RunEndEncoded(
2147 Arc::new(Field::new("run_ends", DataType::Int32, false)),
2148 Arc::new(Field::new("values", value_type, nullable_values)),
2149 )
2150 }
2151
2152 let leaf_schema = Schema::new(vec![Field::new(
2155 "ree",
2156 run_end_type(DataType::Int32, true),
2157 false,
2158 )]);
2159 let converted = ArrowSchemaConverter::new().convert(&leaf_schema).unwrap();
2160 assert_eq!(converted.columns().len(), 1);
2161 assert_eq!(converted.column(0).name(), "ree");
2162 assert_eq!(converted.column(0).physical_type(), PhysicalType::INT32);
2163 assert_eq!(converted.column(0).max_def_level(), 1);
2164
2165 let list_schema = Schema::new(vec![Field::new(
2168 "ree_list",
2169 run_end_type(
2170 DataType::List(Arc::new(Field::new("element", DataType::Int32, true))),
2171 true,
2172 ),
2173 true,
2174 )]);
2175 let converted = ArrowSchemaConverter::new().convert(&list_schema).unwrap();
2176 assert_eq!(converted.columns().len(), 1);
2177 assert_eq!(converted.column(0).physical_type(), PhysicalType::INT32);
2178 assert!(
2179 converted.column(0).max_rep_level() >= 1,
2180 "list element should be repeated"
2181 );
2182
2183 let dictionary_schema = Schema::new(vec![Field::new(
2184 "ree_dictionary",
2185 run_end_type(
2186 DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)),
2187 true,
2188 ),
2189 true,
2190 )]);
2191 let converted = ArrowSchemaConverter::new()
2192 .convert(&dictionary_schema)
2193 .unwrap();
2194 assert_eq!(converted.columns().len(), 1);
2195 assert_eq!(
2196 converted.column(0).physical_type(),
2197 PhysicalType::BYTE_ARRAY
2198 );
2199 }
2200
2201 fn map_type_with_key(key: Field) -> DataType {
2202 DataType::Map(
2203 Arc::new(Field::new(
2204 "entries",
2205 DataType::Struct(vec![key, Field::new("value", DataType::Int32, true)].into()),
2206 false,
2207 )),
2208 false,
2209 )
2210 }
2211
2212 #[test]
2213 fn test_map_key_schema_rejects_logically_nullable_keys() {
2214 let ree = DataType::RunEndEncoded(
2215 Arc::new(Field::new("run_ends", DataType::Int32, false)),
2216 Arc::new(Field::new("values", DataType::Int32, true)),
2217 );
2218 let dictionary_ree = DataType::Dictionary(Box::new(DataType::Int8), Box::new(ree.clone()));
2219 let nested_ree = DataType::RunEndEncoded(
2220 Arc::new(Field::new("run_ends", DataType::Int32, false)),
2221 Arc::new(Field::new("values", dictionary_ree.clone(), false)),
2222 );
2223 for key in [
2224 Field::new("key", DataType::Int32, true),
2225 Field::new("key", ree, false),
2226 Field::new("key", dictionary_ree, false),
2227 Field::new("key", nested_ree, false),
2228 ] {
2229 let map = map_type_with_key(key);
2230 for data_type in [
2231 map.clone(),
2232 DataType::Struct(vec![Field::new("nested", map.clone(), true)].into()),
2233 DataType::List(Arc::new(Field::new("item", map.clone(), true))),
2234 DataType::Dictionary(Box::new(DataType::Int8), Box::new(map.clone())),
2235 DataType::RunEndEncoded(
2236 Arc::new(Field::new("run_ends", DataType::Int32, false)),
2237 Arc::new(Field::new("values", map, true)),
2238 ),
2239 ] {
2240 let schema = Schema::new(vec![Field::new("map", data_type, true)]);
2241 for coerce_types in [false, true] {
2242 let err = ArrowSchemaConverter::new()
2243 .with_coerce_types(coerce_types)
2244 .convert(&schema)
2245 .unwrap_err();
2246 assert!(
2247 err.to_string()
2248 .contains("Map key field 'key' must be non-nullable"),
2249 "{err}"
2250 );
2251 }
2252 }
2253 }
2254 }
2255
2256 #[test]
2257 fn test_map_key_schema_preserves_required_keys_and_nullable_members() {
2258 for key_type in [
2259 DataType::Int32,
2260 DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Int32)),
2261 DataType::RunEndEncoded(
2262 Arc::new(Field::new("run_ends", DataType::Int32, false)),
2263 Arc::new(Field::new("values", DataType::Int32, false)),
2264 ),
2265 DataType::Struct(vec![Field::new("member", DataType::Int32, true)].into()),
2266 ] {
2267 let schema = Schema::new(vec![Field::new(
2268 "map",
2269 map_type_with_key(Field::new("key", key_type, false)),
2270 true,
2271 )]);
2272 let parquet = ArrowSchemaConverter::new().convert(&schema).unwrap();
2273 let key = &parquet.root_schema().get_fields()[0].get_fields()[0].get_fields()[0];
2274 assert_eq!(key.get_basic_info().repetition(), Repetition::REQUIRED);
2275
2276 let mut props = WriterProperties::builder().build();
2277 add_encoded_arrow_schema_to_metadata(&schema, &mut props);
2278 let encoded = props
2279 .key_value_metadata()
2280 .unwrap()
2281 .iter()
2282 .find(|kv| kv.key == crate::arrow::ARROW_SCHEMA_META_KEY)
2283 .unwrap()
2284 .value
2285 .as_deref()
2286 .unwrap();
2287 let decoded = get_arrow_schema_from_metadata(encoded).unwrap();
2288 let DataType::Map(entries, _) = decoded.field(0).data_type() else {
2289 unreachable!()
2290 };
2291 let DataType::Struct(fields) = entries.data_type() else {
2292 unreachable!()
2293 };
2294 assert!(!fields[0].is_nullable());
2295 assert!(fields[1].is_nullable());
2296 if let DataType::Struct(members) = fields[0].data_type() {
2297 assert!(members[0].is_nullable());
2298 }
2299 }
2300 }
2301
2302 #[test]
2303 fn test_arrow_metadata_normalization_merges_ree_value_nullability() {
2304 let run_ends = Arc::new(Field::new("run_ends", DataType::Int32, false));
2305 let inner = Arc::new(Field::new("inner", DataType::Int32, true));
2306 let nested = Arc::new(Field::new(
2307 "nested",
2308 DataType::RunEndEncoded(Arc::clone(&run_ends), inner),
2309 false,
2310 ));
2311 let field = Field::new("outer", DataType::RunEndEncoded(run_ends, nested), false);
2312
2313 let normalized = normalize_metadata_field(&field);
2314 assert_eq!(normalized.data_type(), &DataType::Int32);
2315 assert!(normalized.is_nullable());
2316 }
2317
2318 #[test]
2319 fn test_arrow_metadata_normalization_descends_dictionary_values() {
2320 let ree = DataType::RunEndEncoded(
2321 Arc::new(Field::new("run_ends", DataType::Int32, false)),
2322 Arc::new(Field::new("values", DataType::Int64, true)),
2323 );
2324 let field = Field::new(
2325 "dictionary_ree",
2326 DataType::Dictionary(Box::new(DataType::Int8), Box::new(ree)),
2327 false,
2328 );
2329
2330 assert!(needs_metadata_normalization(field.data_type()));
2331 let normalized = normalize_metadata_field(&field);
2332 assert_eq!(
2333 normalized.data_type(),
2334 &DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Int64))
2335 );
2336 assert!(normalized.is_nullable());
2337 assert!(!needs_metadata_normalization(normalized.data_type()));
2338
2339 let mut props = WriterProperties::builder().build();
2340 add_encoded_arrow_schema_to_metadata(&Schema::new(vec![field]), &mut props);
2341 let encoded = props
2342 .key_value_metadata()
2343 .unwrap()
2344 .iter()
2345 .find(|kv| kv.key == crate::arrow::ARROW_SCHEMA_META_KEY)
2346 .unwrap()
2347 .value
2348 .as_deref()
2349 .unwrap();
2350 let decoded = get_arrow_schema_from_metadata(encoded).unwrap();
2351 assert_eq!(decoded.field(0), &normalized);
2352 }
2353
2354 #[test]
2355 fn test_arrow_metadata_normalization_preserves_view_dictionary_siblings() {
2356 let dictionary = |dt| DataType::Dictionary(Box::new(DataType::Int8), Box::new(dt));
2357 let ree = DataType::RunEndEncoded(
2358 Arc::new(Field::new(
2359 Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
2360 DataType::Int32,
2361 false,
2362 )),
2363 Arc::new(Field::new(
2364 Field::REE_VALUES_FIELD_DEFAULT_NAME,
2365 DataType::Int32,
2366 true,
2367 )),
2368 );
2369 let fields = vec![
2370 Field::new(
2372 "nested",
2373 dictionary(DataType::Struct(
2374 vec![Field::new("child", ree, false)].into(),
2375 )),
2376 false,
2377 ),
2378 Field::new("view", dictionary(DataType::Utf8View), true),
2381 Field::new("ordinary", dictionary(DataType::Utf8), true),
2382 Field::new("binary_view", dictionary(DataType::BinaryView), true),
2383 ];
2384 let schema = Schema::new(fields);
2385 assert!(needs_metadata_normalization(schema.field(0).data_type()));
2386 let mut props = WriterProperties::builder().build();
2387 add_encoded_arrow_schema_to_metadata(&schema, &mut props);
2388 let encoded = props
2389 .key_value_metadata()
2390 .unwrap()
2391 .iter()
2392 .find(|kv| kv.key == crate::arrow::ARROW_SCHEMA_META_KEY)
2393 .unwrap()
2394 .value
2395 .as_deref()
2396 .unwrap();
2397 let decoded = get_arrow_schema_from_metadata(encoded).unwrap();
2398 let DataType::Struct(children) = decoded.field(0).data_type() else {
2399 panic!("nested dictionary not erased")
2400 };
2401 assert_eq!(children[0].name(), "child");
2402 assert_eq!(children[0].data_type(), &DataType::Int32);
2403 assert!(children[0].is_nullable());
2404 assert!(!decoded.field(0).is_nullable());
2405 assert_eq!(decoded.field(1), schema.field(1));
2406 assert_eq!(decoded.field(2), schema.field(2));
2407 assert_eq!(decoded.field(3), schema.field(3));
2408 assert!(!needs_metadata_normalization(schema.field(1).data_type()));
2409 assert!(!needs_metadata_normalization(schema.field(3).data_type()));
2410 let parquet = ArrowSchemaConverter::new().convert(&schema).unwrap();
2411 assert_eq!(parquet.column(0).path().parts(), &["nested", "child"]);
2412 assert_eq!(parquet.column(0).max_def_level(), 1);
2413 }
2414
2415 #[test]
2416 fn test_metadata() {
2417 let message_type = "
2418 message test_schema {
2419 OPTIONAL BINARY string (STRING);
2420 }
2421 ";
2422 let parquet_group_type = parse_message_type(message_type).unwrap();
2423
2424 let key_value_metadata = vec![
2425 KeyValue::new("foo".to_owned(), Some("bar".to_owned())),
2426 KeyValue::new("baz".to_owned(), None),
2427 ];
2428
2429 let mut expected_metadata: HashMap<String, String> = HashMap::new();
2430 expected_metadata.insert("foo".to_owned(), "bar".to_owned());
2431
2432 let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_group_type));
2433 let converted_arrow_schema =
2434 parquet_to_arrow_schema(&parquet_schema, Some(&key_value_metadata)).unwrap();
2435
2436 assert_eq!(converted_arrow_schema.metadata(), &expected_metadata);
2437 }
2438
2439 #[test]
2440 fn test_arrow_schema_roundtrip() -> Result<()> {
2441 let meta = |a: &[(&str, &str)]| -> HashMap<String, String> {
2442 a.iter()
2443 .map(|(a, b)| (a.to_string(), b.to_string()))
2444 .collect()
2445 };
2446
2447 let schema = Schema::new_with_metadata(
2448 vec![
2449 Field::new("c1", DataType::Utf8, false)
2450 .with_metadata(meta(&[("Key", "Foo"), (PARQUET_FIELD_ID_META_KEY, "2")])),
2451 Field::new("c2", DataType::Binary, false),
2452 Field::new("c3", DataType::FixedSizeBinary(3), false),
2453 Field::new("c4", DataType::Boolean, false),
2454 Field::new("c5", DataType::Date32, false),
2455 Field::new("c6", DataType::Date64, false),
2456 Field::new("c7", DataType::Time32(TimeUnit::Second), false),
2457 Field::new("c8", DataType::Time32(TimeUnit::Millisecond), false),
2458 Field::new("c13", DataType::Time64(TimeUnit::Microsecond), false),
2459 Field::new("c14", DataType::Time64(TimeUnit::Nanosecond), false),
2460 Field::new("c15", DataType::Timestamp(TimeUnit::Second, None), false),
2461 Field::new(
2462 "c16",
2463 DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())),
2464 false,
2465 ),
2466 Field::new(
2467 "c17",
2468 DataType::Timestamp(TimeUnit::Microsecond, Some("Africa/Johannesburg".into())),
2469 false,
2470 ),
2471 Field::new(
2472 "c18",
2473 DataType::Timestamp(TimeUnit::Nanosecond, None),
2474 false,
2475 ),
2476 Field::new("c19", DataType::Interval(IntervalUnit::DayTime), false),
2477 Field::new("c20", DataType::Interval(IntervalUnit::YearMonth), false),
2478 Field::new_list(
2479 "c21",
2480 Field::new_list_field(DataType::Boolean, true)
2481 .with_metadata(meta(&[("Key", "Bar"), (PARQUET_FIELD_ID_META_KEY, "5")])),
2482 false,
2483 )
2484 .with_metadata(meta(&[(PARQUET_FIELD_ID_META_KEY, "4")])),
2485 Field::new(
2486 "c22",
2487 DataType::FixedSizeList(
2488 Arc::new(Field::new_list_field(DataType::Boolean, true)),
2489 5,
2490 ),
2491 false,
2492 ),
2493 Field::new_list(
2494 "c23",
2495 Field::new_large_list(
2496 "inner",
2497 Field::new_list_field(
2498 DataType::Struct(
2499 vec![
2500 Field::new("a", DataType::Int16, true),
2501 Field::new("b", DataType::Float64, false),
2502 Field::new("c", DataType::Float32, false),
2503 Field::new("d", DataType::Float16, false),
2504 ]
2505 .into(),
2506 ),
2507 false,
2508 ),
2509 true,
2510 ),
2511 false,
2512 ),
2513 Field::new(
2514 "c24",
2515 DataType::Struct(Fields::from(vec![
2516 Field::new("a", DataType::Utf8, false),
2517 Field::new("b", DataType::UInt16, false),
2518 ])),
2519 false,
2520 ),
2521 Field::new("c25", DataType::Interval(IntervalUnit::YearMonth), true),
2522 Field::new("c26", DataType::Interval(IntervalUnit::DayTime), true),
2523 #[expect(deprecated)]
2529 Field::new_dict(
2530 "c31",
2531 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
2532 true,
2533 123,
2534 true,
2535 )
2536 .with_metadata(meta(&[(PARQUET_FIELD_ID_META_KEY, "6")])),
2537 Field::new("c32", DataType::LargeBinary, true),
2538 Field::new("c33", DataType::LargeUtf8, true),
2539 Field::new_large_list(
2540 "c34",
2541 Field::new_list(
2542 "inner",
2543 Field::new_list_field(
2544 DataType::Struct(
2545 vec![
2546 Field::new("a", DataType::Int16, true),
2547 Field::new("b", DataType::Float64, true),
2548 ]
2549 .into(),
2550 ),
2551 true,
2552 ),
2553 true,
2554 ),
2555 true,
2556 ),
2557 Field::new("c35", DataType::Null, true),
2558 Field::new("c36", DataType::Decimal128(2, 1), false),
2559 Field::new("c37", DataType::Decimal256(50, 20), false),
2560 Field::new("c38", DataType::Decimal128(18, 12), true),
2561 Field::new_map(
2562 "c39",
2563 "key_value",
2564 Field::new("key", DataType::Utf8, false),
2565 Field::new_list("value", Field::new("element", DataType::Utf8, true), true),
2566 false, true,
2568 ),
2569 Field::new_map(
2570 "c40",
2571 "my_entries",
2572 Field::new("my_key", DataType::Utf8, false)
2573 .with_metadata(meta(&[(PARQUET_FIELD_ID_META_KEY, "8")])),
2574 Field::new_list(
2575 "my_value",
2576 Field::new_list_field(DataType::Utf8, true)
2577 .with_metadata(meta(&[(PARQUET_FIELD_ID_META_KEY, "10")])),
2578 true,
2579 )
2580 .with_metadata(meta(&[(PARQUET_FIELD_ID_META_KEY, "9")])),
2581 false, true,
2583 )
2584 .with_metadata(meta(&[(PARQUET_FIELD_ID_META_KEY, "7")])),
2585 Field::new_map(
2586 "c41",
2587 "my_entries",
2588 Field::new("my_key", DataType::Utf8, false),
2589 Field::new_list(
2590 "my_value",
2591 Field::new_list_field(DataType::Utf8, true)
2592 .with_metadata(meta(&[(PARQUET_FIELD_ID_META_KEY, "11")])),
2593 true,
2594 ),
2595 false, false,
2597 ),
2598 Field::new("c42", DataType::Decimal32(5, 2), false),
2599 Field::new("c43", DataType::Decimal64(18, 12), true),
2600 ],
2601 meta(&[("Key", "Value")]),
2602 );
2603
2604 let file = tempfile::tempfile().unwrap();
2606 let writer =
2607 ArrowWriter::try_new(file.try_clone().unwrap(), Arc::new(schema.clone()), None)?;
2608 writer.close()?;
2609
2610 let arrow_reader = ParquetRecordBatchReaderBuilder::try_new(file).unwrap();
2612
2613 let read_schema = arrow_reader.schema();
2615 assert_eq!(&schema, read_schema.as_ref());
2616
2617 let mut stack = Vec::with_capacity(10);
2619 let mut out = Vec::with_capacity(10);
2620
2621 let root = arrow_reader.parquet_schema().root_schema_ptr();
2622 stack.push((root.name().to_string(), root));
2623
2624 while let Some((p, t)) = stack.pop() {
2625 if t.is_group() {
2626 for f in t.get_fields() {
2627 stack.push((format!("{p}.{}", f.name()), f.clone()))
2628 }
2629 }
2630
2631 let info = t.get_basic_info();
2632 if info.has_id() {
2633 out.push(format!("{p} -> {}", info.id()))
2634 }
2635 }
2636 out.sort_unstable();
2637 let out: Vec<_> = out.iter().map(|x| x.as_str()).collect();
2638
2639 assert_eq!(
2640 &out,
2641 &[
2642 "arrow_schema.c1 -> 2",
2643 "arrow_schema.c21 -> 4",
2644 "arrow_schema.c21.list.item -> 5",
2645 "arrow_schema.c31 -> 6",
2646 "arrow_schema.c40 -> 7",
2647 "arrow_schema.c40.my_entries.my_key -> 8",
2648 "arrow_schema.c40.my_entries.my_value -> 9",
2649 "arrow_schema.c40.my_entries.my_value.list.item -> 10",
2650 "arrow_schema.c41.my_entries.my_value.list.item -> 11",
2651 ]
2652 );
2653
2654 Ok(())
2655 }
2656
2657 #[test]
2658 fn test_read_parquet_field_ids_raw() -> Result<()> {
2659 let meta = |a: &[(&str, &str)]| -> HashMap<String, String> {
2660 a.iter()
2661 .map(|(a, b)| (a.to_string(), b.to_string()))
2662 .collect()
2663 };
2664 let schema = Schema::new_with_metadata(
2665 vec![
2666 Field::new("c1", DataType::Utf8, true)
2667 .with_metadata(meta(&[(PARQUET_FIELD_ID_META_KEY, "1")])),
2668 Field::new("c2", DataType::Utf8, true)
2669 .with_metadata(meta(&[(PARQUET_FIELD_ID_META_KEY, "2")])),
2670 ],
2671 HashMap::new(),
2672 );
2673
2674 let writer = ArrowWriter::try_new(vec![], Arc::new(schema.clone()), None)?;
2675 let parquet_bytes = writer.into_inner()?;
2676
2677 let reader =
2678 crate::file::reader::SerializedFileReader::new(bytes::Bytes::from(parquet_bytes))?;
2679 let schema_descriptor = reader.metadata().file_metadata().schema_descr_ptr();
2680
2681 let arrow_schema = crate::arrow::parquet_to_arrow_schema(&schema_descriptor, None)?;
2683
2684 let parq_schema_descr = ArrowSchemaConverter::new()
2685 .with_coerce_types(true)
2686 .convert(&arrow_schema)?;
2687 let parq_fields = parq_schema_descr.root_schema().get_fields();
2688 assert_eq!(parq_fields.len(), 2);
2689 assert_eq!(parq_fields[0].get_basic_info().id(), 1);
2690 assert_eq!(parq_fields[1].get_basic_info().id(), 2);
2691
2692 Ok(())
2693 }
2694
2695 #[test]
2696 fn test_arrow_schema_roundtrip_lists() -> Result<()> {
2697 let metadata = HashMap::from([("Key".to_string(), "Value".to_string())]);
2698
2699 let schema = Schema::new_with_metadata(
2700 vec![
2701 Field::new_list("c21", Field::new("array", DataType::Boolean, true), false),
2702 Field::new(
2703 "c22",
2704 DataType::FixedSizeList(
2705 Arc::new(Field::new("items", DataType::Boolean, false)),
2706 5,
2707 ),
2708 false,
2709 ),
2710 Field::new_list(
2711 "c23",
2712 Field::new_large_list(
2713 "items",
2714 Field::new_struct(
2715 "items",
2716 vec![
2717 Field::new("a", DataType::Int16, true),
2718 Field::new("b", DataType::Float64, false),
2719 ],
2720 true,
2721 ),
2722 true,
2723 ),
2724 true,
2725 ),
2726 ],
2727 metadata,
2728 );
2729
2730 let file = tempfile::tempfile().unwrap();
2732 let writer =
2733 ArrowWriter::try_new(file.try_clone().unwrap(), Arc::new(schema.clone()), None)?;
2734 writer.close()?;
2735
2736 let arrow_reader = ParquetRecordBatchReaderBuilder::try_new(file).unwrap();
2738 let read_schema = arrow_reader.schema();
2739 assert_eq!(&schema, read_schema.as_ref());
2740 Ok(())
2741 }
2742
2743 #[test]
2744 fn test_get_arrow_schema_from_metadata() {
2745 assert!(get_arrow_schema_from_metadata("").is_err());
2746 }
2747
2748 #[test]
2749 #[cfg(feature = "arrow_canonical_extension_types")]
2750 fn arrow_uuid_to_parquet_uuid() -> Result<()> {
2751 use arrow_schema::extension::Uuid;
2752 let arrow_schema = Schema::new(vec![
2753 Field::new("uuid", DataType::FixedSizeBinary(16), false).with_extension_type(Uuid),
2754 ]);
2755
2756 let parquet_schema = ArrowSchemaConverter::new().convert(&arrow_schema)?;
2757
2758 assert_eq!(
2759 parquet_schema.column(0).logical_type_ref(),
2760 Some(&LogicalType::Uuid)
2761 );
2762
2763 let arrow_schema = parquet_to_arrow_schema(&parquet_schema, None)?;
2764 assert_eq!(arrow_schema.field(0).try_extension_type::<Uuid>()?, Uuid);
2765
2766 Ok(())
2767 }
2768
2769 #[test]
2770 #[cfg(feature = "arrow_canonical_extension_types")]
2771 fn arrow_json_to_parquet_json() -> Result<()> {
2772 use arrow_schema::extension::Json;
2773 let arrow_schema = Schema::new(vec![
2774 Field::new("json", DataType::Utf8, false).with_extension_type(Json::default()),
2775 ]);
2776
2777 let parquet_schema = ArrowSchemaConverter::new().convert(&arrow_schema)?;
2778
2779 assert_eq!(
2780 parquet_schema.column(0).logical_type_ref(),
2781 Some(&LogicalType::Json)
2782 );
2783
2784 let arrow_schema = parquet_to_arrow_schema(&parquet_schema, None)?;
2785 assert_eq!(
2786 arrow_schema.field(0).try_extension_type::<Json>()?,
2787 Json::default()
2788 );
2789
2790 Ok(())
2791 }
2792
2793 #[test]
2794 #[cfg(feature = "variant_experimental")]
2795 fn arrow_variant_to_parquet_variant() -> Result<()> {
2796 use parquet_variant_compute::VariantType;
2797 let variant_fields = Fields::from(vec![
2798 Field::new("metadata", DataType::BinaryView, false),
2799 Field::new("value", DataType::BinaryView, true),
2800 ]);
2801 let arrow_schema = Schema::new(vec![
2802 Field::new("v", DataType::Struct(variant_fields), true)
2803 .with_extension_type(VariantType),
2804 ]);
2805
2806 let parquet_schema = ArrowSchemaConverter::new().convert(&arrow_schema)?;
2807
2808 let root = parquet_schema.root_schema();
2811 assert_eq!(
2812 root.get_fields()[0].get_basic_info().logical_type_ref(),
2813 Some(&LogicalType::variant(Some(1)))
2814 );
2815
2816 let arrow_schema = parquet_to_arrow_schema(&parquet_schema, None)?;
2817 assert!(
2818 arrow_schema
2819 .field(0)
2820 .has_valid_extension_type::<VariantType>()
2821 );
2822
2823 Ok(())
2824 }
2825
2826 #[test]
2827 fn test_parquet_to_arrow_field_levels_with_virtual_rejects_non_virtual() {
2828 let message_type = "
2829 message test_schema {
2830 REQUIRED INT32 id;
2831 }
2832 ";
2833 let parquet_schema = Arc::new(parse_message_type(message_type).unwrap());
2834 let descriptor = SchemaDescriptor::new(parquet_schema);
2835
2836 let regular_field = Arc::new(Field::new("regular_column", DataType::Int64, false));
2838 let result = parquet_to_arrow_field_levels_with_virtual(
2839 &descriptor,
2840 ProjectionMask::all(),
2841 None,
2842 &[regular_field],
2843 );
2844
2845 assert!(result.is_err());
2846 assert!(
2847 result
2848 .unwrap_err()
2849 .to_string()
2850 .contains("is not a virtual column")
2851 );
2852 }
2853}