1use std::vec::IntoIter;
8use std::{collections::HashMap, fmt, sync::Arc};
9
10use crate::file::metadata::HeapSize;
11use crate::file::metadata::thrift::SchemaElement;
12
13use crate::basic::{
14 ColumnOrder, ConvertedType, IntType, LogicalType, Repetition, SortOrder, TimeType, TimeUnit,
15 Type as PhysicalType,
16};
17use crate::errors::{ParquetError, Result};
18
19pub type TypePtr = Arc<Type>;
24pub type SchemaDescPtr = Arc<SchemaDescriptor>;
26pub type ColumnDescPtr = Arc<ColumnDescriptor>;
28
29#[derive(Clone, Debug, PartialEq)]
36pub enum Type {
37 PrimitiveType {
39 basic_info: BasicTypeInfo,
41 physical_type: PhysicalType,
43 type_length: i32,
45 scale: i32,
47 precision: i32,
49 },
50 GroupType {
52 basic_info: BasicTypeInfo,
54 fields: Vec<TypePtr>,
56 },
57}
58
59impl HeapSize for Type {
60 fn heap_size(&self) -> usize {
61 match self {
62 Type::PrimitiveType { basic_info, .. } => basic_info.heap_size(),
63 Type::GroupType { basic_info, fields } => basic_info.heap_size() + fields.heap_size(),
64 }
65 }
66}
67
68impl Type {
69 pub fn primitive_type_builder(
71 name: &str,
72 physical_type: PhysicalType,
73 ) -> PrimitiveTypeBuilder<'_> {
74 PrimitiveTypeBuilder::new(name, physical_type)
75 }
76
77 pub fn group_type_builder(name: &str) -> GroupTypeBuilder<'_> {
79 GroupTypeBuilder::new(name)
80 }
81
82 pub fn get_basic_info(&self) -> &BasicTypeInfo {
84 match *self {
85 Type::PrimitiveType { ref basic_info, .. } => basic_info,
86 Type::GroupType { ref basic_info, .. } => basic_info,
87 }
88 }
89
90 pub fn name(&self) -> &str {
92 self.get_basic_info().name()
93 }
94
95 pub fn get_fields(&self) -> &[TypePtr] {
102 match *self {
103 Type::GroupType { ref fields, .. } => &fields[..],
104 Type::PrimitiveType { .. } => panic!("Cannot call get_fields() on a non-group type"),
105 }
106 }
107
108 pub fn get_physical_type(&self) -> PhysicalType {
114 match *self {
115 Type::PrimitiveType {
116 basic_info: _,
117 physical_type,
118 ..
119 } => physical_type,
120 Type::GroupType { .. } => {
121 panic!("Cannot call get_physical_type() on a non-primitive type")
122 }
123 }
124 }
125
126 pub fn get_precision(&self) -> i32 {
132 match *self {
133 Type::PrimitiveType { precision, .. } => precision,
134 Type::GroupType { .. } => panic!("Cannot call get_precision() on non-primitive type"),
135 }
136 }
137
138 pub fn get_scale(&self) -> i32 {
144 match *self {
145 Type::PrimitiveType { scale, .. } => scale,
146 Type::GroupType { .. } => panic!("Cannot call get_scale() on non-primitive type"),
147 }
148 }
149
150 pub fn check_contains(&self, sub_type: &Type) -> bool {
153 let basic_match = self.get_basic_info().name() == sub_type.get_basic_info().name()
155 && (self.is_schema() && sub_type.is_schema()
156 || !self.is_schema()
157 && !sub_type.is_schema()
158 && self.get_basic_info().repetition()
159 == sub_type.get_basic_info().repetition());
160
161 match *self {
162 Type::PrimitiveType { .. } if basic_match && sub_type.is_primitive() => {
163 self.get_physical_type() == sub_type.get_physical_type()
164 }
165 Type::GroupType { .. } if basic_match && sub_type.is_group() => {
166 let mut field_map = HashMap::new();
168 for field in self.get_fields() {
169 field_map.insert(field.name(), field);
170 }
171
172 for field in sub_type.get_fields() {
173 if !field_map
174 .get(field.name())
175 .map(|type_| type_.check_contains(field))
176 .unwrap_or(false)
177 {
178 return false;
179 }
180 }
181 true
182 }
183 _ => false,
184 }
185 }
186
187 pub fn is_primitive(&self) -> bool {
189 matches!(*self, Type::PrimitiveType { .. })
190 }
191
192 pub fn is_group(&self) -> bool {
194 matches!(*self, Type::GroupType { .. })
195 }
196
197 pub fn is_schema(&self) -> bool {
199 match *self {
200 Type::GroupType { ref basic_info, .. } => !basic_info.has_repetition(),
201 Type::PrimitiveType { .. } => false,
202 }
203 }
204
205 pub fn is_optional(&self) -> bool {
208 self.get_basic_info().has_repetition()
209 && self.get_basic_info().repetition() != Repetition::REQUIRED
210 }
211
212 pub(crate) fn is_list(&self) -> bool {
214 if self.is_group() {
215 let basic_info = self.get_basic_info();
216 if let Some(logical_type) = basic_info.logical_type_ref() {
217 return logical_type == &LogicalType::List;
218 }
219 return basic_info.converted_type() == ConvertedType::LIST;
220 }
221 false
222 }
223
224 pub(crate) fn has_single_repeated_child(&self) -> bool {
226 if self.is_group() {
227 let children = self.get_fields();
228 return children.len() == 1
229 && children[0].get_basic_info().has_repetition()
230 && children[0].get_basic_info().repetition() == Repetition::REPEATED;
231 }
232 false
233 }
234}
235
236pub struct PrimitiveTypeBuilder<'a> {
240 name: &'a str,
241 repetition: Repetition,
242 physical_type: PhysicalType,
243 converted_type: ConvertedType,
244 logical_type: Option<LogicalType>,
245 length: i32,
246 precision: i32,
247 scale: i32,
248 id: Option<i32>,
249}
250
251impl<'a> PrimitiveTypeBuilder<'a> {
252 pub fn new(name: &'a str, physical_type: PhysicalType) -> Self {
254 Self {
255 name,
256 repetition: Repetition::OPTIONAL,
257 physical_type,
258 converted_type: ConvertedType::NONE,
259 logical_type: None,
260 length: -1,
261 precision: -1,
262 scale: -1,
263 id: None,
264 }
265 }
266
267 pub fn with_repetition(self, repetition: Repetition) -> Self {
269 Self { repetition, ..self }
270 }
271
272 pub fn with_converted_type(self, converted_type: ConvertedType) -> Self {
274 Self {
275 converted_type,
276 ..self
277 }
278 }
279
280 pub fn with_logical_type(self, logical_type: Option<LogicalType>) -> Self {
284 Self {
285 logical_type,
286 ..self
287 }
288 }
289
290 pub fn with_length(self, length: i32) -> Self {
295 Self { length, ..self }
296 }
297
298 pub fn with_precision(self, precision: i32) -> Self {
301 Self { precision, ..self }
302 }
303
304 pub fn with_scale(self, scale: i32) -> Self {
307 Self { scale, ..self }
308 }
309
310 pub fn with_id(self, id: Option<i32>) -> Self {
312 Self { id, ..self }
313 }
314
315 pub fn build(self) -> Result<Type> {
318 let sort_order = ColumnOrder::column_order_for_type(
319 self.logical_type.as_ref(),
320 self.converted_type,
321 self.physical_type,
322 )
323 .sort_order();
324 let mut basic_info = BasicTypeInfo {
325 name: String::from(self.name),
326 repetition: Some(self.repetition),
327 converted_type: self.converted_type,
328 logical_type: self.logical_type.clone(),
329 id: self.id,
330 sort_order,
331 };
332
333 if self.physical_type == PhysicalType::FIXED_LEN_BYTE_ARRAY && self.length < 0 {
335 return Err(general_err!(
336 "Invalid FIXED_LEN_BYTE_ARRAY length: {} for field '{}'",
337 self.length,
338 self.name
339 ));
340 }
341
342 if let Some(logical_type) = &self.logical_type {
343 if self.converted_type != ConvertedType::NONE {
346 if ConvertedType::from(self.logical_type.clone()) != self.converted_type {
347 return Err(general_err!(
348 "Logical type {:?} is incompatible with converted type {} for field '{}'",
349 logical_type,
350 self.converted_type,
351 self.name
352 ));
353 }
354 } else {
355 basic_info.converted_type = self.logical_type.clone().into();
357 }
358 match (logical_type, self.physical_type) {
360 (LogicalType::Map | LogicalType::List | LogicalType::File, _) => {
361 return Err(general_err!(
362 "{:?} cannot be applied to a primitive type for field '{}'",
363 logical_type,
364 self.name
365 ));
366 }
367 (LogicalType::Enum, PhysicalType::BYTE_ARRAY) => {}
368 (LogicalType::Decimal(decimal), _) => {
369 if decimal.scale != self.scale {
371 return Err(general_err!(
372 "DECIMAL logical type scale {} must match self.scale {} for field '{}'",
373 decimal.scale,
374 self.scale,
375 self.name
376 ));
377 }
378 if decimal.precision != self.precision {
379 return Err(general_err!(
380 "DECIMAL logical type precision {} must match self.precision {} for field '{}'",
381 decimal.precision,
382 self.precision,
383 self.name
384 ));
385 }
386 self.check_decimal_precision_scale()?;
387 }
388 (LogicalType::Date, PhysicalType::INT32) => {}
389 (
390 LogicalType::Time(TimeType {
391 unit: TimeUnit::MILLIS,
392 ..
393 }),
394 PhysicalType::INT32,
395 ) => {}
396 (LogicalType::Time(time), PhysicalType::INT64) => {
397 if time.unit == TimeUnit::MILLIS {
398 return Err(general_err!(
399 "Cannot use millisecond unit on INT64 type for field '{}'",
400 self.name
401 ));
402 }
403 }
404 (LogicalType::Timestamp(_), PhysicalType::INT64) => {}
405 (LogicalType::Integer(int), PhysicalType::INT32) if int.bit_width <= 32 => {}
406 (LogicalType::Integer(int), PhysicalType::INT64) if int.bit_width == 64 => {}
407 (LogicalType::Unknown, _) => {}
409 (LogicalType::String, PhysicalType::BYTE_ARRAY) => {}
410 (LogicalType::Json, PhysicalType::BYTE_ARRAY) => {}
411 (LogicalType::Bson, PhysicalType::BYTE_ARRAY) => {}
412 (LogicalType::Geometry(_), PhysicalType::BYTE_ARRAY) => {}
413 (LogicalType::Geography(_), PhysicalType::BYTE_ARRAY) => {}
414 (LogicalType::Uuid, PhysicalType::FIXED_LEN_BYTE_ARRAY) if self.length == 16 => {}
415 (LogicalType::Uuid, PhysicalType::FIXED_LEN_BYTE_ARRAY) => {
416 return Err(general_err!(
417 "UUID cannot annotate field '{}' because it is not a FIXED_LEN_BYTE_ARRAY(16) field",
418 self.name
419 ));
420 }
421 (LogicalType::Float16, PhysicalType::FIXED_LEN_BYTE_ARRAY) if self.length == 2 => {}
422 (LogicalType::Float16, PhysicalType::FIXED_LEN_BYTE_ARRAY) => {
423 return Err(general_err!(
424 "FLOAT16 cannot annotate field '{}' because it is not a FIXED_LEN_BYTE_ARRAY(2) field",
425 self.name
426 ));
427 }
428 (LogicalType::_Unknown { .. }, _) => {}
430 (a, b) => {
431 return Err(general_err!(
432 "Cannot annotate {:?} from {} for field '{}'",
433 a,
434 b,
435 self.name
436 ));
437 }
438 }
439 }
440
441 match self.converted_type {
442 ConvertedType::NONE => {}
443 ConvertedType::UTF8 | ConvertedType::BSON | ConvertedType::JSON => {
444 if self.physical_type != PhysicalType::BYTE_ARRAY {
445 return Err(general_err!(
446 "{} cannot annotate field '{}' because it is not a BYTE_ARRAY field",
447 self.converted_type,
448 self.name
449 ));
450 }
451 }
452 ConvertedType::DECIMAL => {
453 self.check_decimal_precision_scale()?;
454 }
455 ConvertedType::DATE
456 | ConvertedType::TIME_MILLIS
457 | ConvertedType::UINT_8
458 | ConvertedType::UINT_16
459 | ConvertedType::UINT_32
460 | ConvertedType::INT_8
461 | ConvertedType::INT_16
462 | ConvertedType::INT_32 => {
463 if self.physical_type != PhysicalType::INT32 {
464 return Err(general_err!(
465 "{} cannot annotate field '{}' because it is not a INT32 field",
466 self.converted_type,
467 self.name
468 ));
469 }
470 }
471 ConvertedType::TIME_MICROS
472 | ConvertedType::TIMESTAMP_MILLIS
473 | ConvertedType::TIMESTAMP_MICROS
474 | ConvertedType::UINT_64
475 | ConvertedType::INT_64 => {
476 if self.physical_type != PhysicalType::INT64 {
477 return Err(general_err!(
478 "{} cannot annotate field '{}' because it is not a INT64 field",
479 self.converted_type,
480 self.name
481 ));
482 }
483 }
484 ConvertedType::INTERVAL => {
485 if self.physical_type != PhysicalType::FIXED_LEN_BYTE_ARRAY || self.length != 12 {
486 return Err(general_err!(
487 "INTERVAL cannot annotate field '{}' because it is not a FIXED_LEN_BYTE_ARRAY(12) field",
488 self.name
489 ));
490 }
491 }
492 ConvertedType::ENUM => {
493 if self.physical_type != PhysicalType::BYTE_ARRAY {
494 return Err(general_err!(
495 "ENUM cannot annotate field '{}' because it is not a BYTE_ARRAY field",
496 self.name
497 ));
498 }
499 }
500 _ => {
501 return Err(general_err!(
502 "{} cannot be applied to primitive field '{}'",
503 self.converted_type,
504 self.name
505 ));
506 }
507 }
508
509 Ok(Type::PrimitiveType {
510 basic_info,
511 physical_type: self.physical_type,
512 type_length: self.length,
513 scale: self.scale,
514 precision: self.precision,
515 })
516 }
517
518 #[inline]
519 fn check_decimal_precision_scale(&self) -> Result<()> {
520 match self.physical_type {
521 PhysicalType::INT32
522 | PhysicalType::INT64
523 | PhysicalType::BYTE_ARRAY
524 | PhysicalType::FIXED_LEN_BYTE_ARRAY => (),
525 _ => {
526 return Err(general_err!(
527 "DECIMAL can only annotate INT32, INT64, BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY"
528 ));
529 }
530 }
531
532 if self.precision < 1 {
534 return Err(general_err!(
535 "Invalid DECIMAL precision: {}",
536 self.precision
537 ));
538 }
539
540 if self.scale < 0 {
542 return Err(general_err!("Invalid DECIMAL scale: {}", self.scale));
543 }
544
545 if self.scale > self.precision {
546 return Err(general_err!(
547 "Invalid DECIMAL: scale ({}) cannot be greater than precision \
548 ({})",
549 self.scale,
550 self.precision
551 ));
552 }
553
554 match self.physical_type {
556 PhysicalType::INT32 => {
557 if self.precision > 9 {
558 return Err(general_err!(
559 "Cannot represent INT32 as DECIMAL with precision {}",
560 self.precision
561 ));
562 }
563 }
564 PhysicalType::INT64 => {
565 if self.precision > 18 {
566 return Err(general_err!(
567 "Cannot represent INT64 as DECIMAL with precision {}",
568 self.precision
569 ));
570 }
571 }
572 PhysicalType::FIXED_LEN_BYTE_ARRAY => {
573 let length = self
574 .length
575 .checked_mul(8)
576 .ok_or(general_err!("Invalid length {} for Decimal", self.length))?;
577 let max_precision = (2f64.powi(length - 1) - 1f64).log10().floor() as i32;
578
579 if self.precision > max_precision {
580 return Err(general_err!(
581 "Cannot represent FIXED_LEN_BYTE_ARRAY as DECIMAL with length {} and \
582 precision {}. The max precision can only be {}",
583 self.length,
584 self.precision,
585 max_precision
586 ));
587 }
588 }
589 _ => (), }
591
592 Ok(())
593 }
594}
595
596pub struct GroupTypeBuilder<'a> {
600 name: &'a str,
601 repetition: Option<Repetition>,
602 converted_type: ConvertedType,
603 logical_type: Option<LogicalType>,
604 fields: Vec<TypePtr>,
605 id: Option<i32>,
606}
607
608impl<'a> GroupTypeBuilder<'a> {
609 pub fn new(name: &'a str) -> Self {
611 Self {
612 name,
613 repetition: None,
614 converted_type: ConvertedType::NONE,
615 logical_type: None,
616 fields: Vec::new(),
617 id: None,
618 }
619 }
620
621 pub fn with_repetition(mut self, repetition: Repetition) -> Self {
623 self.repetition = Some(repetition);
624 self
625 }
626
627 pub fn with_converted_type(self, converted_type: ConvertedType) -> Self {
629 Self {
630 converted_type,
631 ..self
632 }
633 }
634
635 pub fn with_logical_type(self, logical_type: Option<LogicalType>) -> Self {
637 Self {
638 logical_type,
639 ..self
640 }
641 }
642
643 pub fn with_fields(self, fields: Vec<TypePtr>) -> Self {
646 Self { fields, ..self }
647 }
648
649 pub fn with_id(self, id: Option<i32>) -> Self {
651 Self { id, ..self }
652 }
653
654 pub fn build(self) -> Result<Type> {
656 if matches!(&self.logical_type, Some(LogicalType::File)) {
657 validate_file_type_fields(self.name, &self.fields)?;
658 }
659 let mut basic_info = BasicTypeInfo {
660 name: String::from(self.name),
661 repetition: self.repetition,
662 converted_type: self.converted_type,
663 logical_type: self.logical_type.clone(),
664 id: self.id,
665 sort_order: SortOrder::UNDEFINED,
666 };
667 if self.logical_type.is_some() && self.converted_type == ConvertedType::NONE {
669 basic_info.converted_type = self.logical_type.into();
670 }
671 Ok(Type::GroupType {
672 basic_info,
673 fields: self.fields,
674 })
675 }
676}
677
678fn validate_file_type_fields(name: &str, fields: &[TypePtr]) -> Result<()> {
698 const VALID_FIELDS: &[(&str, PhysicalType, Option<LogicalType>)] = &[
700 ("uri", PhysicalType::BYTE_ARRAY, Some(LogicalType::String)),
701 ("offset", PhysicalType::INT64, None),
702 ("size", PhysicalType::INT64, None),
703 (
704 "content_type",
705 PhysicalType::BYTE_ARRAY,
706 Some(LogicalType::String),
707 ),
708 (
709 "checksum",
710 PhysicalType::BYTE_ARRAY,
711 Some(LogicalType::String),
712 ),
713 ("inline", PhysicalType::BYTE_ARRAY, None),
714 ];
715
716 for field in fields {
717 let field_name = field.get_basic_info().name();
718 let Some((_, expected_physical, expected_logical)) =
719 VALID_FIELDS.iter().find(|(n, _, _)| *n == field_name)
720 else {
721 return Err(general_err!(
722 "FILE type group '{}' contains unrecognized field '{}'. \
723 Valid fields are: uri, offset, size, content_type, checksum, inline",
724 name,
725 field_name
726 ));
727 };
728
729 let is_optional = field.get_basic_info().has_repetition()
731 && field.get_basic_info().repetition() == Repetition::OPTIONAL;
732 if !is_optional {
733 return Err(general_err!(
734 "FILE type field '{}' must be OPTIONAL in group '{}'",
735 field_name,
736 name
737 ));
738 }
739
740 if field.is_group() {
742 return Err(general_err!(
743 "FILE type field '{}' in group '{}' must be a primitive type",
744 field_name,
745 name
746 ));
747 }
748 if field.get_physical_type() != *expected_physical {
749 return Err(general_err!(
750 "FILE type field '{}' in group '{}' must have physical type {:?}",
751 field_name,
752 name,
753 expected_physical
754 ));
755 }
756 if field.get_basic_info().logical_type_ref() != expected_logical.as_ref() {
757 return Err(general_err!(
758 "FILE type field '{}' in group '{}' must have logical type {:?}",
759 field_name,
760 name,
761 expected_logical
762 ));
763 }
764 }
765 Ok(())
766}
767
768#[derive(Clone, Debug, PartialEq, Eq)]
771pub struct BasicTypeInfo {
772 name: String,
773 repetition: Option<Repetition>,
774 converted_type: ConvertedType,
775 logical_type: Option<LogicalType>,
776 id: Option<i32>,
777 sort_order: SortOrder,
778}
779
780impl HeapSize for BasicTypeInfo {
781 fn heap_size(&self) -> usize {
782 self.name.heap_size()
784 }
785}
786
787impl BasicTypeInfo {
788 pub fn name(&self) -> &str {
790 &self.name
791 }
792
793 pub fn has_repetition(&self) -> bool {
797 self.repetition.is_some()
798 }
799
800 pub fn repetition(&self) -> Repetition {
806 assert!(self.repetition.is_some());
807 self.repetition.unwrap()
808 }
809
810 pub fn converted_type(&self) -> ConvertedType {
812 self.converted_type
813 }
814
815 pub fn logical_type_ref(&self) -> Option<&LogicalType> {
817 self.logical_type.as_ref()
818 }
819
820 pub fn has_id(&self) -> bool {
822 self.id.is_some()
823 }
824
825 pub fn id(&self) -> i32 {
831 assert!(self.id.is_some());
832 self.id.unwrap()
833 }
834
835 pub fn sort_order(&self) -> SortOrder {
837 self.sort_order
838 }
839}
840
841#[derive(Clone, PartialEq, Debug, Eq, Hash)]
863pub struct ColumnPath {
864 parts: Vec<String>,
865}
866
867impl HeapSize for ColumnPath {
868 fn heap_size(&self) -> usize {
869 self.parts.heap_size()
870 }
871}
872
873impl ColumnPath {
874 pub fn new(parts: Vec<String>) -> Self {
876 ColumnPath { parts }
877 }
878
879 pub fn string(&self) -> String {
887 self.parts.join(".")
888 }
889
890 pub fn append(&mut self, mut tail: Vec<String>) {
902 self.parts.append(&mut tail);
903 }
904
905 pub fn parts(&self) -> &[String] {
907 &self.parts
908 }
909}
910
911impl fmt::Display for ColumnPath {
912 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
913 write!(f, "{:?}", self.string())
914 }
915}
916
917impl From<Vec<String>> for ColumnPath {
918 fn from(parts: Vec<String>) -> Self {
919 ColumnPath { parts }
920 }
921}
922
923impl From<&str> for ColumnPath {
924 fn from(single_path: &str) -> Self {
925 let s = String::from(single_path);
926 ColumnPath::from(s)
927 }
928}
929
930impl From<String> for ColumnPath {
931 fn from(single_path: String) -> Self {
932 let v = vec![single_path];
933 ColumnPath { parts: v }
934 }
935}
936
937impl AsRef<[String]> for ColumnPath {
938 fn as_ref(&self) -> &[String] {
939 &self.parts
940 }
941}
942
943#[derive(Debug, PartialEq)]
948pub struct ColumnDescriptor {
949 primitive_type: TypePtr,
951
952 max_def_level: i16,
954
955 max_rep_level: i16,
957
958 repeated_ancestor_def_level: i16,
960
961 path: ColumnPath,
963}
964
965impl HeapSize for ColumnDescriptor {
966 fn heap_size(&self) -> usize {
967 self.path.heap_size()
970 }
971}
972
973impl ColumnDescriptor {
974 pub fn new(
976 primitive_type: TypePtr,
977 max_def_level: i16,
978 max_rep_level: i16,
979 path: ColumnPath,
980 ) -> Self {
981 Self::new_with_repeated_ancestor(primitive_type, max_def_level, max_rep_level, path, 0)
982 }
983
984 pub(crate) fn new_with_repeated_ancestor(
985 primitive_type: TypePtr,
986 max_def_level: i16,
987 max_rep_level: i16,
988 path: ColumnPath,
989 repeated_ancestor_def_level: i16,
990 ) -> Self {
991 Self {
992 primitive_type,
993 max_def_level,
994 max_rep_level,
995 repeated_ancestor_def_level,
996 path,
997 }
998 }
999
1000 #[inline]
1002 pub fn max_def_level(&self) -> i16 {
1003 self.max_def_level
1004 }
1005
1006 #[inline]
1008 pub fn max_rep_level(&self) -> i16 {
1009 self.max_rep_level
1010 }
1011
1012 #[inline]
1014 pub fn repeated_ancestor_def_level(&self) -> i16 {
1015 self.repeated_ancestor_def_level
1016 }
1017
1018 pub fn path(&self) -> &ColumnPath {
1020 &self.path
1021 }
1022
1023 pub fn self_type(&self) -> &Type {
1025 self.primitive_type.as_ref()
1026 }
1027
1028 pub fn self_type_ptr(&self) -> TypePtr {
1031 self.primitive_type.clone()
1032 }
1033
1034 pub fn get_basic_info(&self) -> &BasicTypeInfo {
1036 self.primitive_type.get_basic_info()
1037 }
1038
1039 pub fn name(&self) -> &str {
1041 self.primitive_type.name()
1042 }
1043
1044 pub fn converted_type(&self) -> ConvertedType {
1046 self.primitive_type.get_basic_info().converted_type()
1047 }
1048
1049 pub fn logical_type_ref(&self) -> Option<&LogicalType> {
1051 self.primitive_type.get_basic_info().logical_type_ref()
1052 }
1053
1054 pub fn physical_type(&self) -> PhysicalType {
1060 match self.primitive_type.as_ref() {
1061 Type::PrimitiveType { physical_type, .. } => *physical_type,
1062 Type::GroupType { .. } => panic!("Expected primitive type!"),
1063 }
1064 }
1065
1066 pub fn type_length(&self) -> i32 {
1072 match self.primitive_type.as_ref() {
1073 Type::PrimitiveType { type_length, .. } => *type_length,
1074 Type::GroupType { .. } => panic!("Expected primitive type!"),
1075 }
1076 }
1077
1078 pub fn type_precision(&self) -> i32 {
1084 match self.primitive_type.as_ref() {
1085 Type::PrimitiveType { precision, .. } => *precision,
1086 Type::GroupType { .. } => panic!("Expected primitive type!"),
1087 }
1088 }
1089
1090 pub fn type_scale(&self) -> i32 {
1096 match self.primitive_type.as_ref() {
1097 Type::PrimitiveType { scale, .. } => *scale,
1098 Type::GroupType { .. } => panic!("Expected primitive type!"),
1099 }
1100 }
1101
1102 pub fn sort_order(&self) -> SortOrder {
1107 self.primitive_type.get_basic_info().sort_order()
1108 }
1109}
1110
1111#[derive(PartialEq, Clone)]
1142pub struct SchemaDescriptor {
1143 schema: TypePtr,
1148
1149 leaves: Vec<ColumnDescPtr>,
1153
1154 leaf_to_base: Vec<usize>,
1165}
1166
1167impl fmt::Debug for SchemaDescriptor {
1168 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1169 f.debug_struct("SchemaDescriptor")
1171 .field("schema", &self.schema)
1172 .finish()
1173 }
1174}
1175
1176impl HeapSize for SchemaDescriptor {
1178 fn heap_size(&self) -> usize {
1179 self.schema.heap_size() + self.leaves.heap_size() + self.leaf_to_base.heap_size()
1180 }
1181}
1182
1183impl SchemaDescriptor {
1184 pub fn new(tp: TypePtr) -> Self {
1186 const INIT_SCHEMA_DEPTH: usize = 16;
1187 assert!(tp.is_group(), "SchemaDescriptor should take a GroupType");
1188 let n_leaves = num_leaves(&tp).unwrap();
1190 let mut leaves = Vec::with_capacity(n_leaves);
1191 let mut leaf_to_base = Vec::with_capacity(n_leaves);
1192 let mut path = Vec::with_capacity(INIT_SCHEMA_DEPTH);
1193 for (root_idx, f) in tp.get_fields().iter().enumerate() {
1194 path.clear();
1195 build_tree(
1196 f,
1197 root_idx,
1198 0,
1199 0,
1200 0,
1201 &mut leaves,
1202 &mut leaf_to_base,
1203 &mut path,
1204 );
1205 }
1206
1207 Self {
1208 schema: tp,
1209 leaves,
1210 leaf_to_base,
1211 }
1212 }
1213
1214 pub fn column(&self, i: usize) -> ColumnDescPtr {
1220 assert!(
1221 i < self.leaves.len(),
1222 "Index out of bound: {} not in [0, {})",
1223 i,
1224 self.leaves.len()
1225 );
1226 self.leaves[i].clone()
1227 }
1228
1229 pub fn columns(&self) -> &[ColumnDescPtr] {
1231 &self.leaves
1232 }
1233
1234 pub fn num_columns(&self) -> usize {
1236 self.leaves.len()
1237 }
1238
1239 pub fn get_column_root(&self, i: usize) -> &Type {
1241 let result = self.column_root_of(i);
1242 result.as_ref()
1243 }
1244
1245 pub fn get_column_root_ptr(&self, i: usize) -> TypePtr {
1251 let result = self.column_root_of(i);
1252 result.clone()
1253 }
1254
1255 pub fn get_column_root_idx(&self, leaf: usize) -> usize {
1261 assert!(
1262 leaf < self.leaves.len(),
1263 "Index out of bound: {} not in [0, {})",
1264 leaf,
1265 self.leaves.len()
1266 );
1267
1268 *self
1269 .leaf_to_base
1270 .get(leaf)
1271 .unwrap_or_else(|| panic!("Expected a value for index {leaf} but found None"))
1272 }
1273
1274 fn column_root_of(&self, i: usize) -> &TypePtr {
1275 &self.schema.get_fields()[self.get_column_root_idx(i)]
1276 }
1277
1278 pub fn root_schema(&self) -> &Type {
1280 self.schema.as_ref()
1281 }
1282
1283 pub fn root_schema_ptr(&self) -> TypePtr {
1285 self.schema.clone()
1286 }
1287
1288 pub fn name(&self) -> &str {
1290 self.schema.name()
1291 }
1292}
1293
1294pub(crate) fn num_nodes(tp: &TypePtr) -> Result<usize> {
1296 if !tp.is_group() {
1297 return Err(general_err!("Root schema must be Group type"));
1298 }
1299 let mut n_nodes = 1usize; for f in tp.get_fields() {
1301 count_nodes(f, &mut n_nodes);
1302 }
1303 Ok(n_nodes)
1304}
1305
1306pub(crate) fn count_nodes(tp: &TypePtr, n_nodes: &mut usize) {
1307 *n_nodes += 1;
1308 if let Type::GroupType { fields, .. } = tp.as_ref() {
1309 for f in fields {
1310 count_nodes(f, n_nodes);
1311 }
1312 }
1313}
1314
1315fn num_leaves(tp: &TypePtr) -> Result<usize> {
1317 if !tp.is_group() {
1318 return Err(general_err!("Root schema must be Group type"));
1319 }
1320 let mut n_leaves = 0usize;
1321 for f in tp.get_fields() {
1322 count_leaves(f, &mut n_leaves);
1323 }
1324 Ok(n_leaves)
1325}
1326
1327fn count_leaves(tp: &TypePtr, n_leaves: &mut usize) {
1328 match tp.as_ref() {
1329 Type::PrimitiveType { .. } => *n_leaves += 1,
1330 Type::GroupType { fields, .. } => {
1331 for f in fields {
1332 count_leaves(f, n_leaves);
1333 }
1334 }
1335 }
1336}
1337
1338#[expect(clippy::too_many_arguments)]
1339fn build_tree<'a>(
1340 tp: &'a TypePtr,
1341 root_idx: usize,
1342 mut max_rep_level: i16,
1343 mut max_def_level: i16,
1344 mut repeated_ancestor_def_level: i16,
1345 leaves: &mut Vec<ColumnDescPtr>,
1346 leaf_to_base: &mut Vec<usize>,
1347 path_so_far: &mut Vec<&'a str>,
1348) {
1349 assert!(tp.get_basic_info().has_repetition());
1350
1351 path_so_far.push(tp.name());
1352 match tp.get_basic_info().repetition() {
1353 Repetition::OPTIONAL => {
1354 max_def_level += 1;
1355 }
1356 Repetition::REPEATED => {
1357 max_def_level += 1;
1358 max_rep_level += 1;
1359 repeated_ancestor_def_level = max_def_level;
1360 }
1361 Repetition::REQUIRED => {}
1362 }
1363
1364 match tp.as_ref() {
1365 Type::PrimitiveType { .. } => {
1366 let mut path: Vec<String> = vec![];
1367 path.extend(path_so_far.iter().copied().map(String::from));
1368 let desc = ColumnDescriptor::new_with_repeated_ancestor(
1369 tp.clone(),
1370 max_def_level,
1371 max_rep_level,
1372 ColumnPath::new(path),
1373 repeated_ancestor_def_level,
1374 );
1375 leaves.push(Arc::new(desc));
1376 leaf_to_base.push(root_idx);
1377 }
1378 Type::GroupType { fields, .. } => {
1379 for f in fields {
1380 build_tree(
1381 f,
1382 root_idx,
1383 max_rep_level,
1384 max_def_level,
1385 repeated_ancestor_def_level,
1386 leaves,
1387 leaf_to_base,
1388 path_so_far,
1389 );
1390 path_so_far.pop();
1391 }
1392 }
1393 }
1394}
1395
1396fn check_logical_type(logical_type: Option<&LogicalType>) -> Result<()> {
1398 if let Some(LogicalType::Integer(IntType { bit_width, .. })) = logical_type
1399 && *bit_width != 8
1400 && *bit_width != 16
1401 && *bit_width != 32
1402 && *bit_width != 64
1403 {
1404 return Err(general_err!(
1405 "Bit width must be 8, 16, 32, or 64 for Integer logical type"
1406 ));
1407 }
1408 Ok(())
1409}
1410
1411pub(crate) fn parquet_schema_from_array(elements: Vec<SchemaElement<'_>>) -> Result<TypePtr> {
1414 let mut index = 0;
1415 let num_elements = elements.len();
1416 let mut schema_nodes = Vec::with_capacity(1); let mut elements = elements.into_iter();
1420
1421 while index < num_elements {
1422 let t = schema_from_array_helper(&mut elements, num_elements, index)?;
1423 index = t.0;
1424 schema_nodes.push(t.1);
1425 }
1426 if schema_nodes.len() != 1 {
1427 return Err(general_err!(
1428 "Expected exactly one root node, but found {}",
1429 schema_nodes.len()
1430 ));
1431 }
1432
1433 if !schema_nodes[0].is_group() {
1434 return Err(general_err!("Expected root node to be a group type"));
1435 }
1436
1437 Ok(schema_nodes.remove(0))
1438}
1439
1440fn schema_from_array_helper(
1442 elements: &mut IntoIter<SchemaElement<'_>>,
1443 num_elements: usize,
1444 index: usize,
1445) -> Result<(usize, TypePtr)> {
1446 let is_root_node = index == 0;
1449
1450 if index >= num_elements {
1451 return Err(general_err!(
1452 "Index out of bound, index = {}, len = {}",
1453 index,
1454 num_elements
1455 ));
1456 }
1457 let element = elements.next().expect("schema vector should not be empty");
1458
1459 if let (true, None | Some(0)) = (is_root_node, element.num_children) {
1461 let builder = Type::group_type_builder(element.name);
1462 return Ok((index + 1, Arc::new(builder.build().unwrap())));
1463 }
1464
1465 let converted_type = element.converted_type.unwrap_or(ConvertedType::NONE);
1466
1467 let logical_type = element.logical_type;
1469
1470 check_logical_type(logical_type.as_ref())?;
1471
1472 let field_id = element.field_id;
1473 match element.num_children {
1474 None | Some(0) => {
1480 if element.repetition_type.is_none() {
1482 return Err(general_err!(
1483 "Repetition level must be defined for a primitive type"
1484 ));
1485 }
1486 let repetition = element.repetition_type.unwrap();
1487 if let Some(physical_type) = element.r#type {
1488 let length = element.type_length.unwrap_or(-1);
1489 let scale = element.scale.unwrap_or(-1);
1490 let precision = element.precision.unwrap_or(-1);
1491 let name = element.name;
1492 let builder = Type::primitive_type_builder(name, physical_type)
1493 .with_repetition(repetition)
1494 .with_converted_type(converted_type)
1495 .with_logical_type(logical_type)
1496 .with_length(length)
1497 .with_precision(precision)
1498 .with_scale(scale)
1499 .with_id(field_id);
1500 Ok((index + 1, Arc::new(builder.build()?)))
1501 } else {
1502 let mut builder = Type::group_type_builder(element.name)
1503 .with_converted_type(converted_type)
1504 .with_logical_type(logical_type)
1505 .with_id(field_id);
1506 if !is_root_node {
1507 builder = builder.with_repetition(repetition);
1515 }
1516 Ok((index + 1, Arc::new(builder.build().unwrap())))
1517 }
1518 }
1519 Some(n) => {
1520 let repetition = element.repetition_type;
1521
1522 let num_children = usize::try_from(n)?;
1523 if num_children > elements.len() {
1524 return Err(general_err!(
1525 "Schema children count {} exceeds remaining schema elements {}",
1526 num_children,
1527 elements.len()
1528 ));
1529 }
1530 let mut fields = Vec::with_capacity(num_children);
1531 let mut next_index = index + 1;
1532 for _ in 0..num_children {
1533 let child_result = schema_from_array_helper(elements, num_elements, next_index)?;
1534 next_index = child_result.0;
1535 fields.push(child_result.1);
1536 }
1537
1538 let mut builder = Type::group_type_builder(element.name)
1539 .with_converted_type(converted_type)
1540 .with_logical_type(logical_type)
1541 .with_fields(fields)
1542 .with_id(field_id);
1543
1544 if !is_root_node {
1552 let Some(rep) = repetition else {
1553 return Err(general_err!(
1554 "Repetition level must be defined for non-root types"
1555 ));
1556 };
1557 builder = builder.with_repetition(rep);
1558 }
1559 Ok((next_index, Arc::new(builder.build()?)))
1560 }
1561 }
1562}
1563
1564#[cfg(test)]
1565mod tests {
1566 use super::*;
1567
1568 use crate::{
1569 file::metadata::thrift::tests::{buf_to_schema_list, roundtrip_schema, schema_to_buf},
1570 schema::parser::parse_message_type,
1571 };
1572
1573 #[test]
1576 fn test_primitive_type() {
1577 let mut result = Type::primitive_type_builder("foo", PhysicalType::INT32)
1578 .with_logical_type(Some(LogicalType::integer(32, true)))
1579 .with_id(Some(0))
1580 .build();
1581 assert!(result.is_ok());
1582
1583 if let Ok(tp) = result {
1584 assert!(tp.is_primitive());
1585 assert!(!tp.is_group());
1586 let basic_info = tp.get_basic_info();
1587 assert_eq!(basic_info.repetition(), Repetition::OPTIONAL);
1588 assert_eq!(
1589 basic_info.logical_type_ref(),
1590 Some(&LogicalType::integer(32, true))
1591 );
1592 assert_eq!(basic_info.converted_type(), ConvertedType::INT_32);
1593 assert_eq!(basic_info.id(), 0);
1594 match tp {
1595 Type::PrimitiveType { physical_type, .. } => {
1596 assert_eq!(physical_type, PhysicalType::INT32);
1597 }
1598 Type::GroupType { .. } => panic!(),
1599 }
1600 }
1601
1602 result = Type::primitive_type_builder("foo", PhysicalType::INT64)
1604 .with_repetition(Repetition::REPEATED)
1605 .with_logical_type(Some(LogicalType::integer(8, true)))
1606 .build();
1607 assert!(result.is_err());
1608 if let Err(e) = result {
1609 assert_eq!(
1610 format!("{e}"),
1611 "Parquet error: Cannot annotate Integer(IntType { bit_width: 8, is_signed: true }) from INT64 for field 'foo'"
1612 );
1613 }
1614
1615 result = Type::primitive_type_builder("foo", PhysicalType::INT64)
1617 .with_repetition(Repetition::REPEATED)
1618 .with_converted_type(ConvertedType::BSON)
1619 .build();
1620 assert!(result.is_err());
1621 if let Err(e) = result {
1622 assert_eq!(
1623 format!("{e}"),
1624 "Parquet error: BSON cannot annotate field 'foo' because it is not a BYTE_ARRAY field"
1625 );
1626 }
1627
1628 result = Type::primitive_type_builder("foo", PhysicalType::INT96)
1629 .with_repetition(Repetition::REQUIRED)
1630 .with_converted_type(ConvertedType::DECIMAL)
1631 .with_precision(-1)
1632 .with_scale(-1)
1633 .build();
1634 assert!(result.is_err());
1635 if let Err(e) = result {
1636 assert_eq!(
1637 format!("{e}"),
1638 "Parquet error: DECIMAL can only annotate INT32, INT64, BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY"
1639 );
1640 }
1641
1642 result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
1643 .with_repetition(Repetition::REQUIRED)
1644 .with_logical_type(Some(LogicalType::decimal(32, 12)))
1645 .with_precision(-1)
1646 .with_scale(-1)
1647 .build();
1648 assert!(result.is_err());
1649 if let Err(e) = result {
1650 assert_eq!(
1651 format!("{e}"),
1652 "Parquet error: DECIMAL logical type scale 32 must match self.scale -1 for field 'foo'"
1653 );
1654 }
1655
1656 result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
1657 .with_repetition(Repetition::REQUIRED)
1658 .with_converted_type(ConvertedType::DECIMAL)
1659 .with_precision(-1)
1660 .with_scale(-1)
1661 .build();
1662 assert!(result.is_err());
1663 if let Err(e) = result {
1664 assert_eq!(
1665 format!("{e}"),
1666 "Parquet error: Invalid DECIMAL precision: -1"
1667 );
1668 }
1669
1670 result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
1671 .with_repetition(Repetition::REQUIRED)
1672 .with_converted_type(ConvertedType::DECIMAL)
1673 .with_precision(0)
1674 .with_scale(-1)
1675 .build();
1676 assert!(result.is_err());
1677 if let Err(e) = result {
1678 assert_eq!(
1679 format!("{e}"),
1680 "Parquet error: Invalid DECIMAL precision: 0"
1681 );
1682 }
1683
1684 result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
1685 .with_repetition(Repetition::REQUIRED)
1686 .with_converted_type(ConvertedType::DECIMAL)
1687 .with_precision(1)
1688 .with_scale(-1)
1689 .build();
1690 assert!(result.is_err());
1691 if let Err(e) = result {
1692 assert_eq!(format!("{e}"), "Parquet error: Invalid DECIMAL scale: -1");
1693 }
1694
1695 result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
1696 .with_repetition(Repetition::REQUIRED)
1697 .with_converted_type(ConvertedType::DECIMAL)
1698 .with_precision(1)
1699 .with_scale(2)
1700 .build();
1701 assert!(result.is_err());
1702 if let Err(e) = result {
1703 assert_eq!(
1704 format!("{e}"),
1705 "Parquet error: Invalid DECIMAL: scale (2) cannot be greater than precision (1)"
1706 );
1707 }
1708
1709 result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
1711 .with_repetition(Repetition::REQUIRED)
1712 .with_converted_type(ConvertedType::DECIMAL)
1713 .with_precision(1)
1714 .with_scale(1)
1715 .build();
1716 assert!(result.is_ok());
1717
1718 result = Type::primitive_type_builder("foo", PhysicalType::INT32)
1719 .with_repetition(Repetition::REQUIRED)
1720 .with_converted_type(ConvertedType::DECIMAL)
1721 .with_precision(18)
1722 .with_scale(2)
1723 .build();
1724 assert!(result.is_err());
1725 if let Err(e) = result {
1726 assert_eq!(
1727 format!("{e}"),
1728 "Parquet error: Cannot represent INT32 as DECIMAL with precision 18"
1729 );
1730 }
1731
1732 result = Type::primitive_type_builder("foo", PhysicalType::INT64)
1733 .with_repetition(Repetition::REQUIRED)
1734 .with_converted_type(ConvertedType::DECIMAL)
1735 .with_precision(32)
1736 .with_scale(2)
1737 .build();
1738 assert!(result.is_err());
1739 if let Err(e) = result {
1740 assert_eq!(
1741 format!("{e}"),
1742 "Parquet error: Cannot represent INT64 as DECIMAL with precision 32"
1743 );
1744 }
1745
1746 result = Type::primitive_type_builder("foo", PhysicalType::FIXED_LEN_BYTE_ARRAY)
1747 .with_repetition(Repetition::REQUIRED)
1748 .with_converted_type(ConvertedType::DECIMAL)
1749 .with_length(5)
1750 .with_precision(12)
1751 .with_scale(2)
1752 .build();
1753 assert!(result.is_err());
1754 if let Err(e) = result {
1755 assert_eq!(
1756 format!("{e}"),
1757 "Parquet error: Cannot represent FIXED_LEN_BYTE_ARRAY as DECIMAL with length 5 and precision 12. The max precision can only be 11"
1758 );
1759 }
1760
1761 result = Type::primitive_type_builder("foo", PhysicalType::INT64)
1762 .with_repetition(Repetition::REQUIRED)
1763 .with_converted_type(ConvertedType::UINT_8)
1764 .build();
1765 assert!(result.is_err());
1766 if let Err(e) = result {
1767 assert_eq!(
1768 format!("{e}"),
1769 "Parquet error: UINT_8 cannot annotate field 'foo' because it is not a INT32 field"
1770 );
1771 }
1772
1773 result = Type::primitive_type_builder("foo", PhysicalType::INT32)
1774 .with_repetition(Repetition::REQUIRED)
1775 .with_converted_type(ConvertedType::TIME_MICROS)
1776 .build();
1777 assert!(result.is_err());
1778 if let Err(e) = result {
1779 assert_eq!(
1780 format!("{e}"),
1781 "Parquet error: TIME_MICROS cannot annotate field 'foo' because it is not a INT64 field"
1782 );
1783 }
1784
1785 result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
1786 .with_repetition(Repetition::REQUIRED)
1787 .with_converted_type(ConvertedType::INTERVAL)
1788 .build();
1789 assert!(result.is_err());
1790 if let Err(e) = result {
1791 assert_eq!(
1792 format!("{e}"),
1793 "Parquet error: INTERVAL cannot annotate field 'foo' because it is not a FIXED_LEN_BYTE_ARRAY(12) field"
1794 );
1795 }
1796
1797 result = Type::primitive_type_builder("foo", PhysicalType::FIXED_LEN_BYTE_ARRAY)
1798 .with_repetition(Repetition::REQUIRED)
1799 .with_converted_type(ConvertedType::INTERVAL)
1800 .with_length(1)
1801 .build();
1802 assert!(result.is_err());
1803 if let Err(e) = result {
1804 assert_eq!(
1805 format!("{e}"),
1806 "Parquet error: INTERVAL cannot annotate field 'foo' because it is not a FIXED_LEN_BYTE_ARRAY(12) field"
1807 );
1808 }
1809
1810 result = Type::primitive_type_builder("foo", PhysicalType::INT32)
1811 .with_repetition(Repetition::REQUIRED)
1812 .with_converted_type(ConvertedType::ENUM)
1813 .build();
1814 assert!(result.is_err());
1815 if let Err(e) = result {
1816 assert_eq!(
1817 format!("{e}"),
1818 "Parquet error: ENUM cannot annotate field 'foo' because it is not a BYTE_ARRAY field"
1819 );
1820 }
1821
1822 result = Type::primitive_type_builder("foo", PhysicalType::INT32)
1823 .with_repetition(Repetition::REQUIRED)
1824 .with_converted_type(ConvertedType::MAP)
1825 .build();
1826 assert!(result.is_err());
1827 if let Err(e) = result {
1828 assert_eq!(
1829 format!("{e}"),
1830 "Parquet error: MAP cannot be applied to primitive field 'foo'"
1831 );
1832 }
1833
1834 result = Type::primitive_type_builder("foo", PhysicalType::FIXED_LEN_BYTE_ARRAY)
1835 .with_repetition(Repetition::REQUIRED)
1836 .with_converted_type(ConvertedType::DECIMAL)
1837 .with_length(-1)
1838 .build();
1839 assert!(result.is_err());
1840 if let Err(e) = result {
1841 assert_eq!(
1842 format!("{e}"),
1843 "Parquet error: Invalid FIXED_LEN_BYTE_ARRAY length: -1 for field 'foo'"
1844 );
1845 }
1846
1847 result = Type::primitive_type_builder("foo", PhysicalType::FIXED_LEN_BYTE_ARRAY)
1848 .with_repetition(Repetition::REQUIRED)
1849 .with_logical_type(Some(LogicalType::Float16))
1850 .with_length(2)
1851 .build();
1852 assert!(result.is_ok());
1853
1854 result = Type::primitive_type_builder("foo", PhysicalType::FLOAT)
1856 .with_repetition(Repetition::REQUIRED)
1857 .with_logical_type(Some(LogicalType::Float16))
1858 .with_length(2)
1859 .build();
1860 assert!(result.is_err());
1861 if let Err(e) = result {
1862 assert_eq!(
1863 format!("{e}"),
1864 "Parquet error: Cannot annotate Float16 from FLOAT for field 'foo'"
1865 );
1866 }
1867
1868 result = Type::primitive_type_builder("foo", PhysicalType::FIXED_LEN_BYTE_ARRAY)
1870 .with_repetition(Repetition::REQUIRED)
1871 .with_logical_type(Some(LogicalType::Float16))
1872 .with_length(4)
1873 .build();
1874 assert!(result.is_err());
1875 if let Err(e) = result {
1876 assert_eq!(
1877 format!("{e}"),
1878 "Parquet error: FLOAT16 cannot annotate field 'foo' because it is not a FIXED_LEN_BYTE_ARRAY(2) field"
1879 );
1880 }
1881
1882 result = Type::primitive_type_builder("foo", PhysicalType::FIXED_LEN_BYTE_ARRAY)
1884 .with_repetition(Repetition::REQUIRED)
1885 .with_logical_type(Some(LogicalType::Uuid))
1886 .with_length(15)
1887 .build();
1888 assert!(result.is_err());
1889 if let Err(e) = result {
1890 assert_eq!(
1891 format!("{e}"),
1892 "Parquet error: UUID cannot annotate field 'foo' because it is not a FIXED_LEN_BYTE_ARRAY(16) field"
1893 );
1894 }
1895
1896 result = Type::primitive_type_builder("foo", PhysicalType::BYTE_ARRAY)
1898 .with_logical_type(Some(LogicalType::_Unknown { field_id: 100 }))
1899 .build();
1900 assert!(result.is_ok());
1901 }
1902
1903 #[test]
1904 fn test_group_type() {
1905 let f1 = Type::primitive_type_builder("f1", PhysicalType::INT32)
1906 .with_converted_type(ConvertedType::INT_32)
1907 .with_id(Some(0))
1908 .build();
1909 assert!(f1.is_ok());
1910 let f2 = Type::primitive_type_builder("f2", PhysicalType::BYTE_ARRAY)
1911 .with_converted_type(ConvertedType::UTF8)
1912 .with_id(Some(1))
1913 .build();
1914 assert!(f2.is_ok());
1915
1916 let fields = vec![Arc::new(f1.unwrap()), Arc::new(f2.unwrap())];
1917
1918 let result = Type::group_type_builder("foo")
1919 .with_repetition(Repetition::REPEATED)
1920 .with_logical_type(Some(LogicalType::List))
1921 .with_fields(fields)
1922 .with_id(Some(1))
1923 .build();
1924 assert!(result.is_ok());
1925
1926 let tp = result.unwrap();
1927 let basic_info = tp.get_basic_info();
1928 assert!(tp.is_group());
1929 assert!(!tp.is_primitive());
1930 assert_eq!(basic_info.repetition(), Repetition::REPEATED);
1931 assert_eq!(basic_info.logical_type_ref(), Some(&LogicalType::List));
1932 assert_eq!(basic_info.converted_type(), ConvertedType::LIST);
1933 assert_eq!(basic_info.id(), 1);
1934 assert_eq!(tp.get_fields().len(), 2);
1935 assert_eq!(tp.get_fields()[0].name(), "f1");
1936 assert_eq!(tp.get_fields()[1].name(), "f2");
1937 }
1938
1939 #[test]
1940 fn test_column_descriptor() {
1941 let result = test_column_descriptor_helper();
1942 assert!(
1943 result.is_ok(),
1944 "Expected result to be OK but got err:\n {}",
1945 result.unwrap_err()
1946 );
1947 }
1948
1949 fn test_column_descriptor_helper() -> Result<()> {
1950 let tp = Type::primitive_type_builder("name", PhysicalType::BYTE_ARRAY)
1951 .with_converted_type(ConvertedType::UTF8)
1952 .build()?;
1953
1954 let descr = ColumnDescriptor::new(Arc::new(tp), 4, 1, ColumnPath::from("name"));
1955
1956 assert_eq!(descr.path(), &ColumnPath::from("name"));
1957 assert_eq!(descr.converted_type(), ConvertedType::UTF8);
1958 assert_eq!(descr.physical_type(), PhysicalType::BYTE_ARRAY);
1959 assert_eq!(descr.max_def_level(), 4);
1960 assert_eq!(descr.max_rep_level(), 1);
1961 assert_eq!(descr.name(), "name");
1962 assert_eq!(descr.type_length(), -1);
1963 assert_eq!(descr.type_precision(), -1);
1964 assert_eq!(descr.type_scale(), -1);
1965
1966 Ok(())
1967 }
1968
1969 #[test]
1970 fn test_schema_descriptor() {
1971 let result = test_schema_descriptor_helper();
1972 assert!(
1973 result.is_ok(),
1974 "Expected result to be OK but got err:\n {}",
1975 result.unwrap_err()
1976 );
1977 }
1978
1979 fn test_schema_descriptor_helper() -> Result<()> {
1981 let mut fields = vec![];
1982
1983 let inta = Type::primitive_type_builder("a", PhysicalType::INT32)
1984 .with_repetition(Repetition::REQUIRED)
1985 .with_converted_type(ConvertedType::INT_32)
1986 .build()?;
1987 fields.push(Arc::new(inta));
1988 let intb = Type::primitive_type_builder("b", PhysicalType::INT64)
1989 .with_converted_type(ConvertedType::INT_64)
1990 .build()?;
1991 fields.push(Arc::new(intb));
1992 let intc = Type::primitive_type_builder("c", PhysicalType::BYTE_ARRAY)
1993 .with_repetition(Repetition::REPEATED)
1994 .with_converted_type(ConvertedType::UTF8)
1995 .build()?;
1996 fields.push(Arc::new(intc));
1997
1998 let item1 = Type::primitive_type_builder("item1", PhysicalType::INT64)
2000 .with_repetition(Repetition::REQUIRED)
2001 .with_converted_type(ConvertedType::INT_64)
2002 .build()?;
2003 let item2 = Type::primitive_type_builder("item2", PhysicalType::BOOLEAN).build()?;
2004 let item3 = Type::primitive_type_builder("item3", PhysicalType::INT32)
2005 .with_repetition(Repetition::REPEATED)
2006 .with_converted_type(ConvertedType::INT_32)
2007 .build()?;
2008 let list = Type::group_type_builder("records")
2009 .with_repetition(Repetition::REPEATED)
2010 .with_converted_type(ConvertedType::LIST)
2011 .with_fields(vec![Arc::new(item1), Arc::new(item2), Arc::new(item3)])
2012 .build()?;
2013 let bag = Type::group_type_builder("bag")
2014 .with_repetition(Repetition::OPTIONAL)
2015 .with_fields(vec![Arc::new(list)])
2016 .build()?;
2017 fields.push(Arc::new(bag));
2018
2019 let schema = Type::group_type_builder("schema")
2020 .with_repetition(Repetition::REPEATED)
2021 .with_fields(fields)
2022 .build()?;
2023 let descr = SchemaDescriptor::new(Arc::new(schema));
2024
2025 let nleaves = 6;
2026 assert_eq!(descr.num_columns(), nleaves);
2027
2028 let ex_max_def_levels = [0, 1, 1, 2, 3, 3];
2038 let ex_max_rep_levels = [0, 0, 1, 1, 1, 2];
2039
2040 for i in 0..nleaves {
2041 let col = descr.column(i);
2042 assert_eq!(col.max_def_level(), ex_max_def_levels[i], "{i}");
2043 assert_eq!(col.max_rep_level(), ex_max_rep_levels[i], "{i}");
2044 }
2045
2046 assert_eq!(descr.column(0).path().string(), "a");
2047 assert_eq!(descr.column(1).path().string(), "b");
2048 assert_eq!(descr.column(2).path().string(), "c");
2049 assert_eq!(descr.column(3).path().string(), "bag.records.item1");
2050 assert_eq!(descr.column(4).path().string(), "bag.records.item2");
2051 assert_eq!(descr.column(5).path().string(), "bag.records.item3");
2052
2053 assert_eq!(descr.get_column_root(0).name(), "a");
2054 assert_eq!(descr.get_column_root(3).name(), "bag");
2055 assert_eq!(descr.get_column_root(4).name(), "bag");
2056 assert_eq!(descr.get_column_root(5).name(), "bag");
2057
2058 Ok(())
2059 }
2060
2061 #[test]
2062 fn test_schema_build_tree_def_rep_levels() {
2063 let message_type = "
2064 message spark_schema {
2065 REQUIRED INT32 a;
2066 OPTIONAL group b {
2067 OPTIONAL INT32 _1;
2068 OPTIONAL INT32 _2;
2069 }
2070 OPTIONAL group c (LIST) {
2071 REPEATED group list {
2072 OPTIONAL INT32 element;
2073 }
2074 }
2075 }
2076 ";
2077 let schema = parse_message_type(message_type).expect("should parse schema");
2078 let descr = SchemaDescriptor::new(Arc::new(schema));
2079 assert_eq!(descr.column(0).max_def_level(), 0);
2081 assert_eq!(descr.column(0).max_rep_level(), 0);
2082 assert_eq!(descr.column(1).max_def_level(), 2);
2084 assert_eq!(descr.column(1).max_rep_level(), 0);
2085 assert_eq!(descr.column(2).max_def_level(), 2);
2087 assert_eq!(descr.column(2).max_rep_level(), 0);
2088 assert_eq!(descr.column(3).max_def_level(), 3);
2090 assert_eq!(descr.column(3).max_rep_level(), 1);
2091 }
2092
2093 #[test]
2094 fn test_schema_build_tree_repeated_ancestor_def_level() {
2095 let message_type = "
2097 message m {
2098 REQUIRED INT32 a;
2099 OPTIONAL INT32 b;
2100 OPTIONAL group s {
2101 OPTIONAL INT32 x;
2102 }
2103 }
2104 ";
2105 let schema = parse_message_type(message_type).expect("should parse schema");
2106 let descr = SchemaDescriptor::new(Arc::new(schema));
2107 assert_eq!(descr.column(0).repeated_ancestor_def_level(), 0); assert_eq!(descr.column(1).repeated_ancestor_def_level(), 0); assert_eq!(descr.column(2).repeated_ancestor_def_level(), 0); let message_type = "
2114 message m {
2115 OPTIONAL group c (LIST) {
2116 REPEATED group list {
2117 OPTIONAL INT32 element;
2118 }
2119 }
2120 }
2121 ";
2122 let schema = parse_message_type(message_type).expect("should parse schema");
2123 let descr = SchemaDescriptor::new(Arc::new(schema));
2124 assert_eq!(descr.column(0).max_def_level(), 3);
2126 assert_eq!(descr.column(0).max_rep_level(), 1);
2127 assert_eq!(descr.column(0).repeated_ancestor_def_level(), 2);
2128
2129 let message_type = "
2132 message m {
2133 REQUIRED group c (LIST) {
2134 REPEATED group list {
2135 REQUIRED INT32 element;
2136 }
2137 }
2138 }
2139 ";
2140 let schema = parse_message_type(message_type).expect("should parse schema");
2141 let descr = SchemaDescriptor::new(Arc::new(schema));
2142 assert_eq!(descr.column(0).max_def_level(), 1);
2144 assert_eq!(descr.column(0).max_rep_level(), 1);
2145 assert_eq!(descr.column(0).repeated_ancestor_def_level(), 1);
2146
2147 let message_type = "
2149 message m {
2150 OPTIONAL group outer (LIST) {
2151 REPEATED group list {
2152 OPTIONAL group inner (LIST) {
2153 REPEATED group list2 {
2154 OPTIONAL INT32 element;
2155 }
2156 }
2157 }
2158 }
2159 }
2160 ";
2161 let schema = parse_message_type(message_type).expect("should parse schema");
2162 let descr = SchemaDescriptor::new(Arc::new(schema));
2163 assert_eq!(descr.column(0).max_def_level(), 5);
2165 assert_eq!(descr.column(0).max_rep_level(), 2);
2166 assert_eq!(descr.column(0).repeated_ancestor_def_level(), 4);
2167
2168 let message_type = "
2170 message m {
2171 OPTIONAL group bag (LIST) {
2172 REPEATED group list {
2173 REQUIRED group item {
2174 OPTIONAL INT32 x;
2175 REQUIRED INT32 y;
2176 }
2177 }
2178 }
2179 }
2180 ";
2181 let schema = parse_message_type(message_type).expect("should parse schema");
2182 let descr = SchemaDescriptor::new(Arc::new(schema));
2183 assert_eq!(descr.column(0).repeated_ancestor_def_level(), 2); assert_eq!(descr.column(1).repeated_ancestor_def_level(), 2); let message_type = "
2190 message m {
2191 OPTIONAL group my_map (MAP) {
2192 REPEATED group key_value {
2193 REQUIRED BYTE_ARRAY key (UTF8);
2194 OPTIONAL INT32 value;
2195 }
2196 }
2197 }
2198 ";
2199 let schema = parse_message_type(message_type).expect("should parse schema");
2200 let descr = SchemaDescriptor::new(Arc::new(schema));
2201 assert_eq!(descr.column(0).max_def_level(), 2);
2203 assert_eq!(descr.column(0).repeated_ancestor_def_level(), 2); assert_eq!(descr.column(1).max_def_level(), 3);
2206 assert_eq!(descr.column(1).repeated_ancestor_def_level(), 2); }
2208
2209 #[test]
2210 #[should_panic(expected = "Cannot call get_physical_type() on a non-primitive type")]
2211 fn test_get_physical_type_panic() {
2212 let list = Type::group_type_builder("records")
2213 .with_repetition(Repetition::REPEATED)
2214 .build()
2215 .unwrap();
2216 list.get_physical_type();
2217 }
2218
2219 #[test]
2220 fn test_get_physical_type_primitive() {
2221 let f = Type::primitive_type_builder("f", PhysicalType::INT64)
2222 .build()
2223 .unwrap();
2224 assert_eq!(f.get_physical_type(), PhysicalType::INT64);
2225
2226 let f = Type::primitive_type_builder("f", PhysicalType::BYTE_ARRAY)
2227 .build()
2228 .unwrap();
2229 assert_eq!(f.get_physical_type(), PhysicalType::BYTE_ARRAY);
2230 }
2231
2232 #[test]
2233 fn test_check_contains_primitive_primitive() {
2234 let f1 = Type::primitive_type_builder("f", PhysicalType::INT32)
2236 .build()
2237 .unwrap();
2238 let f2 = Type::primitive_type_builder("f", PhysicalType::INT32)
2239 .build()
2240 .unwrap();
2241 assert!(f1.check_contains(&f2));
2242
2243 let f1 = Type::primitive_type_builder("f", PhysicalType::INT32)
2245 .with_converted_type(ConvertedType::UINT_8)
2246 .build()
2247 .unwrap();
2248 let f2 = Type::primitive_type_builder("f", PhysicalType::INT32)
2249 .with_converted_type(ConvertedType::UINT_16)
2250 .build()
2251 .unwrap();
2252 assert!(f1.check_contains(&f2));
2253
2254 let f1 = Type::primitive_type_builder("f1", PhysicalType::INT32)
2256 .build()
2257 .unwrap();
2258 let f2 = Type::primitive_type_builder("f2", PhysicalType::INT32)
2259 .build()
2260 .unwrap();
2261 assert!(!f1.check_contains(&f2));
2262
2263 let f1 = Type::primitive_type_builder("f", PhysicalType::INT32)
2265 .build()
2266 .unwrap();
2267 let f2 = Type::primitive_type_builder("f", PhysicalType::INT64)
2268 .build()
2269 .unwrap();
2270 assert!(!f1.check_contains(&f2));
2271
2272 let f1 = Type::primitive_type_builder("f", PhysicalType::INT32)
2274 .with_repetition(Repetition::REQUIRED)
2275 .build()
2276 .unwrap();
2277 let f2 = Type::primitive_type_builder("f", PhysicalType::INT32)
2278 .with_repetition(Repetition::OPTIONAL)
2279 .build()
2280 .unwrap();
2281 assert!(!f1.check_contains(&f2));
2282 }
2283
2284 fn test_new_group_type(name: &str, repetition: Repetition, types: Vec<Type>) -> Type {
2286 Type::group_type_builder(name)
2287 .with_repetition(repetition)
2288 .with_fields(types.into_iter().map(Arc::new).collect())
2289 .build()
2290 .unwrap()
2291 }
2292
2293 #[test]
2294 fn test_check_contains_group_group() {
2295 let f1 = Type::group_type_builder("f").build().unwrap();
2297 let f2 = Type::group_type_builder("f").build().unwrap();
2298 assert!(f1.check_contains(&f2));
2299 assert!(!f1.is_optional());
2300
2301 let f1 = test_new_group_type(
2303 "f",
2304 Repetition::REPEATED,
2305 vec![
2306 Type::primitive_type_builder("f1", PhysicalType::INT32)
2307 .build()
2308 .unwrap(),
2309 Type::primitive_type_builder("f2", PhysicalType::INT64)
2310 .build()
2311 .unwrap(),
2312 ],
2313 );
2314 let f2 = test_new_group_type(
2315 "f",
2316 Repetition::REPEATED,
2317 vec![
2318 Type::primitive_type_builder("f1", PhysicalType::INT32)
2319 .build()
2320 .unwrap(),
2321 Type::primitive_type_builder("f2", PhysicalType::INT64)
2322 .build()
2323 .unwrap(),
2324 ],
2325 );
2326 assert!(f1.check_contains(&f2));
2327
2328 let f1 = test_new_group_type(
2330 "f",
2331 Repetition::REPEATED,
2332 vec![
2333 Type::primitive_type_builder("f1", PhysicalType::INT32)
2334 .build()
2335 .unwrap(),
2336 Type::primitive_type_builder("f2", PhysicalType::INT64)
2337 .build()
2338 .unwrap(),
2339 ],
2340 );
2341 let f2 = test_new_group_type(
2342 "f",
2343 Repetition::REPEATED,
2344 vec![
2345 Type::primitive_type_builder("f2", PhysicalType::INT64)
2346 .build()
2347 .unwrap(),
2348 ],
2349 );
2350 assert!(f1.check_contains(&f2));
2351
2352 let f1 = Type::group_type_builder("f1").build().unwrap();
2354 let f2 = Type::group_type_builder("f2").build().unwrap();
2355 assert!(!f1.check_contains(&f2));
2356
2357 let f1 = Type::group_type_builder("f")
2359 .with_repetition(Repetition::OPTIONAL)
2360 .build()
2361 .unwrap();
2362 let f2 = Type::group_type_builder("f")
2363 .with_repetition(Repetition::REPEATED)
2364 .build()
2365 .unwrap();
2366 assert!(!f1.check_contains(&f2));
2367
2368 let f1 = test_new_group_type(
2370 "f",
2371 Repetition::REPEATED,
2372 vec![
2373 Type::primitive_type_builder("f1", PhysicalType::INT32)
2374 .build()
2375 .unwrap(),
2376 Type::primitive_type_builder("f2", PhysicalType::INT64)
2377 .build()
2378 .unwrap(),
2379 ],
2380 );
2381 let f2 = test_new_group_type(
2382 "f",
2383 Repetition::REPEATED,
2384 vec![
2385 Type::primitive_type_builder("f1", PhysicalType::INT32)
2386 .build()
2387 .unwrap(),
2388 Type::primitive_type_builder("f2", PhysicalType::BOOLEAN)
2389 .build()
2390 .unwrap(),
2391 ],
2392 );
2393 assert!(!f1.check_contains(&f2));
2394
2395 let f1 = test_new_group_type(
2397 "f",
2398 Repetition::REPEATED,
2399 vec![
2400 Type::primitive_type_builder("f1", PhysicalType::INT32)
2401 .build()
2402 .unwrap(),
2403 Type::primitive_type_builder("f2", PhysicalType::INT64)
2404 .build()
2405 .unwrap(),
2406 ],
2407 );
2408 let f2 = test_new_group_type(
2409 "f",
2410 Repetition::REPEATED,
2411 vec![
2412 Type::primitive_type_builder("f3", PhysicalType::INT32)
2413 .build()
2414 .unwrap(),
2415 ],
2416 );
2417 assert!(!f1.check_contains(&f2));
2418 }
2419
2420 #[test]
2421 fn test_check_contains_group_primitive() {
2422 let f1 = Type::group_type_builder("f").build().unwrap();
2424 let f2 = Type::primitive_type_builder("f", PhysicalType::INT64)
2425 .build()
2426 .unwrap();
2427 assert!(!f1.check_contains(&f2));
2428 assert!(!f2.check_contains(&f1));
2429
2430 let f1 = test_new_group_type(
2432 "f",
2433 Repetition::REPEATED,
2434 vec![
2435 Type::primitive_type_builder("f1", PhysicalType::INT32)
2436 .build()
2437 .unwrap(),
2438 ],
2439 );
2440 let f2 = Type::primitive_type_builder("f1", PhysicalType::INT32)
2441 .build()
2442 .unwrap();
2443 assert!(!f1.check_contains(&f2));
2444 assert!(!f2.check_contains(&f1));
2445
2446 let f1 = test_new_group_type(
2448 "a",
2449 Repetition::REPEATED,
2450 vec![
2451 test_new_group_type(
2452 "b",
2453 Repetition::REPEATED,
2454 vec![
2455 Type::primitive_type_builder("c", PhysicalType::INT32)
2456 .build()
2457 .unwrap(),
2458 ],
2459 ),
2460 Type::primitive_type_builder("d", PhysicalType::INT64)
2461 .build()
2462 .unwrap(),
2463 Type::primitive_type_builder("e", PhysicalType::BOOLEAN)
2464 .build()
2465 .unwrap(),
2466 ],
2467 );
2468 let f2 = test_new_group_type(
2469 "a",
2470 Repetition::REPEATED,
2471 vec![test_new_group_type(
2472 "b",
2473 Repetition::REPEATED,
2474 vec![
2475 Type::primitive_type_builder("c", PhysicalType::INT32)
2476 .build()
2477 .unwrap(),
2478 ],
2479 )],
2480 );
2481 assert!(f1.check_contains(&f2)); assert!(!f2.check_contains(&f1)); }
2484
2485 #[test]
2486 fn test_schema_type_thrift_conversion_err() {
2487 let schema = Type::primitive_type_builder("col", PhysicalType::INT32)
2488 .build()
2489 .unwrap();
2490 let schema = Arc::new(schema);
2491 let thrift_schema = schema_to_buf(&schema);
2492 assert!(thrift_schema.is_err());
2493 if let Err(e) = thrift_schema {
2494 assert_eq!(
2495 format!("{e}"),
2496 "Parquet error: Root schema must be Group type"
2497 );
2498 }
2499 }
2500
2501 #[test]
2502 fn test_schema_type_thrift_conversion() {
2503 let message_type = "
2504 message conversions {
2505 REQUIRED INT64 id;
2506 OPTIONAL FIXED_LEN_BYTE_ARRAY (2) f16 (FLOAT16);
2507 OPTIONAL group int_array_Array (LIST) {
2508 REPEATED group list {
2509 OPTIONAL group element (LIST) {
2510 REPEATED group list {
2511 OPTIONAL INT32 element;
2512 }
2513 }
2514 }
2515 }
2516 OPTIONAL group int_map (MAP) {
2517 REPEATED group map (MAP_KEY_VALUE) {
2518 REQUIRED BYTE_ARRAY key (UTF8);
2519 OPTIONAL INT32 value;
2520 }
2521 }
2522 OPTIONAL group int_Map_Array (LIST) {
2523 REPEATED group list {
2524 OPTIONAL group g (MAP) {
2525 REPEATED group map (MAP_KEY_VALUE) {
2526 REQUIRED BYTE_ARRAY key (UTF8);
2527 OPTIONAL group value {
2528 OPTIONAL group H {
2529 OPTIONAL group i (LIST) {
2530 REPEATED group list {
2531 OPTIONAL DOUBLE element;
2532 }
2533 }
2534 }
2535 }
2536 }
2537 }
2538 }
2539 }
2540 OPTIONAL group nested_struct {
2541 OPTIONAL INT32 A;
2542 OPTIONAL group b (LIST) {
2543 REPEATED group list {
2544 REQUIRED FIXED_LEN_BYTE_ARRAY (16) element;
2545 }
2546 }
2547 }
2548 }
2549 ";
2550 let expected_schema = parse_message_type(message_type).unwrap();
2551 let result_schema = roundtrip_schema(Arc::new(expected_schema.clone())).unwrap();
2552 assert_eq!(result_schema, Arc::new(expected_schema));
2553 }
2554
2555 #[test]
2556 fn test_schema_type_thrift_conversion_decimal() {
2557 let message_type = "
2558 message decimals {
2559 OPTIONAL INT32 field0;
2560 OPTIONAL INT64 field1 (DECIMAL (18, 2));
2561 OPTIONAL FIXED_LEN_BYTE_ARRAY (16) field2 (DECIMAL (38, 18));
2562 OPTIONAL BYTE_ARRAY field3 (DECIMAL (9));
2563 }
2564 ";
2565 let expected_schema = parse_message_type(message_type).unwrap();
2566 let result_schema = roundtrip_schema(Arc::new(expected_schema.clone())).unwrap();
2567 assert_eq!(result_schema, Arc::new(expected_schema));
2568 }
2569
2570 #[test]
2573 fn test_schema_from_thrift_with_num_children_set() {
2574 let message_type = "
2576 message schema {
2577 OPTIONAL BYTE_ARRAY id (UTF8);
2578 OPTIONAL BYTE_ARRAY name (UTF8);
2579 OPTIONAL BYTE_ARRAY message (UTF8);
2580 OPTIONAL INT32 type (UINT_8);
2581 OPTIONAL INT64 author_time (TIMESTAMP_MILLIS);
2582 OPTIONAL INT64 __index_level_0__;
2583 }
2584 ";
2585
2586 let expected_schema = Arc::new(parse_message_type(message_type).unwrap());
2587 let mut buf = schema_to_buf(&expected_schema).unwrap();
2588 let mut thrift_schema = buf_to_schema_list(&mut buf).unwrap();
2589
2590 for elem in &mut thrift_schema[..] {
2592 if elem.num_children.is_none() {
2593 elem.num_children = Some(0);
2594 }
2595 }
2596
2597 let result_schema = parquet_schema_from_array(thrift_schema).unwrap();
2598 assert_eq!(result_schema, expected_schema);
2599 }
2600
2601 #[test]
2604 fn test_schema_from_thrift_root_has_repetition() {
2605 let message_type = "
2607 message schema {
2608 OPTIONAL BYTE_ARRAY a (UTF8);
2609 OPTIONAL INT32 b (UINT_8);
2610 }
2611 ";
2612
2613 let expected_schema = Arc::new(parse_message_type(message_type).unwrap());
2614 let mut buf = schema_to_buf(&expected_schema).unwrap();
2615 let mut thrift_schema = buf_to_schema_list(&mut buf).unwrap();
2616 thrift_schema[0].repetition_type = Some(Repetition::REQUIRED);
2617
2618 let result_schema = parquet_schema_from_array(thrift_schema).unwrap();
2619 assert_eq!(result_schema, expected_schema);
2620 }
2621
2622 #[test]
2623 fn test_schema_from_thrift_group_has_no_child() {
2624 let message_type = "message schema {}";
2625
2626 let expected_schema = Arc::new(parse_message_type(message_type).unwrap());
2627 let mut buf = schema_to_buf(&expected_schema).unwrap();
2628 let mut thrift_schema = buf_to_schema_list(&mut buf).unwrap();
2629 thrift_schema[0].repetition_type = Some(Repetition::REQUIRED);
2630
2631 let result_schema = parquet_schema_from_array(thrift_schema).unwrap();
2632 assert_eq!(result_schema, expected_schema);
2633 }
2634
2635 fn file_field(name: &str, physical: PhysicalType, logical: Option<LogicalType>) -> TypePtr {
2637 Arc::new(
2638 Type::primitive_type_builder(name, physical)
2639 .with_repetition(Repetition::OPTIONAL)
2640 .with_logical_type(logical)
2641 .build()
2642 .unwrap(),
2643 )
2644 }
2645
2646 fn all_file_fields() -> Vec<TypePtr> {
2648 vec![
2649 file_field("uri", PhysicalType::BYTE_ARRAY, Some(LogicalType::String)),
2650 file_field("offset", PhysicalType::INT64, None),
2651 file_field("size", PhysicalType::INT64, None),
2652 file_field(
2653 "content_type",
2654 PhysicalType::BYTE_ARRAY,
2655 Some(LogicalType::String),
2656 ),
2657 file_field(
2658 "checksum",
2659 PhysicalType::BYTE_ARRAY,
2660 Some(LogicalType::String),
2661 ),
2662 file_field("inline", PhysicalType::BYTE_ARRAY, None),
2663 ]
2664 }
2665
2666 #[test]
2667 fn test_file_logical_type_roundtrip() {
2668 let file_group = Arc::new(
2669 Type::group_type_builder("f")
2670 .with_repetition(Repetition::REQUIRED)
2671 .with_logical_type(Some(LogicalType::File))
2672 .with_fields(all_file_fields())
2673 .build()
2674 .unwrap(),
2675 );
2676 let schema = Arc::new(
2677 Type::group_type_builder("example")
2678 .with_fields(vec![file_group])
2679 .build()
2680 .unwrap(),
2681 );
2682 let result = roundtrip_schema(schema.clone()).unwrap();
2683 assert_eq!(result, schema);
2684 assert_eq!(
2685 result.get_fields()[0].get_basic_info().logical_type_ref(),
2686 Some(&LogicalType::File)
2687 );
2688 }
2689
2690 #[test]
2691 fn test_file_logical_type_all_fields() {
2692 let result = Type::group_type_builder("file_field")
2693 .with_repetition(Repetition::REQUIRED)
2694 .with_logical_type(Some(LogicalType::File))
2695 .with_fields(all_file_fields())
2696 .build();
2697 assert!(result.is_ok());
2698 assert_eq!(result.unwrap().get_fields().len(), 6);
2699 }
2700
2701 #[test]
2702 fn test_file_logical_type_uri_only() {
2703 let result = Type::group_type_builder("file_field")
2705 .with_repetition(Repetition::REQUIRED)
2706 .with_logical_type(Some(LogicalType::File))
2707 .with_fields(vec![file_field(
2708 "uri",
2709 PhysicalType::BYTE_ARRAY,
2710 Some(LogicalType::String),
2711 )])
2712 .build();
2713 assert!(result.is_ok());
2714 assert_eq!(
2715 result.unwrap().get_basic_info().logical_type_ref(),
2716 Some(&LogicalType::File)
2717 );
2718 }
2719
2720 #[test]
2721 fn test_file_logical_type_inline_only() {
2722 let result = Type::group_type_builder("inline_file")
2724 .with_repetition(Repetition::REQUIRED)
2725 .with_logical_type(Some(LogicalType::File))
2726 .with_fields(vec![file_field("inline", PhysicalType::BYTE_ARRAY, None)])
2727 .build();
2728 assert!(result.is_ok());
2729 }
2730
2731 #[test]
2732 fn test_file_logical_type_empty_group_is_allowed() {
2733 let result = Type::group_type_builder("empty_file")
2735 .with_repetition(Repetition::REQUIRED)
2736 .with_logical_type(Some(LogicalType::File))
2737 .with_fields(vec![])
2738 .build();
2739 assert!(result.is_ok());
2740 }
2741
2742 #[test]
2743 fn test_file_logical_type_rejects_unrecognized_field() {
2744 let unknown_field = file_field("unknown_field", PhysicalType::BYTE_ARRAY, None);
2745 let result = Type::group_type_builder("bad_file")
2746 .with_repetition(Repetition::REQUIRED)
2747 .with_logical_type(Some(LogicalType::File))
2748 .with_fields(vec![
2749 file_field("uri", PhysicalType::BYTE_ARRAY, Some(LogicalType::String)),
2750 unknown_field,
2751 ])
2752 .build();
2753 assert_eq!(
2754 result.unwrap_err().to_string(),
2755 "Parquet error: FILE type group 'bad_file' contains unrecognized field \
2756 'unknown_field'. Valid fields are: uri, offset, size, content_type, \
2757 checksum, inline"
2758 );
2759 }
2760
2761 #[test]
2762 fn test_file_logical_type_requires_optional_fields() {
2763 let uri_field = Arc::new(
2765 Type::primitive_type_builder("uri", PhysicalType::BYTE_ARRAY)
2766 .with_repetition(Repetition::REQUIRED)
2767 .with_logical_type(Some(LogicalType::String))
2768 .build()
2769 .unwrap(),
2770 );
2771 let result = Type::group_type_builder("required_uri")
2772 .with_repetition(Repetition::REQUIRED)
2773 .with_logical_type(Some(LogicalType::File))
2774 .with_fields(vec![uri_field])
2775 .build();
2776 assert_eq!(
2777 result.unwrap_err().to_string(),
2778 "Parquet error: FILE type field 'uri' must be OPTIONAL in group 'required_uri'"
2779 );
2780 }
2781
2782 #[test]
2783 fn test_file_logical_type_rejects_wrong_physical_type() {
2784 let bad_size = file_field("size", PhysicalType::BYTE_ARRAY, None);
2786 let result = Type::group_type_builder("bad_size")
2787 .with_repetition(Repetition::REQUIRED)
2788 .with_logical_type(Some(LogicalType::File))
2789 .with_fields(vec![bad_size])
2790 .build();
2791 assert_eq!(
2792 result.unwrap_err().to_string(),
2793 "Parquet error: FILE type field 'size' in group 'bad_size' must have physical type INT64"
2794 );
2795 }
2796
2797 #[test]
2798 fn test_file_logical_type_rejects_wrong_logical_type() {
2799 let bad_uri = file_field("uri", PhysicalType::BYTE_ARRAY, None);
2801 let result = Type::group_type_builder("bad_uri")
2802 .with_repetition(Repetition::REQUIRED)
2803 .with_logical_type(Some(LogicalType::File))
2804 .with_fields(vec![bad_uri])
2805 .build();
2806 assert_eq!(
2807 result.unwrap_err().to_string(),
2808 "Parquet error: FILE type field 'uri' in group 'bad_uri' must have logical type \
2809 Some(String)"
2810 );
2811 }
2812
2813 #[test]
2814 fn test_file_logical_type_not_allowed_on_primitive() {
2815 let result = Type::primitive_type_builder("bad", PhysicalType::BYTE_ARRAY)
2816 .with_repetition(Repetition::REQUIRED)
2817 .with_logical_type(Some(LogicalType::File))
2818 .build();
2819 assert!(result.is_err());
2820 }
2821
2822 #[test]
2823 fn test_parquet_schema_from_array_rejects_huge_num_children() {
2824 let schema = Arc::new(parse_message_type("message schema {}").unwrap());
2825 let mut buf = schema_to_buf(&schema).unwrap();
2826 let mut elements = buf_to_schema_list(&mut buf).unwrap();
2827 elements[0].num_children = Some(i32::MAX);
2828 let err = parquet_schema_from_array(elements).unwrap_err().to_string();
2829 assert!(
2830 err.contains("Schema children count 2147483647 exceeds remaining schema elements 0"),
2831 "{err}"
2832 );
2833 }
2834
2835 #[test]
2836 fn test_parquet_schema_from_array_rejects_children_exceeding_remaining_elements() {
2837 let schema = Arc::new(
2838 parse_message_type("message schema { REQUIRED group nested { REQUIRED INT32 leaf; } }")
2839 .unwrap(),
2840 );
2841 let mut buf = schema_to_buf(&schema).unwrap();
2842 let mut elements = buf_to_schema_list(&mut buf).unwrap();
2843 elements[1].num_children = Some(2);
2846 let err = parquet_schema_from_array(elements).unwrap_err().to_string();
2847 assert!(
2848 err.contains("Schema children count 2 exceeds remaining schema elements 1"),
2849 "{err}"
2850 );
2851 }
2852
2853 #[test]
2854 fn test_parquet_schema_from_array_rejects_negative_num_children() {
2855 let elements = vec![SchemaElement {
2856 r#type: None,
2857 type_length: None,
2858 repetition_type: Some(Repetition::REQUIRED),
2859 name: "schema",
2860 num_children: Some(-1),
2861 converted_type: None,
2862 scale: None,
2863 precision: None,
2864 field_id: None,
2865 logical_type: None,
2866 }];
2867 let result = parquet_schema_from_array(elements);
2868 assert!(result.unwrap_err().to_string().contains("Integer overflow"));
2869 }
2870}