parquet/schema/
parser.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Parquet schema parser.
6//! Provides methods to parse and validate string message type into Parquet
7//! [`Type`].
8//!
9//! # Example
10//!
11//! ```rust
12//! use parquet::schema::parser::parse_message_type;
13//!
14//! let message_type = "
15//!   message spark_schema {
16//!     OPTIONAL BYTE_ARRAY a (UTF8);
17//!     REQUIRED INT32 b;
18//!     REQUIRED DOUBLE c;
19//!     REQUIRED BOOLEAN d;
20//!     OPTIONAL group e (LIST) {
21//!       REPEATED group list {
22//!         REQUIRED INT32 element;
23//!       }
24//!     }
25//!   }
26//! ";
27//!
28//! let schema = parse_message_type(message_type).expect("Expected valid schema");
29//! println!("{:?}", schema);
30//! ```
31
32use std::sync::Arc;
33
34use crate::basic::{ConvertedType, LogicalType, Repetition, TimeUnit, Type as PhysicalType};
35use crate::errors::{ParquetError, Result};
36use crate::schema::types::{Type, TypePtr};
37
38/// Parses message type as string into a Parquet [`Type`]
39/// which, for example, could be used to extract individual columns. Returns Parquet
40/// general error when parsing or validation fails.
41pub fn parse_message_type(message_type: &str) -> Result<Type> {
42    let mut parser = Parser {
43        tokenizer: &mut Tokenizer::from_str(message_type),
44    };
45    parser.parse_message_type()
46}
47
48/// Tokenizer to split message type string into tokens that are separated using characters
49/// defined in `is_schema_delim` method. Tokenizer also preserves delimiters as tokens.
50/// Tokenizer provides Iterator interface to process tokens; it also allows to step back
51/// to reprocess previous tokens.
52struct Tokenizer<'a> {
53    // List of all tokens for a string
54    tokens: Vec<&'a str>,
55    // Current index of vector
56    index: usize,
57}
58
59impl<'a> Tokenizer<'a> {
60    // Create tokenizer from message type string
61    pub fn from_str(string: &'a str) -> Self {
62        let vec = string
63            .split_whitespace()
64            .flat_map(Self::split_token)
65            .collect();
66        Tokenizer {
67            tokens: vec,
68            index: 0,
69        }
70    }
71
72    // List of all special characters in schema
73    fn is_schema_delim(c: char) -> bool {
74        c == ';' || c == '{' || c == '}' || c == '(' || c == ')' || c == '=' || c == ','
75    }
76
77    /// Splits string into tokens; input string can already be token or can contain
78    /// delimiters, e.g. required" -> Vec("required") and
79    /// "(UTF8);" -> Vec("(", "UTF8", ")", ";")
80    fn split_token(string: &str) -> Vec<&str> {
81        let mut buffer: Vec<&str> = Vec::new();
82        let mut tail = string;
83        while let Some(index) = tail.find(Self::is_schema_delim) {
84            let (h, t) = tail.split_at(index);
85            if !h.is_empty() {
86                buffer.push(h);
87            }
88            buffer.push(&t[0..1]);
89            tail = &t[1..];
90        }
91        if !tail.is_empty() {
92            buffer.push(tail);
93        }
94        buffer
95    }
96
97    // Move pointer to a previous element
98    fn backtrack(&mut self) {
99        self.index -= 1;
100    }
101}
102
103impl<'a> Iterator for Tokenizer<'a> {
104    type Item = &'a str;
105
106    fn next(&mut self) -> Option<&'a str> {
107        if self.index < self.tokens.len() {
108            self.index += 1;
109            Some(self.tokens[self.index - 1])
110        } else {
111            None
112        }
113    }
114}
115
116/// Internal Schema parser.
117/// Traverses message type using tokenizer and parses each group/primitive type
118/// recursively.
119struct Parser<'a> {
120    tokenizer: &'a mut Tokenizer<'a>,
121}
122
123// Utility function to assert token on validity.
124fn assert_token(token: Option<&str>, expected: &str) -> Result<()> {
125    match token {
126        Some(value) if value == expected => Ok(()),
127        Some(other) => Err(general_err!(
128            "Expected '{}', found token '{}'",
129            expected,
130            other
131        )),
132        None => Err(general_err!(
133            "Expected '{}', but no token found (None)",
134            expected
135        )),
136    }
137}
138
139// Utility function to parse i32 or return general error.
140#[inline]
141fn parse_i32(value: Option<&str>, not_found_msg: &str, parse_fail_msg: &str) -> Result<i32> {
142    value
143        .ok_or_else(|| general_err!(not_found_msg))?
144        .parse::<i32>()
145        .map_err(|_| general_err!(parse_fail_msg))
146}
147
148// Utility function to parse boolean or return general error.
149#[inline]
150fn parse_bool(value: Option<&str>, not_found_msg: &str, parse_fail_msg: &str) -> Result<bool> {
151    value
152        .ok_or_else(|| general_err!(not_found_msg))?
153        .to_lowercase()
154        .parse::<bool>()
155        .map_err(|_| general_err!(parse_fail_msg))
156}
157
158// Utility function to parse TimeUnit or return general error.
159fn parse_timeunit(
160    value: Option<&str>,
161    not_found_msg: &str,
162    parse_fail_msg: &str,
163) -> Result<TimeUnit> {
164    let v = value.ok_or_else(|| general_err!(not_found_msg))?;
165    match v.to_uppercase().as_str() {
166        "MILLIS" => Ok(TimeUnit::MILLIS),
167        "MICROS" => Ok(TimeUnit::MICROS),
168        "NANOS" => Ok(TimeUnit::NANOS),
169        _ => Err(general_err!(parse_fail_msg)),
170    }
171}
172
173impl Parser<'_> {
174    // Entry function to parse message type, uses internal tokenizer.
175    fn parse_message_type(&mut self) -> Result<Type> {
176        // Check that message type starts with "message".
177        match self.tokenizer.next() {
178            Some("message") => {
179                let name = self
180                    .tokenizer
181                    .next()
182                    .ok_or_else(|| general_err!("Expected name, found None"))?;
183                Type::group_type_builder(name)
184                    .with_fields(self.parse_child_types()?)
185                    .build()
186            }
187            _ => Err(general_err!("Message type does not start with 'message'")),
188        }
189    }
190
191    // Parses child types for a current group type.
192    // This is only invoked on root and group types.
193    fn parse_child_types(&mut self) -> Result<Vec<TypePtr>> {
194        assert_token(self.tokenizer.next(), "{")?;
195        let mut vec = Vec::new();
196        while let Some(value) = self.tokenizer.next() {
197            if value == "}" {
198                break;
199            }
200            self.tokenizer.backtrack();
201            vec.push(Arc::new(self.add_type()?));
202        }
203        Ok(vec)
204    }
205
206    fn add_type(&mut self) -> Result<Type> {
207        // Parse repetition
208        let repetition = self
209            .tokenizer
210            .next()
211            .ok_or_else(|| general_err!("Expected repetition, found None"))
212            .and_then(|v| v.to_uppercase().parse::<Repetition>())?;
213
214        match self.tokenizer.next() {
215            Some(group) if group.to_uppercase() == "GROUP" => self.add_group_type(Some(repetition)),
216            Some(type_string) => {
217                let physical_type = type_string.to_uppercase().parse::<PhysicalType>()?;
218                self.add_primitive_type(repetition, physical_type)
219            }
220            None => Err(general_err!("Invalid type, could not extract next token")),
221        }
222    }
223
224    fn add_group_type(&mut self, repetition: Option<Repetition>) -> Result<Type> {
225        // Parse name of the group type
226        let name = self
227            .tokenizer
228            .next()
229            .ok_or_else(|| general_err!("Expected name, found None"))?;
230
231        // Parse logical or converted type if exists
232        let (logical_type, converted_type) = if self.tokenizer.next() == Some("(") {
233            let type_ = self
234                .tokenizer
235                .next()
236                .ok_or_else(|| general_err!("Expected converted type, found None"))
237                .and_then(|v| {
238                    // Try logical type first
239                    let upper = v.to_uppercase();
240                    let logical = upper.parse::<LogicalType>();
241                    match logical {
242                        Ok(logical) => {
243                            Ok((Some(logical.clone()), ConvertedType::from(Some(logical))))
244                        }
245                        Err(_) => Ok((None, upper.parse::<ConvertedType>()?)),
246                    }
247                })?;
248            assert_token(self.tokenizer.next(), ")")?;
249            type_
250        } else {
251            self.tokenizer.backtrack();
252            (None, ConvertedType::NONE)
253        };
254
255        // Parse optional id
256        let id = if self.tokenizer.next() == Some("=") {
257            self.tokenizer.next().and_then(|v| v.parse::<i32>().ok())
258        } else {
259            self.tokenizer.backtrack();
260            None
261        };
262
263        let mut builder = Type::group_type_builder(name)
264            .with_logical_type(logical_type)
265            .with_converted_type(converted_type)
266            .with_fields(self.parse_child_types()?)
267            .with_id(id);
268        if let Some(rep) = repetition {
269            builder = builder.with_repetition(rep);
270        }
271        builder.build()
272    }
273
274    fn add_primitive_type(
275        &mut self,
276        repetition: Repetition,
277        physical_type: PhysicalType,
278    ) -> Result<Type> {
279        // Read type length if the type is FIXED_LEN_BYTE_ARRAY.
280        let mut length: i32 = -1;
281        if physical_type == PhysicalType::FIXED_LEN_BYTE_ARRAY {
282            assert_token(self.tokenizer.next(), "(")?;
283            length = parse_i32(
284                self.tokenizer.next(),
285                "Expected length for FIXED_LEN_BYTE_ARRAY, found None",
286                "Failed to parse length for FIXED_LEN_BYTE_ARRAY",
287            )?;
288            assert_token(self.tokenizer.next(), ")")?;
289        }
290
291        // Parse name of the primitive type
292        let name = self
293            .tokenizer
294            .next()
295            .ok_or_else(|| general_err!("Expected name, found None"))?;
296
297        // Parse converted type
298        let (logical_type, converted_type, precision, scale) = if self.tokenizer.next() == Some("(")
299        {
300            let (mut logical, mut converted) = self
301                .tokenizer
302                .next()
303                .ok_or_else(|| general_err!("Expected logical or converted type, found None"))
304                .and_then(|v| {
305                    let upper = v.to_uppercase();
306                    let logical = upper.parse::<LogicalType>();
307                    match logical {
308                        Ok(logical) => {
309                            Ok((Some(logical.clone()), ConvertedType::from(Some(logical))))
310                        }
311                        Err(_) => Ok((None, upper.parse::<ConvertedType>()?)),
312                    }
313                })?;
314
315            // Parse precision and scale for decimals
316            let mut precision: i32 = -1;
317            let mut scale: i32 = -1;
318
319            // Parse the concrete logical type
320            if let Some(type_) = &logical {
321                match type_ {
322                    LogicalType::Decimal { .. } => {
323                        if self.tokenizer.next() == Some("(") {
324                            precision = parse_i32(
325                                self.tokenizer.next(),
326                                "Expected precision, found None",
327                                "Failed to parse precision for DECIMAL type",
328                            )?;
329                            if self.tokenizer.next() == Some(",") {
330                                scale = parse_i32(
331                                    self.tokenizer.next(),
332                                    "Expected scale, found None",
333                                    "Failed to parse scale for DECIMAL type",
334                                )?;
335                                assert_token(self.tokenizer.next(), ")")?;
336                            } else {
337                                scale = 0
338                            }
339                            logical = Some(LogicalType::decimal(scale, precision));
340                            converted = ConvertedType::from(logical.clone());
341                        }
342                    }
343                    LogicalType::Time { .. } => {
344                        if self.tokenizer.next() == Some("(") {
345                            let unit = parse_timeunit(
346                                self.tokenizer.next(),
347                                "Invalid timeunit found",
348                                "Failed to parse timeunit for TIME type",
349                            )?;
350                            if self.tokenizer.next() == Some(",") {
351                                let is_adjusted_to_u_t_c = parse_bool(
352                                    self.tokenizer.next(),
353                                    "Invalid boolean found",
354                                    "Failed to parse timezone info for TIME type",
355                                )?;
356                                assert_token(self.tokenizer.next(), ")")?;
357                                logical = Some(LogicalType::time(is_adjusted_to_u_t_c, unit));
358                                converted = ConvertedType::from(logical.clone());
359                            } else {
360                                // Invalid token for unit
361                                self.tokenizer.backtrack();
362                            }
363                        }
364                    }
365                    LogicalType::Timestamp { .. } => {
366                        if self.tokenizer.next() == Some("(") {
367                            let unit = parse_timeunit(
368                                self.tokenizer.next(),
369                                "Invalid timeunit found",
370                                "Failed to parse timeunit for TIMESTAMP type",
371                            )?;
372                            if self.tokenizer.next() == Some(",") {
373                                let is_adjusted_to_u_t_c = parse_bool(
374                                    self.tokenizer.next(),
375                                    "Invalid boolean found",
376                                    "Failed to parse timezone info for TIMESTAMP type",
377                                )?;
378                                assert_token(self.tokenizer.next(), ")")?;
379                                logical = Some(LogicalType::timestamp(is_adjusted_to_u_t_c, unit));
380                                converted = ConvertedType::from(logical.clone());
381                            } else {
382                                // Invalid token for unit
383                                self.tokenizer.backtrack();
384                            }
385                        }
386                    }
387                    LogicalType::Integer { .. } if self.tokenizer.next() == Some("(") => {
388                        let bit_width = parse_i32(
389                            self.tokenizer.next(),
390                            "Invalid bit_width found",
391                            "Failed to parse bit_width for INTEGER type",
392                        )? as i8;
393                        match physical_type {
394                            PhysicalType::INT32 => match bit_width {
395                                8 | 16 | 32 => {}
396                                _ => {
397                                    return Err(general_err!(
398                                        "Incorrect bit width {} for INT32",
399                                        bit_width
400                                    ));
401                                }
402                            },
403                            PhysicalType::INT64 => {
404                                if bit_width != 64 {
405                                    return Err(general_err!(
406                                        "Incorrect bit width {} for INT64",
407                                        bit_width
408                                    ));
409                                }
410                            }
411                            _ => {
412                                return Err(general_err!(
413                                    "Logical type Integer cannot be used with physical type {}",
414                                    physical_type
415                                ));
416                            }
417                        }
418                        if self.tokenizer.next() == Some(",") {
419                            let is_signed = parse_bool(
420                                self.tokenizer.next(),
421                                "Invalid boolean found",
422                                "Failed to parse is_signed for INTEGER type",
423                            )?;
424                            assert_token(self.tokenizer.next(), ")")?;
425                            logical = Some(LogicalType::integer(bit_width, is_signed));
426                            converted = ConvertedType::from(logical.clone());
427                        } else {
428                            // Invalid token for unit
429                            self.tokenizer.backtrack();
430                        }
431                    }
432                    _ => {}
433                }
434            } else if converted == ConvertedType::DECIMAL {
435                if self.tokenizer.next() == Some("(") {
436                    // Parse precision
437                    precision = parse_i32(
438                        self.tokenizer.next(),
439                        "Expected precision, found None",
440                        "Failed to parse precision for DECIMAL type",
441                    )?;
442
443                    // Parse scale
444                    scale = if self.tokenizer.next() == Some(",") {
445                        parse_i32(
446                            self.tokenizer.next(),
447                            "Expected scale, found None",
448                            "Failed to parse scale for DECIMAL type",
449                        )?
450                    } else {
451                        // Scale is not provided, set it to 0.
452                        self.tokenizer.backtrack();
453                        0
454                    };
455
456                    assert_token(self.tokenizer.next(), ")")?;
457                } else {
458                    self.tokenizer.backtrack();
459                }
460            }
461
462            assert_token(self.tokenizer.next(), ")")?;
463            (logical, converted, precision, scale)
464        } else {
465            self.tokenizer.backtrack();
466            (None, ConvertedType::NONE, -1, -1)
467        };
468
469        // Parse optional id
470        let id = if self.tokenizer.next() == Some("=") {
471            self.tokenizer.next().and_then(|v| v.parse::<i32>().ok())
472        } else {
473            self.tokenizer.backtrack();
474            None
475        };
476        assert_token(self.tokenizer.next(), ";")?;
477
478        Type::primitive_type_builder(name, physical_type)
479            .with_repetition(repetition)
480            .with_logical_type(logical_type)
481            .with_converted_type(converted_type)
482            .with_length(length)
483            .with_precision(precision)
484            .with_scale(scale)
485            .with_id(id)
486            .build()
487    }
488}
489
490#[cfg(test)]
491mod tests {
492    use super::*;
493
494    #[test]
495    fn test_tokenize_empty_string() {
496        assert_eq!(Tokenizer::from_str("").next(), None);
497    }
498
499    #[test]
500    fn test_tokenize_delimiters() {
501        let mut iter = Tokenizer::from_str(",;{}()=");
502        assert_eq!(iter.next(), Some(","));
503        assert_eq!(iter.next(), Some(";"));
504        assert_eq!(iter.next(), Some("{"));
505        assert_eq!(iter.next(), Some("}"));
506        assert_eq!(iter.next(), Some("("));
507        assert_eq!(iter.next(), Some(")"));
508        assert_eq!(iter.next(), Some("="));
509        assert_eq!(iter.next(), None);
510    }
511
512    #[test]
513    fn test_tokenize_delimiters_with_whitespaces() {
514        let mut iter = Tokenizer::from_str(" , ; { } ( ) = ");
515        assert_eq!(iter.next(), Some(","));
516        assert_eq!(iter.next(), Some(";"));
517        assert_eq!(iter.next(), Some("{"));
518        assert_eq!(iter.next(), Some("}"));
519        assert_eq!(iter.next(), Some("("));
520        assert_eq!(iter.next(), Some(")"));
521        assert_eq!(iter.next(), Some("="));
522        assert_eq!(iter.next(), None);
523    }
524
525    #[test]
526    fn test_tokenize_words() {
527        let mut iter = Tokenizer::from_str("abc def ghi jkl mno");
528        assert_eq!(iter.next(), Some("abc"));
529        assert_eq!(iter.next(), Some("def"));
530        assert_eq!(iter.next(), Some("ghi"));
531        assert_eq!(iter.next(), Some("jkl"));
532        assert_eq!(iter.next(), Some("mno"));
533        assert_eq!(iter.next(), None);
534    }
535
536    #[test]
537    fn test_tokenize_backtrack() {
538        let mut iter = Tokenizer::from_str("abc;");
539        assert_eq!(iter.next(), Some("abc"));
540        assert_eq!(iter.next(), Some(";"));
541        iter.backtrack();
542        assert_eq!(iter.next(), Some(";"));
543        assert_eq!(iter.next(), None);
544    }
545
546    #[test]
547    fn test_tokenize_message_type() {
548        let schema = "
549    message schema {
550      required int32 a;
551      optional binary c (UTF8);
552      required group d {
553        required int32 a;
554        optional binary c (UTF8);
555      }
556      required group e (LIST) {
557        repeated group list {
558          required int32 element;
559        }
560      }
561    }
562    ";
563        let iter = Tokenizer::from_str(schema);
564        let mut res = Vec::new();
565        for token in iter {
566            res.push(token);
567        }
568        assert_eq!(
569            res,
570            vec![
571                "message", "schema", "{", "required", "int32", "a", ";", "optional", "binary", "c",
572                "(", "UTF8", ")", ";", "required", "group", "d", "{", "required", "int32", "a",
573                ";", "optional", "binary", "c", "(", "UTF8", ")", ";", "}", "required", "group",
574                "e", "(", "LIST", ")", "{", "repeated", "group", "list", "{", "required", "int32",
575                "element", ";", "}", "}", "}"
576            ]
577        );
578    }
579
580    #[test]
581    fn test_assert_token() {
582        assert!(assert_token(Some("a"), "a").is_ok());
583        assert!(assert_token(Some("a"), "b").is_err());
584        assert!(assert_token(None, "b").is_err());
585    }
586
587    fn parse(schema: &str) -> Result<Type, ParquetError> {
588        let mut iter = Tokenizer::from_str(schema);
589        Parser {
590            tokenizer: &mut iter,
591        }
592        .parse_message_type()
593    }
594
595    #[test]
596    fn test_parse_message_type_invalid() {
597        assert_eq!(
598            parse("test").unwrap_err().to_string(),
599            "Parquet error: Message type does not start with 'message'"
600        );
601    }
602
603    #[test]
604    fn test_parse_message_type_no_name() {
605        assert_eq!(
606            parse("message").unwrap_err().to_string(),
607            "Parquet error: Expected name, found None"
608        );
609    }
610
611    #[test]
612    fn test_parse_message_type_fixed_byte_array() {
613        let schema = "
614            message schema {
615              REQUIRED FIXED_LEN_BYTE_ARRAY col;
616            }
617        ";
618        assert_eq!(
619            parse(schema).unwrap_err().to_string(),
620            "Parquet error: Expected '(', found token 'col'"
621        );
622
623        let schema = "
624            message schema {
625              REQUIRED FIXED_LEN_BYTE_ARRAY(16) col;
626            }
627        ";
628        parse(schema).unwrap();
629    }
630
631    #[test]
632    fn test_parse_message_type_integer() {
633        // Invalid integer syntax
634        let schema = "
635            message root {
636              optional int64 f1 (INTEGER());
637            }
638        ";
639        assert_eq!(
640            parse(schema).unwrap_err().to_string(),
641            "Parquet error: Failed to parse bit_width for INTEGER type"
642        );
643
644        // Invalid integer syntax, needs both bit-width and UTC sign
645        let schema = "
646    message root {
647      optional int64 f1 (INTEGER(32,));
648    }
649    ";
650        assert_eq!(
651            parse(schema).unwrap_err().to_string(),
652            "Parquet error: Incorrect bit width 32 for INT64"
653        );
654
655        // Invalid integer because of non-numeric bit width
656        let schema = "
657            message root {
658              optional int32 f1 (INTEGER(eight,true));
659            }
660        ";
661        assert_eq!(
662            parse(schema).unwrap_err().to_string(),
663            "Parquet error: Failed to parse bit_width for INTEGER type"
664        );
665
666        // Valid types
667        let schema = "
668            message root {
669              optional int32 f1 (INTEGER(8,false));
670              optional int32 f2 (INTEGER(8,true));
671              optional int32 f3 (INTEGER(16,false));
672              optional int32 f4 (INTEGER(16,true));
673              optional int32 f5 (INTEGER(32,false));
674              optional int32 f6 (INTEGER(32,true));
675              optional int64 f7 (INTEGER(64,false));
676              optional int64 f7 (INTEGER(64,true));
677            }
678        ";
679        parse(schema).unwrap();
680    }
681
682    #[test]
683    fn test_parse_message_type_temporal() {
684        // Invalid timestamp syntax
685        let schema = "
686            message root {
687              optional int64 f1 (TIMESTAMP();
688            }
689        ";
690        assert_eq!(
691            parse(schema).unwrap_err().to_string(),
692            "Parquet error: Failed to parse timeunit for TIMESTAMP type"
693        );
694
695        // Invalid timestamp syntax, needs both unit and UTC adjustment
696        let schema = "
697            message root {
698              optional int64 f1 (TIMESTAMP(MILLIS,));
699            }
700        ";
701        assert_eq!(
702            parse(schema).unwrap_err().to_string(),
703            "Parquet error: Failed to parse timezone info for TIMESTAMP type"
704        );
705
706        // Invalid timestamp because of unknown unit
707        let schema = "
708            message root {
709              optional int64 f1 (TIMESTAMP(YOCTOS,));
710            }
711        ";
712
713        assert_eq!(
714            parse(schema).unwrap_err().to_string(),
715            "Parquet error: Failed to parse timeunit for TIMESTAMP type"
716        );
717
718        // Valid types
719        let schema = "
720            message root {
721              optional int32 f1 (DATE);
722              optional int32 f2 (TIME(MILLIS,true));
723              optional int64 f3 (TIME(MICROS,false));
724              optional int64 f4 (TIME(NANOS,true));
725              optional int64 f5 (TIMESTAMP(MILLIS,true));
726              optional int64 f6 (TIMESTAMP(MICROS,true));
727              optional int64 f7 (TIMESTAMP(NANOS,false));
728            }
729        ";
730        parse(schema).unwrap();
731    }
732
733    #[test]
734    fn test_parse_message_type_decimal() {
735        // It is okay for decimal to omit precision and scale with right syntax.
736        // Here we test wrong syntax of decimal type
737
738        // Invalid decimal syntax
739        let schema = "
740            message root {
741              optional int32 f1 (DECIMAL();
742            }
743        ";
744        assert_eq!(
745            parse(schema).unwrap_err().to_string(),
746            "Parquet error: Failed to parse precision for DECIMAL type"
747        );
748
749        // Invalid decimal, need precision and scale
750        let schema = "
751            message root {
752              optional int32 f1 (DECIMAL());
753            }
754        ";
755        assert_eq!(
756            parse(schema).unwrap_err().to_string(),
757            "Parquet error: Failed to parse precision for DECIMAL type"
758        );
759
760        // Invalid decimal because of `,` - has precision, needs scale
761        let schema = "
762            message root {
763              optional int32 f1 (DECIMAL(8,));
764            }
765        ";
766        assert_eq!(
767            parse(schema).unwrap_err().to_string(),
768            "Parquet error: Failed to parse scale for DECIMAL type"
769        );
770
771        // Invalid decimal because, we always require either precision or scale to be
772        // specified as part of converted type
773        let schema = "
774            message root {
775              optional int32 f3 (DECIMAL);
776            }
777        ";
778        assert_eq!(
779            parse(schema).unwrap_err().to_string(),
780            "Parquet error: Expected ')', found token ';'"
781        );
782
783        // Valid decimal (precision, scale)
784        let schema = "
785            message root {
786              optional int32 f1 (DECIMAL(8, 3));
787              optional int32 f2 (DECIMAL(8));
788            }
789        ";
790        parse(schema).unwrap();
791    }
792
793    #[test]
794    fn test_parse_message_type_compare_1() {
795        let schema = "
796            message root {
797              optional fixed_len_byte_array(5) f1 (DECIMAL(9, 3));
798              optional fixed_len_byte_array (16) f2 (DECIMAL (38, 18));
799              optional fixed_len_byte_array (2) f3 (FLOAT16);
800            }
801        ";
802        let message = parse(schema).unwrap();
803
804        let expected = Type::group_type_builder("root")
805            .with_fields(vec![
806                Arc::new(
807                    Type::primitive_type_builder("f1", PhysicalType::FIXED_LEN_BYTE_ARRAY)
808                        .with_logical_type(Some(LogicalType::decimal(3, 9)))
809                        .with_converted_type(ConvertedType::DECIMAL)
810                        .with_length(5)
811                        .with_precision(9)
812                        .with_scale(3)
813                        .build()
814                        .unwrap(),
815                ),
816                Arc::new(
817                    Type::primitive_type_builder("f2", PhysicalType::FIXED_LEN_BYTE_ARRAY)
818                        .with_logical_type(Some(LogicalType::decimal(18, 38)))
819                        .with_converted_type(ConvertedType::DECIMAL)
820                        .with_length(16)
821                        .with_precision(38)
822                        .with_scale(18)
823                        .build()
824                        .unwrap(),
825                ),
826                Arc::new(
827                    Type::primitive_type_builder("f3", PhysicalType::FIXED_LEN_BYTE_ARRAY)
828                        .with_logical_type(Some(LogicalType::Float16))
829                        .with_length(2)
830                        .build()
831                        .unwrap(),
832                ),
833            ])
834            .build()
835            .unwrap();
836
837        assert_eq!(message, expected);
838    }
839
840    #[test]
841    fn test_parse_message_type_compare_2() {
842        let schema = "
843            message root {
844              required group a0 {
845                optional group a1 (LIST) {
846                  repeated binary a2 (UTF8);
847                }
848
849                optional group b1 (LIST) {
850                  repeated group b2 {
851                    optional int32 b3;
852                    optional double b4;
853                  }
854                }
855              }
856            }
857        ";
858        let message = parse(schema).unwrap();
859
860        let expected = Type::group_type_builder("root")
861            .with_fields(vec![Arc::new(
862                Type::group_type_builder("a0")
863                    .with_repetition(Repetition::REQUIRED)
864                    .with_fields(vec![
865                        Arc::new(
866                            Type::group_type_builder("a1")
867                                .with_repetition(Repetition::OPTIONAL)
868                                .with_logical_type(Some(LogicalType::List))
869                                .with_converted_type(ConvertedType::LIST)
870                                .with_fields(vec![Arc::new(
871                                    Type::primitive_type_builder("a2", PhysicalType::BYTE_ARRAY)
872                                        .with_repetition(Repetition::REPEATED)
873                                        .with_converted_type(ConvertedType::UTF8)
874                                        .build()
875                                        .unwrap(),
876                                )])
877                                .build()
878                                .unwrap(),
879                        ),
880                        Arc::new(
881                            Type::group_type_builder("b1")
882                                .with_repetition(Repetition::OPTIONAL)
883                                .with_logical_type(Some(LogicalType::List))
884                                .with_converted_type(ConvertedType::LIST)
885                                .with_fields(vec![Arc::new(
886                                    Type::group_type_builder("b2")
887                                        .with_repetition(Repetition::REPEATED)
888                                        .with_fields(vec![
889                                            Arc::new(
890                                                Type::primitive_type_builder(
891                                                    "b3",
892                                                    PhysicalType::INT32,
893                                                )
894                                                .build()
895                                                .unwrap(),
896                                            ),
897                                            Arc::new(
898                                                Type::primitive_type_builder(
899                                                    "b4",
900                                                    PhysicalType::DOUBLE,
901                                                )
902                                                .build()
903                                                .unwrap(),
904                                            ),
905                                        ])
906                                        .build()
907                                        .unwrap(),
908                                )])
909                                .build()
910                                .unwrap(),
911                        ),
912                    ])
913                    .build()
914                    .unwrap(),
915            )])
916            .build()
917            .unwrap();
918
919        assert_eq!(message, expected);
920    }
921
922    #[test]
923    fn test_parse_message_type_compare_3() {
924        let schema = "
925            message root {
926              required int32 _1 (INT_8);
927              required int32 _2 (INT_16);
928              required float _3;
929              required double _4;
930              optional int32 _5 (DATE);
931              optional binary _6 (UTF8);
932            }
933        ";
934        let message = parse(schema).unwrap();
935
936        let fields = vec![
937            Arc::new(
938                Type::primitive_type_builder("_1", PhysicalType::INT32)
939                    .with_repetition(Repetition::REQUIRED)
940                    .with_converted_type(ConvertedType::INT_8)
941                    .build()
942                    .unwrap(),
943            ),
944            Arc::new(
945                Type::primitive_type_builder("_2", PhysicalType::INT32)
946                    .with_repetition(Repetition::REQUIRED)
947                    .with_converted_type(ConvertedType::INT_16)
948                    .build()
949                    .unwrap(),
950            ),
951            Arc::new(
952                Type::primitive_type_builder("_3", PhysicalType::FLOAT)
953                    .with_repetition(Repetition::REQUIRED)
954                    .build()
955                    .unwrap(),
956            ),
957            Arc::new(
958                Type::primitive_type_builder("_4", PhysicalType::DOUBLE)
959                    .with_repetition(Repetition::REQUIRED)
960                    .build()
961                    .unwrap(),
962            ),
963            Arc::new(
964                Type::primitive_type_builder("_5", PhysicalType::INT32)
965                    .with_logical_type(Some(LogicalType::Date))
966                    .with_converted_type(ConvertedType::DATE)
967                    .build()
968                    .unwrap(),
969            ),
970            Arc::new(
971                Type::primitive_type_builder("_6", PhysicalType::BYTE_ARRAY)
972                    .with_converted_type(ConvertedType::UTF8)
973                    .build()
974                    .unwrap(),
975            ),
976        ];
977
978        let expected = Type::group_type_builder("root")
979            .with_fields(fields)
980            .build()
981            .unwrap();
982        assert_eq!(message, expected);
983    }
984
985    #[test]
986    fn test_parse_message_type_compare_4() {
987        let schema = "
988            message root {
989              required int32 _1 (INTEGER(8,true));
990              required int32 _2 (INTEGER(16,false));
991              required float _3;
992              required double _4;
993              optional int32 _5 (DATE);
994              optional int32 _6 (TIME(MILLIS,false));
995              optional int64 _7 (TIME(MICROS,true));
996              optional int64 _8 (TIMESTAMP(MILLIS,true));
997              optional int64 _9 (TIMESTAMP(NANOS,false));
998              optional binary _10 (STRING);
999            }
1000        ";
1001        let message = parse(schema).unwrap();
1002
1003        let fields = vec![
1004            Arc::new(
1005                Type::primitive_type_builder("_1", PhysicalType::INT32)
1006                    .with_repetition(Repetition::REQUIRED)
1007                    .with_logical_type(Some(LogicalType::integer(8, true)))
1008                    .build()
1009                    .unwrap(),
1010            ),
1011            Arc::new(
1012                Type::primitive_type_builder("_2", PhysicalType::INT32)
1013                    .with_repetition(Repetition::REQUIRED)
1014                    .with_logical_type(Some(LogicalType::integer(16, false)))
1015                    .build()
1016                    .unwrap(),
1017            ),
1018            Arc::new(
1019                Type::primitive_type_builder("_3", PhysicalType::FLOAT)
1020                    .with_repetition(Repetition::REQUIRED)
1021                    .build()
1022                    .unwrap(),
1023            ),
1024            Arc::new(
1025                Type::primitive_type_builder("_4", PhysicalType::DOUBLE)
1026                    .with_repetition(Repetition::REQUIRED)
1027                    .build()
1028                    .unwrap(),
1029            ),
1030            Arc::new(
1031                Type::primitive_type_builder("_5", PhysicalType::INT32)
1032                    .with_logical_type(Some(LogicalType::Date))
1033                    .build()
1034                    .unwrap(),
1035            ),
1036            Arc::new(
1037                Type::primitive_type_builder("_6", PhysicalType::INT32)
1038                    .with_logical_type(Some(LogicalType::time(false, TimeUnit::MILLIS)))
1039                    .build()
1040                    .unwrap(),
1041            ),
1042            Arc::new(
1043                Type::primitive_type_builder("_7", PhysicalType::INT64)
1044                    .with_logical_type(Some(LogicalType::time(true, TimeUnit::MICROS)))
1045                    .build()
1046                    .unwrap(),
1047            ),
1048            Arc::new(
1049                Type::primitive_type_builder("_8", PhysicalType::INT64)
1050                    .with_logical_type(Some(LogicalType::timestamp(true, TimeUnit::MILLIS)))
1051                    .build()
1052                    .unwrap(),
1053            ),
1054            Arc::new(
1055                Type::primitive_type_builder("_9", PhysicalType::INT64)
1056                    .with_logical_type(Some(LogicalType::timestamp(false, TimeUnit::NANOS)))
1057                    .build()
1058                    .unwrap(),
1059            ),
1060            Arc::new(
1061                Type::primitive_type_builder("_10", PhysicalType::BYTE_ARRAY)
1062                    .with_logical_type(Some(LogicalType::String))
1063                    .build()
1064                    .unwrap(),
1065            ),
1066        ];
1067
1068        let expected = Type::group_type_builder("root")
1069            .with_fields(fields)
1070            .build()
1071            .unwrap();
1072        assert_eq!(message, expected);
1073    }
1074}
parquet/schema/parser.rs

parquet/schema/
parser.rs
