parquet/schema/
parser.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Parquet schema parser.
19//! Provides methods to parse and validate string message type into Parquet
20//! [`Type`].
21//!
22//! # Example
23//!
24//! ```rust
25//! use parquet::schema::parser::parse_message_type;
26//!
27//! let message_type = "
28//!   message spark_schema {
29//!     OPTIONAL BYTE_ARRAY a (UTF8);
30//!     REQUIRED INT32 b;
31//!     REQUIRED DOUBLE c;
32//!     REQUIRED BOOLEAN d;
33//!     OPTIONAL group e (LIST) {
34//!       REPEATED group list {
35//!         REQUIRED INT32 element;
36//!       }
37//!     }
38//!   }
39//! ";
40//!
41//! let schema = parse_message_type(message_type).expect("Expected valid schema");
42//! println!("{:?}", schema);
43//! ```
44
45use std::sync::Arc;
46
47use crate::basic::{ConvertedType, LogicalType, Repetition, TimeUnit, Type as PhysicalType};
48use crate::errors::{ParquetError, Result};
49use crate::schema::types::{Type, TypePtr};
50
51/// Parses message type as string into a Parquet [`Type`]
52/// which, for example, could be used to extract individual columns. Returns Parquet
53/// general error when parsing or validation fails.
54pub fn parse_message_type(message_type: &str) -> Result<Type> {
55    let mut parser = Parser {
56        tokenizer: &mut Tokenizer::from_str(message_type),
57    };
58    parser.parse_message_type()
59}
60
61/// Tokenizer to split message type string into tokens that are separated using characters
62/// defined in `is_schema_delim` method. Tokenizer also preserves delimiters as tokens.
63/// Tokenizer provides Iterator interface to process tokens; it also allows to step back
64/// to reprocess previous tokens.
65struct Tokenizer<'a> {
66    // List of all tokens for a string
67    tokens: Vec<&'a str>,
68    // Current index of vector
69    index: usize,
70}
71
72impl<'a> Tokenizer<'a> {
73    // Create tokenizer from message type string
74    pub fn from_str(string: &'a str) -> Self {
75        let vec = string
76            .split_whitespace()
77            .flat_map(Self::split_token)
78            .collect();
79        Tokenizer {
80            tokens: vec,
81            index: 0,
82        }
83    }
84
85    // List of all special characters in schema
86    fn is_schema_delim(c: char) -> bool {
87        c == ';' || c == '{' || c == '}' || c == '(' || c == ')' || c == '=' || c == ','
88    }
89
90    /// Splits string into tokens; input string can already be token or can contain
91    /// delimiters, e.g. required" -> Vec("required") and
92    /// "(UTF8);" -> Vec("(", "UTF8", ")", ";")
93    fn split_token(string: &str) -> Vec<&str> {
94        let mut buffer: Vec<&str> = Vec::new();
95        let mut tail = string;
96        while let Some(index) = tail.find(Self::is_schema_delim) {
97            let (h, t) = tail.split_at(index);
98            if !h.is_empty() {
99                buffer.push(h);
100            }
101            buffer.push(&t[0..1]);
102            tail = &t[1..];
103        }
104        if !tail.is_empty() {
105            buffer.push(tail);
106        }
107        buffer
108    }
109
110    // Move pointer to a previous element
111    fn backtrack(&mut self) {
112        self.index -= 1;
113    }
114}
115
116impl<'a> Iterator for Tokenizer<'a> {
117    type Item = &'a str;
118
119    fn next(&mut self) -> Option<&'a str> {
120        if self.index < self.tokens.len() {
121            self.index += 1;
122            Some(self.tokens[self.index - 1])
123        } else {
124            None
125        }
126    }
127}
128
129/// Internal Schema parser.
130/// Traverses message type using tokenizer and parses each group/primitive type
131/// recursively.
132struct Parser<'a> {
133    tokenizer: &'a mut Tokenizer<'a>,
134}
135
136// Utility function to assert token on validity.
137fn assert_token(token: Option<&str>, expected: &str) -> Result<()> {
138    match token {
139        Some(value) if value == expected => Ok(()),
140        Some(other) => Err(general_err!(
141            "Expected '{}', found token '{}'",
142            expected,
143            other
144        )),
145        None => Err(general_err!(
146            "Expected '{}', but no token found (None)",
147            expected
148        )),
149    }
150}
151
152// Utility function to parse i32 or return general error.
153#[inline]
154fn parse_i32(value: Option<&str>, not_found_msg: &str, parse_fail_msg: &str) -> Result<i32> {
155    value
156        .ok_or_else(|| general_err!(not_found_msg))
157        .and_then(|v| v.parse::<i32>().map_err(|_| general_err!(parse_fail_msg)))
158}
159
160// Utility function to parse boolean or return general error.
161#[inline]
162fn parse_bool(value: Option<&str>, not_found_msg: &str, parse_fail_msg: &str) -> Result<bool> {
163    value
164        .ok_or_else(|| general_err!(not_found_msg))
165        .and_then(|v| {
166            v.to_lowercase()
167                .parse::<bool>()
168                .map_err(|_| general_err!(parse_fail_msg))
169        })
170}
171
172// Utility function to parse TimeUnit or return general error.
173fn parse_timeunit(
174    value: Option<&str>,
175    not_found_msg: &str,
176    parse_fail_msg: &str,
177) -> Result<TimeUnit> {
178    value
179        .ok_or_else(|| general_err!(not_found_msg))
180        .and_then(|v| match v.to_uppercase().as_str() {
181            "MILLIS" => Ok(TimeUnit::MILLIS),
182            "MICROS" => Ok(TimeUnit::MICROS),
183            "NANOS" => Ok(TimeUnit::NANOS),
184            _ => Err(general_err!(parse_fail_msg)),
185        })
186}
187
188impl Parser<'_> {
189    // Entry function to parse message type, uses internal tokenizer.
190    fn parse_message_type(&mut self) -> Result<Type> {
191        // Check that message type starts with "message".
192        match self.tokenizer.next() {
193            Some("message") => {
194                let name = self
195                    .tokenizer
196                    .next()
197                    .ok_or_else(|| general_err!("Expected name, found None"))?;
198                Type::group_type_builder(name)
199                    .with_fields(self.parse_child_types()?)
200                    .build()
201            }
202            _ => Err(general_err!("Message type does not start with 'message'")),
203        }
204    }
205
206    // Parses child types for a current group type.
207    // This is only invoked on root and group types.
208    fn parse_child_types(&mut self) -> Result<Vec<TypePtr>> {
209        assert_token(self.tokenizer.next(), "{")?;
210        let mut vec = Vec::new();
211        while let Some(value) = self.tokenizer.next() {
212            if value == "}" {
213                break;
214            } else {
215                self.tokenizer.backtrack();
216                vec.push(Arc::new(self.add_type()?));
217            }
218        }
219        Ok(vec)
220    }
221
222    fn add_type(&mut self) -> Result<Type> {
223        // Parse repetition
224        let repetition = self
225            .tokenizer
226            .next()
227            .ok_or_else(|| general_err!("Expected repetition, found None"))
228            .and_then(|v| v.to_uppercase().parse::<Repetition>())?;
229
230        match self.tokenizer.next() {
231            Some(group) if group.to_uppercase() == "GROUP" => self.add_group_type(Some(repetition)),
232            Some(type_string) => {
233                let physical_type = type_string.to_uppercase().parse::<PhysicalType>()?;
234                self.add_primitive_type(repetition, physical_type)
235            }
236            None => Err(general_err!("Invalid type, could not extract next token")),
237        }
238    }
239
240    fn add_group_type(&mut self, repetition: Option<Repetition>) -> Result<Type> {
241        // Parse name of the group type
242        let name = self
243            .tokenizer
244            .next()
245            .ok_or_else(|| general_err!("Expected name, found None"))?;
246
247        // Parse logical or converted type if exists
248        let (logical_type, converted_type) = if self.tokenizer.next() == Some("(") {
249            let tpe = self
250                .tokenizer
251                .next()
252                .ok_or_else(|| general_err!("Expected converted type, found None"))
253                .and_then(|v| {
254                    // Try logical type first
255                    let upper = v.to_uppercase();
256                    let logical = upper.parse::<LogicalType>();
257                    match logical {
258                        Ok(logical) => {
259                            Ok((Some(logical.clone()), ConvertedType::from(Some(logical))))
260                        }
261                        Err(_) => Ok((None, upper.parse::<ConvertedType>()?)),
262                    }
263                })?;
264            assert_token(self.tokenizer.next(), ")")?;
265            tpe
266        } else {
267            self.tokenizer.backtrack();
268            (None, ConvertedType::NONE)
269        };
270
271        // Parse optional id
272        let id = if self.tokenizer.next() == Some("=") {
273            self.tokenizer.next().and_then(|v| v.parse::<i32>().ok())
274        } else {
275            self.tokenizer.backtrack();
276            None
277        };
278
279        let mut builder = Type::group_type_builder(name)
280            .with_logical_type(logical_type)
281            .with_converted_type(converted_type)
282            .with_fields(self.parse_child_types()?)
283            .with_id(id);
284        if let Some(rep) = repetition {
285            builder = builder.with_repetition(rep);
286        }
287        builder.build()
288    }
289
290    fn add_primitive_type(
291        &mut self,
292        repetition: Repetition,
293        physical_type: PhysicalType,
294    ) -> Result<Type> {
295        // Read type length if the type is FIXED_LEN_BYTE_ARRAY.
296        let mut length: i32 = -1;
297        if physical_type == PhysicalType::FIXED_LEN_BYTE_ARRAY {
298            assert_token(self.tokenizer.next(), "(")?;
299            length = parse_i32(
300                self.tokenizer.next(),
301                "Expected length for FIXED_LEN_BYTE_ARRAY, found None",
302                "Failed to parse length for FIXED_LEN_BYTE_ARRAY",
303            )?;
304            assert_token(self.tokenizer.next(), ")")?;
305        }
306
307        // Parse name of the primitive type
308        let name = self
309            .tokenizer
310            .next()
311            .ok_or_else(|| general_err!("Expected name, found None"))?;
312
313        // Parse converted type
314        let (logical_type, converted_type, precision, scale) = if self.tokenizer.next() == Some("(")
315        {
316            let (mut logical, mut converted) = self
317                .tokenizer
318                .next()
319                .ok_or_else(|| general_err!("Expected logical or converted type, found None"))
320                .and_then(|v| {
321                    let upper = v.to_uppercase();
322                    let logical = upper.parse::<LogicalType>();
323                    match logical {
324                        Ok(logical) => {
325                            Ok((Some(logical.clone()), ConvertedType::from(Some(logical))))
326                        }
327                        Err(_) => Ok((None, upper.parse::<ConvertedType>()?)),
328                    }
329                })?;
330
331            // Parse precision and scale for decimals
332            let mut precision: i32 = -1;
333            let mut scale: i32 = -1;
334
335            // Parse the concrete logical type
336            if let Some(tpe) = &logical {
337                match tpe {
338                    LogicalType::Decimal { .. } => {
339                        if self.tokenizer.next() == Some("(") {
340                            precision = parse_i32(
341                                self.tokenizer.next(),
342                                "Expected precision, found None",
343                                "Failed to parse precision for DECIMAL type",
344                            )?;
345                            if self.tokenizer.next() == Some(",") {
346                                scale = parse_i32(
347                                    self.tokenizer.next(),
348                                    "Expected scale, found None",
349                                    "Failed to parse scale for DECIMAL type",
350                                )?;
351                                assert_token(self.tokenizer.next(), ")")?;
352                            } else {
353                                scale = 0
354                            }
355                            logical = Some(LogicalType::decimal(scale, precision));
356                            converted = ConvertedType::from(logical.clone());
357                        }
358                    }
359                    LogicalType::Time { .. } => {
360                        if self.tokenizer.next() == Some("(") {
361                            let unit = parse_timeunit(
362                                self.tokenizer.next(),
363                                "Invalid timeunit found",
364                                "Failed to parse timeunit for TIME type",
365                            )?;
366                            if self.tokenizer.next() == Some(",") {
367                                let is_adjusted_to_u_t_c = parse_bool(
368                                    self.tokenizer.next(),
369                                    "Invalid boolean found",
370                                    "Failed to parse timezone info for TIME type",
371                                )?;
372                                assert_token(self.tokenizer.next(), ")")?;
373                                logical = Some(LogicalType::time(is_adjusted_to_u_t_c, unit));
374                                converted = ConvertedType::from(logical.clone());
375                            } else {
376                                // Invalid token for unit
377                                self.tokenizer.backtrack();
378                            }
379                        }
380                    }
381                    LogicalType::Timestamp { .. } => {
382                        if self.tokenizer.next() == Some("(") {
383                            let unit = parse_timeunit(
384                                self.tokenizer.next(),
385                                "Invalid timeunit found",
386                                "Failed to parse timeunit for TIMESTAMP type",
387                            )?;
388                            if self.tokenizer.next() == Some(",") {
389                                let is_adjusted_to_u_t_c = parse_bool(
390                                    self.tokenizer.next(),
391                                    "Invalid boolean found",
392                                    "Failed to parse timezone info for TIMESTAMP type",
393                                )?;
394                                assert_token(self.tokenizer.next(), ")")?;
395                                logical = Some(LogicalType::timestamp(is_adjusted_to_u_t_c, unit));
396                                converted = ConvertedType::from(logical.clone());
397                            } else {
398                                // Invalid token for unit
399                                self.tokenizer.backtrack();
400                            }
401                        }
402                    }
403                    LogicalType::Integer { .. } if self.tokenizer.next() == Some("(") => {
404                        let bit_width = parse_i32(
405                            self.tokenizer.next(),
406                            "Invalid bit_width found",
407                            "Failed to parse bit_width for INTEGER type",
408                        )? as i8;
409                        match physical_type {
410                            PhysicalType::INT32 => match bit_width {
411                                8 | 16 | 32 => {}
412                                _ => {
413                                    return Err(general_err!(
414                                        "Incorrect bit width {} for INT32",
415                                        bit_width
416                                    ));
417                                }
418                            },
419                            PhysicalType::INT64 => {
420                                if bit_width != 64 {
421                                    return Err(general_err!(
422                                        "Incorrect bit width {} for INT64",
423                                        bit_width
424                                    ));
425                                }
426                            }
427                            _ => {
428                                return Err(general_err!(
429                                    "Logical type Integer cannot be used with physical type {}",
430                                    physical_type
431                                ));
432                            }
433                        }
434                        if self.tokenizer.next() == Some(",") {
435                            let is_signed = parse_bool(
436                                self.tokenizer.next(),
437                                "Invalid boolean found",
438                                "Failed to parse is_signed for INTEGER type",
439                            )?;
440                            assert_token(self.tokenizer.next(), ")")?;
441                            logical = Some(LogicalType::integer(bit_width, is_signed));
442                            converted = ConvertedType::from(logical.clone());
443                        } else {
444                            // Invalid token for unit
445                            self.tokenizer.backtrack();
446                        }
447                    }
448                    _ => {}
449                }
450            } else if converted == ConvertedType::DECIMAL {
451                if self.tokenizer.next() == Some("(") {
452                    // Parse precision
453                    precision = parse_i32(
454                        self.tokenizer.next(),
455                        "Expected precision, found None",
456                        "Failed to parse precision for DECIMAL type",
457                    )?;
458
459                    // Parse scale
460                    scale = if self.tokenizer.next() == Some(",") {
461                        parse_i32(
462                            self.tokenizer.next(),
463                            "Expected scale, found None",
464                            "Failed to parse scale for DECIMAL type",
465                        )?
466                    } else {
467                        // Scale is not provided, set it to 0.
468                        self.tokenizer.backtrack();
469                        0
470                    };
471
472                    assert_token(self.tokenizer.next(), ")")?;
473                } else {
474                    self.tokenizer.backtrack();
475                }
476            }
477
478            assert_token(self.tokenizer.next(), ")")?;
479            (logical, converted, precision, scale)
480        } else {
481            self.tokenizer.backtrack();
482            (None, ConvertedType::NONE, -1, -1)
483        };
484
485        // Parse optional id
486        let id = if self.tokenizer.next() == Some("=") {
487            self.tokenizer.next().and_then(|v| v.parse::<i32>().ok())
488        } else {
489            self.tokenizer.backtrack();
490            None
491        };
492        assert_token(self.tokenizer.next(), ";")?;
493
494        Type::primitive_type_builder(name, physical_type)
495            .with_repetition(repetition)
496            .with_logical_type(logical_type)
497            .with_converted_type(converted_type)
498            .with_length(length)
499            .with_precision(precision)
500            .with_scale(scale)
501            .with_id(id)
502            .build()
503    }
504}
505
506#[cfg(test)]
507mod tests {
508    use super::*;
509
510    #[test]
511    fn test_tokenize_empty_string() {
512        assert_eq!(Tokenizer::from_str("").next(), None);
513    }
514
515    #[test]
516    fn test_tokenize_delimiters() {
517        let mut iter = Tokenizer::from_str(",;{}()=");
518        assert_eq!(iter.next(), Some(","));
519        assert_eq!(iter.next(), Some(";"));
520        assert_eq!(iter.next(), Some("{"));
521        assert_eq!(iter.next(), Some("}"));
522        assert_eq!(iter.next(), Some("("));
523        assert_eq!(iter.next(), Some(")"));
524        assert_eq!(iter.next(), Some("="));
525        assert_eq!(iter.next(), None);
526    }
527
528    #[test]
529    fn test_tokenize_delimiters_with_whitespaces() {
530        let mut iter = Tokenizer::from_str(" , ; { } ( ) = ");
531        assert_eq!(iter.next(), Some(","));
532        assert_eq!(iter.next(), Some(";"));
533        assert_eq!(iter.next(), Some("{"));
534        assert_eq!(iter.next(), Some("}"));
535        assert_eq!(iter.next(), Some("("));
536        assert_eq!(iter.next(), Some(")"));
537        assert_eq!(iter.next(), Some("="));
538        assert_eq!(iter.next(), None);
539    }
540
541    #[test]
542    fn test_tokenize_words() {
543        let mut iter = Tokenizer::from_str("abc def ghi jkl mno");
544        assert_eq!(iter.next(), Some("abc"));
545        assert_eq!(iter.next(), Some("def"));
546        assert_eq!(iter.next(), Some("ghi"));
547        assert_eq!(iter.next(), Some("jkl"));
548        assert_eq!(iter.next(), Some("mno"));
549        assert_eq!(iter.next(), None);
550    }
551
552    #[test]
553    fn test_tokenize_backtrack() {
554        let mut iter = Tokenizer::from_str("abc;");
555        assert_eq!(iter.next(), Some("abc"));
556        assert_eq!(iter.next(), Some(";"));
557        iter.backtrack();
558        assert_eq!(iter.next(), Some(";"));
559        assert_eq!(iter.next(), None);
560    }
561
562    #[test]
563    fn test_tokenize_message_type() {
564        let schema = "
565    message schema {
566      required int32 a;
567      optional binary c (UTF8);
568      required group d {
569        required int32 a;
570        optional binary c (UTF8);
571      }
572      required group e (LIST) {
573        repeated group list {
574          required int32 element;
575        }
576      }
577    }
578    ";
579        let iter = Tokenizer::from_str(schema);
580        let mut res = Vec::new();
581        for token in iter {
582            res.push(token);
583        }
584        assert_eq!(
585            res,
586            vec![
587                "message", "schema", "{", "required", "int32", "a", ";", "optional", "binary", "c",
588                "(", "UTF8", ")", ";", "required", "group", "d", "{", "required", "int32", "a",
589                ";", "optional", "binary", "c", "(", "UTF8", ")", ";", "}", "required", "group",
590                "e", "(", "LIST", ")", "{", "repeated", "group", "list", "{", "required", "int32",
591                "element", ";", "}", "}", "}"
592            ]
593        );
594    }
595
596    #[test]
597    fn test_assert_token() {
598        assert!(assert_token(Some("a"), "a").is_ok());
599        assert!(assert_token(Some("a"), "b").is_err());
600        assert!(assert_token(None, "b").is_err());
601    }
602
603    fn parse(schema: &str) -> Result<Type, ParquetError> {
604        let mut iter = Tokenizer::from_str(schema);
605        Parser {
606            tokenizer: &mut iter,
607        }
608        .parse_message_type()
609    }
610
611    #[test]
612    fn test_parse_message_type_invalid() {
613        assert_eq!(
614            parse("test").unwrap_err().to_string(),
615            "Parquet error: Message type does not start with 'message'"
616        );
617    }
618
619    #[test]
620    fn test_parse_message_type_no_name() {
621        assert_eq!(
622            parse("message").unwrap_err().to_string(),
623            "Parquet error: Expected name, found None"
624        );
625    }
626
627    #[test]
628    fn test_parse_message_type_fixed_byte_array() {
629        let schema = "
630            message schema {
631              REQUIRED FIXED_LEN_BYTE_ARRAY col;
632            }
633        ";
634        assert_eq!(
635            parse(schema).unwrap_err().to_string(),
636            "Parquet error: Expected '(', found token 'col'"
637        );
638
639        let schema = "
640            message schema {
641              REQUIRED FIXED_LEN_BYTE_ARRAY(16) col;
642            }
643        ";
644        parse(schema).unwrap();
645    }
646
647    #[test]
648    fn test_parse_message_type_integer() {
649        // Invalid integer syntax
650        let schema = "
651            message root {
652              optional int64 f1 (INTEGER());
653            }
654        ";
655        assert_eq!(
656            parse(schema).unwrap_err().to_string(),
657            "Parquet error: Failed to parse bit_width for INTEGER type"
658        );
659
660        // Invalid integer syntax, needs both bit-width and UTC sign
661        let schema = "
662    message root {
663      optional int64 f1 (INTEGER(32,));
664    }
665    ";
666        assert_eq!(
667            parse(schema).unwrap_err().to_string(),
668            "Parquet error: Incorrect bit width 32 for INT64"
669        );
670
671        // Invalid integer because of non-numeric bit width
672        let schema = "
673            message root {
674              optional int32 f1 (INTEGER(eight,true));
675            }
676        ";
677        assert_eq!(
678            parse(schema).unwrap_err().to_string(),
679            "Parquet error: Failed to parse bit_width for INTEGER type"
680        );
681
682        // Valid types
683        let schema = "
684            message root {
685              optional int32 f1 (INTEGER(8,false));
686              optional int32 f2 (INTEGER(8,true));
687              optional int32 f3 (INTEGER(16,false));
688              optional int32 f4 (INTEGER(16,true));
689              optional int32 f5 (INTEGER(32,false));
690              optional int32 f6 (INTEGER(32,true));
691              optional int64 f7 (INTEGER(64,false));
692              optional int64 f7 (INTEGER(64,true));
693            }
694        ";
695        parse(schema).unwrap();
696    }
697
698    #[test]
699    fn test_parse_message_type_temporal() {
700        // Invalid timestamp syntax
701        let schema = "
702            message root {
703              optional int64 f1 (TIMESTAMP();
704            }
705        ";
706        assert_eq!(
707            parse(schema).unwrap_err().to_string(),
708            "Parquet error: Failed to parse timeunit for TIMESTAMP type"
709        );
710
711        // Invalid timestamp syntax, needs both unit and UTC adjustment
712        let schema = "
713            message root {
714              optional int64 f1 (TIMESTAMP(MILLIS,));
715            }
716        ";
717        assert_eq!(
718            parse(schema).unwrap_err().to_string(),
719            "Parquet error: Failed to parse timezone info for TIMESTAMP type"
720        );
721
722        // Invalid timestamp because of unknown unit
723        let schema = "
724            message root {
725              optional int64 f1 (TIMESTAMP(YOCTOS,));
726            }
727        ";
728
729        assert_eq!(
730            parse(schema).unwrap_err().to_string(),
731            "Parquet error: Failed to parse timeunit for TIMESTAMP type"
732        );
733
734        // Valid types
735        let schema = "
736            message root {
737              optional int32 f1 (DATE);
738              optional int32 f2 (TIME(MILLIS,true));
739              optional int64 f3 (TIME(MICROS,false));
740              optional int64 f4 (TIME(NANOS,true));
741              optional int64 f5 (TIMESTAMP(MILLIS,true));
742              optional int64 f6 (TIMESTAMP(MICROS,true));
743              optional int64 f7 (TIMESTAMP(NANOS,false));
744            }
745        ";
746        parse(schema).unwrap();
747    }
748
749    #[test]
750    fn test_parse_message_type_decimal() {
751        // It is okay for decimal to omit precision and scale with right syntax.
752        // Here we test wrong syntax of decimal type
753
754        // Invalid decimal syntax
755        let schema = "
756            message root {
757              optional int32 f1 (DECIMAL();
758            }
759        ";
760        assert_eq!(
761            parse(schema).unwrap_err().to_string(),
762            "Parquet error: Failed to parse precision for DECIMAL type"
763        );
764
765        // Invalid decimal, need precision and scale
766        let schema = "
767            message root {
768              optional int32 f1 (DECIMAL());
769            }
770        ";
771        assert_eq!(
772            parse(schema).unwrap_err().to_string(),
773            "Parquet error: Failed to parse precision for DECIMAL type"
774        );
775
776        // Invalid decimal because of `,` - has precision, needs scale
777        let schema = "
778            message root {
779              optional int32 f1 (DECIMAL(8,));
780            }
781        ";
782        assert_eq!(
783            parse(schema).unwrap_err().to_string(),
784            "Parquet error: Failed to parse scale for DECIMAL type"
785        );
786
787        // Invalid decimal because, we always require either precision or scale to be
788        // specified as part of converted type
789        let schema = "
790            message root {
791              optional int32 f3 (DECIMAL);
792            }
793        ";
794        assert_eq!(
795            parse(schema).unwrap_err().to_string(),
796            "Parquet error: Expected ')', found token ';'"
797        );
798
799        // Valid decimal (precision, scale)
800        let schema = "
801            message root {
802              optional int32 f1 (DECIMAL(8, 3));
803              optional int32 f2 (DECIMAL(8));
804            }
805        ";
806        parse(schema).unwrap();
807    }
808
809    #[test]
810    fn test_parse_message_type_compare_1() {
811        let schema = "
812            message root {
813              optional fixed_len_byte_array(5) f1 (DECIMAL(9, 3));
814              optional fixed_len_byte_array (16) f2 (DECIMAL (38, 18));
815              optional fixed_len_byte_array (2) f3 (FLOAT16);
816            }
817        ";
818        let message = parse(schema).unwrap();
819
820        let expected = Type::group_type_builder("root")
821            .with_fields(vec![
822                Arc::new(
823                    Type::primitive_type_builder("f1", PhysicalType::FIXED_LEN_BYTE_ARRAY)
824                        .with_logical_type(Some(LogicalType::decimal(3, 9)))
825                        .with_converted_type(ConvertedType::DECIMAL)
826                        .with_length(5)
827                        .with_precision(9)
828                        .with_scale(3)
829                        .build()
830                        .unwrap(),
831                ),
832                Arc::new(
833                    Type::primitive_type_builder("f2", PhysicalType::FIXED_LEN_BYTE_ARRAY)
834                        .with_logical_type(Some(LogicalType::decimal(18, 38)))
835                        .with_converted_type(ConvertedType::DECIMAL)
836                        .with_length(16)
837                        .with_precision(38)
838                        .with_scale(18)
839                        .build()
840                        .unwrap(),
841                ),
842                Arc::new(
843                    Type::primitive_type_builder("f3", PhysicalType::FIXED_LEN_BYTE_ARRAY)
844                        .with_logical_type(Some(LogicalType::Float16))
845                        .with_length(2)
846                        .build()
847                        .unwrap(),
848                ),
849            ])
850            .build()
851            .unwrap();
852
853        assert_eq!(message, expected);
854    }
855
856    #[test]
857    fn test_parse_message_type_compare_2() {
858        let schema = "
859            message root {
860              required group a0 {
861                optional group a1 (LIST) {
862                  repeated binary a2 (UTF8);
863                }
864
865                optional group b1 (LIST) {
866                  repeated group b2 {
867                    optional int32 b3;
868                    optional double b4;
869                  }
870                }
871              }
872            }
873        ";
874        let message = parse(schema).unwrap();
875
876        let expected = Type::group_type_builder("root")
877            .with_fields(vec![Arc::new(
878                Type::group_type_builder("a0")
879                    .with_repetition(Repetition::REQUIRED)
880                    .with_fields(vec![
881                        Arc::new(
882                            Type::group_type_builder("a1")
883                                .with_repetition(Repetition::OPTIONAL)
884                                .with_logical_type(Some(LogicalType::List))
885                                .with_converted_type(ConvertedType::LIST)
886                                .with_fields(vec![Arc::new(
887                                    Type::primitive_type_builder("a2", PhysicalType::BYTE_ARRAY)
888                                        .with_repetition(Repetition::REPEATED)
889                                        .with_converted_type(ConvertedType::UTF8)
890                                        .build()
891                                        .unwrap(),
892                                )])
893                                .build()
894                                .unwrap(),
895                        ),
896                        Arc::new(
897                            Type::group_type_builder("b1")
898                                .with_repetition(Repetition::OPTIONAL)
899                                .with_logical_type(Some(LogicalType::List))
900                                .with_converted_type(ConvertedType::LIST)
901                                .with_fields(vec![Arc::new(
902                                    Type::group_type_builder("b2")
903                                        .with_repetition(Repetition::REPEATED)
904                                        .with_fields(vec![
905                                            Arc::new(
906                                                Type::primitive_type_builder(
907                                                    "b3",
908                                                    PhysicalType::INT32,
909                                                )
910                                                .build()
911                                                .unwrap(),
912                                            ),
913                                            Arc::new(
914                                                Type::primitive_type_builder(
915                                                    "b4",
916                                                    PhysicalType::DOUBLE,
917                                                )
918                                                .build()
919                                                .unwrap(),
920                                            ),
921                                        ])
922                                        .build()
923                                        .unwrap(),
924                                )])
925                                .build()
926                                .unwrap(),
927                        ),
928                    ])
929                    .build()
930                    .unwrap(),
931            )])
932            .build()
933            .unwrap();
934
935        assert_eq!(message, expected);
936    }
937
938    #[test]
939    fn test_parse_message_type_compare_3() {
940        let schema = "
941            message root {
942              required int32 _1 (INT_8);
943              required int32 _2 (INT_16);
944              required float _3;
945              required double _4;
946              optional int32 _5 (DATE);
947              optional binary _6 (UTF8);
948            }
949        ";
950        let message = parse(schema).unwrap();
951
952        let fields = vec![
953            Arc::new(
954                Type::primitive_type_builder("_1", PhysicalType::INT32)
955                    .with_repetition(Repetition::REQUIRED)
956                    .with_converted_type(ConvertedType::INT_8)
957                    .build()
958                    .unwrap(),
959            ),
960            Arc::new(
961                Type::primitive_type_builder("_2", PhysicalType::INT32)
962                    .with_repetition(Repetition::REQUIRED)
963                    .with_converted_type(ConvertedType::INT_16)
964                    .build()
965                    .unwrap(),
966            ),
967            Arc::new(
968                Type::primitive_type_builder("_3", PhysicalType::FLOAT)
969                    .with_repetition(Repetition::REQUIRED)
970                    .build()
971                    .unwrap(),
972            ),
973            Arc::new(
974                Type::primitive_type_builder("_4", PhysicalType::DOUBLE)
975                    .with_repetition(Repetition::REQUIRED)
976                    .build()
977                    .unwrap(),
978            ),
979            Arc::new(
980                Type::primitive_type_builder("_5", PhysicalType::INT32)
981                    .with_logical_type(Some(LogicalType::Date))
982                    .with_converted_type(ConvertedType::DATE)
983                    .build()
984                    .unwrap(),
985            ),
986            Arc::new(
987                Type::primitive_type_builder("_6", PhysicalType::BYTE_ARRAY)
988                    .with_converted_type(ConvertedType::UTF8)
989                    .build()
990                    .unwrap(),
991            ),
992        ];
993
994        let expected = Type::group_type_builder("root")
995            .with_fields(fields)
996            .build()
997            .unwrap();
998        assert_eq!(message, expected);
999    }
1000
1001    #[test]
1002    fn test_parse_message_type_compare_4() {
1003        let schema = "
1004            message root {
1005              required int32 _1 (INTEGER(8,true));
1006              required int32 _2 (INTEGER(16,false));
1007              required float _3;
1008              required double _4;
1009              optional int32 _5 (DATE);
1010              optional int32 _6 (TIME(MILLIS,false));
1011              optional int64 _7 (TIME(MICROS,true));
1012              optional int64 _8 (TIMESTAMP(MILLIS,true));
1013              optional int64 _9 (TIMESTAMP(NANOS,false));
1014              optional binary _10 (STRING);
1015            }
1016        ";
1017        let message = parse(schema).unwrap();
1018
1019        let fields = vec![
1020            Arc::new(
1021                Type::primitive_type_builder("_1", PhysicalType::INT32)
1022                    .with_repetition(Repetition::REQUIRED)
1023                    .with_logical_type(Some(LogicalType::integer(8, true)))
1024                    .build()
1025                    .unwrap(),
1026            ),
1027            Arc::new(
1028                Type::primitive_type_builder("_2", PhysicalType::INT32)
1029                    .with_repetition(Repetition::REQUIRED)
1030                    .with_logical_type(Some(LogicalType::integer(16, false)))
1031                    .build()
1032                    .unwrap(),
1033            ),
1034            Arc::new(
1035                Type::primitive_type_builder("_3", PhysicalType::FLOAT)
1036                    .with_repetition(Repetition::REQUIRED)
1037                    .build()
1038                    .unwrap(),
1039            ),
1040            Arc::new(
1041                Type::primitive_type_builder("_4", PhysicalType::DOUBLE)
1042                    .with_repetition(Repetition::REQUIRED)
1043                    .build()
1044                    .unwrap(),
1045            ),
1046            Arc::new(
1047                Type::primitive_type_builder("_5", PhysicalType::INT32)
1048                    .with_logical_type(Some(LogicalType::Date))
1049                    .build()
1050                    .unwrap(),
1051            ),
1052            Arc::new(
1053                Type::primitive_type_builder("_6", PhysicalType::INT32)
1054                    .with_logical_type(Some(LogicalType::time(false, TimeUnit::MILLIS)))
1055                    .build()
1056                    .unwrap(),
1057            ),
1058            Arc::new(
1059                Type::primitive_type_builder("_7", PhysicalType::INT64)
1060                    .with_logical_type(Some(LogicalType::time(true, TimeUnit::MICROS)))
1061                    .build()
1062                    .unwrap(),
1063            ),
1064            Arc::new(
1065                Type::primitive_type_builder("_8", PhysicalType::INT64)
1066                    .with_logical_type(Some(LogicalType::timestamp(true, TimeUnit::MILLIS)))
1067                    .build()
1068                    .unwrap(),
1069            ),
1070            Arc::new(
1071                Type::primitive_type_builder("_9", PhysicalType::INT64)
1072                    .with_logical_type(Some(LogicalType::timestamp(false, TimeUnit::NANOS)))
1073                    .build()
1074                    .unwrap(),
1075            ),
1076            Arc::new(
1077                Type::primitive_type_builder("_10", PhysicalType::BYTE_ARRAY)
1078                    .with_logical_type(Some(LogicalType::String))
1079                    .build()
1080                    .unwrap(),
1081            ),
1082        ];
1083
1084        let expected = Type::group_type_builder("root")
1085            .with_fields(fields)
1086            .build()
1087            .unwrap();
1088        assert_eq!(message, expected);
1089    }
1090}
parquet/schema/parser.rs

parquet/schema/
parser.rs
