arrow_schema/
datatype_parse.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::{fmt::Display, iter::Peekable, str::Chars, sync::Arc};
19
20use crate::{
21    ArrowError, DECIMAL32_MAX_PRECISION, DECIMAL64_MAX_PRECISION, DECIMAL128_MAX_PRECISION,
22    DECIMAL256_MAX_PRECISION, DataType, Field, Fields, IntervalUnit, TimeUnit, UnionFields,
23    UnionMode,
24};
25
26/// Parses a DataType from a string representation
27///
28/// For example, the string "Int32" would be parsed into [`DataType::Int32`]
29pub(crate) fn parse_data_type(val: &str) -> ArrowResult<DataType> {
30    Parser::new(val).parse()
31}
32
33type ArrowResult<T> = Result<T, ArrowError>;
34
35fn make_error(val: &str, msg: &str) -> ArrowError {
36    let msg = format!(
37        "Unsupported type '{val}'. Must be a supported arrow type name such as 'Int32' or 'Timestamp(ns)'. Error {msg}"
38    );
39    ArrowError::ParseError(msg)
40}
41
42fn make_error_expected(val: &str, expected: &Token, actual: &Token) -> ArrowError {
43    make_error(val, &format!("Expected '{expected}', got '{actual}'"))
44}
45
46/// Implementation of `parse_data_type`, modeled after <https://github.com/sqlparser-rs/sqlparser-rs>
47#[derive(Debug)]
48struct Parser<'a> {
49    val: &'a str,
50    tokenizer: Peekable<Tokenizer<'a>>,
51}
52
53impl<'a> Parser<'a> {
54    fn new(val: &'a str) -> Self {
55        Self {
56            val,
57            tokenizer: Tokenizer::new(val).peekable(),
58        }
59    }
60
61    fn parse(mut self) -> ArrowResult<DataType> {
62        let data_type = self.parse_next_type()?;
63        // ensure that there is no trailing content
64        if self.tokenizer.next().is_some() {
65            Err(make_error(
66                self.val,
67                &format!("checking trailing content after parsing '{data_type}'"),
68            ))
69        } else {
70            Ok(data_type)
71        }
72    }
73
74    /// parses the next full DataType
75    fn parse_next_type(&mut self) -> ArrowResult<DataType> {
76        match self.next_token()? {
77            Token::SimpleType(data_type) => Ok(data_type),
78            Token::Timestamp => self.parse_timestamp(),
79            Token::Time32 => self.parse_time32(),
80            Token::Time64 => self.parse_time64(),
81            Token::Duration => self.parse_duration(),
82            Token::Interval => self.parse_interval(),
83            Token::FixedSizeBinary => self.parse_fixed_size_binary(),
84            Token::Decimal32 => self.parse_decimal_32(),
85            Token::Decimal64 => self.parse_decimal_64(),
86            Token::Decimal128 => self.parse_decimal_128(),
87            Token::Decimal256 => self.parse_decimal_256(),
88            Token::Dictionary => self.parse_dictionary(),
89            Token::List => self.parse_list(),
90            Token::ListView => self.parse_list_view(),
91            Token::LargeList => self.parse_large_list(),
92            Token::LargeListView => self.parse_large_list_view(),
93            Token::FixedSizeList => self.parse_fixed_size_list(),
94            Token::Struct => self.parse_struct(),
95            Token::Union => self.parse_union(),
96            Token::Map => self.parse_map(),
97            Token::RunEndEncoded => self.parse_run_end_encoded(),
98            tok => Err(make_error(
99                self.val,
100                &format!("finding next type, got unexpected '{tok}'"),
101            )),
102        }
103    }
104
105    /// parses Field, this is the inversion of `format_field` in `datatype_display.rs`.
106    /// E.g: "a": non-null Int64
107    ///
108    /// TODO: support metadata: `"a": non-null Int64 metadata: {"foo": "value"}`
109    fn parse_field(&mut self) -> ArrowResult<Field> {
110        let name = self.parse_double_quoted_string("Field")?;
111        self.expect_token(Token::Colon)?;
112        let nullable = self.parse_opt_nullable();
113        let data_type = self.parse_next_type()?;
114        Ok(Field::new(name, data_type, nullable))
115    }
116
117    /// Parses field inside a list. Use `Field::LIST_FIELD_DEFAULT_NAME`
118    /// if no field name is specified.
119    /// E.g: `non-null Int64, field: 'foo'` or `non-null Int64`
120    ///
121    /// TODO: support metadata: `non-ull Int64, metadata: {"foo2": "value"}`
122    fn parse_list_field(&mut self, context: &str) -> ArrowResult<Field> {
123        let nullable = self.parse_opt_nullable();
124        let data_type = self.parse_next_type()?;
125
126        // the field name (if exists) must be after a comma
127        let field_name = if self
128            .tokenizer
129            .next_if(|next| matches!(next, Ok(Token::Comma)))
130            .is_none()
131        {
132            Field::LIST_FIELD_DEFAULT_NAME.into()
133        } else {
134            // expects: `field: 'field_name'`.
135            self.expect_token(Token::Field)?;
136            self.expect_token(Token::Colon)?;
137            self.parse_single_quoted_string(context)?
138        };
139
140        Ok(Field::new(field_name, data_type, nullable))
141    }
142
143    /// Parses the List type (called after `List` has been consumed)
144    /// E.g: List(non-null Int64, field: 'foo')
145    fn parse_list(&mut self) -> ArrowResult<DataType> {
146        self.expect_token(Token::LParen)?;
147        let field = self.parse_list_field("List")?;
148        self.expect_token(Token::RParen)?;
149        Ok(DataType::List(Arc::new(field)))
150    }
151
152    /// Parses the ListView type (called after `ListView` has been consumed)
153    /// E.g: ListView(non-null Int64, field: 'foo')
154    fn parse_list_view(&mut self) -> ArrowResult<DataType> {
155        self.expect_token(Token::LParen)?;
156        let field = self.parse_list_field("ListView")?;
157        self.expect_token(Token::RParen)?;
158        Ok(DataType::ListView(Arc::new(field)))
159    }
160
161    /// Parses the LargeList type (called after `LargeList` has been consumed)
162    /// E.g: LargeList(non-null Int64, field: 'foo')
163    fn parse_large_list(&mut self) -> ArrowResult<DataType> {
164        self.expect_token(Token::LParen)?;
165        let field = self.parse_list_field("LargeList")?;
166        self.expect_token(Token::RParen)?;
167        Ok(DataType::LargeList(Arc::new(field)))
168    }
169
170    /// Parses the LargeListView type (called after `LargeListView` has been consumed)
171    /// E.g: LargeListView(non-null Int64, field: 'foo')
172    fn parse_large_list_view(&mut self) -> ArrowResult<DataType> {
173        self.expect_token(Token::LParen)?;
174        let field = self.parse_list_field("LargeListView")?;
175        self.expect_token(Token::RParen)?;
176        Ok(DataType::LargeListView(Arc::new(field)))
177    }
178
179    /// Parses the FixedSizeList type (called after `FixedSizeList` has been consumed)
180    ///
181    /// Examples:
182    /// * `FixedSizeList(5 x non-null Int64, field: 'foo')`
183    /// * `FixedSizeList(4, Int64)`
184    ///
185    fn parse_fixed_size_list(&mut self) -> ArrowResult<DataType> {
186        self.expect_token(Token::LParen)?;
187        let length = self.parse_i32("FixedSizeList")?;
188        if length < 0 {
189            return Err(make_error(
190                self.val,
191                &format!("FixedSizeList length must be non-negative, got {length}"),
192            ));
193        }
194        match self.next_token()? {
195            // `FixedSizeList(5 x non-null Int64, field: 'foo')` format
196            Token::X => {
197                let field = self.parse_list_field("FixedSizeList")?;
198                self.expect_token(Token::RParen)?;
199                Ok(DataType::FixedSizeList(Arc::new(field), length))
200            }
201            // `FixedSizeList(4, Int64)` format
202            Token::Comma => {
203                let data_type = self.parse_next_type()?;
204                self.expect_token(Token::RParen)?;
205                Ok(DataType::FixedSizeList(
206                    Arc::new(Field::new_list_field(data_type, true)),
207                    length,
208                ))
209            }
210            tok => Err(make_error(
211                self.val,
212                &format!("Expected 'x' or ',' after length for FixedSizeList, got '{tok}'"),
213            )),
214        }
215    }
216
217    /// Parses the next timeunit
218    fn parse_time_unit(&mut self, context: &str) -> ArrowResult<TimeUnit> {
219        match self.next_token()? {
220            Token::TimeUnit(time_unit) => Ok(time_unit),
221            tok => Err(make_error(
222                self.val,
223                &format!("finding TimeUnit for {context}, got {tok}"),
224            )),
225        }
226    }
227
228    /// Parses the next double quoted string
229    fn parse_double_quoted_string(&mut self, context: &str) -> ArrowResult<String> {
230        let token = self.next_token()?;
231        if let Token::DoubleQuotedString(string) = token {
232            Ok(string)
233        } else {
234            Err(make_error(
235                self.val,
236                &format!("expected double quoted string for {context}, got '{token}'"),
237            ))
238        }
239    }
240
241    /// Parses the next single quoted string
242    fn parse_single_quoted_string(&mut self, context: &str) -> ArrowResult<String> {
243        let token = self.next_token()?;
244        if let Token::SingleQuotedString(string) = token {
245            Ok(string)
246        } else {
247            Err(make_error(
248                self.val,
249                &format!("expected single quoted string for {context}, got '{token}'"),
250            ))
251        }
252    }
253
254    /// Parses the next integer value
255    fn parse_i64(&mut self, context: &str) -> ArrowResult<i64> {
256        match self.next_token()? {
257            Token::Integer(v) => Ok(v),
258            tok => Err(make_error(
259                self.val,
260                &format!("finding i64 for {context}, got '{tok}'"),
261            )),
262        }
263    }
264
265    /// Parses the next i32 integer value
266    fn parse_i32(&mut self, context: &str) -> ArrowResult<i32> {
267        let length = self.parse_i64(context)?;
268        length.try_into().map_err(|e| {
269            make_error(
270                self.val,
271                &format!("converting {length} into i32 for {context}: {e}"),
272            )
273        })
274    }
275
276    /// Parses the next i8 integer value
277    fn parse_i8(&mut self, context: &str) -> ArrowResult<i8> {
278        let length = self.parse_i64(context)?;
279        length.try_into().map_err(|e| {
280            make_error(
281                self.val,
282                &format!("converting {length} into i8 for {context}: {e}"),
283            )
284        })
285    }
286
287    /// Parses the next u8 integer value
288    fn parse_u8(&mut self, context: &str) -> ArrowResult<u8> {
289        let length = self.parse_i64(context)?;
290        length.try_into().map_err(|e| {
291            make_error(
292                self.val,
293                &format!("converting {length} into u8 for {context}: {e}"),
294            )
295        })
296    }
297
298    /// Parses the next timestamp (called after `Timestamp` has been consumed)
299    fn parse_timestamp(&mut self) -> ArrowResult<DataType> {
300        self.expect_token(Token::LParen)?;
301        let time_unit = self.parse_time_unit("Timestamp")?;
302
303        let timezone;
304        match self.next_token()? {
305            Token::Comma => {
306                match self.next_token()? {
307                    // Support old style `Timestamp(Nanosecond, None)`
308                    Token::None => {
309                        timezone = None;
310                    }
311                    // Support old style `Timestamp(Nanosecond, Some("Timezone"))`
312                    Token::Some => {
313                        self.expect_token(Token::LParen)?;
314                        timezone = Some(self.parse_double_quoted_string("Timezone")?);
315                        self.expect_token(Token::RParen)?;
316                    }
317                    Token::DoubleQuotedString(tz) => {
318                        // Support new style `Timestamp(Nanosecond, "Timezone")`
319                        timezone = Some(tz);
320                    }
321                    tok => {
322                        return Err(make_error(
323                            self.val,
324                            &format!("Expected None, Some, or a timezone string, got {tok:?}"),
325                        ));
326                    }
327                }
328                self.expect_token(Token::RParen)?;
329            }
330            // No timezone (e.g `Timestamp(ns)`)
331            Token::RParen => {
332                timezone = None;
333            }
334            next_token => {
335                return Err(make_error(
336                    self.val,
337                    &format!("Expected comma followed by a timezone, or an ), got {next_token:?}"),
338                ));
339            }
340        }
341        Ok(DataType::Timestamp(time_unit, timezone.map(Into::into)))
342    }
343
344    /// Parses the next Time32 (called after `Time32` has been consumed)
345    fn parse_time32(&mut self) -> ArrowResult<DataType> {
346        self.expect_token(Token::LParen)?;
347        let time_unit = self.parse_time_unit("Time32")?;
348        match time_unit {
349            TimeUnit::Second | TimeUnit::Millisecond => (),
350            TimeUnit::Microsecond | TimeUnit::Nanosecond => {
351                return Err(make_error(
352                    self.val,
353                    &format!("Time32 time unit must be 's' or 'ms', got '{time_unit}'"),
354                ));
355            }
356        }
357        self.expect_token(Token::RParen)?;
358        Ok(DataType::Time32(time_unit))
359    }
360
361    /// Parses the next Time64 (called after `Time64` has been consumed)
362    fn parse_time64(&mut self) -> ArrowResult<DataType> {
363        self.expect_token(Token::LParen)?;
364        let time_unit = self.parse_time_unit("Time64")?;
365        match time_unit {
366            TimeUnit::Microsecond | TimeUnit::Nanosecond => (),
367            TimeUnit::Second | TimeUnit::Millisecond => {
368                return Err(make_error(
369                    self.val,
370                    &format!("Time64 time unit must be 'µs' or 'ns', got '{time_unit}'"),
371                ));
372            }
373        }
374        self.expect_token(Token::RParen)?;
375        Ok(DataType::Time64(time_unit))
376    }
377
378    /// Parses the next Duration (called after `Duration` has been consumed)
379    fn parse_duration(&mut self) -> ArrowResult<DataType> {
380        self.expect_token(Token::LParen)?;
381        let time_unit = self.parse_time_unit("Duration")?;
382        self.expect_token(Token::RParen)?;
383        Ok(DataType::Duration(time_unit))
384    }
385
386    /// Parses the next Interval (called after `Interval` has been consumed)
387    fn parse_interval(&mut self) -> ArrowResult<DataType> {
388        self.expect_token(Token::LParen)?;
389        let interval_unit = match self.next_token()? {
390            Token::IntervalUnit(interval_unit) => interval_unit,
391            tok => {
392                return Err(make_error(
393                    self.val,
394                    &format!("finding IntervalUnit for Interval, got {tok}"),
395                ));
396            }
397        };
398        self.expect_token(Token::RParen)?;
399        Ok(DataType::Interval(interval_unit))
400    }
401
402    /// Parses the next FixedSizeBinary (called after `FixedSizeBinary` has been consumed)
403    fn parse_fixed_size_binary(&mut self) -> ArrowResult<DataType> {
404        self.expect_token(Token::LParen)?;
405        let length = self.parse_i32("FixedSizeBinary")?;
406        if length < 0 {
407            return Err(make_error(
408                self.val,
409                &format!("FixedSizeBinary length must be non-negative, got {length}"),
410            ));
411        }
412        self.expect_token(Token::RParen)?;
413        Ok(DataType::FixedSizeBinary(length))
414    }
415
416    fn validate_decimal(
417        &self,
418        precision: u8,
419        scale: i8,
420        type_name: &str,
421        max_precision: u8,
422    ) -> ArrowResult<()> {
423        if precision == 0 || precision > max_precision {
424            return Err(make_error(
425                self.val,
426                &format!(
427                    "{type_name} precision must be in range [1, {max_precision}], got '{precision}'"
428                ),
429            ));
430        }
431        if scale > 0 && scale as u8 > precision {
432            return Err(make_error(
433                self.val,
434                &format!(
435                    "{type_name} scale '{scale}' cannot be greater than precision '{precision}'"
436                ),
437            ));
438        }
439        Ok(())
440    }
441
442    /// Parses the next Decimal32 (called after `Decimal32` has been consumed)
443    fn parse_decimal_32(&mut self) -> ArrowResult<DataType> {
444        self.expect_token(Token::LParen)?;
445        let precision = self.parse_u8("Decimal32")?;
446        self.expect_token(Token::Comma)?;
447        let scale = self.parse_i8("Decimal32")?;
448        self.expect_token(Token::RParen)?;
449        self.validate_decimal(precision, scale, "Decimal32", DECIMAL32_MAX_PRECISION)?;
450        Ok(DataType::Decimal32(precision, scale))
451    }
452
453    /// Parses the next Decimal64 (called after `Decimal64` has been consumed)
454    fn parse_decimal_64(&mut self) -> ArrowResult<DataType> {
455        self.expect_token(Token::LParen)?;
456        let precision = self.parse_u8("Decimal64")?;
457        self.expect_token(Token::Comma)?;
458        let scale = self.parse_i8("Decimal64")?;
459        self.expect_token(Token::RParen)?;
460        self.validate_decimal(precision, scale, "Decimal64", DECIMAL64_MAX_PRECISION)?;
461        Ok(DataType::Decimal64(precision, scale))
462    }
463
464    /// Parses the next Decimal128 (called after `Decimal128` has been consumed)
465    fn parse_decimal_128(&mut self) -> ArrowResult<DataType> {
466        self.expect_token(Token::LParen)?;
467        let precision = self.parse_u8("Decimal128")?;
468        self.expect_token(Token::Comma)?;
469        let scale = self.parse_i8("Decimal128")?;
470        self.expect_token(Token::RParen)?;
471        self.validate_decimal(precision, scale, "Decimal128", DECIMAL128_MAX_PRECISION)?;
472        Ok(DataType::Decimal128(precision, scale))
473    }
474
475    /// Parses the next Decimal256 (called after `Decimal256` has been consumed)
476    fn parse_decimal_256(&mut self) -> ArrowResult<DataType> {
477        self.expect_token(Token::LParen)?;
478        let precision = self.parse_u8("Decimal256")?;
479        self.expect_token(Token::Comma)?;
480        let scale = self.parse_i8("Decimal256")?;
481        self.expect_token(Token::RParen)?;
482        self.validate_decimal(precision, scale, "Decimal256", DECIMAL256_MAX_PRECISION)?;
483        Ok(DataType::Decimal256(precision, scale))
484    }
485
486    /// Parses the next Dictionary (called after `Dictionary` has been consumed)
487    fn parse_dictionary(&mut self) -> ArrowResult<DataType> {
488        self.expect_token(Token::LParen)?;
489        let key_type = self.parse_next_type()?;
490        self.expect_token(Token::Comma)?;
491        let value_type = self.parse_next_type()?;
492        self.expect_token(Token::RParen)?;
493        Ok(DataType::Dictionary(
494            Box::new(key_type),
495            Box::new(value_type),
496        ))
497    }
498
499    /// Parses the next Struct (called after `Struct` has been consumed)
500    fn parse_struct(&mut self) -> ArrowResult<DataType> {
501        self.expect_token(Token::LParen)?;
502        let mut fields = Vec::new();
503        loop {
504            if self
505                .tokenizer
506                .next_if(|next| matches!(next, Ok(Token::RParen)))
507                .is_some()
508            {
509                break;
510            }
511
512            let field = self.parse_field()?;
513            fields.push(Arc::new(field));
514            match self.next_token()? {
515                Token::Comma => {}
516                Token::RParen => break,
517                tok => {
518                    return Err(make_error(
519                        self.val,
520                        &format!(
521                            "Unexpected token while parsing Struct fields. Expected ',' or ')', but got '{tok}'"
522                        ),
523                    ));
524                }
525            }
526        }
527        Ok(DataType::Struct(Fields::from(fields)))
528    }
529
530    /// Parses the next Union (called after `Union` has been consumed)
531    /// E.g: Union(Sparse, 0: ("a": Int32), 1: ("b": non-null Utf8))
532    fn parse_union(&mut self) -> ArrowResult<DataType> {
533        self.expect_token(Token::LParen)?;
534        let union_mode = self.parse_union_mode()?;
535        let mut type_ids = vec![];
536        let mut fields = vec![];
537        loop {
538            if self
539                .tokenizer
540                .next_if(|next| matches!(next, Ok(Token::RParen)))
541                .is_some()
542            {
543                break;
544            }
545            self.expect_token(Token::Comma)?;
546            let (type_id, field) = self.parse_union_field()?;
547            type_ids.push(type_id);
548            fields.push(field);
549        }
550        Ok(DataType::Union(
551            UnionFields::try_new(type_ids, fields)?,
552            union_mode,
553        ))
554    }
555
556    /// Parses the next UnionMode
557    fn parse_union_mode(&mut self) -> ArrowResult<UnionMode> {
558        match self.next_token()? {
559            Token::UnionMode(union_mode) => Ok(union_mode),
560            tok => Err(make_error(
561                self.val,
562                &format!("finding UnionMode for Union, got {tok}"),
563            )),
564        }
565    }
566
567    /// Parses the next UnionField
568    /// 0: ("a": non-null Int32)
569    fn parse_union_field(&mut self) -> ArrowResult<(i8, Field)> {
570        let type_id = self.parse_i8("UnionField")?;
571        self.expect_token(Token::Colon)?;
572        self.expect_token(Token::LParen)?;
573        let field = self.parse_field()?;
574        self.expect_token(Token::RParen)?;
575        Ok((type_id, field))
576    }
577
578    /// Parses the next Map (called after `Map` has been consumed)
579    /// E.g: Map("entries": Struct("key": Utf8, "value": non-null Int32), sorted)
580    fn parse_map(&mut self) -> ArrowResult<DataType> {
581        self.expect_token(Token::LParen)?;
582        let field = self.parse_field()?;
583        self.expect_token(Token::Comma)?;
584        let sorted = self.parse_map_sorted()?;
585        self.expect_token(Token::RParen)?;
586        Ok(DataType::Map(Arc::new(field), sorted))
587    }
588
589    /// Parses map's sorted
590    fn parse_map_sorted(&mut self) -> ArrowResult<bool> {
591        match self.next_token()? {
592            Token::MapSorted(sorted) => Ok(sorted),
593            tok => Err(make_error(
594                self.val,
595                &format!("Expected sorted or unsorted for a map; got {tok:?}"),
596            )),
597        }
598    }
599
600    /// Parses the next RunEndEncoded (called after `RunEndEncoded` has been consumed)
601    /// E.g: RunEndEncoded("run_ends": UInt32, "values": nonnull Int32)
602    fn parse_run_end_encoded(&mut self) -> ArrowResult<DataType> {
603        self.expect_token(Token::LParen)?;
604        let run_ends = self.parse_field()?;
605        self.expect_token(Token::Comma)?;
606        let values = self.parse_field()?;
607        self.expect_token(Token::RParen)?;
608        Ok(DataType::RunEndEncoded(
609            Arc::new(run_ends),
610            Arc::new(values),
611        ))
612    }
613
614    /// consume the next token and return `false` if the field is `nonnull`.
615    fn parse_opt_nullable(&mut self) -> bool {
616        let tok = self
617            .tokenizer
618            .next_if(|next| matches!(next, Ok(Token::NonNull | Token::Nullable)));
619        !matches!(tok, Some(Ok(Token::NonNull)))
620    }
621
622    /// return the next token, or an error if there are none left
623    fn next_token(&mut self) -> ArrowResult<Token> {
624        match self.tokenizer.next() {
625            None => Err(make_error(self.val, "finding next token")),
626            Some(token) => token,
627        }
628    }
629
630    /// consume the next token, returning OK(()) if it matches tok, and Err if not
631    fn expect_token(&mut self, tok: Token) -> ArrowResult<()> {
632        let next_token = self.next_token()?;
633        if next_token == tok {
634            Ok(())
635        } else {
636            Err(make_error_expected(self.val, &tok, &next_token))
637        }
638    }
639}
640
641/// returns true if this character is a separator
642fn is_separator(c: char) -> bool {
643    c == '(' || c == ')' || c == ',' || c == ':' || c == ' '
644}
645
646enum QuoteType {
647    Double,
648    Single,
649}
650
651#[derive(Debug)]
652/// Splits a strings like Dictionary(Int32, Int64) into tokens suitable for parsing
653///
654/// For example the string "Timestamp(ns)" would be parsed into:
655///
656/// * Token::Timestamp
657/// * Token::Lparen
658/// * Token::IntervalUnit(IntervalUnit::Nanosecond)
659/// * Token::Rparen,
660struct Tokenizer<'a> {
661    val: &'a str,
662    chars: Peekable<Chars<'a>>,
663    // temporary buffer for parsing words
664    word: String,
665}
666
667impl<'a> Tokenizer<'a> {
668    fn new(val: &'a str) -> Self {
669        Self {
670            val,
671            chars: val.chars().peekable(),
672            word: String::new(),
673        }
674    }
675
676    /// returns the next char, without consuming it
677    fn peek_next_char(&mut self) -> Option<char> {
678        self.chars.peek().copied()
679    }
680
681    /// returns the next char, and consuming it
682    fn next_char(&mut self) -> Option<char> {
683        self.chars.next()
684    }
685
686    /// parse the characters in val starting at pos, until the next
687    /// `,`, `(`, or `)` or end of line
688    fn parse_word(&mut self) -> ArrowResult<Token> {
689        // reset temp space
690        self.word.clear();
691        loop {
692            match self.peek_next_char() {
693                None => break,
694                Some(c) if is_separator(c) => break,
695                Some(c) => {
696                    self.next_char();
697                    self.word.push(c);
698                }
699            }
700        }
701
702        if let Some(c) = self.word.chars().next() {
703            // if it started with a number, try parsing it as an integer
704            if c == '-' || c.is_numeric() {
705                let val: i64 = self.word.parse().map_err(|e| {
706                    make_error(self.val, &format!("parsing {} as integer: {e}", self.word))
707                })?;
708                return Ok(Token::Integer(val));
709            }
710        }
711
712        // figure out what the word was
713        let token = match self.word.as_str() {
714            "Null" => Token::SimpleType(DataType::Null),
715            "Boolean" => Token::SimpleType(DataType::Boolean),
716
717            "Int8" => Token::SimpleType(DataType::Int8),
718            "Int16" => Token::SimpleType(DataType::Int16),
719            "Int32" => Token::SimpleType(DataType::Int32),
720            "Int64" => Token::SimpleType(DataType::Int64),
721
722            "UInt8" => Token::SimpleType(DataType::UInt8),
723            "UInt16" => Token::SimpleType(DataType::UInt16),
724            "UInt32" => Token::SimpleType(DataType::UInt32),
725            "UInt64" => Token::SimpleType(DataType::UInt64),
726
727            "Utf8" => Token::SimpleType(DataType::Utf8),
728            "LargeUtf8" => Token::SimpleType(DataType::LargeUtf8),
729            "Utf8View" => Token::SimpleType(DataType::Utf8View),
730            "Binary" => Token::SimpleType(DataType::Binary),
731            "BinaryView" => Token::SimpleType(DataType::BinaryView),
732            "LargeBinary" => Token::SimpleType(DataType::LargeBinary),
733
734            "Float16" => Token::SimpleType(DataType::Float16),
735            "Float32" => Token::SimpleType(DataType::Float32),
736            "Float64" => Token::SimpleType(DataType::Float64),
737
738            "Date32" => Token::SimpleType(DataType::Date32),
739            "Date64" => Token::SimpleType(DataType::Date64),
740
741            "List" => Token::List,
742            "ListView" => Token::ListView,
743            "LargeList" => Token::LargeList,
744            "LargeListView" => Token::LargeListView,
745            "FixedSizeList" => Token::FixedSizeList,
746
747            "s" | "Second" => Token::TimeUnit(TimeUnit::Second),
748            "ms" | "Millisecond" => Token::TimeUnit(TimeUnit::Millisecond),
749            "µs" | "us" | "Microsecond" => Token::TimeUnit(TimeUnit::Microsecond),
750            "ns" | "Nanosecond" => Token::TimeUnit(TimeUnit::Nanosecond),
751
752            "Timestamp" => Token::Timestamp,
753            "Time32" => Token::Time32,
754            "Time64" => Token::Time64,
755            "Duration" => Token::Duration,
756            "Interval" => Token::Interval,
757            "Dictionary" => Token::Dictionary,
758
759            "FixedSizeBinary" => Token::FixedSizeBinary,
760
761            "Decimal32" => Token::Decimal32,
762            "Decimal64" => Token::Decimal64,
763            "Decimal128" => Token::Decimal128,
764            "Decimal256" => Token::Decimal256,
765
766            "YearMonth" => Token::IntervalUnit(IntervalUnit::YearMonth),
767            "DayTime" => Token::IntervalUnit(IntervalUnit::DayTime),
768            "MonthDayNano" => Token::IntervalUnit(IntervalUnit::MonthDayNano),
769
770            "Some" => Token::Some,
771            "None" => Token::None,
772
773            "non-null" => Token::NonNull,
774            "nullable" => Token::Nullable,
775            "field" => Token::Field,
776            "x" => Token::X,
777
778            "Struct" => Token::Struct,
779
780            "Union" => Token::Union,
781            "Sparse" => Token::UnionMode(UnionMode::Sparse),
782            "Dense" => Token::UnionMode(UnionMode::Dense),
783
784            "Map" => Token::Map,
785            "sorted" => Token::MapSorted(true),
786            "unsorted" => Token::MapSorted(false),
787
788            "RunEndEncoded" => Token::RunEndEncoded,
789
790            token => {
791                return Err(make_error(self.val, &format!("unknown token: {token}")));
792            }
793        };
794        Ok(token)
795    }
796
797    /// Parses e.g. `"foo bar"`, `'foo bar'`
798    fn parse_quoted_string(&mut self, quote_type: QuoteType) -> ArrowResult<Token> {
799        let quote = match quote_type {
800            QuoteType::Double => '\"',
801            QuoteType::Single => '\'',
802        };
803
804        if self.next_char() != Some(quote) {
805            return Err(make_error(self.val, "Expected \""));
806        }
807
808        // reset temp space
809        self.word.clear();
810
811        let mut is_escaped = false;
812
813        loop {
814            match self.next_char() {
815                None => {
816                    return Err(ArrowError::ParseError(format!(
817                        "Unterminated string at: \"{}",
818                        self.word
819                    )));
820                }
821                Some(c) => match c {
822                    '\\' => {
823                        is_escaped = true;
824                        self.word.push(c);
825                    }
826                    c if c == quote => {
827                        if is_escaped {
828                            self.word.push(c);
829                            is_escaped = false;
830                        } else {
831                            break;
832                        }
833                    }
834                    c => {
835                        self.word.push(c);
836                    }
837                },
838            }
839        }
840
841        let val: String = self.word.parse().map_err(|err| {
842            ArrowError::ParseError(format!("Failed to parse string: \"{}\": {err}", self.word))
843        })?;
844
845        if val.is_empty() {
846            // Using empty strings as field names is just asking for trouble
847            return Err(make_error(self.val, "empty strings aren't allowed"));
848        }
849
850        match quote_type {
851            QuoteType::Double => Ok(Token::DoubleQuotedString(val)),
852            QuoteType::Single => Ok(Token::SingleQuotedString(val)),
853        }
854    }
855}
856
857impl Iterator for Tokenizer<'_> {
858    type Item = ArrowResult<Token>;
859
860    fn next(&mut self) -> Option<Self::Item> {
861        loop {
862            match self.peek_next_char()? {
863                ' ' => {
864                    // skip whitespace
865                    self.next_char();
866                }
867                '"' => {
868                    return Some(self.parse_quoted_string(QuoteType::Double));
869                }
870                '\'' => {
871                    return Some(self.parse_quoted_string(QuoteType::Single));
872                }
873                '(' => {
874                    self.next_char();
875                    return Some(Ok(Token::LParen));
876                }
877                ')' => {
878                    self.next_char();
879                    return Some(Ok(Token::RParen));
880                }
881                ',' => {
882                    self.next_char();
883                    return Some(Ok(Token::Comma));
884                }
885                ':' => {
886                    self.next_char();
887                    return Some(Ok(Token::Colon));
888                }
889                _ => return Some(self.parse_word()),
890            }
891        }
892    }
893}
894
895/// Grammar is
896///
897#[derive(Debug, PartialEq)]
898enum Token {
899    // Null, or Int32
900    SimpleType(DataType),
901    Timestamp,
902    Time32,
903    Time64,
904    Duration,
905    Interval,
906    FixedSizeBinary,
907    Decimal32,
908    Decimal64,
909    Decimal128,
910    Decimal256,
911    Dictionary,
912    TimeUnit(TimeUnit),
913    IntervalUnit(IntervalUnit),
914    LParen,
915    RParen,
916    Comma,
917    Colon,
918    Some,
919    None,
920    Integer(i64),
921    DoubleQuotedString(String),
922    SingleQuotedString(String),
923    List,
924    ListView,
925    LargeList,
926    LargeListView,
927    FixedSizeList,
928    Struct,
929    Union,
930    UnionMode(UnionMode),
931    Map,
932    MapSorted(bool),
933    RunEndEncoded,
934    NonNull,
935    Nullable,
936    Field,
937    X,
938}
939
940impl Display for Token {
941    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
942        match self {
943            Token::SimpleType(t) => write!(f, "{t}"),
944            Token::List => write!(f, "List"),
945            Token::ListView => write!(f, "ListView"),
946            Token::LargeList => write!(f, "LargeList"),
947            Token::LargeListView => write!(f, "LargeListView"),
948            Token::FixedSizeList => write!(f, "FixedSizeList"),
949            Token::Timestamp => write!(f, "Timestamp"),
950            Token::Time32 => write!(f, "Time32"),
951            Token::Time64 => write!(f, "Time64"),
952            Token::Duration => write!(f, "Duration"),
953            Token::Interval => write!(f, "Interval"),
954            Token::TimeUnit(u) => write!(f, "TimeUnit({u:?})"),
955            Token::IntervalUnit(u) => write!(f, "IntervalUnit({u:?})"),
956            Token::LParen => write!(f, "("),
957            Token::RParen => write!(f, ")"),
958            Token::Comma => write!(f, ","),
959            Token::Colon => write!(f, ":"),
960            Token::Some => write!(f, "Some"),
961            Token::None => write!(f, "None"),
962            Token::FixedSizeBinary => write!(f, "FixedSizeBinary"),
963            Token::Decimal32 => write!(f, "Decimal32"),
964            Token::Decimal64 => write!(f, "Decimal64"),
965            Token::Decimal128 => write!(f, "Decimal128"),
966            Token::Decimal256 => write!(f, "Decimal256"),
967            Token::Dictionary => write!(f, "Dictionary"),
968            Token::Integer(v) => write!(f, "Integer({v})"),
969            Token::DoubleQuotedString(s) => write!(f, "DoubleQuotedString({s})"),
970            Token::SingleQuotedString(s) => write!(f, "SingleQuotedString({s})"),
971            Token::Struct => write!(f, "Struct"),
972            Token::Union => write!(f, "Union"),
973            Token::UnionMode(m) => write!(f, "{m:?}"),
974            Token::Map => write!(f, "Map"),
975            Token::MapSorted(sorted) => {
976                write!(f, "{}", if *sorted { "sorted" } else { "unsorted" })
977            }
978            Token::RunEndEncoded => write!(f, "RunEndEncoded"),
979            Token::NonNull => write!(f, "non-null"),
980            Token::Nullable => write!(f, "nullable"),
981            Token::Field => write!(f, "field"),
982            Token::X => write!(f, "x"),
983        }
984    }
985}
986
987#[cfg(test)]
988mod test {
989    use super::*;
990
991    #[test]
992    fn test_parse_data_type() {
993        // this ensures types can be parsed correctly from their string representations
994        for dt in list_datatypes() {
995            round_trip(dt)
996        }
997    }
998
999    /// Ensure we converting data_type to a string, and then parse it as a type
1000    /// verifying it is the same
1001    fn round_trip(data_type: DataType) {
1002        let data_type_string = data_type.to_string();
1003        println!("Input '{data_type_string}' ({data_type:?})");
1004        let parsed_type = parse_data_type(&data_type_string).unwrap();
1005        assert_eq!(
1006            data_type, parsed_type,
1007            "Mismatch parsing {data_type_string}"
1008        );
1009    }
1010
1011    fn list_datatypes() -> Vec<DataType> {
1012        vec![
1013            // ---------
1014            // Non Nested types
1015            // ---------
1016            DataType::Null,
1017            DataType::Boolean,
1018            DataType::Int8,
1019            DataType::Int16,
1020            DataType::Int32,
1021            DataType::Int64,
1022            DataType::UInt8,
1023            DataType::UInt16,
1024            DataType::UInt32,
1025            DataType::UInt64,
1026            DataType::Float16,
1027            DataType::Float32,
1028            DataType::Float64,
1029            DataType::Timestamp(TimeUnit::Second, None),
1030            DataType::Timestamp(TimeUnit::Millisecond, None),
1031            DataType::Timestamp(TimeUnit::Microsecond, None),
1032            DataType::Timestamp(TimeUnit::Nanosecond, None),
1033            // we can't cover all possible timezones, here we only test utc and +08:00
1034            DataType::Timestamp(TimeUnit::Nanosecond, Some("+00:00".into())),
1035            DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
1036            DataType::Timestamp(TimeUnit::Millisecond, Some("+00:00".into())),
1037            DataType::Timestamp(TimeUnit::Second, Some("+00:00".into())),
1038            DataType::Timestamp(TimeUnit::Nanosecond, Some("+08:00".into())),
1039            DataType::Timestamp(TimeUnit::Microsecond, Some("+08:00".into())),
1040            DataType::Timestamp(TimeUnit::Millisecond, Some("+08:00".into())),
1041            DataType::Timestamp(TimeUnit::Second, Some("+08:00".into())),
1042            DataType::Date32,
1043            DataType::Date64,
1044            DataType::Time32(TimeUnit::Second),
1045            DataType::Time32(TimeUnit::Millisecond),
1046            DataType::Time64(TimeUnit::Microsecond),
1047            DataType::Time64(TimeUnit::Nanosecond),
1048            DataType::Duration(TimeUnit::Second),
1049            DataType::Duration(TimeUnit::Millisecond),
1050            DataType::Duration(TimeUnit::Microsecond),
1051            DataType::Duration(TimeUnit::Nanosecond),
1052            DataType::Interval(IntervalUnit::YearMonth),
1053            DataType::Interval(IntervalUnit::DayTime),
1054            DataType::Interval(IntervalUnit::MonthDayNano),
1055            DataType::Binary,
1056            DataType::BinaryView,
1057            DataType::FixedSizeBinary(0),
1058            DataType::FixedSizeBinary(1234),
1059            DataType::LargeBinary,
1060            DataType::Utf8,
1061            DataType::Utf8View,
1062            DataType::LargeUtf8,
1063            DataType::Decimal32(7, 6),
1064            DataType::Decimal64(6, 5),
1065            DataType::Decimal128(7, 6),
1066            DataType::Decimal256(6, 5),
1067            // ---------
1068            // Nested types
1069            // ---------
1070            DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
1071            DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)),
1072            DataType::Dictionary(
1073                Box::new(DataType::Int8),
1074                Box::new(DataType::Timestamp(TimeUnit::Nanosecond, None)),
1075            ),
1076            DataType::Dictionary(
1077                Box::new(DataType::Int8),
1078                Box::new(DataType::FixedSizeBinary(23)),
1079            ),
1080            DataType::Dictionary(
1081                Box::new(DataType::Int8),
1082                Box::new(
1083                    // nested dictionaries are probably a bad idea but they are possible
1084                    DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)),
1085                ),
1086            ),
1087            DataType::Struct(Fields::from(vec![
1088                Field::new("f1", DataType::Int64, true),
1089                Field::new("f2", DataType::Float64, true),
1090                Field::new(
1091                    "f3",
1092                    DataType::Timestamp(TimeUnit::Second, Some("+08:00".into())),
1093                    true,
1094                ),
1095                Field::new(
1096                    "f4",
1097                    DataType::Dictionary(
1098                        Box::new(DataType::Int8),
1099                        Box::new(DataType::FixedSizeBinary(23)),
1100                    ),
1101                    true,
1102                ),
1103            ])),
1104            DataType::Struct(Fields::from(vec![
1105                Field::new("Int64", DataType::Int64, true),
1106                Field::new("Float64", DataType::Float64, true),
1107            ])),
1108            DataType::Struct(Fields::from(vec![
1109                Field::new("f1", DataType::Int64, true),
1110                Field::new(
1111                    "nested_struct",
1112                    DataType::Struct(Fields::from(vec![Field::new("n1", DataType::Int64, true)])),
1113                    true,
1114                ),
1115            ])),
1116            DataType::Struct(Fields::from(vec![Field::new("f1", DataType::Int64, true)])),
1117            DataType::Struct(Fields::empty()),
1118            DataType::List(Arc::new(Field::new_list_field(DataType::Int64, true))),
1119            DataType::List(Arc::new(Field::new_list_field(DataType::Int64, false))),
1120            DataType::List(Arc::new(Field::new("Int64", DataType::Int64, true))),
1121            DataType::List(Arc::new(Field::new("Int64", DataType::Int64, false))),
1122            DataType::List(Arc::new(Field::new(
1123                "nested_list",
1124                DataType::List(Arc::new(Field::new("Int64", DataType::Int64, true))),
1125                true,
1126            ))),
1127            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int64, true))),
1128            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int64, false))),
1129            DataType::ListView(Arc::new(Field::new("Int64", DataType::Int64, true))),
1130            DataType::ListView(Arc::new(Field::new("Int64", DataType::Int64, false))),
1131            DataType::ListView(Arc::new(Field::new(
1132                "nested_list_view",
1133                DataType::ListView(Arc::new(Field::new("Int64", DataType::Int64, true))),
1134                true,
1135            ))),
1136            DataType::LargeList(Arc::new(Field::new_list_field(DataType::Int64, true))),
1137            DataType::LargeList(Arc::new(Field::new_list_field(DataType::Int64, false))),
1138            DataType::LargeList(Arc::new(Field::new("Int64", DataType::Int64, true))),
1139            DataType::LargeList(Arc::new(Field::new("Int64", DataType::Int64, false))),
1140            DataType::LargeList(Arc::new(Field::new(
1141                "nested_large_list",
1142                DataType::LargeList(Arc::new(Field::new("Int64", DataType::Int64, true))),
1143                true,
1144            ))),
1145            DataType::LargeListView(Arc::new(Field::new_list_field(DataType::Int64, true))),
1146            DataType::LargeListView(Arc::new(Field::new_list_field(DataType::Int64, false))),
1147            DataType::LargeListView(Arc::new(Field::new("Int64", DataType::Int64, true))),
1148            DataType::LargeListView(Arc::new(Field::new("Int64", DataType::Int64, false))),
1149            DataType::LargeListView(Arc::new(Field::new(
1150                "nested_large_list_view",
1151                DataType::LargeListView(Arc::new(Field::new("Int64", DataType::Int64, true))),
1152                true,
1153            ))),
1154            DataType::FixedSizeList(Arc::new(Field::new_list_field(DataType::Int64, true)), 2),
1155            DataType::FixedSizeList(Arc::new(Field::new_list_field(DataType::Int64, false)), 2),
1156            DataType::FixedSizeList(Arc::new(Field::new("Int64", DataType::Int64, true)), 2),
1157            DataType::FixedSizeList(Arc::new(Field::new("Int64", DataType::Int64, false)), 2),
1158            DataType::FixedSizeList(
1159                Arc::new(Field::new(
1160                    "nested_fixed_size_list",
1161                    DataType::FixedSizeList(
1162                        Arc::new(Field::new("Int64", DataType::Int64, true)),
1163                        2,
1164                    ),
1165                    true,
1166                )),
1167                2,
1168            ),
1169            DataType::Union(
1170                UnionFields::from_fields(vec![
1171                    Field::new("Int32", DataType::Int32, false),
1172                    Field::new("Utf8", DataType::Utf8, true),
1173                ]),
1174                UnionMode::Sparse,
1175            ),
1176            DataType::Union(
1177                UnionFields::from_fields(vec![
1178                    Field::new("Int32", DataType::Int32, false),
1179                    Field::new("Utf8", DataType::Utf8, true),
1180                ]),
1181                UnionMode::Dense,
1182            ),
1183            DataType::Union(
1184                UnionFields::from_fields(vec![
1185                    Field::new_union(
1186                        "nested_union",
1187                        vec![0, 1],
1188                        vec![
1189                            Field::new("Int32", DataType::Int32, false),
1190                            Field::new("Utf8", DataType::Utf8, true),
1191                        ],
1192                        UnionMode::Dense,
1193                    ),
1194                    Field::new("Utf8", DataType::Utf8, true),
1195                ]),
1196                UnionMode::Sparse,
1197            ),
1198            DataType::Union(
1199                UnionFields::from_fields(vec![Field::new("Int32", DataType::Int32, false)]),
1200                UnionMode::Dense,
1201            ),
1202            DataType::Union(
1203                UnionFields::try_new(Vec::<i8>::new(), Vec::<Field>::new()).unwrap(),
1204                UnionMode::Sparse,
1205            ),
1206            DataType::Map(Arc::new(Field::new("Int64", DataType::Int64, true)), true),
1207            DataType::Map(Arc::new(Field::new("Int64", DataType::Int64, true)), false),
1208            DataType::Map(
1209                Arc::new(Field::new_map(
1210                    "nested_map",
1211                    Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
1212                    Field::new(Field::MAP_KEY_FIELD_DEFAULT_NAME, DataType::Utf8, false),
1213                    Field::new(Field::MAP_VALUE_FIELD_DEFAULT_NAME, DataType::Int32, true),
1214                    false,
1215                    true,
1216                )),
1217                true,
1218            ),
1219            DataType::RunEndEncoded(
1220                Arc::new(Field::new("run_ends", DataType::UInt32, false)),
1221                Arc::new(Field::new("values", DataType::Int32, true)),
1222            ),
1223            DataType::RunEndEncoded(
1224                Arc::new(Field::new(
1225                    "nested_run_end_encoded",
1226                    DataType::RunEndEncoded(
1227                        Arc::new(Field::new("run_ends", DataType::UInt32, false)),
1228                        Arc::new(Field::new("values", DataType::Int32, true)),
1229                    ),
1230                    true,
1231                )),
1232                Arc::new(Field::new("values", DataType::Int32, true)),
1233            ),
1234        ]
1235    }
1236
1237    #[test]
1238    fn test_parse_data_type_whitespace_tolerance() {
1239        // (string to parse, expected DataType)
1240        let cases = [
1241            ("Int8", DataType::Int8),
1242            (
1243                "Timestamp        (ns)",
1244                DataType::Timestamp(TimeUnit::Nanosecond, None),
1245            ),
1246            (
1247                "Timestamp        (ns)  ",
1248                DataType::Timestamp(TimeUnit::Nanosecond, None),
1249            ),
1250            (
1251                "          Timestamp        (ns               )",
1252                DataType::Timestamp(TimeUnit::Nanosecond, None),
1253            ),
1254            (
1255                "Timestamp        (ns               )  ",
1256                DataType::Timestamp(TimeUnit::Nanosecond, None),
1257            ),
1258        ];
1259
1260        for (data_type_string, expected_data_type) in cases {
1261            let parsed_data_type = parse_data_type(data_type_string).unwrap();
1262            assert_eq!(
1263                parsed_data_type, expected_data_type,
1264                "Parsing '{data_type_string}', expecting '{expected_data_type}'"
1265            );
1266        }
1267    }
1268
1269    /// Ensure that old style types can still be parsed
1270    #[test]
1271    fn test_parse_data_type_backwards_compatibility() {
1272        use DataType::*;
1273        use IntervalUnit::*;
1274        use TimeUnit::*;
1275        // List below created with:
1276        for t in list_datatypes() {
1277            println!(r#"("{t}", {t:?}),"#);
1278        }
1279        // (string to parse, expected DataType)
1280        let cases = [
1281            ("Timestamp(Nanosecond, None)", Timestamp(Nanosecond, None)),
1282            ("Timestamp(Microsecond, None)", Timestamp(Microsecond, None)),
1283            ("Timestamp(Millisecond, None)", Timestamp(Millisecond, None)),
1284            ("Timestamp(Second, None)", Timestamp(Second, None)),
1285            ("Timestamp(Nanosecond, None)", Timestamp(Nanosecond, None)),
1286            // Timezones
1287            (
1288                r#"Timestamp(Nanosecond, Some("+00:00"))"#,
1289                Timestamp(Nanosecond, Some("+00:00".into())),
1290            ),
1291            (
1292                r#"Timestamp(Microsecond, Some("+00:00"))"#,
1293                Timestamp(Microsecond, Some("+00:00".into())),
1294            ),
1295            (
1296                r#"Timestamp(Millisecond, Some("+00:00"))"#,
1297                Timestamp(Millisecond, Some("+00:00".into())),
1298            ),
1299            (
1300                r#"Timestamp(Second, Some("+00:00"))"#,
1301                Timestamp(Second, Some("+00:00".into())),
1302            ),
1303            ("Null", Null),
1304            ("Boolean", Boolean),
1305            ("Int8", Int8),
1306            ("Int16", Int16),
1307            ("Int32", Int32),
1308            ("Int64", Int64),
1309            ("UInt8", UInt8),
1310            ("UInt16", UInt16),
1311            ("UInt32", UInt32),
1312            ("UInt64", UInt64),
1313            ("Float16", Float16),
1314            ("Float32", Float32),
1315            ("Float64", Float64),
1316            ("Timestamp(s)", Timestamp(Second, None)),
1317            ("Timestamp(ms)", Timestamp(Millisecond, None)),
1318            ("Timestamp(µs)", Timestamp(Microsecond, None)),
1319            ("Timestamp(ns)", Timestamp(Nanosecond, None)),
1320            (
1321                r#"Timestamp(ns, "+00:00")"#,
1322                Timestamp(Nanosecond, Some("+00:00".into())),
1323            ),
1324            (
1325                r#"Timestamp(µs, "+00:00")"#,
1326                Timestamp(Microsecond, Some("+00:00".into())),
1327            ),
1328            (
1329                r#"Timestamp(ms, "+00:00")"#,
1330                Timestamp(Millisecond, Some("+00:00".into())),
1331            ),
1332            (
1333                r#"Timestamp(s, "+00:00")"#,
1334                Timestamp(Second, Some("+00:00".into())),
1335            ),
1336            (
1337                r#"Timestamp(ns, "+08:00")"#,
1338                Timestamp(Nanosecond, Some("+08:00".into())),
1339            ),
1340            (
1341                r#"Timestamp(µs, "+08:00")"#,
1342                Timestamp(Microsecond, Some("+08:00".into())),
1343            ),
1344            (
1345                r#"Timestamp(ms, "+08:00")"#,
1346                Timestamp(Millisecond, Some("+08:00".into())),
1347            ),
1348            (
1349                r#"Timestamp(s, "+08:00")"#,
1350                Timestamp(Second, Some("+08:00".into())),
1351            ),
1352            ("Date32", Date32),
1353            ("Date64", Date64),
1354            ("Time32(s)", Time32(Second)),
1355            ("Time32(ms)", Time32(Millisecond)),
1356            ("Time64(µs)", Time64(Microsecond)),
1357            ("Time64(ns)", Time64(Nanosecond)),
1358            ("Duration(s)", Duration(Second)),
1359            ("Duration(ms)", Duration(Millisecond)),
1360            ("Duration(µs)", Duration(Microsecond)),
1361            ("Duration(ns)", Duration(Nanosecond)),
1362            ("Interval(YearMonth)", Interval(YearMonth)),
1363            ("Interval(DayTime)", Interval(DayTime)),
1364            ("Interval(MonthDayNano)", Interval(MonthDayNano)),
1365            ("Binary", Binary),
1366            ("BinaryView", BinaryView),
1367            ("FixedSizeBinary(0)", FixedSizeBinary(0)),
1368            ("FixedSizeBinary(1234)", FixedSizeBinary(1234)),
1369            ("LargeBinary", LargeBinary),
1370            ("Utf8", Utf8),
1371            ("Utf8View", Utf8View),
1372            ("LargeUtf8", LargeUtf8),
1373            ("Decimal32(7, 6)", Decimal32(7, 6)),
1374            ("Decimal64(6, 5)", Decimal64(6, 5)),
1375            ("Decimal128(7, 6)", Decimal128(7, 6)),
1376            ("Decimal256(6, 5)", Decimal256(6, 5)),
1377            (
1378                "Dictionary(Int32, Utf8)",
1379                Dictionary(Box::new(Int32), Box::new(Utf8)),
1380            ),
1381            (
1382                "Dictionary(Int8, Utf8)",
1383                Dictionary(Box::new(Int8), Box::new(Utf8)),
1384            ),
1385            (
1386                "Dictionary(Int8, Timestamp(ns))",
1387                Dictionary(Box::new(Int8), Box::new(Timestamp(Nanosecond, None))),
1388            ),
1389            (
1390                "Dictionary(Int8, FixedSizeBinary(23))",
1391                Dictionary(Box::new(Int8), Box::new(FixedSizeBinary(23))),
1392            ),
1393            (
1394                "Dictionary(Int8, Dictionary(Int8, Utf8))",
1395                Dictionary(
1396                    Box::new(Int8),
1397                    Box::new(Dictionary(Box::new(Int8), Box::new(Utf8))),
1398                ),
1399            ),
1400            (
1401                r#"Struct("f1": nullable Int64, "f2": nullable Float64, "f3": nullable Timestamp(s, "+08:00"), "f4": nullable Dictionary(Int8, FixedSizeBinary(23)))"#,
1402                Struct(Fields::from(vec![
1403                    Field::new("f1", Int64, true),
1404                    Field::new("f2", Float64, true),
1405                    Field::new("f3", Timestamp(Second, Some("+08:00".into())), true),
1406                    Field::new(
1407                        "f4",
1408                        Dictionary(Box::new(Int8), Box::new(FixedSizeBinary(23))),
1409                        true,
1410                    ),
1411                ])),
1412            ),
1413            (
1414                r#"Struct("Int64": nullable Int64, "Float64": nullable Float64)"#,
1415                Struct(Fields::from(vec![
1416                    Field::new("Int64", Int64, true),
1417                    Field::new("Float64", Float64, true),
1418                ])),
1419            ),
1420            (
1421                r#"Struct("f1": nullable Int64, "nested_struct": nullable Struct("n1": nullable Int64))"#,
1422                Struct(Fields::from(vec![
1423                    Field::new("f1", Int64, true),
1424                    Field::new(
1425                        "nested_struct",
1426                        Struct(Fields::from(vec![Field::new("n1", Int64, true)])),
1427                        true,
1428                    ),
1429                ])),
1430            ),
1431            (r"Struct()", Struct(Fields::empty())),
1432            (
1433                "FixedSizeList(4, Int64)",
1434                FixedSizeList(Arc::new(Field::new_list_field(Int64, true)), 4),
1435            ),
1436            (
1437                "List(Int64)",
1438                List(Arc::new(Field::new_list_field(Int64, true))),
1439            ),
1440            (
1441                "LargeList(Int64)",
1442                LargeList(Arc::new(Field::new_list_field(Int64, true))),
1443            ),
1444        ];
1445
1446        for (data_type_string, expected_data_type) in cases {
1447            let parsed_data_type = parse_data_type(data_type_string).unwrap();
1448            assert_eq!(
1449                parsed_data_type, expected_data_type,
1450                "Parsing '{data_type_string}', expecting '{expected_data_type}'"
1451            );
1452        }
1453    }
1454
1455    #[test]
1456    fn parse_data_type_errors() {
1457        // (string to parse, expected error message)
1458        let cases = [
1459            ("", "Unsupported type ''"),
1460            ("", "Error finding next token"),
1461            ("null", "Unsupported type 'null'"),
1462            ("Nu", "Unsupported type 'Nu'"),
1463            (r"Timestamp(ns, +00:00)", "Error unknown token: +00"),
1464            (
1465                r#"Timestamp(ns, "+00:00)"#,
1466                r#"Unterminated string at: "+00:00)"#,
1467            ),
1468            (r#"Timestamp(ns, "")"#, r"empty strings aren't allowed"),
1469            (
1470                r#"Timestamp(ns, "+00:00"")"#,
1471                r#"Parser error: Unterminated string at: ")"#,
1472            ),
1473            ("Timestamp(ns, ", "Error finding next token"),
1474            (
1475                "Float32 Float32",
1476                "trailing content after parsing 'Float32'",
1477            ),
1478            ("Int32, ", "trailing content after parsing 'Int32'"),
1479            ("Int32(3), ", "trailing content after parsing 'Int32'"),
1480            (
1481                "FixedSizeBinary(Int32), ",
1482                "Error finding i64 for FixedSizeBinary, got 'Int32'",
1483            ),
1484            (
1485                "FixedSizeBinary(3.0), ",
1486                "Error parsing 3.0 as integer: invalid digit found in string",
1487            ),
1488            // too large for i32
1489            (
1490                "FixedSizeBinary(4000000000), ",
1491                "Error converting 4000000000 into i32 for FixedSizeBinary:",
1492            ),
1493            // can't have negative width
1494            (
1495                "FixedSizeBinary(-1), ",
1496                "FixedSizeBinary length must be non-negative, got -1",
1497            ),
1498            (
1499                "FixedSizeList(-1, Int64), ",
1500                "FixedSizeList length must be non-negative, got -1",
1501            ),
1502            // can't have negative precision
1503            (
1504                "Decimal32(-3, 5)",
1505                "Error converting -3 into u8 for Decimal32:",
1506            ),
1507            (
1508                "Decimal64(-3, 5)",
1509                "Error converting -3 into u8 for Decimal64:",
1510            ),
1511            (
1512                "Decimal128(-3, 5)",
1513                "Error converting -3 into u8 for Decimal128:",
1514            ),
1515            (
1516                "Decimal256(-3, 5)",
1517                "Error converting -3 into u8 for Decimal256:",
1518            ),
1519            (
1520                "Decimal32(3, 500)",
1521                "Error converting 500 into i8 for Decimal32:",
1522            ),
1523            (
1524                "Decimal64(3, 500)",
1525                "Error converting 500 into i8 for Decimal64:",
1526            ),
1527            (
1528                "Decimal128(3, 500)",
1529                "Error converting 500 into i8 for Decimal128:",
1530            ),
1531            (
1532                "Decimal256(3, 500)",
1533                "Error converting 500 into i8 for Decimal256:",
1534            ),
1535            ("Struct(f1 Int64)", "Error unknown token: f1"),
1536            ("Struct(\"f1\" Int64)", "Expected ':'"),
1537            (
1538                "Struct(\"f1\": )",
1539                "Error finding next type, got unexpected ')'",
1540            ),
1541            // Invalid time combinations
1542            (
1543                "Time32(µs)",
1544                "Error Time32 time unit must be 's' or 'ms', got 'µs'",
1545            ),
1546            (
1547                "Time32(ns)",
1548                "Error Time32 time unit must be 's' or 'ms', got 'ns'",
1549            ),
1550            (
1551                "Time64(s)",
1552                "Error Time64 time unit must be 'µs' or 'ns', got 's'",
1553            ),
1554            (
1555                "Time64(ms)",
1556                "Error Time64 time unit must be 'µs' or 'ns', got 'ms'",
1557            ),
1558            // Decimals can't have scale exceeding precision
1559            (
1560                "Decimal32(5, 6)",
1561                "Error Decimal32 scale '6' cannot be greater than precision '5'",
1562            ),
1563            (
1564                "Decimal64(5, 6)",
1565                "Error Decimal64 scale '6' cannot be greater than precision '5'",
1566            ),
1567            (
1568                "Decimal128(5, 6)",
1569                "Error Decimal128 scale '6' cannot be greater than precision '5'",
1570            ),
1571            (
1572                "Decimal256(5, 6)",
1573                "Error Decimal256 scale '6' cannot be greater than precision '5'",
1574            ),
1575            // Decimals have a max supported precision
1576            (
1577                "Decimal32(10, 0)",
1578                "Error Decimal32 precision must be in range [1, 9], got '10'",
1579            ),
1580            (
1581                "Decimal64(19, 0)",
1582                "Error Decimal64 precision must be in range [1, 18], got '19'",
1583            ),
1584            (
1585                "Decimal128(39, 0)",
1586                "Error Decimal128 precision must be in range [1, 38], got '39'",
1587            ),
1588            (
1589                "Decimal256(77, 0)",
1590                "Error Decimal256 precision must be in range [1, 76], got '77'",
1591            ),
1592            // Decimals precision can't be 0
1593            (
1594                "Decimal32(0, 0)",
1595                "Error Decimal32 precision must be in range [1, 9], got '0'",
1596            ),
1597            (
1598                "Decimal64(0, 0)",
1599                "Error Decimal64 precision must be in range [1, 18], got '0'",
1600            ),
1601            (
1602                "Decimal128(0, 0)",
1603                "Error Decimal128 precision must be in range [1, 38], got '0'",
1604            ),
1605            (
1606                "Decimal256(0, 0)",
1607                "Error Decimal256 precision must be in range [1, 76], got '0'",
1608            ),
1609        ];
1610
1611        for (data_type_string, expected_message) in cases {
1612            println!("Parsing '{data_type_string}', expecting '{expected_message}'");
1613            match parse_data_type(data_type_string) {
1614                Ok(d) => panic!("Expected error while parsing '{data_type_string}', but got '{d}'"),
1615                Err(e) => {
1616                    let message = e.to_string();
1617                    assert!(
1618                        message.contains(expected_message),
1619                        "\n\ndid not find expected in actual.\n\nexpected: {expected_message}\nactual: {message}\n"
1620                    );
1621
1622                    if !message.contains("Unterminated string") {
1623                        // errors should also contain a help message
1624                        assert!(message.contains("Must be a supported arrow type name such as 'Int32' or 'Timestamp(ns)'"), "message: {message}");
1625                    }
1626                }
1627            }
1628        }
1629    }
1630
1631    #[test]
1632    fn parse_error_type() {
1633        let err = parse_data_type("foobar").unwrap_err();
1634        assert!(matches!(err, ArrowError::ParseError(_)));
1635        assert_eq!(
1636            err.to_string(),
1637            "Parser error: Unsupported type 'foobar'. Must be a supported arrow type name such as 'Int32' or 'Timestamp(ns)'. Error unknown token: foobar"
1638        );
1639    }
1640}
arrow_schema/datatype_parse.rs

arrow_schema/
datatype_parse.rs
