arrow_schema/
datatype_parse.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use std::{fmt::Display, iter::Peekable, str::Chars, sync::Arc};
6
7use crate::{
8    ArrowError, DECIMAL32_MAX_PRECISION, DECIMAL64_MAX_PRECISION, DECIMAL128_MAX_PRECISION,
9    DECIMAL256_MAX_PRECISION, DataType, Field, Fields, IntervalUnit, TimeUnit, UnionFields,
10    UnionMode,
11};
12
13/// Parses a DataType from a string representation
14///
15/// For example, the string "Int32" would be parsed into [`DataType::Int32`]
16pub(crate) fn parse_data_type(val: &str) -> ArrowResult<DataType> {
17    Parser::new(val).parse()
18}
19
20type ArrowResult<T> = Result<T, ArrowError>;
21
22fn make_error(val: &str, msg: &str) -> ArrowError {
23    let msg = format!(
24        "Unsupported type '{val}'. Must be a supported arrow type name such as 'Int32' or 'Timestamp(ns)'. Error {msg}"
25    );
26    ArrowError::ParseError(msg)
27}
28
29fn make_error_expected(val: &str, expected: &Token, actual: &Token) -> ArrowError {
30    make_error(val, &format!("Expected '{expected}', got '{actual}'"))
31}
32
33/// Implementation of `parse_data_type`, modeled after <https://github.com/sqlparser-rs/sqlparser-rs>
34#[derive(Debug)]
35struct Parser<'a> {
36    val: &'a str,
37    tokenizer: Peekable<Tokenizer<'a>>,
38}
39
40impl<'a> Parser<'a> {
41    fn new(val: &'a str) -> Self {
42        Self {
43            val,
44            tokenizer: Tokenizer::new(val).peekable(),
45        }
46    }
47
48    fn parse(mut self) -> ArrowResult<DataType> {
49        let data_type = self.parse_next_type()?;
50        // ensure that there is no trailing content
51        if self.tokenizer.next().is_some() {
52            Err(make_error(
53                self.val,
54                &format!("checking trailing content after parsing '{data_type}'"),
55            ))
56        } else {
57            Ok(data_type)
58        }
59    }
60
61    /// parses the next full DataType
62    fn parse_next_type(&mut self) -> ArrowResult<DataType> {
63        match self.next_token()? {
64            Token::SimpleType(data_type) => Ok(data_type),
65            Token::Timestamp => self.parse_timestamp(),
66            Token::Time32 => self.parse_time32(),
67            Token::Time64 => self.parse_time64(),
68            Token::Duration => self.parse_duration(),
69            Token::Interval => self.parse_interval(),
70            Token::FixedSizeBinary => self.parse_fixed_size_binary(),
71            Token::Decimal32 => self.parse_decimal_32(),
72            Token::Decimal64 => self.parse_decimal_64(),
73            Token::Decimal128 => self.parse_decimal_128(),
74            Token::Decimal256 => self.parse_decimal_256(),
75            Token::Dictionary => self.parse_dictionary(),
76            Token::List => self.parse_list(),
77            Token::ListView => self.parse_list_view(),
78            Token::LargeList => self.parse_large_list(),
79            Token::LargeListView => self.parse_large_list_view(),
80            Token::FixedSizeList => self.parse_fixed_size_list(),
81            Token::Struct => self.parse_struct(),
82            Token::Union => self.parse_union(),
83            Token::Map => self.parse_map(),
84            Token::RunEndEncoded => self.parse_run_end_encoded(),
85            tok => Err(make_error(
86                self.val,
87                &format!("finding next type, got unexpected '{tok}'"),
88            )),
89        }
90    }
91
92    /// parses Field, this is the inversion of `format_field` in `datatype_display.rs`.
93    /// E.g: "a": non-null Int64
94    ///
95    /// TODO: support metadata: `"a": non-null Int64 metadata: {"foo": "value"}`
96    fn parse_field(&mut self) -> ArrowResult<Field> {
97        let name = self.parse_double_quoted_string("Field")?;
98        self.expect_token(Token::Colon)?;
99        let nullable = self.parse_opt_nullable();
100        let data_type = self.parse_next_type()?;
101        Ok(Field::new(name, data_type, nullable))
102    }
103
104    /// Parses field inside a list. Use `Field::LIST_FIELD_DEFAULT_NAME`
105    /// if no field name is specified.
106    /// E.g: `non-null Int64, field: 'foo'` or `non-null Int64`
107    ///
108    /// TODO: support metadata: `non-ull Int64, metadata: {"foo2": "value"}`
109    fn parse_list_field(&mut self, context: &str) -> ArrowResult<Field> {
110        let nullable = self.parse_opt_nullable();
111        let data_type = self.parse_next_type()?;
112
113        // the field name (if exists) must be after a comma
114        let field_name = if self
115            .tokenizer
116            .next_if(|next| matches!(next, Ok(Token::Comma)))
117            .is_none()
118        {
119            Field::LIST_FIELD_DEFAULT_NAME.into()
120        } else {
121            // expects: `field: 'field_name'`.
122            self.expect_token(Token::Field)?;
123            self.expect_token(Token::Colon)?;
124            self.parse_single_quoted_string(context)?
125        };
126
127        Ok(Field::new(field_name, data_type, nullable))
128    }
129
130    /// Parses the List type (called after `List` has been consumed)
131    /// E.g: List(non-null Int64, field: 'foo')
132    fn parse_list(&mut self) -> ArrowResult<DataType> {
133        self.expect_token(Token::LParen)?;
134        let field = self.parse_list_field("List")?;
135        self.expect_token(Token::RParen)?;
136        Ok(DataType::List(Arc::new(field)))
137    }
138
139    /// Parses the ListView type (called after `ListView` has been consumed)
140    /// E.g: ListView(non-null Int64, field: 'foo')
141    fn parse_list_view(&mut self) -> ArrowResult<DataType> {
142        self.expect_token(Token::LParen)?;
143        let field = self.parse_list_field("ListView")?;
144        self.expect_token(Token::RParen)?;
145        Ok(DataType::ListView(Arc::new(field)))
146    }
147
148    /// Parses the LargeList type (called after `LargeList` has been consumed)
149    /// E.g: LargeList(non-null Int64, field: 'foo')
150    fn parse_large_list(&mut self) -> ArrowResult<DataType> {
151        self.expect_token(Token::LParen)?;
152        let field = self.parse_list_field("LargeList")?;
153        self.expect_token(Token::RParen)?;
154        Ok(DataType::LargeList(Arc::new(field)))
155    }
156
157    /// Parses the LargeListView type (called after `LargeListView` has been consumed)
158    /// E.g: LargeListView(non-null Int64, field: 'foo')
159    fn parse_large_list_view(&mut self) -> ArrowResult<DataType> {
160        self.expect_token(Token::LParen)?;
161        let field = self.parse_list_field("LargeListView")?;
162        self.expect_token(Token::RParen)?;
163        Ok(DataType::LargeListView(Arc::new(field)))
164    }
165
166    /// Parses the FixedSizeList type (called after `FixedSizeList` has been consumed)
167    ///
168    /// Examples:
169    /// * `FixedSizeList(5 x non-null Int64, field: 'foo')`
170    /// * `FixedSizeList(4, Int64)`
171    ///
172    fn parse_fixed_size_list(&mut self) -> ArrowResult<DataType> {
173        self.expect_token(Token::LParen)?;
174        let length = self.parse_i32("FixedSizeList")?;
175        if length < 0 {
176            return Err(make_error(
177                self.val,
178                &format!("FixedSizeList length must be non-negative, got {length}"),
179            ));
180        }
181        match self.next_token()? {
182            // `FixedSizeList(5 x non-null Int64, field: 'foo')` format
183            Token::X => {
184                let field = self.parse_list_field("FixedSizeList")?;
185                self.expect_token(Token::RParen)?;
186                Ok(DataType::FixedSizeList(Arc::new(field), length))
187            }
188            // `FixedSizeList(4, Int64)` format
189            Token::Comma => {
190                let data_type = self.parse_next_type()?;
191                self.expect_token(Token::RParen)?;
192                Ok(DataType::FixedSizeList(
193                    Arc::new(Field::new_list_field(data_type, true)),
194                    length,
195                ))
196            }
197            tok => Err(make_error(
198                self.val,
199                &format!("Expected 'x' or ',' after length for FixedSizeList, got '{tok}'"),
200            )),
201        }
202    }
203
204    /// Parses the next timeunit
205    fn parse_time_unit(&mut self, context: &str) -> ArrowResult<TimeUnit> {
206        match self.next_token()? {
207            Token::TimeUnit(time_unit) => Ok(time_unit),
208            tok => Err(make_error(
209                self.val,
210                &format!("finding TimeUnit for {context}, got {tok}"),
211            )),
212        }
213    }
214
215    /// Parses the next double quoted string
216    fn parse_double_quoted_string(&mut self, context: &str) -> ArrowResult<String> {
217        let token = self.next_token()?;
218        if let Token::DoubleQuotedString(string) = token {
219            Ok(string)
220        } else {
221            Err(make_error(
222                self.val,
223                &format!("expected double quoted string for {context}, got '{token}'"),
224            ))
225        }
226    }
227
228    /// Parses the next single quoted string
229    fn parse_single_quoted_string(&mut self, context: &str) -> ArrowResult<String> {
230        let token = self.next_token()?;
231        if let Token::SingleQuotedString(string) = token {
232            Ok(string)
233        } else {
234            Err(make_error(
235                self.val,
236                &format!("expected single quoted string for {context}, got '{token}'"),
237            ))
238        }
239    }
240
241    /// Parses the next integer value
242    fn parse_i64(&mut self, context: &str) -> ArrowResult<i64> {
243        match self.next_token()? {
244            Token::Integer(v) => Ok(v),
245            tok => Err(make_error(
246                self.val,
247                &format!("finding i64 for {context}, got '{tok}'"),
248            )),
249        }
250    }
251
252    /// Parses the next i32 integer value
253    fn parse_i32(&mut self, context: &str) -> ArrowResult<i32> {
254        let length = self.parse_i64(context)?;
255        length.try_into().map_err(|e| {
256            make_error(
257                self.val,
258                &format!("converting {length} into i32 for {context}: {e}"),
259            )
260        })
261    }
262
263    /// Parses the next i8 integer value
264    fn parse_i8(&mut self, context: &str) -> ArrowResult<i8> {
265        let length = self.parse_i64(context)?;
266        length.try_into().map_err(|e| {
267            make_error(
268                self.val,
269                &format!("converting {length} into i8 for {context}: {e}"),
270            )
271        })
272    }
273
274    /// Parses the next u8 integer value
275    fn parse_u8(&mut self, context: &str) -> ArrowResult<u8> {
276        let length = self.parse_i64(context)?;
277        length.try_into().map_err(|e| {
278            make_error(
279                self.val,
280                &format!("converting {length} into u8 for {context}: {e}"),
281            )
282        })
283    }
284
285    /// Parses the next timestamp (called after `Timestamp` has been consumed)
286    fn parse_timestamp(&mut self) -> ArrowResult<DataType> {
287        self.expect_token(Token::LParen)?;
288        let time_unit = self.parse_time_unit("Timestamp")?;
289
290        let timezone;
291        match self.next_token()? {
292            Token::Comma => {
293                match self.next_token()? {
294                    // Support old style `Timestamp(Nanosecond, None)`
295                    Token::None => {
296                        timezone = None;
297                    }
298                    // Support old style `Timestamp(Nanosecond, Some("Timezone"))`
299                    Token::Some => {
300                        self.expect_token(Token::LParen)?;
301                        timezone = Some(self.parse_double_quoted_string("Timezone")?);
302                        self.expect_token(Token::RParen)?;
303                    }
304                    Token::DoubleQuotedString(tz) => {
305                        // Support new style `Timestamp(Nanosecond, "Timezone")`
306                        timezone = Some(tz);
307                    }
308                    tok => {
309                        return Err(make_error(
310                            self.val,
311                            &format!("Expected None, Some, or a timezone string, got {tok:?}"),
312                        ));
313                    }
314                }
315                self.expect_token(Token::RParen)?;
316            }
317            // No timezone (e.g `Timestamp(ns)`)
318            Token::RParen => {
319                timezone = None;
320            }
321            next_token => {
322                return Err(make_error(
323                    self.val,
324                    &format!("Expected comma followed by a timezone, or an ), got {next_token:?}"),
325                ));
326            }
327        }
328        Ok(DataType::Timestamp(time_unit, timezone.map(Into::into)))
329    }
330
331    /// Parses the next Time32 (called after `Time32` has been consumed)
332    fn parse_time32(&mut self) -> ArrowResult<DataType> {
333        self.expect_token(Token::LParen)?;
334        let time_unit = self.parse_time_unit("Time32")?;
335        match time_unit {
336            TimeUnit::Second | TimeUnit::Millisecond => (),
337            TimeUnit::Microsecond | TimeUnit::Nanosecond => {
338                return Err(make_error(
339                    self.val,
340                    &format!("Time32 time unit must be 's' or 'ms', got '{time_unit}'"),
341                ));
342            }
343        }
344        self.expect_token(Token::RParen)?;
345        Ok(DataType::Time32(time_unit))
346    }
347
348    /// Parses the next Time64 (called after `Time64` has been consumed)
349    fn parse_time64(&mut self) -> ArrowResult<DataType> {
350        self.expect_token(Token::LParen)?;
351        let time_unit = self.parse_time_unit("Time64")?;
352        match time_unit {
353            TimeUnit::Microsecond | TimeUnit::Nanosecond => (),
354            TimeUnit::Second | TimeUnit::Millisecond => {
355                return Err(make_error(
356                    self.val,
357                    &format!("Time64 time unit must be 'µs' or 'ns', got '{time_unit}'"),
358                ));
359            }
360        }
361        self.expect_token(Token::RParen)?;
362        Ok(DataType::Time64(time_unit))
363    }
364
365    /// Parses the next Duration (called after `Duration` has been consumed)
366    fn parse_duration(&mut self) -> ArrowResult<DataType> {
367        self.expect_token(Token::LParen)?;
368        let time_unit = self.parse_time_unit("Duration")?;
369        self.expect_token(Token::RParen)?;
370        Ok(DataType::Duration(time_unit))
371    }
372
373    /// Parses the next Interval (called after `Interval` has been consumed)
374    fn parse_interval(&mut self) -> ArrowResult<DataType> {
375        self.expect_token(Token::LParen)?;
376        let interval_unit = match self.next_token()? {
377            Token::IntervalUnit(interval_unit) => interval_unit,
378            tok => {
379                return Err(make_error(
380                    self.val,
381                    &format!("finding IntervalUnit for Interval, got {tok}"),
382                ));
383            }
384        };
385        self.expect_token(Token::RParen)?;
386        Ok(DataType::Interval(interval_unit))
387    }
388
389    /// Parses the next FixedSizeBinary (called after `FixedSizeBinary` has been consumed)
390    fn parse_fixed_size_binary(&mut self) -> ArrowResult<DataType> {
391        self.expect_token(Token::LParen)?;
392        let length = self.parse_i32("FixedSizeBinary")?;
393        if length < 0 {
394            return Err(make_error(
395                self.val,
396                &format!("FixedSizeBinary length must be non-negative, got {length}"),
397            ));
398        }
399        self.expect_token(Token::RParen)?;
400        Ok(DataType::FixedSizeBinary(length))
401    }
402
403    fn validate_decimal(
404        &self,
405        precision: u8,
406        scale: i8,
407        type_name: &str,
408        max_precision: u8,
409    ) -> ArrowResult<()> {
410        if precision == 0 || precision > max_precision {
411            return Err(make_error(
412                self.val,
413                &format!(
414                    "{type_name} precision must be in range [1, {max_precision}], got '{precision}'"
415                ),
416            ));
417        }
418        if scale > 0 && scale as u8 > precision {
419            return Err(make_error(
420                self.val,
421                &format!(
422                    "{type_name} scale '{scale}' cannot be greater than precision '{precision}'"
423                ),
424            ));
425        }
426        Ok(())
427    }
428
429    /// Parses the next Decimal32 (called after `Decimal32` has been consumed)
430    fn parse_decimal_32(&mut self) -> ArrowResult<DataType> {
431        self.expect_token(Token::LParen)?;
432        let precision = self.parse_u8("Decimal32")?;
433        self.expect_token(Token::Comma)?;
434        let scale = self.parse_i8("Decimal32")?;
435        self.expect_token(Token::RParen)?;
436        self.validate_decimal(precision, scale, "Decimal32", DECIMAL32_MAX_PRECISION)?;
437        Ok(DataType::Decimal32(precision, scale))
438    }
439
440    /// Parses the next Decimal64 (called after `Decimal64` has been consumed)
441    fn parse_decimal_64(&mut self) -> ArrowResult<DataType> {
442        self.expect_token(Token::LParen)?;
443        let precision = self.parse_u8("Decimal64")?;
444        self.expect_token(Token::Comma)?;
445        let scale = self.parse_i8("Decimal64")?;
446        self.expect_token(Token::RParen)?;
447        self.validate_decimal(precision, scale, "Decimal64", DECIMAL64_MAX_PRECISION)?;
448        Ok(DataType::Decimal64(precision, scale))
449    }
450
451    /// Parses the next Decimal128 (called after `Decimal128` has been consumed)
452    fn parse_decimal_128(&mut self) -> ArrowResult<DataType> {
453        self.expect_token(Token::LParen)?;
454        let precision = self.parse_u8("Decimal128")?;
455        self.expect_token(Token::Comma)?;
456        let scale = self.parse_i8("Decimal128")?;
457        self.expect_token(Token::RParen)?;
458        self.validate_decimal(precision, scale, "Decimal128", DECIMAL128_MAX_PRECISION)?;
459        Ok(DataType::Decimal128(precision, scale))
460    }
461
462    /// Parses the next Decimal256 (called after `Decimal256` has been consumed)
463    fn parse_decimal_256(&mut self) -> ArrowResult<DataType> {
464        self.expect_token(Token::LParen)?;
465        let precision = self.parse_u8("Decimal256")?;
466        self.expect_token(Token::Comma)?;
467        let scale = self.parse_i8("Decimal256")?;
468        self.expect_token(Token::RParen)?;
469        self.validate_decimal(precision, scale, "Decimal256", DECIMAL256_MAX_PRECISION)?;
470        Ok(DataType::Decimal256(precision, scale))
471    }
472
473    /// Parses the next Dictionary (called after `Dictionary` has been consumed)
474    fn parse_dictionary(&mut self) -> ArrowResult<DataType> {
475        self.expect_token(Token::LParen)?;
476        let key_type = self.parse_next_type()?;
477        self.expect_token(Token::Comma)?;
478        let value_type = self.parse_next_type()?;
479        self.expect_token(Token::RParen)?;
480        Ok(DataType::Dictionary(
481            Box::new(key_type),
482            Box::new(value_type),
483        ))
484    }
485
486    /// Parses the next Struct (called after `Struct` has been consumed)
487    fn parse_struct(&mut self) -> ArrowResult<DataType> {
488        self.expect_token(Token::LParen)?;
489        let mut fields = Vec::new();
490        loop {
491            if self
492                .tokenizer
493                .next_if(|next| matches!(next, Ok(Token::RParen)))
494                .is_some()
495            {
496                break;
497            }
498
499            let field = self.parse_field()?;
500            fields.push(Arc::new(field));
501            match self.next_token()? {
502                Token::Comma => {}
503                Token::RParen => break,
504                tok => {
505                    return Err(make_error(
506                        self.val,
507                        &format!(
508                            "Unexpected token while parsing Struct fields. Expected ',' or ')', but got '{tok}'"
509                        ),
510                    ));
511                }
512            }
513        }
514        Ok(DataType::Struct(Fields::from(fields)))
515    }
516
517    /// Parses the next Union (called after `Union` has been consumed)
518    /// E.g: Union(Sparse, 0: ("a": Int32), 1: ("b": non-null Utf8))
519    fn parse_union(&mut self) -> ArrowResult<DataType> {
520        self.expect_token(Token::LParen)?;
521        let union_mode = self.parse_union_mode()?;
522        let mut type_ids = vec![];
523        let mut fields = vec![];
524        loop {
525            if self
526                .tokenizer
527                .next_if(|next| matches!(next, Ok(Token::RParen)))
528                .is_some()
529            {
530                break;
531            }
532            self.expect_token(Token::Comma)?;
533            let (type_id, field) = self.parse_union_field()?;
534            type_ids.push(type_id);
535            fields.push(field);
536        }
537        Ok(DataType::Union(
538            UnionFields::try_new(type_ids, fields)?,
539            union_mode,
540        ))
541    }
542
543    /// Parses the next UnionMode
544    fn parse_union_mode(&mut self) -> ArrowResult<UnionMode> {
545        match self.next_token()? {
546            Token::UnionMode(union_mode) => Ok(union_mode),
547            tok => Err(make_error(
548                self.val,
549                &format!("finding UnionMode for Union, got {tok}"),
550            )),
551        }
552    }
553
554    /// Parses the next UnionField
555    /// 0: ("a": non-null Int32)
556    fn parse_union_field(&mut self) -> ArrowResult<(i8, Field)> {
557        let type_id = self.parse_i8("UnionField")?;
558        self.expect_token(Token::Colon)?;
559        self.expect_token(Token::LParen)?;
560        let field = self.parse_field()?;
561        self.expect_token(Token::RParen)?;
562        Ok((type_id, field))
563    }
564
565    /// Parses the next Map (called after `Map` has been consumed)
566    /// E.g: Map("entries": Struct("key": Utf8, "value": non-null Int32), sorted)
567    fn parse_map(&mut self) -> ArrowResult<DataType> {
568        self.expect_token(Token::LParen)?;
569        let field = self.parse_field()?;
570        if field.is_nullable() {
571            return Err(make_error(self.val, "Map entries field cannot be nullable"));
572        }
573        if let DataType::Struct(fields) = field.data_type() {
574            if fields.len() != 2 {
575                return Err(make_error(
576                    self.val,
577                    &format!(
578                        "Map entries must contain two children, got {}",
579                        fields.len()
580                    ),
581                ));
582            }
583            if fields[0].is_nullable() {
584                return Err(make_error(self.val, "Map key field cannot be nullable"));
585            }
586        } else {
587            return Err(make_error(
588                self.val,
589                &format!(
590                    "Map entries must be a Struct type, got {}",
591                    field.data_type()
592                ),
593            ));
594        }
595        self.expect_token(Token::Comma)?;
596        let sorted = self.parse_map_sorted()?;
597        self.expect_token(Token::RParen)?;
598        Ok(DataType::Map(Arc::new(field), sorted))
599    }
600
601    /// Parses map's sorted
602    fn parse_map_sorted(&mut self) -> ArrowResult<bool> {
603        match self.next_token()? {
604            Token::MapSorted(sorted) => Ok(sorted),
605            tok => Err(make_error(
606                self.val,
607                &format!("Expected sorted or unsorted for a map; got {tok:?}"),
608            )),
609        }
610    }
611
612    /// Parses the next RunEndEncoded (called after `RunEndEncoded` has been consumed).
613    ///
614    /// Compact form (default field names): `RunEndEncoded(non-null Int32, non-null Utf8)`
615    /// Verbose form (custom field names):  `RunEndEncoded("re": Int32, "v": non-null Utf8)`
616    fn parse_run_end_encoded(&mut self) -> ArrowResult<DataType> {
617        self.expect_token(Token::LParen)?;
618
619        // Distinguish compact from verbose by peeking: verbose starts with a double-quoted name.
620        let verbose = matches!(
621            self.tokenizer.peek(),
622            Some(Ok(Token::DoubleQuotedString(_)))
623        );
624
625        let (run_ends, values) = if verbose {
626            let run_ends = self.parse_field()?;
627            if run_ends.is_nullable() {
628                return Err(make_error(
629                    self.val,
630                    "RunEndEncoded run_ends field cannot be nullable",
631                ));
632            }
633            self.expect_token(Token::Comma)?;
634            let values = self.parse_field()?;
635            (run_ends, values)
636        } else {
637            if self.parse_opt_nullable() {
638                return Err(make_error(
639                    self.val,
640                    "RunEndEncoded run_ends field cannot be nullable",
641                ));
642            }
643            let re_type = self.parse_next_type()?;
644            self.expect_token(Token::Comma)?;
645            let v_nullable = self.parse_opt_nullable();
646            let v_type = self.parse_next_type()?;
647            (
648                Field::new(Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME, re_type, false),
649                Field::new(Field::REE_VALUES_FIELD_DEFAULT_NAME, v_type, v_nullable),
650            )
651        };
652
653        self.expect_token(Token::RParen)?;
654        Ok(DataType::RunEndEncoded(
655            Arc::new(run_ends),
656            Arc::new(values),
657        ))
658    }
659
660    /// consume the next token and return `false` if the field is `nonnull`.
661    fn parse_opt_nullable(&mut self) -> bool {
662        let tok = self
663            .tokenizer
664            .next_if(|next| matches!(next, Ok(Token::NonNull | Token::Nullable)));
665        !matches!(tok, Some(Ok(Token::NonNull)))
666    }
667
668    /// return the next token, or an error if there are none left
669    fn next_token(&mut self) -> ArrowResult<Token> {
670        match self.tokenizer.next() {
671            None => Err(make_error(self.val, "finding next token")),
672            Some(token) => token,
673        }
674    }
675
676    /// consume the next token, returning OK(()) if it matches tok, and Err if not
677    fn expect_token(&mut self, tok: Token) -> ArrowResult<()> {
678        let next_token = self.next_token()?;
679        if next_token == tok {
680            Ok(())
681        } else {
682            Err(make_error_expected(self.val, &tok, &next_token))
683        }
684    }
685}
686
687/// returns true if this character is a separator
688fn is_separator(c: char) -> bool {
689    c == '(' || c == ')' || c == ',' || c == ':' || c == ' '
690}
691
692enum QuoteType {
693    Double,
694    Single,
695}
696
697#[derive(Debug)]
698/// Splits a strings like Dictionary(Int32, Int64) into tokens suitable for parsing
699///
700/// For example the string "Timestamp(ns)" would be parsed into:
701///
702/// * Token::Timestamp
703/// * Token::Lparen
704/// * Token::IntervalUnit(IntervalUnit::Nanosecond)
705/// * Token::Rparen,
706struct Tokenizer<'a> {
707    val: &'a str,
708    chars: Peekable<Chars<'a>>,
709    // temporary buffer for parsing words
710    word: String,
711}
712
713impl<'a> Tokenizer<'a> {
714    fn new(val: &'a str) -> Self {
715        Self {
716            val,
717            chars: val.chars().peekable(),
718            word: String::new(),
719        }
720    }
721
722    /// returns the next char, without consuming it
723    fn peek_next_char(&mut self) -> Option<char> {
724        self.chars.peek().copied()
725    }
726
727    /// returns the next char, and consuming it
728    fn next_char(&mut self) -> Option<char> {
729        self.chars.next()
730    }
731
732    /// parse the characters in val starting at pos, until the next
733    /// `,`, `(`, or `)` or end of line
734    fn parse_word(&mut self) -> ArrowResult<Token> {
735        // reset temp space
736        self.word.clear();
737        loop {
738            match self.peek_next_char() {
739                None => break,
740                Some(c) if is_separator(c) => break,
741                Some(c) => {
742                    self.next_char();
743                    self.word.push(c);
744                }
745            }
746        }
747
748        if let Some(c) = self.word.chars().next() {
749            // if it started with a number, try parsing it as an integer
750            if c == '-' || c.is_numeric() {
751                let val: i64 = self.word.parse().map_err(|e| {
752                    make_error(self.val, &format!("parsing {} as integer: {e}", self.word))
753                })?;
754                return Ok(Token::Integer(val));
755            }
756        }
757
758        // figure out what the word was
759        let token = match self.word.as_str() {
760            "Null" => Token::SimpleType(DataType::Null),
761            "Boolean" => Token::SimpleType(DataType::Boolean),
762
763            "Int8" => Token::SimpleType(DataType::Int8),
764            "Int16" => Token::SimpleType(DataType::Int16),
765            "Int32" => Token::SimpleType(DataType::Int32),
766            "Int64" => Token::SimpleType(DataType::Int64),
767
768            "UInt8" => Token::SimpleType(DataType::UInt8),
769            "UInt16" => Token::SimpleType(DataType::UInt16),
770            "UInt32" => Token::SimpleType(DataType::UInt32),
771            "UInt64" => Token::SimpleType(DataType::UInt64),
772
773            "Utf8" => Token::SimpleType(DataType::Utf8),
774            "LargeUtf8" => Token::SimpleType(DataType::LargeUtf8),
775            "Utf8View" => Token::SimpleType(DataType::Utf8View),
776            "Binary" => Token::SimpleType(DataType::Binary),
777            "BinaryView" => Token::SimpleType(DataType::BinaryView),
778            "LargeBinary" => Token::SimpleType(DataType::LargeBinary),
779
780            "Float16" => Token::SimpleType(DataType::Float16),
781            "Float32" => Token::SimpleType(DataType::Float32),
782            "Float64" => Token::SimpleType(DataType::Float64),
783
784            "Date32" => Token::SimpleType(DataType::Date32),
785            "Date64" => Token::SimpleType(DataType::Date64),
786
787            "List" => Token::List,
788            "ListView" => Token::ListView,
789            "LargeList" => Token::LargeList,
790            "LargeListView" => Token::LargeListView,
791            "FixedSizeList" => Token::FixedSizeList,
792
793            "s" | "Second" => Token::TimeUnit(TimeUnit::Second),
794            "ms" | "Millisecond" => Token::TimeUnit(TimeUnit::Millisecond),
795            "µs" | "us" | "Microsecond" => Token::TimeUnit(TimeUnit::Microsecond),
796            "ns" | "Nanosecond" => Token::TimeUnit(TimeUnit::Nanosecond),
797
798            "Timestamp" => Token::Timestamp,
799            "Time32" => Token::Time32,
800            "Time64" => Token::Time64,
801            "Duration" => Token::Duration,
802            "Interval" => Token::Interval,
803            "Dictionary" => Token::Dictionary,
804
805            "FixedSizeBinary" => Token::FixedSizeBinary,
806
807            "Decimal32" => Token::Decimal32,
808            "Decimal64" => Token::Decimal64,
809            "Decimal128" => Token::Decimal128,
810            "Decimal256" => Token::Decimal256,
811
812            "YearMonth" => Token::IntervalUnit(IntervalUnit::YearMonth),
813            "DayTime" => Token::IntervalUnit(IntervalUnit::DayTime),
814            "MonthDayNano" => Token::IntervalUnit(IntervalUnit::MonthDayNano),
815
816            "Some" => Token::Some,
817            "None" => Token::None,
818
819            "non-null" => Token::NonNull,
820            "nullable" => Token::Nullable,
821            "field" => Token::Field,
822            "x" => Token::X,
823
824            "Struct" => Token::Struct,
825
826            "Union" => Token::Union,
827            "Sparse" => Token::UnionMode(UnionMode::Sparse),
828            "Dense" => Token::UnionMode(UnionMode::Dense),
829
830            "Map" => Token::Map,
831            "sorted" => Token::MapSorted(true),
832            "unsorted" => Token::MapSorted(false),
833
834            "RunEndEncoded" => Token::RunEndEncoded,
835
836            token => {
837                return Err(make_error(self.val, &format!("unknown token: {token}")));
838            }
839        };
840        Ok(token)
841    }
842
843    /// Parses e.g. `"foo bar"`, `'foo bar'`
844    fn parse_quoted_string(&mut self, quote_type: QuoteType) -> ArrowResult<Token> {
845        let quote = match quote_type {
846            QuoteType::Double => '\"',
847            QuoteType::Single => '\'',
848        };
849
850        if self.next_char() != Some(quote) {
851            return Err(make_error(self.val, "Expected \""));
852        }
853
854        // reset temp space
855        self.word.clear();
856
857        let mut is_escaped = false;
858
859        loop {
860            match self.next_char() {
861                None => {
862                    return Err(ArrowError::ParseError(format!(
863                        "Unterminated string at: \"{}",
864                        self.word
865                    )));
866                }
867                Some(c) => match c {
868                    '\\' => {
869                        is_escaped = true;
870                        self.word.push(c);
871                    }
872                    c if c == quote => {
873                        if is_escaped {
874                            self.word.push(c);
875                            is_escaped = false;
876                        } else {
877                            break;
878                        }
879                    }
880                    c => {
881                        self.word.push(c);
882                    }
883                },
884            }
885        }
886
887        let val: String = self.word.parse().map_err(|err| {
888            ArrowError::ParseError(format!("Failed to parse string: \"{}\": {err}", self.word))
889        })?;
890
891        if val.is_empty() {
892            // Using empty strings as field names is just asking for trouble
893            return Err(make_error(self.val, "empty strings aren't allowed"));
894        }
895
896        match quote_type {
897            QuoteType::Double => Ok(Token::DoubleQuotedString(val)),
898            QuoteType::Single => Ok(Token::SingleQuotedString(val)),
899        }
900    }
901}
902
903impl Iterator for Tokenizer<'_> {
904    type Item = ArrowResult<Token>;
905
906    fn next(&mut self) -> Option<Self::Item> {
907        loop {
908            match self.peek_next_char()? {
909                ' ' => {
910                    // skip whitespace
911                    self.next_char();
912                }
913                '"' => {
914                    return Some(self.parse_quoted_string(QuoteType::Double));
915                }
916                '\'' => {
917                    return Some(self.parse_quoted_string(QuoteType::Single));
918                }
919                '(' => {
920                    self.next_char();
921                    return Some(Ok(Token::LParen));
922                }
923                ')' => {
924                    self.next_char();
925                    return Some(Ok(Token::RParen));
926                }
927                ',' => {
928                    self.next_char();
929                    return Some(Ok(Token::Comma));
930                }
931                ':' => {
932                    self.next_char();
933                    return Some(Ok(Token::Colon));
934                }
935                _ => return Some(self.parse_word()),
936            }
937        }
938    }
939}
940
941/// Grammar is
942///
943#[derive(Debug, PartialEq)]
944enum Token {
945    // Null, or Int32
946    SimpleType(DataType),
947    Timestamp,
948    Time32,
949    Time64,
950    Duration,
951    Interval,
952    FixedSizeBinary,
953    Decimal32,
954    Decimal64,
955    Decimal128,
956    Decimal256,
957    Dictionary,
958    TimeUnit(TimeUnit),
959    IntervalUnit(IntervalUnit),
960    LParen,
961    RParen,
962    Comma,
963    Colon,
964    Some,
965    None,
966    Integer(i64),
967    DoubleQuotedString(String),
968    SingleQuotedString(String),
969    List,
970    ListView,
971    LargeList,
972    LargeListView,
973    FixedSizeList,
974    Struct,
975    Union,
976    UnionMode(UnionMode),
977    Map,
978    MapSorted(bool),
979    RunEndEncoded,
980    NonNull,
981    Nullable,
982    Field,
983    X,
984}
985
986impl Display for Token {
987    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
988        match self {
989            Token::SimpleType(t) => write!(f, "{t}"),
990            Token::List => write!(f, "List"),
991            Token::ListView => write!(f, "ListView"),
992            Token::LargeList => write!(f, "LargeList"),
993            Token::LargeListView => write!(f, "LargeListView"),
994            Token::FixedSizeList => write!(f, "FixedSizeList"),
995            Token::Timestamp => write!(f, "Timestamp"),
996            Token::Time32 => write!(f, "Time32"),
997            Token::Time64 => write!(f, "Time64"),
998            Token::Duration => write!(f, "Duration"),
999            Token::Interval => write!(f, "Interval"),
1000            Token::TimeUnit(u) => write!(f, "TimeUnit({u:?})"),
1001            Token::IntervalUnit(u) => write!(f, "IntervalUnit({u:?})"),
1002            Token::LParen => write!(f, "("),
1003            Token::RParen => write!(f, ")"),
1004            Token::Comma => write!(f, ","),
1005            Token::Colon => write!(f, ":"),
1006            Token::Some => write!(f, "Some"),
1007            Token::None => write!(f, "None"),
1008            Token::FixedSizeBinary => write!(f, "FixedSizeBinary"),
1009            Token::Decimal32 => write!(f, "Decimal32"),
1010            Token::Decimal64 => write!(f, "Decimal64"),
1011            Token::Decimal128 => write!(f, "Decimal128"),
1012            Token::Decimal256 => write!(f, "Decimal256"),
1013            Token::Dictionary => write!(f, "Dictionary"),
1014            Token::Integer(v) => write!(f, "Integer({v})"),
1015            Token::DoubleQuotedString(s) => write!(f, "DoubleQuotedString({s})"),
1016            Token::SingleQuotedString(s) => write!(f, "SingleQuotedString({s})"),
1017            Token::Struct => write!(f, "Struct"),
1018            Token::Union => write!(f, "Union"),
1019            Token::UnionMode(m) => write!(f, "{m:?}"),
1020            Token::Map => write!(f, "Map"),
1021            Token::MapSorted(sorted) => {
1022                write!(f, "{}", if *sorted { "sorted" } else { "unsorted" })
1023            }
1024            Token::RunEndEncoded => write!(f, "RunEndEncoded"),
1025            Token::NonNull => write!(f, "non-null"),
1026            Token::Nullable => write!(f, "nullable"),
1027            Token::Field => write!(f, "field"),
1028            Token::X => write!(f, "x"),
1029        }
1030    }
1031}
1032
1033#[cfg(test)]
1034mod test {
1035    use super::*;
1036
1037    #[test]
1038    fn test_parse_data_type() {
1039        // this ensures types can be parsed correctly from their string representations
1040        for dt in list_datatypes() {
1041            round_trip(dt)
1042        }
1043    }
1044
1045    /// Ensure we converting data_type to a string, and then parse it as a type
1046    /// verifying it is the same
1047    fn round_trip(data_type: DataType) {
1048        let data_type_string = data_type.to_string();
1049        println!("Input '{data_type_string}' ({data_type:?})");
1050        let parsed_type = parse_data_type(&data_type_string).unwrap();
1051        assert_eq!(
1052            data_type, parsed_type,
1053            "Mismatch parsing {data_type_string}"
1054        );
1055    }
1056
1057    fn list_datatypes() -> Vec<DataType> {
1058        vec![
1059            // ---------
1060            // Non Nested types
1061            // ---------
1062            DataType::Null,
1063            DataType::Boolean,
1064            DataType::Int8,
1065            DataType::Int16,
1066            DataType::Int32,
1067            DataType::Int64,
1068            DataType::UInt8,
1069            DataType::UInt16,
1070            DataType::UInt32,
1071            DataType::UInt64,
1072            DataType::Float16,
1073            DataType::Float32,
1074            DataType::Float64,
1075            DataType::Timestamp(TimeUnit::Second, None),
1076            DataType::Timestamp(TimeUnit::Millisecond, None),
1077            DataType::Timestamp(TimeUnit::Microsecond, None),
1078            DataType::Timestamp(TimeUnit::Nanosecond, None),
1079            // we can't cover all possible timezones, here we only test utc and +08:00
1080            DataType::Timestamp(TimeUnit::Nanosecond, Some("+00:00".into())),
1081            DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
1082            DataType::Timestamp(TimeUnit::Millisecond, Some("+00:00".into())),
1083            DataType::Timestamp(TimeUnit::Second, Some("+00:00".into())),
1084            DataType::Timestamp(TimeUnit::Nanosecond, Some("+08:00".into())),
1085            DataType::Timestamp(TimeUnit::Microsecond, Some("+08:00".into())),
1086            DataType::Timestamp(TimeUnit::Millisecond, Some("+08:00".into())),
1087            DataType::Timestamp(TimeUnit::Second, Some("+08:00".into())),
1088            DataType::Date32,
1089            DataType::Date64,
1090            DataType::Time32(TimeUnit::Second),
1091            DataType::Time32(TimeUnit::Millisecond),
1092            DataType::Time64(TimeUnit::Microsecond),
1093            DataType::Time64(TimeUnit::Nanosecond),
1094            DataType::Duration(TimeUnit::Second),
1095            DataType::Duration(TimeUnit::Millisecond),
1096            DataType::Duration(TimeUnit::Microsecond),
1097            DataType::Duration(TimeUnit::Nanosecond),
1098            DataType::Interval(IntervalUnit::YearMonth),
1099            DataType::Interval(IntervalUnit::DayTime),
1100            DataType::Interval(IntervalUnit::MonthDayNano),
1101            DataType::Binary,
1102            DataType::BinaryView,
1103            DataType::FixedSizeBinary(0),
1104            DataType::FixedSizeBinary(1234),
1105            DataType::LargeBinary,
1106            DataType::Utf8,
1107            DataType::Utf8View,
1108            DataType::LargeUtf8,
1109            DataType::Decimal32(7, 6),
1110            DataType::Decimal64(6, 5),
1111            DataType::Decimal128(7, 6),
1112            DataType::Decimal256(6, 5),
1113            // ---------
1114            // Nested types
1115            // ---------
1116            DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
1117            DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)),
1118            DataType::Dictionary(
1119                Box::new(DataType::Int8),
1120                Box::new(DataType::Timestamp(TimeUnit::Nanosecond, None)),
1121            ),
1122            DataType::Dictionary(
1123                Box::new(DataType::Int8),
1124                Box::new(DataType::FixedSizeBinary(23)),
1125            ),
1126            DataType::Dictionary(
1127                Box::new(DataType::Int8),
1128                Box::new(
1129                    // nested dictionaries are probably a bad idea but they are possible
1130                    DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)),
1131                ),
1132            ),
1133            DataType::Struct(Fields::from(vec![
1134                Field::new("f1", DataType::Int64, true),
1135                Field::new("f2", DataType::Float64, true),
1136                Field::new(
1137                    "f3",
1138                    DataType::Timestamp(TimeUnit::Second, Some("+08:00".into())),
1139                    true,
1140                ),
1141                Field::new(
1142                    "f4",
1143                    DataType::Dictionary(
1144                        Box::new(DataType::Int8),
1145                        Box::new(DataType::FixedSizeBinary(23)),
1146                    ),
1147                    true,
1148                ),
1149            ])),
1150            DataType::Struct(Fields::from(vec![
1151                Field::new("Int64", DataType::Int64, true),
1152                Field::new("Float64", DataType::Float64, true),
1153            ])),
1154            DataType::Struct(Fields::from(vec![
1155                Field::new("f1", DataType::Int64, true),
1156                Field::new(
1157                    "nested_struct",
1158                    DataType::Struct(Fields::from(vec![Field::new("n1", DataType::Int64, true)])),
1159                    true,
1160                ),
1161            ])),
1162            DataType::Struct(Fields::from(vec![Field::new("f1", DataType::Int64, true)])),
1163            DataType::Struct(Fields::empty()),
1164            DataType::List(Arc::new(Field::new_list_field(DataType::Int64, true))),
1165            DataType::List(Arc::new(Field::new_list_field(DataType::Int64, false))),
1166            DataType::List(Arc::new(Field::new("Int64", DataType::Int64, true))),
1167            DataType::List(Arc::new(Field::new("Int64", DataType::Int64, false))),
1168            DataType::List(Arc::new(Field::new(
1169                "nested_list",
1170                DataType::List(Arc::new(Field::new("Int64", DataType::Int64, true))),
1171                true,
1172            ))),
1173            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int64, true))),
1174            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int64, false))),
1175            DataType::ListView(Arc::new(Field::new("Int64", DataType::Int64, true))),
1176            DataType::ListView(Arc::new(Field::new("Int64", DataType::Int64, false))),
1177            DataType::ListView(Arc::new(Field::new(
1178                "nested_list_view",
1179                DataType::ListView(Arc::new(Field::new("Int64", DataType::Int64, true))),
1180                true,
1181            ))),
1182            DataType::LargeList(Arc::new(Field::new_list_field(DataType::Int64, true))),
1183            DataType::LargeList(Arc::new(Field::new_list_field(DataType::Int64, false))),
1184            DataType::LargeList(Arc::new(Field::new("Int64", DataType::Int64, true))),
1185            DataType::LargeList(Arc::new(Field::new("Int64", DataType::Int64, false))),
1186            DataType::LargeList(Arc::new(Field::new(
1187                "nested_large_list",
1188                DataType::LargeList(Arc::new(Field::new("Int64", DataType::Int64, true))),
1189                true,
1190            ))),
1191            DataType::LargeListView(Arc::new(Field::new_list_field(DataType::Int64, true))),
1192            DataType::LargeListView(Arc::new(Field::new_list_field(DataType::Int64, false))),
1193            DataType::LargeListView(Arc::new(Field::new("Int64", DataType::Int64, true))),
1194            DataType::LargeListView(Arc::new(Field::new("Int64", DataType::Int64, false))),
1195            DataType::LargeListView(Arc::new(Field::new(
1196                "nested_large_list_view",
1197                DataType::LargeListView(Arc::new(Field::new("Int64", DataType::Int64, true))),
1198                true,
1199            ))),
1200            DataType::FixedSizeList(Arc::new(Field::new_list_field(DataType::Int64, true)), 2),
1201            DataType::FixedSizeList(Arc::new(Field::new_list_field(DataType::Int64, false)), 2),
1202            DataType::FixedSizeList(Arc::new(Field::new("Int64", DataType::Int64, true)), 2),
1203            DataType::FixedSizeList(Arc::new(Field::new("Int64", DataType::Int64, false)), 2),
1204            DataType::FixedSizeList(
1205                Arc::new(Field::new(
1206                    "nested_fixed_size_list",
1207                    DataType::FixedSizeList(
1208                        Arc::new(Field::new("Int64", DataType::Int64, true)),
1209                        2,
1210                    ),
1211                    true,
1212                )),
1213                2,
1214            ),
1215            DataType::Union(
1216                UnionFields::from_fields(vec![
1217                    Field::new("Int32", DataType::Int32, false),
1218                    Field::new("Utf8", DataType::Utf8, true),
1219                ]),
1220                UnionMode::Sparse,
1221            ),
1222            DataType::Union(
1223                UnionFields::from_fields(vec![
1224                    Field::new("Int32", DataType::Int32, false),
1225                    Field::new("Utf8", DataType::Utf8, true),
1226                ]),
1227                UnionMode::Dense,
1228            ),
1229            DataType::Union(
1230                UnionFields::from_fields(vec![
1231                    Field::new_union(
1232                        "nested_union",
1233                        vec![0, 1],
1234                        vec![
1235                            Field::new("Int32", DataType::Int32, false),
1236                            Field::new("Utf8", DataType::Utf8, true),
1237                        ],
1238                        UnionMode::Dense,
1239                    ),
1240                    Field::new("Utf8", DataType::Utf8, true),
1241                ]),
1242                UnionMode::Sparse,
1243            ),
1244            DataType::Union(
1245                UnionFields::from_fields(vec![Field::new("Int32", DataType::Int32, false)]),
1246                UnionMode::Dense,
1247            ),
1248            DataType::Union(
1249                UnionFields::try_new(Vec::<i8>::new(), Vec::<Field>::new()).unwrap(),
1250                UnionMode::Sparse,
1251            ),
1252            DataType::RunEndEncoded(
1253                Arc::new(Field::new(
1254                    Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
1255                    DataType::UInt32,
1256                    false,
1257                )),
1258                Arc::new(Field::new(
1259                    Field::REE_VALUES_FIELD_DEFAULT_NAME,
1260                    DataType::Int32,
1261                    true,
1262                )),
1263            ),
1264            DataType::RunEndEncoded(
1265                Arc::new(Field::new(
1266                    Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
1267                    DataType::RunEndEncoded(
1268                        Arc::new(Field::new(
1269                            Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
1270                            DataType::UInt32,
1271                            false,
1272                        )),
1273                        Arc::new(Field::new(
1274                            Field::REE_VALUES_FIELD_DEFAULT_NAME,
1275                            DataType::Int32,
1276                            true,
1277                        )),
1278                    ),
1279                    false,
1280                )),
1281                Arc::new(Field::new(
1282                    Field::REE_VALUES_FIELD_DEFAULT_NAME,
1283                    DataType::Int32,
1284                    true,
1285                )),
1286            ),
1287            // non-default field names trigger verbose display form
1288            DataType::RunEndEncoded(
1289                Arc::new(Field::new(
1290                    Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
1291                    DataType::RunEndEncoded(
1292                        Arc::new(Field::new(
1293                            Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
1294                            DataType::UInt32,
1295                            false,
1296                        )),
1297                        Arc::new(Field::new(
1298                            Field::REE_VALUES_FIELD_DEFAULT_NAME,
1299                            DataType::Int32,
1300                            true,
1301                        )),
1302                    ),
1303                    false,
1304                )),
1305                Arc::new(Field::new("named_values", DataType::Int32, false)),
1306            ),
1307            // verbose form with non-null inner values
1308            DataType::RunEndEncoded(
1309                Arc::new(Field::new(
1310                    Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
1311                    DataType::RunEndEncoded(
1312                        Arc::new(Field::new(
1313                            Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
1314                            DataType::UInt32,
1315                            false,
1316                        )),
1317                        Arc::new(Field::new(
1318                            Field::REE_VALUES_FIELD_DEFAULT_NAME,
1319                            DataType::Int32,
1320                            false,
1321                        )),
1322                    ),
1323                    false,
1324                )),
1325                Arc::new(Field::new("named_values", DataType::Int32, false)),
1326            ),
1327        ]
1328    }
1329
1330    #[test]
1331    fn test_parse_data_type_whitespace_tolerance() {
1332        // (string to parse, expected DataType)
1333        let cases = [
1334            ("Int8", DataType::Int8),
1335            (
1336                "Timestamp        (ns)",
1337                DataType::Timestamp(TimeUnit::Nanosecond, None),
1338            ),
1339            (
1340                "Timestamp        (ns)  ",
1341                DataType::Timestamp(TimeUnit::Nanosecond, None),
1342            ),
1343            (
1344                "          Timestamp        (ns               )",
1345                DataType::Timestamp(TimeUnit::Nanosecond, None),
1346            ),
1347            (
1348                "Timestamp        (ns               )  ",
1349                DataType::Timestamp(TimeUnit::Nanosecond, None),
1350            ),
1351        ];
1352
1353        for (data_type_string, expected_data_type) in cases {
1354            let parsed_data_type = parse_data_type(data_type_string).unwrap();
1355            assert_eq!(
1356                parsed_data_type, expected_data_type,
1357                "Parsing '{data_type_string}', expecting '{expected_data_type}'"
1358            );
1359        }
1360    }
1361
1362    /// Ensure that old style types can still be parsed
1363    #[test]
1364    fn test_parse_data_type_backwards_compatibility() {
1365        use DataType::*;
1366        use IntervalUnit::*;
1367        use TimeUnit::*;
1368        // List below created with:
1369        for t in list_datatypes() {
1370            println!(r#"("{t}", {t:?}),"#);
1371        }
1372        // (string to parse, expected DataType)
1373        let cases = [
1374            ("Timestamp(Nanosecond, None)", Timestamp(Nanosecond, None)),
1375            ("Timestamp(Microsecond, None)", Timestamp(Microsecond, None)),
1376            ("Timestamp(Millisecond, None)", Timestamp(Millisecond, None)),
1377            ("Timestamp(Second, None)", Timestamp(Second, None)),
1378            ("Timestamp(Nanosecond, None)", Timestamp(Nanosecond, None)),
1379            // Timezones
1380            (
1381                r#"Timestamp(Nanosecond, Some("+00:00"))"#,
1382                Timestamp(Nanosecond, Some("+00:00".into())),
1383            ),
1384            (
1385                r#"Timestamp(Microsecond, Some("+00:00"))"#,
1386                Timestamp(Microsecond, Some("+00:00".into())),
1387            ),
1388            (
1389                r#"Timestamp(Millisecond, Some("+00:00"))"#,
1390                Timestamp(Millisecond, Some("+00:00".into())),
1391            ),
1392            (
1393                r#"Timestamp(Second, Some("+00:00"))"#,
1394                Timestamp(Second, Some("+00:00".into())),
1395            ),
1396            ("Null", Null),
1397            ("Boolean", Boolean),
1398            ("Int8", Int8),
1399            ("Int16", Int16),
1400            ("Int32", Int32),
1401            ("Int64", Int64),
1402            ("UInt8", UInt8),
1403            ("UInt16", UInt16),
1404            ("UInt32", UInt32),
1405            ("UInt64", UInt64),
1406            ("Float16", Float16),
1407            ("Float32", Float32),
1408            ("Float64", Float64),
1409            ("Timestamp(s)", Timestamp(Second, None)),
1410            ("Timestamp(ms)", Timestamp(Millisecond, None)),
1411            ("Timestamp(µs)", Timestamp(Microsecond, None)),
1412            ("Timestamp(ns)", Timestamp(Nanosecond, None)),
1413            (
1414                r#"Timestamp(ns, "+00:00")"#,
1415                Timestamp(Nanosecond, Some("+00:00".into())),
1416            ),
1417            (
1418                r#"Timestamp(µs, "+00:00")"#,
1419                Timestamp(Microsecond, Some("+00:00".into())),
1420            ),
1421            (
1422                r#"Timestamp(ms, "+00:00")"#,
1423                Timestamp(Millisecond, Some("+00:00".into())),
1424            ),
1425            (
1426                r#"Timestamp(s, "+00:00")"#,
1427                Timestamp(Second, Some("+00:00".into())),
1428            ),
1429            (
1430                r#"Timestamp(ns, "+08:00")"#,
1431                Timestamp(Nanosecond, Some("+08:00".into())),
1432            ),
1433            (
1434                r#"Timestamp(µs, "+08:00")"#,
1435                Timestamp(Microsecond, Some("+08:00".into())),
1436            ),
1437            (
1438                r#"Timestamp(ms, "+08:00")"#,
1439                Timestamp(Millisecond, Some("+08:00".into())),
1440            ),
1441            (
1442                r#"Timestamp(s, "+08:00")"#,
1443                Timestamp(Second, Some("+08:00".into())),
1444            ),
1445            ("Date32", Date32),
1446            ("Date64", Date64),
1447            ("Time32(s)", Time32(Second)),
1448            ("Time32(ms)", Time32(Millisecond)),
1449            ("Time64(µs)", Time64(Microsecond)),
1450            ("Time64(ns)", Time64(Nanosecond)),
1451            ("Duration(s)", Duration(Second)),
1452            ("Duration(ms)", Duration(Millisecond)),
1453            ("Duration(µs)", Duration(Microsecond)),
1454            ("Duration(ns)", Duration(Nanosecond)),
1455            ("Interval(YearMonth)", Interval(YearMonth)),
1456            ("Interval(DayTime)", Interval(DayTime)),
1457            ("Interval(MonthDayNano)", Interval(MonthDayNano)),
1458            ("Binary", Binary),
1459            ("BinaryView", BinaryView),
1460            ("FixedSizeBinary(0)", FixedSizeBinary(0)),
1461            ("FixedSizeBinary(1234)", FixedSizeBinary(1234)),
1462            ("LargeBinary", LargeBinary),
1463            ("Utf8", Utf8),
1464            ("Utf8View", Utf8View),
1465            ("LargeUtf8", LargeUtf8),
1466            ("Decimal32(7, 6)", Decimal32(7, 6)),
1467            ("Decimal64(6, 5)", Decimal64(6, 5)),
1468            ("Decimal128(7, 6)", Decimal128(7, 6)),
1469            ("Decimal256(6, 5)", Decimal256(6, 5)),
1470            (
1471                "Dictionary(Int32, Utf8)",
1472                Dictionary(Box::new(Int32), Box::new(Utf8)),
1473            ),
1474            (
1475                "Dictionary(Int8, Utf8)",
1476                Dictionary(Box::new(Int8), Box::new(Utf8)),
1477            ),
1478            (
1479                "Dictionary(Int8, Timestamp(ns))",
1480                Dictionary(Box::new(Int8), Box::new(Timestamp(Nanosecond, None))),
1481            ),
1482            (
1483                "Dictionary(Int8, FixedSizeBinary(23))",
1484                Dictionary(Box::new(Int8), Box::new(FixedSizeBinary(23))),
1485            ),
1486            (
1487                "Dictionary(Int8, Dictionary(Int8, Utf8))",
1488                Dictionary(
1489                    Box::new(Int8),
1490                    Box::new(Dictionary(Box::new(Int8), Box::new(Utf8))),
1491                ),
1492            ),
1493            (
1494                r#"Struct("f1": nullable Int64, "f2": nullable Float64, "f3": nullable Timestamp(s, "+08:00"), "f4": nullable Dictionary(Int8, FixedSizeBinary(23)))"#,
1495                Struct(Fields::from(vec![
1496                    Field::new("f1", Int64, true),
1497                    Field::new("f2", Float64, true),
1498                    Field::new("f3", Timestamp(Second, Some("+08:00".into())), true),
1499                    Field::new(
1500                        "f4",
1501                        Dictionary(Box::new(Int8), Box::new(FixedSizeBinary(23))),
1502                        true,
1503                    ),
1504                ])),
1505            ),
1506            (
1507                r#"Struct("Int64": nullable Int64, "Float64": nullable Float64)"#,
1508                Struct(Fields::from(vec![
1509                    Field::new("Int64", Int64, true),
1510                    Field::new("Float64", Float64, true),
1511                ])),
1512            ),
1513            (
1514                r#"Struct("f1": nullable Int64, "nested_struct": nullable Struct("n1": nullable Int64))"#,
1515                Struct(Fields::from(vec![
1516                    Field::new("f1", Int64, true),
1517                    Field::new(
1518                        "nested_struct",
1519                        Struct(Fields::from(vec![Field::new("n1", Int64, true)])),
1520                        true,
1521                    ),
1522                ])),
1523            ),
1524            (r"Struct()", Struct(Fields::empty())),
1525            (
1526                "FixedSizeList(4, Int64)",
1527                FixedSizeList(Arc::new(Field::new_list_field(Int64, true)), 4),
1528            ),
1529            (
1530                "List(Int64)",
1531                List(Arc::new(Field::new_list_field(Int64, true))),
1532            ),
1533            (
1534                "LargeList(Int64)",
1535                LargeList(Arc::new(Field::new_list_field(Int64, true))),
1536            ),
1537        ];
1538
1539        for (data_type_string, expected_data_type) in cases {
1540            let parsed_data_type = parse_data_type(data_type_string).unwrap();
1541            assert_eq!(
1542                parsed_data_type, expected_data_type,
1543                "Parsing '{data_type_string}', expecting '{expected_data_type}'"
1544            );
1545        }
1546    }
1547
1548    #[test]
1549    fn parse_data_type_errors() {
1550        // (string to parse, expected error message)
1551        let cases = [
1552            ("", "Unsupported type ''"),
1553            ("", "Error finding next token"),
1554            ("null", "Unsupported type 'null'"),
1555            ("Nu", "Unsupported type 'Nu'"),
1556            (r"Timestamp(ns, +00:00)", "Error unknown token: +00"),
1557            (
1558                r#"Timestamp(ns, "+00:00)"#,
1559                r#"Unterminated string at: "+00:00)"#,
1560            ),
1561            (r#"Timestamp(ns, "")"#, r"empty strings aren't allowed"),
1562            (
1563                r#"Timestamp(ns, "+00:00"")"#,
1564                r#"Parser error: Unterminated string at: ")"#,
1565            ),
1566            ("Timestamp(ns, ", "Error finding next token"),
1567            (
1568                "Float32 Float32",
1569                "trailing content after parsing 'Float32'",
1570            ),
1571            ("Int32, ", "trailing content after parsing 'Int32'"),
1572            ("Int32(3), ", "trailing content after parsing 'Int32'"),
1573            (
1574                "FixedSizeBinary(Int32), ",
1575                "Error finding i64 for FixedSizeBinary, got 'Int32'",
1576            ),
1577            (
1578                "FixedSizeBinary(3.0), ",
1579                "Error parsing 3.0 as integer: invalid digit found in string",
1580            ),
1581            // too large for i32
1582            (
1583                "FixedSizeBinary(4000000000), ",
1584                "Error converting 4000000000 into i32 for FixedSizeBinary:",
1585            ),
1586            // can't have negative width
1587            (
1588                "FixedSizeBinary(-1), ",
1589                "FixedSizeBinary length must be non-negative, got -1",
1590            ),
1591            (
1592                "FixedSizeList(-1, Int64), ",
1593                "FixedSizeList length must be non-negative, got -1",
1594            ),
1595            // can't have negative precision
1596            (
1597                "Decimal32(-3, 5)",
1598                "Error converting -3 into u8 for Decimal32:",
1599            ),
1600            (
1601                "Decimal64(-3, 5)",
1602                "Error converting -3 into u8 for Decimal64:",
1603            ),
1604            (
1605                "Decimal128(-3, 5)",
1606                "Error converting -3 into u8 for Decimal128:",
1607            ),
1608            (
1609                "Decimal256(-3, 5)",
1610                "Error converting -3 into u8 for Decimal256:",
1611            ),
1612            (
1613                "Decimal32(3, 500)",
1614                "Error converting 500 into i8 for Decimal32:",
1615            ),
1616            (
1617                "Decimal64(3, 500)",
1618                "Error converting 500 into i8 for Decimal64:",
1619            ),
1620            (
1621                "Decimal128(3, 500)",
1622                "Error converting 500 into i8 for Decimal128:",
1623            ),
1624            (
1625                "Decimal256(3, 500)",
1626                "Error converting 500 into i8 for Decimal256:",
1627            ),
1628            ("Struct(f1 Int64)", "Error unknown token: f1"),
1629            ("Struct(\"f1\" Int64)", "Expected ':'"),
1630            (
1631                "Struct(\"f1\": )",
1632                "Error finding next type, got unexpected ')'",
1633            ),
1634            // Invalid time combinations
1635            (
1636                "Time32(µs)",
1637                "Error Time32 time unit must be 's' or 'ms', got 'µs'",
1638            ),
1639            (
1640                "Time32(ns)",
1641                "Error Time32 time unit must be 's' or 'ms', got 'ns'",
1642            ),
1643            (
1644                "Time64(s)",
1645                "Error Time64 time unit must be 'µs' or 'ns', got 's'",
1646            ),
1647            (
1648                "Time64(ms)",
1649                "Error Time64 time unit must be 'µs' or 'ns', got 'ms'",
1650            ),
1651            // Decimals can't have scale exceeding precision
1652            (
1653                "Decimal32(5, 6)",
1654                "Error Decimal32 scale '6' cannot be greater than precision '5'",
1655            ),
1656            (
1657                "Decimal64(5, 6)",
1658                "Error Decimal64 scale '6' cannot be greater than precision '5'",
1659            ),
1660            (
1661                "Decimal128(5, 6)",
1662                "Error Decimal128 scale '6' cannot be greater than precision '5'",
1663            ),
1664            (
1665                "Decimal256(5, 6)",
1666                "Error Decimal256 scale '6' cannot be greater than precision '5'",
1667            ),
1668            // Decimals have a max supported precision
1669            (
1670                "Decimal32(10, 0)",
1671                "Error Decimal32 precision must be in range [1, 9], got '10'",
1672            ),
1673            (
1674                "Decimal64(19, 0)",
1675                "Error Decimal64 precision must be in range [1, 18], got '19'",
1676            ),
1677            (
1678                "Decimal128(39, 0)",
1679                "Error Decimal128 precision must be in range [1, 38], got '39'",
1680            ),
1681            (
1682                "Decimal256(77, 0)",
1683                "Error Decimal256 precision must be in range [1, 76], got '77'",
1684            ),
1685            // Decimals precision can't be 0
1686            (
1687                "Decimal32(0, 0)",
1688                "Error Decimal32 precision must be in range [1, 9], got '0'",
1689            ),
1690            (
1691                "Decimal64(0, 0)",
1692                "Error Decimal64 precision must be in range [1, 18], got '0'",
1693            ),
1694            (
1695                "Decimal128(0, 0)",
1696                "Error Decimal128 precision must be in range [1, 38], got '0'",
1697            ),
1698            (
1699                "Decimal256(0, 0)",
1700                "Error Decimal256 precision must be in range [1, 76], got '0'",
1701            ),
1702            // REE run_ends cannot be nullable
1703            (
1704                r#"RunEndEncoded("re": nullable Int32, "v": non-null Utf8)"#,
1705                "RunEndEncoded run_ends field cannot be nullable",
1706            ),
1707            (
1708                r#"RunEndEncoded("re": Int32, "v": non-null Utf8)"#,
1709                "RunEndEncoded run_ends field cannot be nullable",
1710            ),
1711            (
1712                "RunEndEncoded(nullable Int32, non-null Utf8)",
1713                "RunEndEncoded run_ends field cannot be nullable",
1714            ),
1715            // Map entries field cannot be nullable
1716            (
1717                r#"Map("entries": Struct("key": non-null Utf8, "value": nullable Int32), unsorted)"#,
1718                "Map entries field cannot be nullable",
1719            ),
1720            // Map key cannot be nullable
1721            (
1722                r#"Map("entries": non-null Struct("key": nullable Utf8, "value": nullable Int32), unsorted)"#,
1723                "Map key field cannot be nullable",
1724            ),
1725            (
1726                r#"Map("entries": non-null Struct("key": Utf8, "value": nullable Int32), unsorted)"#,
1727                "Map key field cannot be nullable",
1728            ),
1729        ];
1730
1731        for (data_type_string, expected_message) in cases {
1732            println!("Parsing '{data_type_string}', expecting '{expected_message}'");
1733            match parse_data_type(data_type_string) {
1734                Ok(d) => panic!("Expected error while parsing '{data_type_string}', but got '{d}'"),
1735                Err(e) => {
1736                    let message = e.to_string();
1737                    assert!(
1738                        message.contains(expected_message),
1739                        "\n\ndid not find expected in actual.\n\nexpected: {expected_message}\nactual: {message}\n"
1740                    );
1741
1742                    if !message.contains("Unterminated string") {
1743                        // errors should also contain a help message
1744                        assert!(message.contains("Must be a supported arrow type name such as 'Int32' or 'Timestamp(ns)'"), "message: {message}");
1745                    }
1746                }
1747            }
1748        }
1749    }
1750
1751    #[test]
1752    fn parse_error_type() {
1753        let err = parse_data_type("foobar").unwrap_err();
1754        assert!(matches!(err, ArrowError::ParseError(_)));
1755        assert_eq!(
1756            err.to_string(),
1757            "Parser error: Unsupported type 'foobar'. Must be a supported arrow type name such as 'Int32' or 'Timestamp(ns)'. Error unknown token: foobar"
1758        );
1759    }
1760}
arrow_schema/datatype_parse.rs

arrow_schema/
datatype_parse.rs
