arrow_schema/
datatype_parse.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::{fmt::Display, iter::Peekable, str::Chars, sync::Arc};
19
20use crate::{
21    ArrowError, DECIMAL32_MAX_PRECISION, DECIMAL64_MAX_PRECISION, DECIMAL128_MAX_PRECISION,
22    DECIMAL256_MAX_PRECISION, DataType, Field, Fields, IntervalUnit, TimeUnit, UnionFields,
23    UnionMode,
24};
25
26/// Parses a DataType from a string representation
27///
28/// For example, the string "Int32" would be parsed into [`DataType::Int32`]
29pub(crate) fn parse_data_type(val: &str) -> ArrowResult<DataType> {
30    Parser::new(val).parse()
31}
32
33type ArrowResult<T> = Result<T, ArrowError>;
34
35fn make_error(val: &str, msg: &str) -> ArrowError {
36    let msg = format!(
37        "Unsupported type '{val}'. Must be a supported arrow type name such as 'Int32' or 'Timestamp(ns)'. Error {msg}"
38    );
39    ArrowError::ParseError(msg)
40}
41
42fn make_error_expected(val: &str, expected: &Token, actual: &Token) -> ArrowError {
43    make_error(val, &format!("Expected '{expected}', got '{actual}'"))
44}
45
46/// Implementation of `parse_data_type`, modeled after <https://github.com/sqlparser-rs/sqlparser-rs>
47#[derive(Debug)]
48struct Parser<'a> {
49    val: &'a str,
50    tokenizer: Peekable<Tokenizer<'a>>,
51}
52
53impl<'a> Parser<'a> {
54    fn new(val: &'a str) -> Self {
55        Self {
56            val,
57            tokenizer: Tokenizer::new(val).peekable(),
58        }
59    }
60
61    fn parse(mut self) -> ArrowResult<DataType> {
62        let data_type = self.parse_next_type()?;
63        // ensure that there is no trailing content
64        if self.tokenizer.next().is_some() {
65            Err(make_error(
66                self.val,
67                &format!("checking trailing content after parsing '{data_type}'"),
68            ))
69        } else {
70            Ok(data_type)
71        }
72    }
73
74    /// parses the next full DataType
75    fn parse_next_type(&mut self) -> ArrowResult<DataType> {
76        match self.next_token()? {
77            Token::SimpleType(data_type) => Ok(data_type),
78            Token::Timestamp => self.parse_timestamp(),
79            Token::Time32 => self.parse_time32(),
80            Token::Time64 => self.parse_time64(),
81            Token::Duration => self.parse_duration(),
82            Token::Interval => self.parse_interval(),
83            Token::FixedSizeBinary => self.parse_fixed_size_binary(),
84            Token::Decimal32 => self.parse_decimal_32(),
85            Token::Decimal64 => self.parse_decimal_64(),
86            Token::Decimal128 => self.parse_decimal_128(),
87            Token::Decimal256 => self.parse_decimal_256(),
88            Token::Dictionary => self.parse_dictionary(),
89            Token::List => self.parse_list(),
90            Token::ListView => self.parse_list_view(),
91            Token::LargeList => self.parse_large_list(),
92            Token::LargeListView => self.parse_large_list_view(),
93            Token::FixedSizeList => self.parse_fixed_size_list(),
94            Token::Struct => self.parse_struct(),
95            Token::Union => self.parse_union(),
96            Token::Map => self.parse_map(),
97            Token::RunEndEncoded => self.parse_run_end_encoded(),
98            tok => Err(make_error(
99                self.val,
100                &format!("finding next type, got unexpected '{tok}'"),
101            )),
102        }
103    }
104
105    /// parses Field, this is the inversion of `format_field` in `datatype_display.rs`.
106    /// E.g: "a": non-null Int64
107    ///
108    /// TODO: support metadata: `"a": non-null Int64 metadata: {"foo": "value"}`
109    fn parse_field(&mut self) -> ArrowResult<Field> {
110        let name = self.parse_double_quoted_string("Field")?;
111        self.expect_token(Token::Colon)?;
112        let nullable = self.parse_opt_nullable();
113        let data_type = self.parse_next_type()?;
114        Ok(Field::new(name, data_type, nullable))
115    }
116
117    /// Parses field inside a list. Use `Field::LIST_FIELD_DEFAULT_NAME`
118    /// if no field name is specified.
119    /// E.g: `non-null Int64, field: 'foo'` or `non-null Int64`
120    ///
121    /// TODO: support metadata: `non-ull Int64, metadata: {"foo2": "value"}`
122    fn parse_list_field(&mut self, context: &str) -> ArrowResult<Field> {
123        let nullable = self.parse_opt_nullable();
124        let data_type = self.parse_next_type()?;
125
126        // the field name (if exists) must be after a comma
127        let field_name = if self
128            .tokenizer
129            .next_if(|next| matches!(next, Ok(Token::Comma)))
130            .is_none()
131        {
132            Field::LIST_FIELD_DEFAULT_NAME.into()
133        } else {
134            // expects: `field: 'field_name'`.
135            self.expect_token(Token::Field)?;
136            self.expect_token(Token::Colon)?;
137            self.parse_single_quoted_string(context)?
138        };
139
140        Ok(Field::new(field_name, data_type, nullable))
141    }
142
143    /// Parses the List type (called after `List` has been consumed)
144    /// E.g: List(non-null Int64, field: 'foo')
145    fn parse_list(&mut self) -> ArrowResult<DataType> {
146        self.expect_token(Token::LParen)?;
147        let field = self.parse_list_field("List")?;
148        self.expect_token(Token::RParen)?;
149        Ok(DataType::List(Arc::new(field)))
150    }
151
152    /// Parses the ListView type (called after `ListView` has been consumed)
153    /// E.g: ListView(non-null Int64, field: 'foo')
154    fn parse_list_view(&mut self) -> ArrowResult<DataType> {
155        self.expect_token(Token::LParen)?;
156        let field = self.parse_list_field("ListView")?;
157        self.expect_token(Token::RParen)?;
158        Ok(DataType::ListView(Arc::new(field)))
159    }
160
161    /// Parses the LargeList type (called after `LargeList` has been consumed)
162    /// E.g: LargeList(non-null Int64, field: 'foo')
163    fn parse_large_list(&mut self) -> ArrowResult<DataType> {
164        self.expect_token(Token::LParen)?;
165        let field = self.parse_list_field("LargeList")?;
166        self.expect_token(Token::RParen)?;
167        Ok(DataType::LargeList(Arc::new(field)))
168    }
169
170    /// Parses the LargeListView type (called after `LargeListView` has been consumed)
171    /// E.g: LargeListView(non-null Int64, field: 'foo')
172    fn parse_large_list_view(&mut self) -> ArrowResult<DataType> {
173        self.expect_token(Token::LParen)?;
174        let field = self.parse_list_field("LargeListView")?;
175        self.expect_token(Token::RParen)?;
176        Ok(DataType::LargeListView(Arc::new(field)))
177    }
178
179    /// Parses the FixedSizeList type (called after `FixedSizeList` has been consumed)
180    ///
181    /// Examples:
182    /// * `FixedSizeList(5 x non-null Int64, field: 'foo')`
183    /// * `FixedSizeList(4, Int64)`
184    ///
185    fn parse_fixed_size_list(&mut self) -> ArrowResult<DataType> {
186        self.expect_token(Token::LParen)?;
187        let length = self.parse_i32("FixedSizeList")?;
188        if length < 0 {
189            return Err(make_error(
190                self.val,
191                &format!("FixedSizeList length must be non-negative, got {length}"),
192            ));
193        }
194        match self.next_token()? {
195            // `FixedSizeList(5 x non-null Int64, field: 'foo')` format
196            Token::X => {
197                let field = self.parse_list_field("FixedSizeList")?;
198                self.expect_token(Token::RParen)?;
199                Ok(DataType::FixedSizeList(Arc::new(field), length))
200            }
201            // `FixedSizeList(4, Int64)` format
202            Token::Comma => {
203                let data_type = self.parse_next_type()?;
204                self.expect_token(Token::RParen)?;
205                Ok(DataType::FixedSizeList(
206                    Arc::new(Field::new_list_field(data_type, true)),
207                    length,
208                ))
209            }
210            tok => Err(make_error(
211                self.val,
212                &format!("Expected 'x' or ',' after length for FixedSizeList, got '{tok}'"),
213            )),
214        }
215    }
216
217    /// Parses the next timeunit
218    fn parse_time_unit(&mut self, context: &str) -> ArrowResult<TimeUnit> {
219        match self.next_token()? {
220            Token::TimeUnit(time_unit) => Ok(time_unit),
221            tok => Err(make_error(
222                self.val,
223                &format!("finding TimeUnit for {context}, got {tok}"),
224            )),
225        }
226    }
227
228    /// Parses the next double quoted string
229    fn parse_double_quoted_string(&mut self, context: &str) -> ArrowResult<String> {
230        let token = self.next_token()?;
231        if let Token::DoubleQuotedString(string) = token {
232            Ok(string)
233        } else {
234            Err(make_error(
235                self.val,
236                &format!("expected double quoted string for {context}, got '{token}'"),
237            ))
238        }
239    }
240
241    /// Parses the next single quoted string
242    fn parse_single_quoted_string(&mut self, context: &str) -> ArrowResult<String> {
243        let token = self.next_token()?;
244        if let Token::SingleQuotedString(string) = token {
245            Ok(string)
246        } else {
247            Err(make_error(
248                self.val,
249                &format!("expected single quoted string for {context}, got '{token}'"),
250            ))
251        }
252    }
253
254    /// Parses the next integer value
255    fn parse_i64(&mut self, context: &str) -> ArrowResult<i64> {
256        match self.next_token()? {
257            Token::Integer(v) => Ok(v),
258            tok => Err(make_error(
259                self.val,
260                &format!("finding i64 for {context}, got '{tok}'"),
261            )),
262        }
263    }
264
265    /// Parses the next i32 integer value
266    fn parse_i32(&mut self, context: &str) -> ArrowResult<i32> {
267        let length = self.parse_i64(context)?;
268        length.try_into().map_err(|e| {
269            make_error(
270                self.val,
271                &format!("converting {length} into i32 for {context}: {e}"),
272            )
273        })
274    }
275
276    /// Parses the next i8 integer value
277    fn parse_i8(&mut self, context: &str) -> ArrowResult<i8> {
278        let length = self.parse_i64(context)?;
279        length.try_into().map_err(|e| {
280            make_error(
281                self.val,
282                &format!("converting {length} into i8 for {context}: {e}"),
283            )
284        })
285    }
286
287    /// Parses the next u8 integer value
288    fn parse_u8(&mut self, context: &str) -> ArrowResult<u8> {
289        let length = self.parse_i64(context)?;
290        length.try_into().map_err(|e| {
291            make_error(
292                self.val,
293                &format!("converting {length} into u8 for {context}: {e}"),
294            )
295        })
296    }
297
298    /// Parses the next timestamp (called after `Timestamp` has been consumed)
299    fn parse_timestamp(&mut self) -> ArrowResult<DataType> {
300        self.expect_token(Token::LParen)?;
301        let time_unit = self.parse_time_unit("Timestamp")?;
302
303        let timezone;
304        match self.next_token()? {
305            Token::Comma => {
306                match self.next_token()? {
307                    // Support old style `Timestamp(Nanosecond, None)`
308                    Token::None => {
309                        timezone = None;
310                    }
311                    // Support old style `Timestamp(Nanosecond, Some("Timezone"))`
312                    Token::Some => {
313                        self.expect_token(Token::LParen)?;
314                        timezone = Some(self.parse_double_quoted_string("Timezone")?);
315                        self.expect_token(Token::RParen)?;
316                    }
317                    Token::DoubleQuotedString(tz) => {
318                        // Support new style `Timestamp(Nanosecond, "Timezone")`
319                        timezone = Some(tz);
320                    }
321                    tok => {
322                        return Err(make_error(
323                            self.val,
324                            &format!("Expected None, Some, or a timezone string, got {tok:?}"),
325                        ));
326                    }
327                }
328                self.expect_token(Token::RParen)?;
329            }
330            // No timezone (e.g `Timestamp(ns)`)
331            Token::RParen => {
332                timezone = None;
333            }
334            next_token => {
335                return Err(make_error(
336                    self.val,
337                    &format!("Expected comma followed by a timezone, or an ), got {next_token:?}"),
338                ));
339            }
340        }
341        Ok(DataType::Timestamp(time_unit, timezone.map(Into::into)))
342    }
343
344    /// Parses the next Time32 (called after `Time32` has been consumed)
345    fn parse_time32(&mut self) -> ArrowResult<DataType> {
346        self.expect_token(Token::LParen)?;
347        let time_unit = self.parse_time_unit("Time32")?;
348        match time_unit {
349            TimeUnit::Second | TimeUnit::Millisecond => (),
350            TimeUnit::Microsecond | TimeUnit::Nanosecond => {
351                return Err(make_error(
352                    self.val,
353                    &format!("Time32 time unit must be 's' or 'ms', got '{time_unit}'"),
354                ));
355            }
356        }
357        self.expect_token(Token::RParen)?;
358        Ok(DataType::Time32(time_unit))
359    }
360
361    /// Parses the next Time64 (called after `Time64` has been consumed)
362    fn parse_time64(&mut self) -> ArrowResult<DataType> {
363        self.expect_token(Token::LParen)?;
364        let time_unit = self.parse_time_unit("Time64")?;
365        match time_unit {
366            TimeUnit::Microsecond | TimeUnit::Nanosecond => (),
367            TimeUnit::Second | TimeUnit::Millisecond => {
368                return Err(make_error(
369                    self.val,
370                    &format!("Time64 time unit must be 'µs' or 'ns', got '{time_unit}'"),
371                ));
372            }
373        }
374        self.expect_token(Token::RParen)?;
375        Ok(DataType::Time64(time_unit))
376    }
377
378    /// Parses the next Duration (called after `Duration` has been consumed)
379    fn parse_duration(&mut self) -> ArrowResult<DataType> {
380        self.expect_token(Token::LParen)?;
381        let time_unit = self.parse_time_unit("Duration")?;
382        self.expect_token(Token::RParen)?;
383        Ok(DataType::Duration(time_unit))
384    }
385
386    /// Parses the next Interval (called after `Interval` has been consumed)
387    fn parse_interval(&mut self) -> ArrowResult<DataType> {
388        self.expect_token(Token::LParen)?;
389        let interval_unit = match self.next_token()? {
390            Token::IntervalUnit(interval_unit) => interval_unit,
391            tok => {
392                return Err(make_error(
393                    self.val,
394                    &format!("finding IntervalUnit for Interval, got {tok}"),
395                ));
396            }
397        };
398        self.expect_token(Token::RParen)?;
399        Ok(DataType::Interval(interval_unit))
400    }
401
402    /// Parses the next FixedSizeBinary (called after `FixedSizeBinary` has been consumed)
403    fn parse_fixed_size_binary(&mut self) -> ArrowResult<DataType> {
404        self.expect_token(Token::LParen)?;
405        let length = self.parse_i32("FixedSizeBinary")?;
406        if length < 0 {
407            return Err(make_error(
408                self.val,
409                &format!("FixedSizeBinary length must be non-negative, got {length}"),
410            ));
411        }
412        self.expect_token(Token::RParen)?;
413        Ok(DataType::FixedSizeBinary(length))
414    }
415
416    fn validate_decimal(
417        &self,
418        precision: u8,
419        scale: i8,
420        type_name: &str,
421        max_precision: u8,
422    ) -> ArrowResult<()> {
423        if precision == 0 || precision > max_precision {
424            return Err(make_error(
425                self.val,
426                &format!(
427                    "{type_name} precision must be in range [1, {max_precision}], got '{precision}'"
428                ),
429            ));
430        }
431        if scale > 0 && scale as u8 > precision {
432            return Err(make_error(
433                self.val,
434                &format!(
435                    "{type_name} scale '{scale}' cannot be greater than precision '{precision}'"
436                ),
437            ));
438        }
439        Ok(())
440    }
441
442    /// Parses the next Decimal32 (called after `Decimal32` has been consumed)
443    fn parse_decimal_32(&mut self) -> ArrowResult<DataType> {
444        self.expect_token(Token::LParen)?;
445        let precision = self.parse_u8("Decimal32")?;
446        self.expect_token(Token::Comma)?;
447        let scale = self.parse_i8("Decimal32")?;
448        self.expect_token(Token::RParen)?;
449        self.validate_decimal(precision, scale, "Decimal32", DECIMAL32_MAX_PRECISION)?;
450        Ok(DataType::Decimal32(precision, scale))
451    }
452
453    /// Parses the next Decimal64 (called after `Decimal64` has been consumed)
454    fn parse_decimal_64(&mut self) -> ArrowResult<DataType> {
455        self.expect_token(Token::LParen)?;
456        let precision = self.parse_u8("Decimal64")?;
457        self.expect_token(Token::Comma)?;
458        let scale = self.parse_i8("Decimal64")?;
459        self.expect_token(Token::RParen)?;
460        self.validate_decimal(precision, scale, "Decimal64", DECIMAL64_MAX_PRECISION)?;
461        Ok(DataType::Decimal64(precision, scale))
462    }
463
464    /// Parses the next Decimal128 (called after `Decimal128` has been consumed)
465    fn parse_decimal_128(&mut self) -> ArrowResult<DataType> {
466        self.expect_token(Token::LParen)?;
467        let precision = self.parse_u8("Decimal128")?;
468        self.expect_token(Token::Comma)?;
469        let scale = self.parse_i8("Decimal128")?;
470        self.expect_token(Token::RParen)?;
471        self.validate_decimal(precision, scale, "Decimal128", DECIMAL128_MAX_PRECISION)?;
472        Ok(DataType::Decimal128(precision, scale))
473    }
474
475    /// Parses the next Decimal256 (called after `Decimal256` has been consumed)
476    fn parse_decimal_256(&mut self) -> ArrowResult<DataType> {
477        self.expect_token(Token::LParen)?;
478        let precision = self.parse_u8("Decimal256")?;
479        self.expect_token(Token::Comma)?;
480        let scale = self.parse_i8("Decimal256")?;
481        self.expect_token(Token::RParen)?;
482        self.validate_decimal(precision, scale, "Decimal256", DECIMAL256_MAX_PRECISION)?;
483        Ok(DataType::Decimal256(precision, scale))
484    }
485
486    /// Parses the next Dictionary (called after `Dictionary` has been consumed)
487    fn parse_dictionary(&mut self) -> ArrowResult<DataType> {
488        self.expect_token(Token::LParen)?;
489        let key_type = self.parse_next_type()?;
490        self.expect_token(Token::Comma)?;
491        let value_type = self.parse_next_type()?;
492        self.expect_token(Token::RParen)?;
493        Ok(DataType::Dictionary(
494            Box::new(key_type),
495            Box::new(value_type),
496        ))
497    }
498
499    /// Parses the next Struct (called after `Struct` has been consumed)
500    fn parse_struct(&mut self) -> ArrowResult<DataType> {
501        self.expect_token(Token::LParen)?;
502        let mut fields = Vec::new();
503        loop {
504            if self
505                .tokenizer
506                .next_if(|next| matches!(next, Ok(Token::RParen)))
507                .is_some()
508            {
509                break;
510            }
511
512            let field = self.parse_field()?;
513            fields.push(Arc::new(field));
514            match self.next_token()? {
515                Token::Comma => {}
516                Token::RParen => break,
517                tok => {
518                    return Err(make_error(
519                        self.val,
520                        &format!(
521                            "Unexpected token while parsing Struct fields. Expected ',' or ')', but got '{tok}'"
522                        ),
523                    ));
524                }
525            }
526        }
527        Ok(DataType::Struct(Fields::from(fields)))
528    }
529
530    /// Parses the next Union (called after `Union` has been consumed)
531    /// E.g: Union(Sparse, 0: ("a": Int32), 1: ("b": non-null Utf8))
532    fn parse_union(&mut self) -> ArrowResult<DataType> {
533        self.expect_token(Token::LParen)?;
534        let union_mode = self.parse_union_mode()?;
535        let mut type_ids = vec![];
536        let mut fields = vec![];
537        loop {
538            if self
539                .tokenizer
540                .next_if(|next| matches!(next, Ok(Token::RParen)))
541                .is_some()
542            {
543                break;
544            }
545            self.expect_token(Token::Comma)?;
546            let (type_id, field) = self.parse_union_field()?;
547            type_ids.push(type_id);
548            fields.push(field);
549        }
550        Ok(DataType::Union(
551            UnionFields::try_new(type_ids, fields)?,
552            union_mode,
553        ))
554    }
555
556    /// Parses the next UnionMode
557    fn parse_union_mode(&mut self) -> ArrowResult<UnionMode> {
558        match self.next_token()? {
559            Token::UnionMode(union_mode) => Ok(union_mode),
560            tok => Err(make_error(
561                self.val,
562                &format!("finding UnionMode for Union, got {tok}"),
563            )),
564        }
565    }
566
567    /// Parses the next UnionField
568    /// 0: ("a": non-null Int32)
569    fn parse_union_field(&mut self) -> ArrowResult<(i8, Field)> {
570        let type_id = self.parse_i8("UnionField")?;
571        self.expect_token(Token::Colon)?;
572        self.expect_token(Token::LParen)?;
573        let field = self.parse_field()?;
574        self.expect_token(Token::RParen)?;
575        Ok((type_id, field))
576    }
577
578    /// Parses the next Map (called after `Map` has been consumed)
579    /// E.g: Map("entries": Struct("key": Utf8, "value": non-null Int32), sorted)
580    fn parse_map(&mut self) -> ArrowResult<DataType> {
581        self.expect_token(Token::LParen)?;
582        let field = self.parse_field()?;
583        if field.is_nullable() {
584            return Err(make_error(self.val, "Map entries field cannot be nullable"));
585        }
586        if let DataType::Struct(fields) = field.data_type() {
587            if fields.len() != 2 {
588                return Err(make_error(
589                    self.val,
590                    &format!(
591                        "Map entries must contain two children, got {}",
592                        fields.len()
593                    ),
594                ));
595            }
596            if fields[0].is_nullable() {
597                return Err(make_error(self.val, "Map key field cannot be nullable"));
598            }
599        } else {
600            return Err(make_error(
601                self.val,
602                &format!(
603                    "Map entries must be a Struct type, got {}",
604                    field.data_type()
605                ),
606            ));
607        }
608        self.expect_token(Token::Comma)?;
609        let sorted = self.parse_map_sorted()?;
610        self.expect_token(Token::RParen)?;
611        Ok(DataType::Map(Arc::new(field), sorted))
612    }
613
614    /// Parses map's sorted
615    fn parse_map_sorted(&mut self) -> ArrowResult<bool> {
616        match self.next_token()? {
617            Token::MapSorted(sorted) => Ok(sorted),
618            tok => Err(make_error(
619                self.val,
620                &format!("Expected sorted or unsorted for a map; got {tok:?}"),
621            )),
622        }
623    }
624
625    /// Parses the next RunEndEncoded (called after `RunEndEncoded` has been consumed).
626    ///
627    /// Compact form (default field names): `RunEndEncoded(non-null Int32, non-null Utf8)`
628    /// Verbose form (custom field names):  `RunEndEncoded("re": Int32, "v": non-null Utf8)`
629    fn parse_run_end_encoded(&mut self) -> ArrowResult<DataType> {
630        self.expect_token(Token::LParen)?;
631
632        // Distinguish compact from verbose by peeking: verbose starts with a double-quoted name.
633        let verbose = matches!(
634            self.tokenizer.peek(),
635            Some(Ok(Token::DoubleQuotedString(_)))
636        );
637
638        let (run_ends, values) = if verbose {
639            let run_ends = self.parse_field()?;
640            if run_ends.is_nullable() {
641                return Err(make_error(
642                    self.val,
643                    "RunEndEncoded run_ends field cannot be nullable",
644                ));
645            }
646            self.expect_token(Token::Comma)?;
647            let values = self.parse_field()?;
648            (run_ends, values)
649        } else {
650            if self.parse_opt_nullable() {
651                return Err(make_error(
652                    self.val,
653                    "RunEndEncoded run_ends field cannot be nullable",
654                ));
655            }
656            let re_type = self.parse_next_type()?;
657            self.expect_token(Token::Comma)?;
658            let v_nullable = self.parse_opt_nullable();
659            let v_type = self.parse_next_type()?;
660            (
661                Field::new(Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME, re_type, false),
662                Field::new(Field::REE_VALUES_FIELD_DEFAULT_NAME, v_type, v_nullable),
663            )
664        };
665
666        self.expect_token(Token::RParen)?;
667        Ok(DataType::RunEndEncoded(
668            Arc::new(run_ends),
669            Arc::new(values),
670        ))
671    }
672
673    /// consume the next token and return `false` if the field is `nonnull`.
674    fn parse_opt_nullable(&mut self) -> bool {
675        let tok = self
676            .tokenizer
677            .next_if(|next| matches!(next, Ok(Token::NonNull | Token::Nullable)));
678        !matches!(tok, Some(Ok(Token::NonNull)))
679    }
680
681    /// return the next token, or an error if there are none left
682    fn next_token(&mut self) -> ArrowResult<Token> {
683        match self.tokenizer.next() {
684            None => Err(make_error(self.val, "finding next token")),
685            Some(token) => token,
686        }
687    }
688
689    /// consume the next token, returning OK(()) if it matches tok, and Err if not
690    fn expect_token(&mut self, tok: Token) -> ArrowResult<()> {
691        let next_token = self.next_token()?;
692        if next_token == tok {
693            Ok(())
694        } else {
695            Err(make_error_expected(self.val, &tok, &next_token))
696        }
697    }
698}
699
700/// returns true if this character is a separator
701fn is_separator(c: char) -> bool {
702    c == '(' || c == ')' || c == ',' || c == ':' || c == ' '
703}
704
705enum QuoteType {
706    Double,
707    Single,
708}
709
710#[derive(Debug)]
711/// Splits a strings like Dictionary(Int32, Int64) into tokens suitable for parsing
712///
713/// For example the string "Timestamp(ns)" would be parsed into:
714///
715/// * Token::Timestamp
716/// * Token::Lparen
717/// * Token::IntervalUnit(IntervalUnit::Nanosecond)
718/// * Token::Rparen,
719struct Tokenizer<'a> {
720    val: &'a str,
721    chars: Peekable<Chars<'a>>,
722    // temporary buffer for parsing words
723    word: String,
724}
725
726impl<'a> Tokenizer<'a> {
727    fn new(val: &'a str) -> Self {
728        Self {
729            val,
730            chars: val.chars().peekable(),
731            word: String::new(),
732        }
733    }
734
735    /// returns the next char, without consuming it
736    fn peek_next_char(&mut self) -> Option<char> {
737        self.chars.peek().copied()
738    }
739
740    /// returns the next char, and consuming it
741    fn next_char(&mut self) -> Option<char> {
742        self.chars.next()
743    }
744
745    /// parse the characters in val starting at pos, until the next
746    /// `,`, `(`, or `)` or end of line
747    fn parse_word(&mut self) -> ArrowResult<Token> {
748        // reset temp space
749        self.word.clear();
750        loop {
751            match self.peek_next_char() {
752                None => break,
753                Some(c) if is_separator(c) => break,
754                Some(c) => {
755                    self.next_char();
756                    self.word.push(c);
757                }
758            }
759        }
760
761        if let Some(c) = self.word.chars().next() {
762            // if it started with a number, try parsing it as an integer
763            if c == '-' || c.is_numeric() {
764                let val: i64 = self.word.parse().map_err(|e| {
765                    make_error(self.val, &format!("parsing {} as integer: {e}", self.word))
766                })?;
767                return Ok(Token::Integer(val));
768            }
769        }
770
771        // figure out what the word was
772        let token = match self.word.as_str() {
773            "Null" => Token::SimpleType(DataType::Null),
774            "Boolean" => Token::SimpleType(DataType::Boolean),
775
776            "Int8" => Token::SimpleType(DataType::Int8),
777            "Int16" => Token::SimpleType(DataType::Int16),
778            "Int32" => Token::SimpleType(DataType::Int32),
779            "Int64" => Token::SimpleType(DataType::Int64),
780
781            "UInt8" => Token::SimpleType(DataType::UInt8),
782            "UInt16" => Token::SimpleType(DataType::UInt16),
783            "UInt32" => Token::SimpleType(DataType::UInt32),
784            "UInt64" => Token::SimpleType(DataType::UInt64),
785
786            "Utf8" => Token::SimpleType(DataType::Utf8),
787            "LargeUtf8" => Token::SimpleType(DataType::LargeUtf8),
788            "Utf8View" => Token::SimpleType(DataType::Utf8View),
789            "Binary" => Token::SimpleType(DataType::Binary),
790            "BinaryView" => Token::SimpleType(DataType::BinaryView),
791            "LargeBinary" => Token::SimpleType(DataType::LargeBinary),
792
793            "Float16" => Token::SimpleType(DataType::Float16),
794            "Float32" => Token::SimpleType(DataType::Float32),
795            "Float64" => Token::SimpleType(DataType::Float64),
796
797            "Date32" => Token::SimpleType(DataType::Date32),
798            "Date64" => Token::SimpleType(DataType::Date64),
799
800            "List" => Token::List,
801            "ListView" => Token::ListView,
802            "LargeList" => Token::LargeList,
803            "LargeListView" => Token::LargeListView,
804            "FixedSizeList" => Token::FixedSizeList,
805
806            "s" | "Second" => Token::TimeUnit(TimeUnit::Second),
807            "ms" | "Millisecond" => Token::TimeUnit(TimeUnit::Millisecond),
808            "µs" | "us" | "Microsecond" => Token::TimeUnit(TimeUnit::Microsecond),
809            "ns" | "Nanosecond" => Token::TimeUnit(TimeUnit::Nanosecond),
810
811            "Timestamp" => Token::Timestamp,
812            "Time32" => Token::Time32,
813            "Time64" => Token::Time64,
814            "Duration" => Token::Duration,
815            "Interval" => Token::Interval,
816            "Dictionary" => Token::Dictionary,
817
818            "FixedSizeBinary" => Token::FixedSizeBinary,
819
820            "Decimal32" => Token::Decimal32,
821            "Decimal64" => Token::Decimal64,
822            "Decimal128" => Token::Decimal128,
823            "Decimal256" => Token::Decimal256,
824
825            "YearMonth" => Token::IntervalUnit(IntervalUnit::YearMonth),
826            "DayTime" => Token::IntervalUnit(IntervalUnit::DayTime),
827            "MonthDayNano" => Token::IntervalUnit(IntervalUnit::MonthDayNano),
828
829            "Some" => Token::Some,
830            "None" => Token::None,
831
832            "non-null" => Token::NonNull,
833            "nullable" => Token::Nullable,
834            "field" => Token::Field,
835            "x" => Token::X,
836
837            "Struct" => Token::Struct,
838
839            "Union" => Token::Union,
840            "Sparse" => Token::UnionMode(UnionMode::Sparse),
841            "Dense" => Token::UnionMode(UnionMode::Dense),
842
843            "Map" => Token::Map,
844            "sorted" => Token::MapSorted(true),
845            "unsorted" => Token::MapSorted(false),
846
847            "RunEndEncoded" => Token::RunEndEncoded,
848
849            token => {
850                return Err(make_error(self.val, &format!("unknown token: {token}")));
851            }
852        };
853        Ok(token)
854    }
855
856    /// Parses e.g. `"foo bar"`, `'foo bar'`
857    fn parse_quoted_string(&mut self, quote_type: QuoteType) -> ArrowResult<Token> {
858        let quote = match quote_type {
859            QuoteType::Double => '\"',
860            QuoteType::Single => '\'',
861        };
862
863        if self.next_char() != Some(quote) {
864            return Err(make_error(self.val, "Expected \""));
865        }
866
867        // reset temp space
868        self.word.clear();
869
870        let mut is_escaped = false;
871
872        loop {
873            match self.next_char() {
874                None => {
875                    return Err(ArrowError::ParseError(format!(
876                        "Unterminated string at: \"{}",
877                        self.word
878                    )));
879                }
880                Some(c) => match c {
881                    '\\' => {
882                        is_escaped = true;
883                        self.word.push(c);
884                    }
885                    c if c == quote => {
886                        if is_escaped {
887                            self.word.push(c);
888                            is_escaped = false;
889                        } else {
890                            break;
891                        }
892                    }
893                    c => {
894                        self.word.push(c);
895                    }
896                },
897            }
898        }
899
900        let val: String = self.word.parse().map_err(|err| {
901            ArrowError::ParseError(format!("Failed to parse string: \"{}\": {err}", self.word))
902        })?;
903
904        if val.is_empty() {
905            // Using empty strings as field names is just asking for trouble
906            return Err(make_error(self.val, "empty strings aren't allowed"));
907        }
908
909        match quote_type {
910            QuoteType::Double => Ok(Token::DoubleQuotedString(val)),
911            QuoteType::Single => Ok(Token::SingleQuotedString(val)),
912        }
913    }
914}
915
916impl Iterator for Tokenizer<'_> {
917    type Item = ArrowResult<Token>;
918
919    fn next(&mut self) -> Option<Self::Item> {
920        loop {
921            match self.peek_next_char()? {
922                ' ' => {
923                    // skip whitespace
924                    self.next_char();
925                }
926                '"' => {
927                    return Some(self.parse_quoted_string(QuoteType::Double));
928                }
929                '\'' => {
930                    return Some(self.parse_quoted_string(QuoteType::Single));
931                }
932                '(' => {
933                    self.next_char();
934                    return Some(Ok(Token::LParen));
935                }
936                ')' => {
937                    self.next_char();
938                    return Some(Ok(Token::RParen));
939                }
940                ',' => {
941                    self.next_char();
942                    return Some(Ok(Token::Comma));
943                }
944                ':' => {
945                    self.next_char();
946                    return Some(Ok(Token::Colon));
947                }
948                _ => return Some(self.parse_word()),
949            }
950        }
951    }
952}
953
954/// Grammar is
955///
956#[derive(Debug, PartialEq)]
957enum Token {
958    // Null, or Int32
959    SimpleType(DataType),
960    Timestamp,
961    Time32,
962    Time64,
963    Duration,
964    Interval,
965    FixedSizeBinary,
966    Decimal32,
967    Decimal64,
968    Decimal128,
969    Decimal256,
970    Dictionary,
971    TimeUnit(TimeUnit),
972    IntervalUnit(IntervalUnit),
973    LParen,
974    RParen,
975    Comma,
976    Colon,
977    Some,
978    None,
979    Integer(i64),
980    DoubleQuotedString(String),
981    SingleQuotedString(String),
982    List,
983    ListView,
984    LargeList,
985    LargeListView,
986    FixedSizeList,
987    Struct,
988    Union,
989    UnionMode(UnionMode),
990    Map,
991    MapSorted(bool),
992    RunEndEncoded,
993    NonNull,
994    Nullable,
995    Field,
996    X,
997}
998
999impl Display for Token {
1000    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1001        match self {
1002            Token::SimpleType(t) => write!(f, "{t}"),
1003            Token::List => write!(f, "List"),
1004            Token::ListView => write!(f, "ListView"),
1005            Token::LargeList => write!(f, "LargeList"),
1006            Token::LargeListView => write!(f, "LargeListView"),
1007            Token::FixedSizeList => write!(f, "FixedSizeList"),
1008            Token::Timestamp => write!(f, "Timestamp"),
1009            Token::Time32 => write!(f, "Time32"),
1010            Token::Time64 => write!(f, "Time64"),
1011            Token::Duration => write!(f, "Duration"),
1012            Token::Interval => write!(f, "Interval"),
1013            Token::TimeUnit(u) => write!(f, "TimeUnit({u:?})"),
1014            Token::IntervalUnit(u) => write!(f, "IntervalUnit({u:?})"),
1015            Token::LParen => write!(f, "("),
1016            Token::RParen => write!(f, ")"),
1017            Token::Comma => write!(f, ","),
1018            Token::Colon => write!(f, ":"),
1019            Token::Some => write!(f, "Some"),
1020            Token::None => write!(f, "None"),
1021            Token::FixedSizeBinary => write!(f, "FixedSizeBinary"),
1022            Token::Decimal32 => write!(f, "Decimal32"),
1023            Token::Decimal64 => write!(f, "Decimal64"),
1024            Token::Decimal128 => write!(f, "Decimal128"),
1025            Token::Decimal256 => write!(f, "Decimal256"),
1026            Token::Dictionary => write!(f, "Dictionary"),
1027            Token::Integer(v) => write!(f, "Integer({v})"),
1028            Token::DoubleQuotedString(s) => write!(f, "DoubleQuotedString({s})"),
1029            Token::SingleQuotedString(s) => write!(f, "SingleQuotedString({s})"),
1030            Token::Struct => write!(f, "Struct"),
1031            Token::Union => write!(f, "Union"),
1032            Token::UnionMode(m) => write!(f, "{m:?}"),
1033            Token::Map => write!(f, "Map"),
1034            Token::MapSorted(sorted) => {
1035                write!(f, "{}", if *sorted { "sorted" } else { "unsorted" })
1036            }
1037            Token::RunEndEncoded => write!(f, "RunEndEncoded"),
1038            Token::NonNull => write!(f, "non-null"),
1039            Token::Nullable => write!(f, "nullable"),
1040            Token::Field => write!(f, "field"),
1041            Token::X => write!(f, "x"),
1042        }
1043    }
1044}
1045
1046#[cfg(test)]
1047mod test {
1048    use super::*;
1049
1050    #[test]
1051    fn test_parse_data_type() {
1052        // this ensures types can be parsed correctly from their string representations
1053        for dt in list_datatypes() {
1054            round_trip(dt)
1055        }
1056    }
1057
1058    /// Ensure we converting data_type to a string, and then parse it as a type
1059    /// verifying it is the same
1060    fn round_trip(data_type: DataType) {
1061        let data_type_string = data_type.to_string();
1062        println!("Input '{data_type_string}' ({data_type:?})");
1063        let parsed_type = parse_data_type(&data_type_string).unwrap();
1064        assert_eq!(
1065            data_type, parsed_type,
1066            "Mismatch parsing {data_type_string}"
1067        );
1068    }
1069
1070    fn list_datatypes() -> Vec<DataType> {
1071        vec![
1072            // ---------
1073            // Non Nested types
1074            // ---------
1075            DataType::Null,
1076            DataType::Boolean,
1077            DataType::Int8,
1078            DataType::Int16,
1079            DataType::Int32,
1080            DataType::Int64,
1081            DataType::UInt8,
1082            DataType::UInt16,
1083            DataType::UInt32,
1084            DataType::UInt64,
1085            DataType::Float16,
1086            DataType::Float32,
1087            DataType::Float64,
1088            DataType::Timestamp(TimeUnit::Second, None),
1089            DataType::Timestamp(TimeUnit::Millisecond, None),
1090            DataType::Timestamp(TimeUnit::Microsecond, None),
1091            DataType::Timestamp(TimeUnit::Nanosecond, None),
1092            // we can't cover all possible timezones, here we only test utc and +08:00
1093            DataType::Timestamp(TimeUnit::Nanosecond, Some("+00:00".into())),
1094            DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
1095            DataType::Timestamp(TimeUnit::Millisecond, Some("+00:00".into())),
1096            DataType::Timestamp(TimeUnit::Second, Some("+00:00".into())),
1097            DataType::Timestamp(TimeUnit::Nanosecond, Some("+08:00".into())),
1098            DataType::Timestamp(TimeUnit::Microsecond, Some("+08:00".into())),
1099            DataType::Timestamp(TimeUnit::Millisecond, Some("+08:00".into())),
1100            DataType::Timestamp(TimeUnit::Second, Some("+08:00".into())),
1101            DataType::Date32,
1102            DataType::Date64,
1103            DataType::Time32(TimeUnit::Second),
1104            DataType::Time32(TimeUnit::Millisecond),
1105            DataType::Time64(TimeUnit::Microsecond),
1106            DataType::Time64(TimeUnit::Nanosecond),
1107            DataType::Duration(TimeUnit::Second),
1108            DataType::Duration(TimeUnit::Millisecond),
1109            DataType::Duration(TimeUnit::Microsecond),
1110            DataType::Duration(TimeUnit::Nanosecond),
1111            DataType::Interval(IntervalUnit::YearMonth),
1112            DataType::Interval(IntervalUnit::DayTime),
1113            DataType::Interval(IntervalUnit::MonthDayNano),
1114            DataType::Binary,
1115            DataType::BinaryView,
1116            DataType::FixedSizeBinary(0),
1117            DataType::FixedSizeBinary(1234),
1118            DataType::LargeBinary,
1119            DataType::Utf8,
1120            DataType::Utf8View,
1121            DataType::LargeUtf8,
1122            DataType::Decimal32(7, 6),
1123            DataType::Decimal64(6, 5),
1124            DataType::Decimal128(7, 6),
1125            DataType::Decimal256(6, 5),
1126            // ---------
1127            // Nested types
1128            // ---------
1129            DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
1130            DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)),
1131            DataType::Dictionary(
1132                Box::new(DataType::Int8),
1133                Box::new(DataType::Timestamp(TimeUnit::Nanosecond, None)),
1134            ),
1135            DataType::Dictionary(
1136                Box::new(DataType::Int8),
1137                Box::new(DataType::FixedSizeBinary(23)),
1138            ),
1139            DataType::Dictionary(
1140                Box::new(DataType::Int8),
1141                Box::new(
1142                    // nested dictionaries are probably a bad idea but they are possible
1143                    DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)),
1144                ),
1145            ),
1146            DataType::Struct(Fields::from(vec![
1147                Field::new("f1", DataType::Int64, true),
1148                Field::new("f2", DataType::Float64, true),
1149                Field::new(
1150                    "f3",
1151                    DataType::Timestamp(TimeUnit::Second, Some("+08:00".into())),
1152                    true,
1153                ),
1154                Field::new(
1155                    "f4",
1156                    DataType::Dictionary(
1157                        Box::new(DataType::Int8),
1158                        Box::new(DataType::FixedSizeBinary(23)),
1159                    ),
1160                    true,
1161                ),
1162            ])),
1163            DataType::Struct(Fields::from(vec![
1164                Field::new("Int64", DataType::Int64, true),
1165                Field::new("Float64", DataType::Float64, true),
1166            ])),
1167            DataType::Struct(Fields::from(vec![
1168                Field::new("f1", DataType::Int64, true),
1169                Field::new(
1170                    "nested_struct",
1171                    DataType::Struct(Fields::from(vec![Field::new("n1", DataType::Int64, true)])),
1172                    true,
1173                ),
1174            ])),
1175            DataType::Struct(Fields::from(vec![Field::new("f1", DataType::Int64, true)])),
1176            DataType::Struct(Fields::empty()),
1177            DataType::List(Arc::new(Field::new_list_field(DataType::Int64, true))),
1178            DataType::List(Arc::new(Field::new_list_field(DataType::Int64, false))),
1179            DataType::List(Arc::new(Field::new("Int64", DataType::Int64, true))),
1180            DataType::List(Arc::new(Field::new("Int64", DataType::Int64, false))),
1181            DataType::List(Arc::new(Field::new(
1182                "nested_list",
1183                DataType::List(Arc::new(Field::new("Int64", DataType::Int64, true))),
1184                true,
1185            ))),
1186            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int64, true))),
1187            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int64, false))),
1188            DataType::ListView(Arc::new(Field::new("Int64", DataType::Int64, true))),
1189            DataType::ListView(Arc::new(Field::new("Int64", DataType::Int64, false))),
1190            DataType::ListView(Arc::new(Field::new(
1191                "nested_list_view",
1192                DataType::ListView(Arc::new(Field::new("Int64", DataType::Int64, true))),
1193                true,
1194            ))),
1195            DataType::LargeList(Arc::new(Field::new_list_field(DataType::Int64, true))),
1196            DataType::LargeList(Arc::new(Field::new_list_field(DataType::Int64, false))),
1197            DataType::LargeList(Arc::new(Field::new("Int64", DataType::Int64, true))),
1198            DataType::LargeList(Arc::new(Field::new("Int64", DataType::Int64, false))),
1199            DataType::LargeList(Arc::new(Field::new(
1200                "nested_large_list",
1201                DataType::LargeList(Arc::new(Field::new("Int64", DataType::Int64, true))),
1202                true,
1203            ))),
1204            DataType::LargeListView(Arc::new(Field::new_list_field(DataType::Int64, true))),
1205            DataType::LargeListView(Arc::new(Field::new_list_field(DataType::Int64, false))),
1206            DataType::LargeListView(Arc::new(Field::new("Int64", DataType::Int64, true))),
1207            DataType::LargeListView(Arc::new(Field::new("Int64", DataType::Int64, false))),
1208            DataType::LargeListView(Arc::new(Field::new(
1209                "nested_large_list_view",
1210                DataType::LargeListView(Arc::new(Field::new("Int64", DataType::Int64, true))),
1211                true,
1212            ))),
1213            DataType::FixedSizeList(Arc::new(Field::new_list_field(DataType::Int64, true)), 2),
1214            DataType::FixedSizeList(Arc::new(Field::new_list_field(DataType::Int64, false)), 2),
1215            DataType::FixedSizeList(Arc::new(Field::new("Int64", DataType::Int64, true)), 2),
1216            DataType::FixedSizeList(Arc::new(Field::new("Int64", DataType::Int64, false)), 2),
1217            DataType::FixedSizeList(
1218                Arc::new(Field::new(
1219                    "nested_fixed_size_list",
1220                    DataType::FixedSizeList(
1221                        Arc::new(Field::new("Int64", DataType::Int64, true)),
1222                        2,
1223                    ),
1224                    true,
1225                )),
1226                2,
1227            ),
1228            DataType::Union(
1229                UnionFields::from_fields(vec![
1230                    Field::new("Int32", DataType::Int32, false),
1231                    Field::new("Utf8", DataType::Utf8, true),
1232                ]),
1233                UnionMode::Sparse,
1234            ),
1235            DataType::Union(
1236                UnionFields::from_fields(vec![
1237                    Field::new("Int32", DataType::Int32, false),
1238                    Field::new("Utf8", DataType::Utf8, true),
1239                ]),
1240                UnionMode::Dense,
1241            ),
1242            DataType::Union(
1243                UnionFields::from_fields(vec![
1244                    Field::new_union(
1245                        "nested_union",
1246                        vec![0, 1],
1247                        vec![
1248                            Field::new("Int32", DataType::Int32, false),
1249                            Field::new("Utf8", DataType::Utf8, true),
1250                        ],
1251                        UnionMode::Dense,
1252                    ),
1253                    Field::new("Utf8", DataType::Utf8, true),
1254                ]),
1255                UnionMode::Sparse,
1256            ),
1257            DataType::Union(
1258                UnionFields::from_fields(vec![Field::new("Int32", DataType::Int32, false)]),
1259                UnionMode::Dense,
1260            ),
1261            DataType::Union(
1262                UnionFields::try_new(Vec::<i8>::new(), Vec::<Field>::new()).unwrap(),
1263                UnionMode::Sparse,
1264            ),
1265            DataType::RunEndEncoded(
1266                Arc::new(Field::new(
1267                    Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
1268                    DataType::UInt32,
1269                    false,
1270                )),
1271                Arc::new(Field::new(
1272                    Field::REE_VALUES_FIELD_DEFAULT_NAME,
1273                    DataType::Int32,
1274                    true,
1275                )),
1276            ),
1277            DataType::RunEndEncoded(
1278                Arc::new(Field::new(
1279                    Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
1280                    DataType::RunEndEncoded(
1281                        Arc::new(Field::new(
1282                            Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
1283                            DataType::UInt32,
1284                            false,
1285                        )),
1286                        Arc::new(Field::new(
1287                            Field::REE_VALUES_FIELD_DEFAULT_NAME,
1288                            DataType::Int32,
1289                            true,
1290                        )),
1291                    ),
1292                    false,
1293                )),
1294                Arc::new(Field::new(
1295                    Field::REE_VALUES_FIELD_DEFAULT_NAME,
1296                    DataType::Int32,
1297                    true,
1298                )),
1299            ),
1300            // non-default field names trigger verbose display form
1301            DataType::RunEndEncoded(
1302                Arc::new(Field::new(
1303                    Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
1304                    DataType::RunEndEncoded(
1305                        Arc::new(Field::new(
1306                            Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
1307                            DataType::UInt32,
1308                            false,
1309                        )),
1310                        Arc::new(Field::new(
1311                            Field::REE_VALUES_FIELD_DEFAULT_NAME,
1312                            DataType::Int32,
1313                            true,
1314                        )),
1315                    ),
1316                    false,
1317                )),
1318                Arc::new(Field::new("named_values", DataType::Int32, false)),
1319            ),
1320            // verbose form with non-null inner values
1321            DataType::RunEndEncoded(
1322                Arc::new(Field::new(
1323                    Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
1324                    DataType::RunEndEncoded(
1325                        Arc::new(Field::new(
1326                            Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
1327                            DataType::UInt32,
1328                            false,
1329                        )),
1330                        Arc::new(Field::new(
1331                            Field::REE_VALUES_FIELD_DEFAULT_NAME,
1332                            DataType::Int32,
1333                            false,
1334                        )),
1335                    ),
1336                    false,
1337                )),
1338                Arc::new(Field::new("named_values", DataType::Int32, false)),
1339            ),
1340        ]
1341    }
1342
1343    #[test]
1344    fn test_parse_data_type_whitespace_tolerance() {
1345        // (string to parse, expected DataType)
1346        let cases = [
1347            ("Int8", DataType::Int8),
1348            (
1349                "Timestamp        (ns)",
1350                DataType::Timestamp(TimeUnit::Nanosecond, None),
1351            ),
1352            (
1353                "Timestamp        (ns)  ",
1354                DataType::Timestamp(TimeUnit::Nanosecond, None),
1355            ),
1356            (
1357                "          Timestamp        (ns               )",
1358                DataType::Timestamp(TimeUnit::Nanosecond, None),
1359            ),
1360            (
1361                "Timestamp        (ns               )  ",
1362                DataType::Timestamp(TimeUnit::Nanosecond, None),
1363            ),
1364        ];
1365
1366        for (data_type_string, expected_data_type) in cases {
1367            let parsed_data_type = parse_data_type(data_type_string).unwrap();
1368            assert_eq!(
1369                parsed_data_type, expected_data_type,
1370                "Parsing '{data_type_string}', expecting '{expected_data_type}'"
1371            );
1372        }
1373    }
1374
1375    /// Ensure that old style types can still be parsed
1376    #[test]
1377    fn test_parse_data_type_backwards_compatibility() {
1378        use DataType::*;
1379        use IntervalUnit::*;
1380        use TimeUnit::*;
1381        // List below created with:
1382        for t in list_datatypes() {
1383            println!(r#"("{t}", {t:?}),"#);
1384        }
1385        // (string to parse, expected DataType)
1386        let cases = [
1387            ("Timestamp(Nanosecond, None)", Timestamp(Nanosecond, None)),
1388            ("Timestamp(Microsecond, None)", Timestamp(Microsecond, None)),
1389            ("Timestamp(Millisecond, None)", Timestamp(Millisecond, None)),
1390            ("Timestamp(Second, None)", Timestamp(Second, None)),
1391            ("Timestamp(Nanosecond, None)", Timestamp(Nanosecond, None)),
1392            // Timezones
1393            (
1394                r#"Timestamp(Nanosecond, Some("+00:00"))"#,
1395                Timestamp(Nanosecond, Some("+00:00".into())),
1396            ),
1397            (
1398                r#"Timestamp(Microsecond, Some("+00:00"))"#,
1399                Timestamp(Microsecond, Some("+00:00".into())),
1400            ),
1401            (
1402                r#"Timestamp(Millisecond, Some("+00:00"))"#,
1403                Timestamp(Millisecond, Some("+00:00".into())),
1404            ),
1405            (
1406                r#"Timestamp(Second, Some("+00:00"))"#,
1407                Timestamp(Second, Some("+00:00".into())),
1408            ),
1409            ("Null", Null),
1410            ("Boolean", Boolean),
1411            ("Int8", Int8),
1412            ("Int16", Int16),
1413            ("Int32", Int32),
1414            ("Int64", Int64),
1415            ("UInt8", UInt8),
1416            ("UInt16", UInt16),
1417            ("UInt32", UInt32),
1418            ("UInt64", UInt64),
1419            ("Float16", Float16),
1420            ("Float32", Float32),
1421            ("Float64", Float64),
1422            ("Timestamp(s)", Timestamp(Second, None)),
1423            ("Timestamp(ms)", Timestamp(Millisecond, None)),
1424            ("Timestamp(µs)", Timestamp(Microsecond, None)),
1425            ("Timestamp(ns)", Timestamp(Nanosecond, None)),
1426            (
1427                r#"Timestamp(ns, "+00:00")"#,
1428                Timestamp(Nanosecond, Some("+00:00".into())),
1429            ),
1430            (
1431                r#"Timestamp(µs, "+00:00")"#,
1432                Timestamp(Microsecond, Some("+00:00".into())),
1433            ),
1434            (
1435                r#"Timestamp(ms, "+00:00")"#,
1436                Timestamp(Millisecond, Some("+00:00".into())),
1437            ),
1438            (
1439                r#"Timestamp(s, "+00:00")"#,
1440                Timestamp(Second, Some("+00:00".into())),
1441            ),
1442            (
1443                r#"Timestamp(ns, "+08:00")"#,
1444                Timestamp(Nanosecond, Some("+08:00".into())),
1445            ),
1446            (
1447                r#"Timestamp(µs, "+08:00")"#,
1448                Timestamp(Microsecond, Some("+08:00".into())),
1449            ),
1450            (
1451                r#"Timestamp(ms, "+08:00")"#,
1452                Timestamp(Millisecond, Some("+08:00".into())),
1453            ),
1454            (
1455                r#"Timestamp(s, "+08:00")"#,
1456                Timestamp(Second, Some("+08:00".into())),
1457            ),
1458            ("Date32", Date32),
1459            ("Date64", Date64),
1460            ("Time32(s)", Time32(Second)),
1461            ("Time32(ms)", Time32(Millisecond)),
1462            ("Time64(µs)", Time64(Microsecond)),
1463            ("Time64(ns)", Time64(Nanosecond)),
1464            ("Duration(s)", Duration(Second)),
1465            ("Duration(ms)", Duration(Millisecond)),
1466            ("Duration(µs)", Duration(Microsecond)),
1467            ("Duration(ns)", Duration(Nanosecond)),
1468            ("Interval(YearMonth)", Interval(YearMonth)),
1469            ("Interval(DayTime)", Interval(DayTime)),
1470            ("Interval(MonthDayNano)", Interval(MonthDayNano)),
1471            ("Binary", Binary),
1472            ("BinaryView", BinaryView),
1473            ("FixedSizeBinary(0)", FixedSizeBinary(0)),
1474            ("FixedSizeBinary(1234)", FixedSizeBinary(1234)),
1475            ("LargeBinary", LargeBinary),
1476            ("Utf8", Utf8),
1477            ("Utf8View", Utf8View),
1478            ("LargeUtf8", LargeUtf8),
1479            ("Decimal32(7, 6)", Decimal32(7, 6)),
1480            ("Decimal64(6, 5)", Decimal64(6, 5)),
1481            ("Decimal128(7, 6)", Decimal128(7, 6)),
1482            ("Decimal256(6, 5)", Decimal256(6, 5)),
1483            (
1484                "Dictionary(Int32, Utf8)",
1485                Dictionary(Box::new(Int32), Box::new(Utf8)),
1486            ),
1487            (
1488                "Dictionary(Int8, Utf8)",
1489                Dictionary(Box::new(Int8), Box::new(Utf8)),
1490            ),
1491            (
1492                "Dictionary(Int8, Timestamp(ns))",
1493                Dictionary(Box::new(Int8), Box::new(Timestamp(Nanosecond, None))),
1494            ),
1495            (
1496                "Dictionary(Int8, FixedSizeBinary(23))",
1497                Dictionary(Box::new(Int8), Box::new(FixedSizeBinary(23))),
1498            ),
1499            (
1500                "Dictionary(Int8, Dictionary(Int8, Utf8))",
1501                Dictionary(
1502                    Box::new(Int8),
1503                    Box::new(Dictionary(Box::new(Int8), Box::new(Utf8))),
1504                ),
1505            ),
1506            (
1507                r#"Struct("f1": nullable Int64, "f2": nullable Float64, "f3": nullable Timestamp(s, "+08:00"), "f4": nullable Dictionary(Int8, FixedSizeBinary(23)))"#,
1508                Struct(Fields::from(vec![
1509                    Field::new("f1", Int64, true),
1510                    Field::new("f2", Float64, true),
1511                    Field::new("f3", Timestamp(Second, Some("+08:00".into())), true),
1512                    Field::new(
1513                        "f4",
1514                        Dictionary(Box::new(Int8), Box::new(FixedSizeBinary(23))),
1515                        true,
1516                    ),
1517                ])),
1518            ),
1519            (
1520                r#"Struct("Int64": nullable Int64, "Float64": nullable Float64)"#,
1521                Struct(Fields::from(vec![
1522                    Field::new("Int64", Int64, true),
1523                    Field::new("Float64", Float64, true),
1524                ])),
1525            ),
1526            (
1527                r#"Struct("f1": nullable Int64, "nested_struct": nullable Struct("n1": nullable Int64))"#,
1528                Struct(Fields::from(vec![
1529                    Field::new("f1", Int64, true),
1530                    Field::new(
1531                        "nested_struct",
1532                        Struct(Fields::from(vec![Field::new("n1", Int64, true)])),
1533                        true,
1534                    ),
1535                ])),
1536            ),
1537            (r"Struct()", Struct(Fields::empty())),
1538            (
1539                "FixedSizeList(4, Int64)",
1540                FixedSizeList(Arc::new(Field::new_list_field(Int64, true)), 4),
1541            ),
1542            (
1543                "List(Int64)",
1544                List(Arc::new(Field::new_list_field(Int64, true))),
1545            ),
1546            (
1547                "LargeList(Int64)",
1548                LargeList(Arc::new(Field::new_list_field(Int64, true))),
1549            ),
1550        ];
1551
1552        for (data_type_string, expected_data_type) in cases {
1553            let parsed_data_type = parse_data_type(data_type_string).unwrap();
1554            assert_eq!(
1555                parsed_data_type, expected_data_type,
1556                "Parsing '{data_type_string}', expecting '{expected_data_type}'"
1557            );
1558        }
1559    }
1560
1561    #[test]
1562    fn parse_data_type_errors() {
1563        // (string to parse, expected error message)
1564        let cases = [
1565            ("", "Unsupported type ''"),
1566            ("", "Error finding next token"),
1567            ("null", "Unsupported type 'null'"),
1568            ("Nu", "Unsupported type 'Nu'"),
1569            (r"Timestamp(ns, +00:00)", "Error unknown token: +00"),
1570            (
1571                r#"Timestamp(ns, "+00:00)"#,
1572                r#"Unterminated string at: "+00:00)"#,
1573            ),
1574            (r#"Timestamp(ns, "")"#, r"empty strings aren't allowed"),
1575            (
1576                r#"Timestamp(ns, "+00:00"")"#,
1577                r#"Parser error: Unterminated string at: ")"#,
1578            ),
1579            ("Timestamp(ns, ", "Error finding next token"),
1580            (
1581                "Float32 Float32",
1582                "trailing content after parsing 'Float32'",
1583            ),
1584            ("Int32, ", "trailing content after parsing 'Int32'"),
1585            ("Int32(3), ", "trailing content after parsing 'Int32'"),
1586            (
1587                "FixedSizeBinary(Int32), ",
1588                "Error finding i64 for FixedSizeBinary, got 'Int32'",
1589            ),
1590            (
1591                "FixedSizeBinary(3.0), ",
1592                "Error parsing 3.0 as integer: invalid digit found in string",
1593            ),
1594            // too large for i32
1595            (
1596                "FixedSizeBinary(4000000000), ",
1597                "Error converting 4000000000 into i32 for FixedSizeBinary:",
1598            ),
1599            // can't have negative width
1600            (
1601                "FixedSizeBinary(-1), ",
1602                "FixedSizeBinary length must be non-negative, got -1",
1603            ),
1604            (
1605                "FixedSizeList(-1, Int64), ",
1606                "FixedSizeList length must be non-negative, got -1",
1607            ),
1608            // can't have negative precision
1609            (
1610                "Decimal32(-3, 5)",
1611                "Error converting -3 into u8 for Decimal32:",
1612            ),
1613            (
1614                "Decimal64(-3, 5)",
1615                "Error converting -3 into u8 for Decimal64:",
1616            ),
1617            (
1618                "Decimal128(-3, 5)",
1619                "Error converting -3 into u8 for Decimal128:",
1620            ),
1621            (
1622                "Decimal256(-3, 5)",
1623                "Error converting -3 into u8 for Decimal256:",
1624            ),
1625            (
1626                "Decimal32(3, 500)",
1627                "Error converting 500 into i8 for Decimal32:",
1628            ),
1629            (
1630                "Decimal64(3, 500)",
1631                "Error converting 500 into i8 for Decimal64:",
1632            ),
1633            (
1634                "Decimal128(3, 500)",
1635                "Error converting 500 into i8 for Decimal128:",
1636            ),
1637            (
1638                "Decimal256(3, 500)",
1639                "Error converting 500 into i8 for Decimal256:",
1640            ),
1641            ("Struct(f1 Int64)", "Error unknown token: f1"),
1642            ("Struct(\"f1\" Int64)", "Expected ':'"),
1643            (
1644                "Struct(\"f1\": )",
1645                "Error finding next type, got unexpected ')'",
1646            ),
1647            // Invalid time combinations
1648            (
1649                "Time32(µs)",
1650                "Error Time32 time unit must be 's' or 'ms', got 'µs'",
1651            ),
1652            (
1653                "Time32(ns)",
1654                "Error Time32 time unit must be 's' or 'ms', got 'ns'",
1655            ),
1656            (
1657                "Time64(s)",
1658                "Error Time64 time unit must be 'µs' or 'ns', got 's'",
1659            ),
1660            (
1661                "Time64(ms)",
1662                "Error Time64 time unit must be 'µs' or 'ns', got 'ms'",
1663            ),
1664            // Decimals can't have scale exceeding precision
1665            (
1666                "Decimal32(5, 6)",
1667                "Error Decimal32 scale '6' cannot be greater than precision '5'",
1668            ),
1669            (
1670                "Decimal64(5, 6)",
1671                "Error Decimal64 scale '6' cannot be greater than precision '5'",
1672            ),
1673            (
1674                "Decimal128(5, 6)",
1675                "Error Decimal128 scale '6' cannot be greater than precision '5'",
1676            ),
1677            (
1678                "Decimal256(5, 6)",
1679                "Error Decimal256 scale '6' cannot be greater than precision '5'",
1680            ),
1681            // Decimals have a max supported precision
1682            (
1683                "Decimal32(10, 0)",
1684                "Error Decimal32 precision must be in range [1, 9], got '10'",
1685            ),
1686            (
1687                "Decimal64(19, 0)",
1688                "Error Decimal64 precision must be in range [1, 18], got '19'",
1689            ),
1690            (
1691                "Decimal128(39, 0)",
1692                "Error Decimal128 precision must be in range [1, 38], got '39'",
1693            ),
1694            (
1695                "Decimal256(77, 0)",
1696                "Error Decimal256 precision must be in range [1, 76], got '77'",
1697            ),
1698            // Decimals precision can't be 0
1699            (
1700                "Decimal32(0, 0)",
1701                "Error Decimal32 precision must be in range [1, 9], got '0'",
1702            ),
1703            (
1704                "Decimal64(0, 0)",
1705                "Error Decimal64 precision must be in range [1, 18], got '0'",
1706            ),
1707            (
1708                "Decimal128(0, 0)",
1709                "Error Decimal128 precision must be in range [1, 38], got '0'",
1710            ),
1711            (
1712                "Decimal256(0, 0)",
1713                "Error Decimal256 precision must be in range [1, 76], got '0'",
1714            ),
1715            // REE run_ends cannot be nullable
1716            (
1717                r#"RunEndEncoded("re": nullable Int32, "v": non-null Utf8)"#,
1718                "RunEndEncoded run_ends field cannot be nullable",
1719            ),
1720            (
1721                r#"RunEndEncoded("re": Int32, "v": non-null Utf8)"#,
1722                "RunEndEncoded run_ends field cannot be nullable",
1723            ),
1724            (
1725                "RunEndEncoded(nullable Int32, non-null Utf8)",
1726                "RunEndEncoded run_ends field cannot be nullable",
1727            ),
1728            // Map entries field cannot be nullable
1729            (
1730                r#"Map("entries": Struct("key": non-null Utf8, "value": nullable Int32), unsorted)"#,
1731                "Map entries field cannot be nullable",
1732            ),
1733            // Map key cannot be nullable
1734            (
1735                r#"Map("entries": non-null Struct("key": nullable Utf8, "value": nullable Int32), unsorted)"#,
1736                "Map key field cannot be nullable",
1737            ),
1738            (
1739                r#"Map("entries": non-null Struct("key": Utf8, "value": nullable Int32), unsorted)"#,
1740                "Map key field cannot be nullable",
1741            ),
1742        ];
1743
1744        for (data_type_string, expected_message) in cases {
1745            println!("Parsing '{data_type_string}', expecting '{expected_message}'");
1746            match parse_data_type(data_type_string) {
1747                Ok(d) => panic!("Expected error while parsing '{data_type_string}', but got '{d}'"),
1748                Err(e) => {
1749                    let message = e.to_string();
1750                    assert!(
1751                        message.contains(expected_message),
1752                        "\n\ndid not find expected in actual.\n\nexpected: {expected_message}\nactual: {message}\n"
1753                    );
1754
1755                    if !message.contains("Unterminated string") {
1756                        // errors should also contain a help message
1757                        assert!(message.contains("Must be a supported arrow type name such as 'Int32' or 'Timestamp(ns)'"), "message: {message}");
1758                    }
1759                }
1760            }
1761        }
1762    }
1763
1764    #[test]
1765    fn parse_error_type() {
1766        let err = parse_data_type("foobar").unwrap_err();
1767        assert!(matches!(err, ArrowError::ParseError(_)));
1768        assert_eq!(
1769            err.to_string(),
1770            "Parser error: Unsupported type 'foobar'. Must be a supported arrow type name such as 'Int32' or 'Timestamp(ns)'. Error unknown token: foobar"
1771        );
1772    }
1773}
arrow_schema/datatype_parse.rs

arrow_schema/
datatype_parse.rs
