arrow_schema/datatype.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use std::str::FromStr;
6use std::sync::Arc;
7
8use crate::{ArrowError, Field, FieldRef, Fields, UnionFields};
9
10/// Datatypes supported by this implementation of Apache Arrow.
11///
12/// The variants of this enum include primitive fixed size types as well as
13/// parametric or nested types. See [`Schema.fbs`] for Arrow's specification.
14///
15/// # Examples
16///
17/// Primitive types
18/// ```
19/// # use arrow_schema::DataType;
20/// // create a new 32-bit signed integer
21/// let data_type = DataType::Int32;
22/// ```
23///
24/// Nested Types
25/// ```
26/// # use arrow_schema::{DataType, Field};
27/// # use std::sync::Arc;
28/// // create a new list of 32-bit signed integers directly
29/// let list_data_type = DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true)));
30/// // Create the same list type with constructor
31/// let list_data_type2 = DataType::new_list(DataType::Int32, true);
32/// assert_eq!(list_data_type, list_data_type2);
33/// ```
34///
35/// Dictionary Types
36/// ```
37/// # use arrow_schema::{DataType};
38/// // String Dictionary (key type Int32 and value type Utf8)
39/// let data_type = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
40/// ```
41///
42/// Timestamp Types
43/// ```
44/// # use arrow_schema::{DataType, TimeUnit};
45/// // timestamp with millisecond precision without timezone specified
46/// let data_type = DataType::Timestamp(TimeUnit::Millisecond, None);
47/// // timestamp with nanosecond precision in UTC timezone
48/// let data_type = DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into()));
49///```
50///
51/// # Display and FromStr
52///
53/// The `Display` and `FromStr` implementations for `DataType` are
54/// human-readable, parseable, and reversible.
55///
56/// ```
57/// # use arrow_schema::DataType;
58/// let data_type = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
59/// let data_type_string = data_type.to_string();
60/// assert_eq!(data_type_string, "Dictionary(Int32, Utf8)");
61/// // display can be parsed back into the original type
62/// let parsed_data_type: DataType = data_type.to_string().parse().unwrap();
63/// assert_eq!(data_type, parsed_data_type);
64/// ```
65///
66/// # Nested Support
67/// Currently, the Rust implementation supports the following nested types:
68/// - `List<T>`
69/// - `LargeList<T>`
70/// - `FixedSizeList<T>`
71/// - `Struct<T, U, V, ...>`
72/// - `Union<T, U, V, ...>`
73/// - `Map<K, V>`
74///
75/// Nested types can themselves be nested within other arrays.
76/// For more information on these types please see
77/// [the physical memory layout of Apache Arrow]
78///
79/// [`Schema.fbs`]: https://github.com/apache/arrow/blob/main/format/Schema.fbs
80/// [the physical memory layout of Apache Arrow]: https://arrow.apache.org/docs/format/Columnar.html#physical-memory-layout
81#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
82#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
83pub enum DataType {
84 /// Null type
85 Null,
86 /// A boolean datatype representing the values `true` and `false`.
87 Boolean,
88 /// A signed 8-bit integer.
89 Int8,
90 /// A signed 16-bit integer.
91 Int16,
92 /// A signed 32-bit integer.
93 Int32,
94 /// A signed 64-bit integer.
95 Int64,
96 /// An unsigned 8-bit integer.
97 UInt8,
98 /// An unsigned 16-bit integer.
99 UInt16,
100 /// An unsigned 32-bit integer.
101 UInt32,
102 /// An unsigned 64-bit integer.
103 UInt64,
104 /// A 16-bit floating point number.
105 Float16,
106 /// A 32-bit floating point number.
107 Float32,
108 /// A 64-bit floating point number.
109 Float64,
110 /// A timestamp with an optional timezone.
111 ///
112 /// Time is measured as a Unix epoch, counting the seconds from
113 /// 00:00:00.000 on 1 January 1970, excluding leap seconds,
114 /// as a signed 64-bit integer.
115 ///
116 /// The time zone is a string indicating the name of a time zone, one of:
117 ///
118 /// * As used in the Olson time zone database (the "tz database" or
119 /// "tzdata"), such as "America/New_York"
120 /// * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30
121 ///
122 /// Timestamps with a non-empty timezone
123 /// ------------------------------------
124 ///
125 /// If a Timestamp column has a non-empty timezone value, its epoch is
126 /// 1970-01-01 00:00:00 (January 1st 1970, midnight) in the *UTC* timezone
127 /// (the Unix epoch), regardless of the Timestamp's own timezone.
128 ///
129 /// Therefore, timestamp values with a non-empty timezone correspond to
130 /// physical points in time together with some additional information about
131 /// how the data was obtained and/or how to display it (the timezone).
132 ///
133 /// For example, the timestamp value 0 with the timezone string "Europe/Paris"
134 /// corresponds to "January 1st 1970, 00h00" in the UTC timezone, but the
135 /// application may prefer to display it as "January 1st 1970, 01h00" in
136 /// the Europe/Paris timezone (which is the same physical point in time).
137 ///
138 /// One consequence is that timestamp values with a non-empty timezone
139 /// can be compared and ordered directly, since they all share the same
140 /// well-known point of reference (the Unix epoch).
141 ///
142 /// Timestamps with an unset / empty timezone
143 /// -----------------------------------------
144 ///
145 /// If a Timestamp column has no timezone value, its epoch is
146 /// 1970-01-01 00:00:00 (January 1st 1970, midnight) in an *unknown* timezone.
147 ///
148 /// Therefore, timestamp values without a timezone cannot be meaningfully
149 /// interpreted as physical points in time, but only as calendar / clock
150 /// indications ("wall clock time") in an unspecified timezone.
151 ///
152 /// For example, the timestamp value 0 with an empty timezone string
153 /// corresponds to "January 1st 1970, 00h00" in an unknown timezone: there
154 /// is not enough information to interpret it as a well-defined physical
155 /// point in time.
156 ///
157 /// One consequence is that timestamp values without a timezone cannot
158 /// be reliably compared or ordered, since they may have different points of
159 /// reference. In particular, it is *not* possible to interpret an unset
160 /// or empty timezone as the same as "UTC".
161 ///
162 /// Conversion between timezones
163 /// ----------------------------
164 ///
165 /// If a Timestamp column has a non-empty timezone, changing the timezone
166 /// to a different non-empty value is a metadata-only operation:
167 /// the timestamp values need not change as their point of reference remains
168 /// the same (the Unix epoch).
169 ///
170 /// However, if a Timestamp column has no timezone value, changing it to a
171 /// non-empty value requires to think about the desired semantics.
172 /// One possibility is to assume that the original timestamp values are
173 /// relative to the epoch of the timezone being set; timestamp values should
174 /// then adjusted to the Unix epoch (for example, changing the timezone from
175 /// empty to "Europe/Paris" would require converting the timestamp values
176 /// from "Europe/Paris" to "UTC", which seems counter-intuitive but is
177 /// nevertheless correct).
178 ///
179 /// ```
180 /// # use arrow_schema::{DataType, TimeUnit};
181 /// DataType::Timestamp(TimeUnit::Second, None);
182 /// DataType::Timestamp(TimeUnit::Second, Some("literal".into()));
183 /// DataType::Timestamp(TimeUnit::Second, Some("string".to_string().into()));
184 /// ```
185 ///
186 /// # Timezone representation
187 /// ----------------------------
188 /// It is possible to use either the timezone string representation, such as "UTC", or the absolute time zone offset "+00:00".
189 /// For timezones with fixed offsets, such as "UTC" or "JST", the offset representation is recommended, as it is more explicit and less ambiguous.
190 ///
191 /// Most arrow-rs functionalities use the absolute offset representation,
192 /// such as [`PrimitiveArray::with_timezone_utc`] that applies a
193 /// UTC timezone to timestamp arrays.
194 ///
195 /// [`PrimitiveArray::with_timezone_utc`]: https://docs.rs/arrow/latest/arrow/array/struct.PrimitiveArray.html#method.with_timezone_utc
196 ///
197 /// Timezone string parsing
198 /// -----------------------
199 /// When feature `chrono-tz` is not enabled, allowed timezone strings are fixed offsets of the form "+09:00", "-09" or "+0930".
200 ///
201 /// When feature `chrono-tz` is enabled, additional strings supported by [chrono_tz](https://docs.rs/chrono-tz/latest/chrono_tz/)
202 /// are also allowed, which include [IANA database](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones)
203 /// timezones.
204 Timestamp(TimeUnit, Option<Arc<str>>),
205 /// A signed 32-bit date representing the elapsed time since UNIX epoch (1970-01-01)
206 /// in days.
207 Date32,
208 /// A signed 64-bit date representing the elapsed time since UNIX epoch (1970-01-01)
209 /// in milliseconds.
210 ///
211 /// # Valid Ranges
212 ///
213 /// According to the Arrow specification ([Schema.fbs]), values of Date64
214 /// are treated as the number of *days*, in milliseconds, since the UNIX
215 /// epoch. Therefore, values of this type must be evenly divisible by
216 /// `86_400_000`, the number of milliseconds in a standard day.
217 ///
218 /// It is not valid to store milliseconds that do not represent an exact
219 /// day. The reason for this restriction is compatibility with other
220 /// language's native libraries (specifically Java), which historically
221 /// lacked a dedicated date type and only supported timestamps.
222 ///
223 /// # Validation
224 ///
225 /// This library does not validate or enforce that Date64 values are evenly
226 /// divisible by `86_400_000` for performance and usability reasons. Date64
227 /// values are treated similarly to `Timestamp(TimeUnit::Millisecond,
228 /// None)`: values will be displayed with a time of day if the value does
229 /// not represent an exact day, and arithmetic will be done at the
230 /// millisecond granularity.
231 ///
232 /// # Recommendation
233 ///
234 /// Users should prefer [`Date32`] to cleanly represent the number
235 /// of days, or one of the Timestamp variants to include time as part of the
236 /// representation, depending on their use case.
237 ///
238 /// # Further Reading
239 ///
240 /// For more details, see [#5288](https://github.com/apache/arrow-rs/issues/5288).
241 ///
242 /// [`Date32`]: Self::Date32
243 /// [Schema.fbs]: https://github.com/apache/arrow/blob/main/format/Schema.fbs
244 Date64,
245 /// A signed 32-bit time representing the elapsed time since midnight in the unit of `TimeUnit`.
246 /// Must be either seconds or milliseconds.
247 Time32(TimeUnit),
248 /// A signed 64-bit time representing the elapsed time since midnight in the unit of `TimeUnit`.
249 /// Must be either microseconds or nanoseconds.
250 Time64(TimeUnit),
251 /// Measure of elapsed time in either seconds, milliseconds, microseconds or nanoseconds.
252 Duration(TimeUnit),
253 /// A "calendar" interval which models types that don't necessarily
254 /// have a precise duration without the context of a base timestamp (e.g.
255 /// days can differ in length during day light savings time transitions).
256 Interval(IntervalUnit),
257 /// Opaque binary data of variable length.
258 ///
259 /// A single Binary array can store up to [`i32::MAX`] bytes
260 /// of binary data in total.
261 Binary,
262 /// Opaque binary data of fixed size.
263 ///
264 /// Enum parameter specifies the number of bytes per value, defined by the
265 /// [`byteWidth` field] in the Arrow Spec
266 ///
267 /// [`byteWidth` field]: https://github.com/apache/arrow/blob/2a89d03bbefd620b42126b8e00f8ae57e99cd638/format/Schema.fbs#L211
268 FixedSizeBinary(i32),
269 /// Opaque binary data of variable length and 64-bit offsets.
270 ///
271 /// A single LargeBinary array can store up to [`i64::MAX`] bytes
272 /// of binary data in total.
273 LargeBinary,
274 /// Opaque binary data of variable length.
275 ///
276 /// Logically the same as [`Binary`], but the internal representation uses a view
277 /// struct that contains the string length and either the string's entire data
278 /// inline (for small strings) or an inlined prefix, an index of another buffer,
279 /// and an offset pointing to a slice in that buffer (for non-small strings).
280 ///
281 /// [`Binary`]: Self::Binary
282 BinaryView,
283 /// A variable-length string in Unicode with UTF-8 encoding.
284 ///
285 /// A single Utf8 array can store up to [`i32::MAX`] bytes
286 /// of string data in total.
287 Utf8,
288 /// A variable-length string in Unicode with UFT-8 encoding and 64-bit offsets.
289 ///
290 /// A single LargeUtf8 array can store up to [`i64::MAX`] bytes
291 /// of string data in total.
292 LargeUtf8,
293 /// A variable-length string in Unicode with UTF-8 encoding
294 ///
295 /// Logically the same as [`Utf8`], but the internal representation uses a view
296 /// struct that contains the string length and either the string's entire data
297 /// inline (for small strings) or an inlined prefix, an index of another buffer,
298 /// and an offset pointing to a slice in that buffer (for non-small strings).
299 ///
300 /// [`Utf8`]: Self::Utf8
301 Utf8View,
302 /// A list of some logical data type with variable length.
303 ///
304 /// A single List array can store up to [`i32::MAX`] elements in total.
305 List(FieldRef),
306 /// A list of some logical data type with variable length.
307 ///
308 /// Logically the same as [`List`], but the internal representation differs in how child
309 /// data is referenced, allowing flexibility in how data is laid out.
310 ///
311 /// [`List`]: Self::List
312 ListView(FieldRef),
313 /// A list of some logical data type with fixed length.
314 FixedSizeList(FieldRef, i32),
315 /// A list of some logical data type with variable length and 64-bit offsets.
316 ///
317 /// A single LargeList array can store up to [`i64::MAX`] elements in total.
318 LargeList(FieldRef),
319 /// A list of some logical data type with variable length and 64-bit offsets.
320 ///
321 /// Logically the same as [`LargeList`], but the internal representation differs in how child
322 /// data is referenced, allowing flexibility in how data is laid out.
323 ///
324 /// [`LargeList`]: Self::LargeList
325 LargeListView(FieldRef),
326 /// A nested datatype that contains a number of sub-fields.
327 Struct(Fields),
328 /// A nested datatype that can represent slots of differing types. Components:
329 ///
330 /// 1. [`UnionFields`]
331 /// 2. The type of union (Sparse or Dense)
332 Union(UnionFields, UnionMode),
333 /// A dictionary encoded array (`key_type`, `value_type`), where
334 /// each array element is an index of `key_type` into an
335 /// associated dictionary of `value_type`.
336 ///
337 /// Dictionary arrays are used to store columns of `value_type`
338 /// that contain many repeated values using less memory, but with
339 /// a higher CPU overhead for some operations.
340 ///
341 /// This type mostly used to represent low cardinality string
342 /// arrays or a limited set of primitive types as integers.
343 Dictionary(Box<DataType>, Box<DataType>),
344 /// Exact 32-bit width decimal value with precision and scale
345 ///
346 /// * precision is the maximum number of digits in the unscaled value
347 /// * scale controls the position of the decimal point
348 ///
349 /// The represented value is the unscaled integer multiplied by 10^{-scale}.
350 /// For example, the unscaled value 12345 with precision 5 and scale 2
351 /// represents 123.45.
352 ///
353 /// Scale can also be negative. For example, the unscaled value 12 with
354 /// precision 2 and scale -3 represents 12000.
355 Decimal32(u8, i8),
356 /// Exact 64-bit width decimal value with precision and scale
357 ///
358 /// * precision is the maximum number of digits in the unscaled value
359 /// * scale controls the position of the decimal point
360 ///
361 /// The represented value is the unscaled integer multiplied by 10^{-scale}.
362 /// For example, the unscaled value 12345 with precision 5 and scale 2
363 /// represents 123.45.
364 ///
365 /// Scale can also be negative. For example, the unscaled value 12 with
366 /// precision 2 and scale -3 represents 12000.
367 Decimal64(u8, i8),
368 /// Exact 128-bit width decimal value with precision and scale
369 ///
370 /// * precision is the maximum number of digits in the unscaled value
371 /// * scale controls the position of the decimal point
372 ///
373 /// The represented value is the unscaled integer multiplied by 10^{-scale}.
374 /// For example, the unscaled value 12345 with precision 5 and scale 2
375 /// represents 123.45.
376 ///
377 /// Scale can also be negative. For example, the unscaled value 12 with
378 /// precision 2 and scale -3 represents 12000.
379 Decimal128(u8, i8),
380 /// Exact 256-bit width decimal value with precision and scale
381 ///
382 /// * precision is the maximum number of digits in the unscaled value
383 /// * scale controls the position of the decimal point
384 ///
385 /// The represented value is the unscaled integer multiplied by 10^{-scale}.
386 /// For example, the unscaled value 12345 with precision 5 and scale 2
387 /// represents 123.45.
388 ///
389 /// Scale can also be negative. For example, the unscaled value 12 with
390 /// precision 2 and scale -3 represents 12000.
391 Decimal256(u8, i8),
392 /// A Map is a logical nested type that is represented as
393 ///
394 /// `List<entries: Struct<key: K, value: V>>`
395 ///
396 /// The keys and values are each respectively contiguous.
397 /// The key and value types are not constrained, but keys should be
398 /// hashable and unique.
399 /// Whether the keys are sorted can be set in the `bool` after the `Field`.
400 ///
401 /// In a field with Map type, the field has a child Struct field, which then
402 /// has two children: key type and the second the value type. The names of the
403 /// child fields may be respectively "entries", "key", and "value", but this is
404 /// not enforced.
405 ///
406 /// # Requirements
407 /// - The entries [`Field`] (first argument) must be non-nullable.
408 /// - The entries field must be a [`DataType::Struct`] with exactly 2 children.
409 /// - The first child (key) must be non-nullable.
410 Map(FieldRef, bool),
411 /// A run-end encoding (REE) is a variation of run-length encoding (RLE). These
412 /// encodings are well-suited for representing data containing sequences of the
413 /// same value, called runs. Each run is represented as a value and an integer giving
414 /// the index in the array where the run ends.
415 ///
416 /// A run-end encoded array has no buffers by itself, but has two child arrays. The
417 /// first child array, called the run ends array, holds either 16, 32, or 64-bit
418 /// signed integers. The actual values of each run are held in the second child array.
419 ///
420 /// These child arrays are prescribed the standard names of "run_ends" and "values"
421 /// respectively.
422 ///
423 /// # Requirements
424 /// - The run_ends [`Field`] (first argument) must be non-nullable and of type
425 /// [`DataType::Int16`], [`DataType::Int32`], or [`DataType::Int64`].
426 RunEndEncoded(FieldRef, FieldRef),
427}
428
429/// An absolute length of time in seconds, milliseconds, microseconds or nanoseconds.
430#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
431#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
432pub enum TimeUnit {
433 /// Time in seconds.
434 Second,
435 /// Time in milliseconds.
436 Millisecond,
437 /// Time in microseconds.
438 Microsecond,
439 /// Time in nanoseconds.
440 Nanosecond,
441}
442
443impl std::fmt::Display for TimeUnit {
444 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
445 match self {
446 TimeUnit::Second => write!(f, "s"),
447 TimeUnit::Millisecond => write!(f, "ms"),
448 TimeUnit::Microsecond => write!(f, "µs"),
449 TimeUnit::Nanosecond => write!(f, "ns"),
450 }
451 }
452}
453
454/// YEAR_MONTH, DAY_TIME, MONTH_DAY_NANO interval in SQL style.
455#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
456#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
457pub enum IntervalUnit {
458 /// Indicates the number of elapsed whole months, stored as 4-byte integers.
459 YearMonth,
460 /// Indicates the number of elapsed days and milliseconds,
461 /// stored as 2 contiguous 32-bit integers (days, milliseconds) (8-bytes in total).
462 DayTime,
463 /// A triple of the number of elapsed months, days, and nanoseconds.
464 /// The values are stored contiguously in 16 byte blocks. Months and
465 /// days are encoded as 32 bit integers and nanoseconds is encoded as a
466 /// 64 bit integer. All integers are signed. Each field is independent
467 /// (e.g. there is no constraint that nanoseconds have the same sign
468 /// as days or that the quantity of nanoseconds represents less
469 /// than a day's worth of time).
470 MonthDayNano,
471}
472
473/// Sparse or Dense union layouts
474#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Copy)]
475#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
476pub enum UnionMode {
477 /// Sparse union layout
478 Sparse,
479 /// Dense union layout
480 Dense,
481}
482
483/// Parses `str` into a `DataType`.
484///
485/// This is the reverse of [`DataType`]'s `Display`
486/// impl, and maintains the invariant that
487/// `DataType::try_from(&data_type.to_string()).unwrap() == data_type`
488///
489/// # Example
490/// ```
491/// use arrow_schema::DataType;
492///
493/// let data_type: DataType = "Int32".parse().unwrap();
494/// assert_eq!(data_type, DataType::Int32);
495/// ```
496impl FromStr for DataType {
497 type Err = ArrowError;
498
499 fn from_str(s: &str) -> Result<Self, Self::Err> {
500 crate::datatype_parse::parse_data_type(s)
501 }
502}
503
504impl TryFrom<&str> for DataType {
505 type Error = ArrowError;
506
507 fn try_from(value: &str) -> Result<Self, Self::Error> {
508 value.parse()
509 }
510}
511
512impl DataType {
513 /// Returns true if the type is primitive: (numeric, temporal).
514 #[inline]
515 pub fn is_primitive(&self) -> bool {
516 self.is_numeric() || self.is_temporal()
517 }
518
519 /// Returns true if this type is numeric: (UInt*, Int*, Float*, Decimal*).
520 #[inline]
521 pub fn is_numeric(&self) -> bool {
522 use DataType::*;
523 matches!(
524 self,
525 UInt8
526 | UInt16
527 | UInt32
528 | UInt64
529 | Int8
530 | Int16
531 | Int32
532 | Int64
533 | Float16
534 | Float32
535 | Float64
536 | Decimal32(_, _)
537 | Decimal64(_, _)
538 | Decimal128(_, _)
539 | Decimal256(_, _)
540 )
541 }
542
543 /// Returns true if this type is temporal: (Date*, Time*, Duration, or Interval).
544 #[inline]
545 pub fn is_temporal(&self) -> bool {
546 use DataType::*;
547 matches!(
548 self,
549 Date32 | Date64 | Timestamp(_, _) | Time32(_) | Time64(_) | Duration(_) | Interval(_)
550 )
551 }
552
553 /// Returns true if this type is floating: (Float*).
554 #[inline]
555 pub fn is_floating(&self) -> bool {
556 use DataType::*;
557 matches!(self, Float16 | Float32 | Float64)
558 }
559
560 /// Returns true if this type is integer: (Int*, UInt*).
561 #[inline]
562 pub fn is_integer(&self) -> bool {
563 self.is_signed_integer() || self.is_unsigned_integer()
564 }
565
566 /// Returns true if this type is signed integer: (Int*).
567 #[inline]
568 pub fn is_signed_integer(&self) -> bool {
569 use DataType::*;
570 matches!(self, Int8 | Int16 | Int32 | Int64)
571 }
572
573 /// Returns true if this type is unsigned integer: (UInt*).
574 #[inline]
575 pub fn is_unsigned_integer(&self) -> bool {
576 use DataType::*;
577 matches!(self, UInt8 | UInt16 | UInt32 | UInt64)
578 }
579
580 /// Returns true if this type is decimal: (Decimal*).
581 #[inline]
582 pub fn is_decimal(&self) -> bool {
583 use DataType::*;
584 matches!(
585 self,
586 Decimal32(..) | Decimal64(..) | Decimal128(..) | Decimal256(..)
587 )
588 }
589
590 /// Returns true if this type is valid as a dictionary key
591 #[inline]
592 pub fn is_dictionary_key_type(&self) -> bool {
593 self.is_integer()
594 }
595
596 /// Returns true if this type is valid for run-ends array in RunArray
597 #[inline]
598 pub fn is_run_ends_type(&self) -> bool {
599 use DataType::*;
600 matches!(self, Int16 | Int32 | Int64)
601 }
602
603 /// Returns true if this type is nested (List, FixedSizeList, LargeList, ListView. LargeListView, Struct, Union,
604 /// or Map), or a dictionary of a nested type
605 #[inline]
606 pub fn is_nested(&self) -> bool {
607 use DataType::*;
608 match self {
609 Dictionary(_, v) => DataType::is_nested(v.as_ref()),
610 RunEndEncoded(_, v) => DataType::is_nested(v.data_type()),
611 List(_)
612 | FixedSizeList(_, _)
613 | LargeList(_)
614 | ListView(_)
615 | LargeListView(_)
616 | Struct(_)
617 | Union(_, _)
618 | Map(_, _) => true,
619 _ => false,
620 }
621 }
622
623 /// Returns true if this type is DataType::Null.
624 #[inline]
625 pub fn is_null(&self) -> bool {
626 use DataType::*;
627 matches!(self, Null)
628 }
629
630 /// Returns true if this type is a String type
631 #[inline]
632 pub fn is_string(&self) -> bool {
633 use DataType::*;
634 matches!(self, Utf8 | LargeUtf8 | Utf8View)
635 }
636
637 /// Returns true if this type is a List type.
638 ///
639 /// List types include List, LargeList, FixedSizeList, ListView, and LargeListView.
640 #[inline]
641 pub fn is_list(&self) -> bool {
642 use DataType::*;
643 matches!(
644 self,
645 List(_) | LargeList(_) | FixedSizeList(_, _) | ListView(_) | LargeListView(_)
646 )
647 }
648
649 /// Returns true if this type is a Binary type.
650 ///
651 /// Binary types include Binary, LargeBinary, FixedSizeBinary and BinaryView.
652 #[inline]
653 pub fn is_binary(&self) -> bool {
654 use DataType::*;
655 matches!(self, Binary | LargeBinary | FixedSizeBinary(_) | BinaryView)
656 }
657
658 /// Compares the datatype with another, ignoring nested field names
659 /// and metadata.
660 pub fn equals_datatype(&self, other: &DataType) -> bool {
661 match (&self, other) {
662 (DataType::List(a), DataType::List(b))
663 | (DataType::LargeList(a), DataType::LargeList(b))
664 | (DataType::ListView(a), DataType::ListView(b))
665 | (DataType::LargeListView(a), DataType::LargeListView(b)) => {
666 a.is_nullable() == b.is_nullable() && a.data_type().equals_datatype(b.data_type())
667 }
668 (DataType::FixedSizeList(a, a_size), DataType::FixedSizeList(b, b_size)) => {
669 a_size == b_size
670 && a.is_nullable() == b.is_nullable()
671 && a.data_type().equals_datatype(b.data_type())
672 }
673 (DataType::Struct(a), DataType::Struct(b)) => {
674 a.len() == b.len()
675 && a.iter().zip(b).all(|(a, b)| {
676 a.is_nullable() == b.is_nullable()
677 && a.data_type().equals_datatype(b.data_type())
678 })
679 }
680 (DataType::Map(a_field, a_is_sorted), DataType::Map(b_field, b_is_sorted)) => {
681 a_field.is_nullable() == b_field.is_nullable()
682 && a_field.data_type().equals_datatype(b_field.data_type())
683 && a_is_sorted == b_is_sorted
684 }
685 (DataType::Dictionary(a_key, a_value), DataType::Dictionary(b_key, b_value)) => {
686 a_key.equals_datatype(b_key) && a_value.equals_datatype(b_value)
687 }
688 (
689 DataType::RunEndEncoded(a_run_ends, a_values),
690 DataType::RunEndEncoded(b_run_ends, b_values),
691 ) => {
692 a_run_ends.is_nullable() == b_run_ends.is_nullable()
693 && a_run_ends
694 .data_type()
695 .equals_datatype(b_run_ends.data_type())
696 && a_values.is_nullable() == b_values.is_nullable()
697 && a_values.data_type().equals_datatype(b_values.data_type())
698 }
699 (
700 DataType::Union(a_union_fields, a_union_mode),
701 DataType::Union(b_union_fields, b_union_mode),
702 ) => {
703 a_union_mode == b_union_mode
704 && a_union_fields.len() == b_union_fields.len()
705 && a_union_fields.iter().all(|a| {
706 b_union_fields.iter().any(|b| {
707 a.0 == b.0
708 && a.1.is_nullable() == b.1.is_nullable()
709 && a.1.data_type().equals_datatype(b.1.data_type())
710 })
711 })
712 }
713 _ => self == other,
714 }
715 }
716
717 /// Returns the byte width of this type if it is a primitive type
718 ///
719 /// Returns `None` if not a primitive type
720 #[inline]
721 pub fn primitive_width(&self) -> Option<usize> {
722 match self {
723 DataType::Null => None,
724 DataType::Boolean => None,
725 DataType::Int8 | DataType::UInt8 => Some(1),
726 DataType::Int16 | DataType::UInt16 | DataType::Float16 => Some(2),
727 DataType::Int32 | DataType::UInt32 | DataType::Float32 => Some(4),
728 DataType::Int64 | DataType::UInt64 | DataType::Float64 => Some(8),
729 DataType::Timestamp(_, _) => Some(8),
730 DataType::Date32 | DataType::Time32(_) => Some(4),
731 DataType::Date64 | DataType::Time64(_) => Some(8),
732 DataType::Duration(_) => Some(8),
733 DataType::Interval(IntervalUnit::YearMonth) => Some(4),
734 DataType::Interval(IntervalUnit::DayTime) => Some(8),
735 DataType::Interval(IntervalUnit::MonthDayNano) => Some(16),
736 DataType::Decimal32(_, _) => Some(4),
737 DataType::Decimal64(_, _) => Some(8),
738 DataType::Decimal128(_, _) => Some(16),
739 DataType::Decimal256(_, _) => Some(32),
740 DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => None,
741 DataType::Binary | DataType::LargeBinary | DataType::BinaryView => None,
742 DataType::FixedSizeBinary(_) => None,
743 DataType::List(_)
744 | DataType::ListView(_)
745 | DataType::LargeList(_)
746 | DataType::LargeListView(_)
747 | DataType::Map(_, _) => None,
748 DataType::FixedSizeList(_, _) => None,
749 DataType::Struct(_) => None,
750 DataType::Union(_, _) => None,
751 DataType::Dictionary(_, _) => None,
752 DataType::RunEndEncoded(_, _) => None,
753 }
754 }
755
756 /// Return size of this instance in bytes.
757 ///
758 /// Includes the size of `Self`.
759 pub fn size(&self) -> usize {
760 std::mem::size_of_val(self)
761 + match self {
762 DataType::Null
763 | DataType::Boolean
764 | DataType::Int8
765 | DataType::Int16
766 | DataType::Int32
767 | DataType::Int64
768 | DataType::UInt8
769 | DataType::UInt16
770 | DataType::UInt32
771 | DataType::UInt64
772 | DataType::Float16
773 | DataType::Float32
774 | DataType::Float64
775 | DataType::Date32
776 | DataType::Date64
777 | DataType::Time32(_)
778 | DataType::Time64(_)
779 | DataType::Duration(_)
780 | DataType::Interval(_)
781 | DataType::Binary
782 | DataType::FixedSizeBinary(_)
783 | DataType::LargeBinary
784 | DataType::BinaryView
785 | DataType::Utf8
786 | DataType::LargeUtf8
787 | DataType::Utf8View
788 | DataType::Decimal32(_, _)
789 | DataType::Decimal64(_, _)
790 | DataType::Decimal128(_, _)
791 | DataType::Decimal256(_, _) => 0,
792 DataType::Timestamp(_, s) => s.as_ref().map(|s| s.len()).unwrap_or_default(),
793 DataType::List(field)
794 | DataType::ListView(field)
795 | DataType::FixedSizeList(field, _)
796 | DataType::LargeList(field)
797 | DataType::LargeListView(field)
798 | DataType::Map(field, _) => field.size(),
799 DataType::Struct(fields) => fields.size(),
800 DataType::Union(fields, _) => fields.size(),
801 DataType::Dictionary(dt1, dt2) => dt1.size() + dt2.size(),
802 DataType::RunEndEncoded(run_ends, values) => {
803 run_ends.size() - std::mem::size_of_val(run_ends) + values.size()
804 - std::mem::size_of_val(values)
805 }
806 }
807 }
808
809 /// Check to see if `self` is a superset of `other`
810 ///
811 /// If DataType is a nested type, then it will check to see if the nested type is a superset of the other nested type
812 /// else it will check to see if the DataType is equal to the other DataType
813 pub fn contains(&self, other: &DataType) -> bool {
814 match (self, other) {
815 (DataType::List(f1), DataType::List(f2))
816 | (DataType::LargeList(f1), DataType::LargeList(f2))
817 | (DataType::ListView(f1), DataType::ListView(f2))
818 | (DataType::LargeListView(f1), DataType::LargeListView(f2)) => f1.contains(f2),
819 (DataType::FixedSizeList(f1, s1), DataType::FixedSizeList(f2, s2)) => {
820 s1 == s2 && f1.contains(f2)
821 }
822 (DataType::Map(f1, s1), DataType::Map(f2, s2)) => s1 == s2 && f1.contains(f2),
823 (DataType::Struct(f1), DataType::Struct(f2)) => f1.contains(f2),
824 (DataType::Union(f1, s1), DataType::Union(f2, s2)) => {
825 s1 == s2
826 && f2.iter().all(|(other_id, other_field)| {
827 f1.iter().any(|(self_id, self_field)| {
828 self_id == other_id && self_field.contains(other_field)
829 })
830 })
831 }
832 (DataType::Dictionary(k1, v1), DataType::Dictionary(k2, v2)) => {
833 k1.contains(k2) && v1.contains(v2)
834 }
835 _ => self == other,
836 }
837 }
838
839 /// Create a [`DataType::List`] with elements of the specified type
840 /// and nullability, and conventionally named inner [`Field`] (`"item"`).
841 ///
842 /// To specify field level metadata, construct the inner [`Field`]
843 /// directly via [`Field::new`] or [`Field::new_list_field`].
844 pub fn new_list(data_type: DataType, nullable: bool) -> Self {
845 DataType::List(Arc::new(Field::new_list_field(data_type, nullable)))
846 }
847
848 /// Create a [`DataType::LargeList`] with elements of the specified type
849 /// and nullability, and conventionally named inner [`Field`] (`"item"`).
850 ///
851 /// To specify field level metadata, construct the inner [`Field`]
852 /// directly via [`Field::new`] or [`Field::new_list_field`].
853 pub fn new_large_list(data_type: DataType, nullable: bool) -> Self {
854 DataType::LargeList(Arc::new(Field::new_list_field(data_type, nullable)))
855 }
856
857 /// Create a [`DataType::FixedSizeList`] with elements of the specified type, size
858 /// and nullability, and conventionally named inner [`Field`] (`"item"`).
859 ///
860 /// To specify field level metadata, construct the inner [`Field`]
861 /// directly via [`Field::new`] or [`Field::new_list_field`].
862 pub fn new_fixed_size_list(data_type: DataType, size: i32, nullable: bool) -> Self {
863 DataType::FixedSizeList(Arc::new(Field::new_list_field(data_type, nullable)), size)
864 }
865}
866
867/// The maximum precision for [DataType::Decimal32] values
868pub const DECIMAL32_MAX_PRECISION: u8 = 9;
869
870/// The maximum scale for [DataType::Decimal32] values
871pub const DECIMAL32_MAX_SCALE: i8 = 9;
872
873/// The maximum precision for [DataType::Decimal64] values
874pub const DECIMAL64_MAX_PRECISION: u8 = 18;
875
876/// The maximum scale for [DataType::Decimal64] values
877pub const DECIMAL64_MAX_SCALE: i8 = 18;
878
879/// The maximum precision for [DataType::Decimal128] values
880pub const DECIMAL128_MAX_PRECISION: u8 = 38;
881
882/// The maximum scale for [DataType::Decimal128] values
883pub const DECIMAL128_MAX_SCALE: i8 = 38;
884
885/// The maximum precision for [DataType::Decimal256] values
886pub const DECIMAL256_MAX_PRECISION: u8 = 76;
887
888/// The maximum scale for [DataType::Decimal256] values
889pub const DECIMAL256_MAX_SCALE: i8 = 76;
890
891/// The default scale for [DataType::Decimal32] values
892pub const DECIMAL32_DEFAULT_SCALE: i8 = 2;
893
894/// The default scale for [DataType::Decimal64] values
895pub const DECIMAL64_DEFAULT_SCALE: i8 = 6;
896
897/// The default scale for [DataType::Decimal128] and [DataType::Decimal256]
898/// values
899pub const DECIMAL_DEFAULT_SCALE: i8 = 10;
900
901#[cfg(test)]
902mod tests {
903 use super::*;
904
905 #[test]
906 fn test_union_contains_subset() {
907 for mode in [UnionMode::Sparse, UnionMode::Dense] {
908 let number = Field::new("number", DataType::Int32, false);
909 let text = Field::new("text", DataType::Utf8, false);
910 let superset = DataType::Union(
911 UnionFields::try_new([0, 1], [number.clone(), text.clone()]).unwrap(),
912 mode,
913 );
914 let subset =
915 DataType::Union(UnionFields::try_new([0], [number.clone()]).unwrap(), mode);
916 let reordered =
917 DataType::Union(UnionFields::try_new([1, 0], [text, number]).unwrap(), mode);
918 let empty = DataType::Union(UnionFields::empty(), mode);
919
920 assert!(superset.contains(&subset));
921 assert!(!subset.contains(&superset));
922 assert!(superset.contains(&superset));
923 assert!(superset.contains(&reordered));
924 assert!(reordered.contains(&superset));
925 assert!(superset.contains(&empty));
926 assert!(!empty.contains(&superset));
927 assert!(empty.contains(&empty));
928 }
929 }
930
931 #[test]
932 fn test_union_contains_field_compatibility() {
933 for mode in [UnionMode::Sparse, UnionMode::Dense] {
934 let field = Field::new("number", DataType::Int32, false);
935 let union = DataType::Union(UnionFields::try_new([0], [field.clone()]).unwrap(), mode);
936 let nullable = DataType::Union(
937 UnionFields::try_new([0], [field.clone().with_nullable(true)]).unwrap(),
938 mode,
939 );
940 assert!(nullable.contains(&union));
941 assert!(!union.contains(&nullable));
942
943 let different_id =
944 DataType::Union(UnionFields::try_new([1], [field.clone()]).unwrap(), mode);
945 assert!(!union.contains(&different_id));
946 assert!(!different_id.contains(&union));
947
948 let different_type = DataType::Union(
949 UnionFields::try_new([0], [field.clone().with_data_type(DataType::Utf8)]).unwrap(),
950 mode,
951 );
952 assert!(!union.contains(&different_type));
953 assert!(!different_type.contains(&union));
954
955 let other_mode = match mode {
956 UnionMode::Sparse => UnionMode::Dense,
957 UnionMode::Dense => UnionMode::Sparse,
958 };
959 let different_mode =
960 DataType::Union(UnionFields::try_new([0], [field]).unwrap(), other_mode);
961 assert!(!union.contains(&different_mode));
962 assert!(!different_mode.contains(&union));
963 }
964 }
965
966 #[test]
967 #[cfg(feature = "serde")]
968 fn serde_struct_type() {
969 use std::collections::HashMap;
970
971 let kv_array = [("k".to_string(), "v".to_string())];
972 let field_metadata: HashMap<String, String> = kv_array.iter().cloned().collect();
973
974 // Non-empty map: should be converted as JSON obj { ... }
975 let first_name =
976 Field::new("first_name", DataType::Utf8, false).with_metadata(field_metadata);
977
978 // Empty map: should be omitted.
979 let last_name =
980 Field::new("last_name", DataType::Utf8, false).with_metadata(HashMap::default());
981
982 let person = DataType::Struct(Fields::from(vec![
983 first_name,
984 last_name,
985 Field::new(
986 "address",
987 DataType::Struct(Fields::from(vec![
988 Field::new("street", DataType::Utf8, false),
989 Field::new("zip", DataType::UInt16, false),
990 ])),
991 false,
992 ),
993 ]));
994
995 let serialized = serde_json::to_string(&person).unwrap();
996
997 // NOTE that this is testing the default (derived) serialization format, not the
998 // JSON format specified in metadata.md
999
1000 assert_eq!(
1001 "{\"Struct\":[\
1002 {\"name\":\"first_name\",\"data_type\":\"Utf8\",\"nullable\":false,\"dict_id\":0,\"dict_is_ordered\":false,\"metadata\":{\"k\":\"v\"}},\
1003 {\"name\":\"last_name\",\"data_type\":\"Utf8\",\"nullable\":false,\"dict_id\":0,\"dict_is_ordered\":false,\"metadata\":{}},\
1004 {\"name\":\"address\",\"data_type\":{\"Struct\":\
1005 [{\"name\":\"street\",\"data_type\":\"Utf8\",\"nullable\":false,\"dict_id\":0,\"dict_is_ordered\":false,\"metadata\":{}},\
1006 {\"name\":\"zip\",\"data_type\":\"UInt16\",\"nullable\":false,\"dict_id\":0,\"dict_is_ordered\":false,\"metadata\":{}}\
1007 ]},\"nullable\":false,\"dict_id\":0,\"dict_is_ordered\":false,\"metadata\":{}}]}",
1008 serialized
1009 );
1010
1011 let deserialized = serde_json::from_str(&serialized).unwrap();
1012
1013 assert_eq!(person, deserialized);
1014 }
1015
1016 #[test]
1017 fn test_list_datatype_equality() {
1018 // tests that list type equality is checked while ignoring list names
1019 let list_a = DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true)));
1020 let list_b = DataType::List(Arc::new(Field::new("array", DataType::Int32, true)));
1021 let list_c = DataType::List(Arc::new(Field::new_list_field(DataType::Int32, false)));
1022 let list_d = DataType::List(Arc::new(Field::new_list_field(DataType::UInt32, true)));
1023 assert!(list_a.equals_datatype(&list_b));
1024 assert!(!list_a.equals_datatype(&list_c));
1025 assert!(!list_b.equals_datatype(&list_c));
1026 assert!(!list_a.equals_datatype(&list_d));
1027
1028 let list_e =
1029 DataType::FixedSizeList(Arc::new(Field::new_list_field(list_a.clone(), false)), 3);
1030 let list_f =
1031 DataType::FixedSizeList(Arc::new(Field::new("array", list_b.clone(), false)), 3);
1032 let list_g = DataType::FixedSizeList(
1033 Arc::new(Field::new_list_field(DataType::FixedSizeBinary(3), true)),
1034 3,
1035 );
1036 assert!(list_e.equals_datatype(&list_f));
1037 assert!(!list_e.equals_datatype(&list_g));
1038 assert!(!list_f.equals_datatype(&list_g));
1039
1040 let list_h = DataType::Struct(Fields::from(vec![Field::new("f1", list_e, true)]));
1041 let list_i = DataType::Struct(Fields::from(vec![Field::new("f1", list_f.clone(), true)]));
1042 let list_j = DataType::Struct(Fields::from(vec![Field::new("f1", list_f.clone(), false)]));
1043 let list_k = DataType::Struct(Fields::from(vec![
1044 Field::new("f1", list_f.clone(), false),
1045 Field::new("f2", list_g.clone(), false),
1046 Field::new("f3", DataType::Utf8, true),
1047 ]));
1048 let list_l = DataType::Struct(Fields::from(vec![
1049 Field::new("ff1", list_f.clone(), false),
1050 Field::new("ff2", list_g.clone(), false),
1051 Field::new("ff3", DataType::LargeUtf8, true),
1052 ]));
1053 let list_m = DataType::Struct(Fields::from(vec![
1054 Field::new("ff1", list_f, false),
1055 Field::new("ff2", list_g, false),
1056 Field::new("ff3", DataType::Utf8, true),
1057 ]));
1058 assert!(list_h.equals_datatype(&list_i));
1059 assert!(!list_h.equals_datatype(&list_j));
1060 assert!(!list_k.equals_datatype(&list_l));
1061 assert!(list_k.equals_datatype(&list_m));
1062
1063 let list_n = DataType::Map(Arc::new(Field::new("f1", list_a.clone(), true)), true);
1064 let list_o = DataType::Map(Arc::new(Field::new("f2", list_b.clone(), true)), true);
1065 let list_p = DataType::Map(Arc::new(Field::new("f2", list_b.clone(), true)), false);
1066 let list_q = DataType::Map(Arc::new(Field::new("f2", list_c.clone(), true)), true);
1067 let list_r = DataType::Map(Arc::new(Field::new("f1", list_a.clone(), false)), true);
1068
1069 assert!(list_n.equals_datatype(&list_o));
1070 assert!(!list_n.equals_datatype(&list_p));
1071 assert!(!list_n.equals_datatype(&list_q));
1072 assert!(!list_n.equals_datatype(&list_r));
1073
1074 let list_s = DataType::Dictionary(Box::new(DataType::UInt8), Box::new(list_a));
1075 let list_t = DataType::Dictionary(Box::new(DataType::UInt8), Box::new(list_b.clone()));
1076 let list_u = DataType::Dictionary(Box::new(DataType::Int8), Box::new(list_b));
1077 let list_v = DataType::Dictionary(Box::new(DataType::UInt8), Box::new(list_c));
1078
1079 assert!(list_s.equals_datatype(&list_t));
1080 assert!(!list_s.equals_datatype(&list_u));
1081 assert!(!list_s.equals_datatype(&list_v));
1082
1083 let union_a = DataType::Union(
1084 UnionFields::try_new(
1085 vec![1, 2],
1086 vec![
1087 Field::new("f1", DataType::Utf8, false),
1088 Field::new("f2", DataType::UInt8, false),
1089 ],
1090 )
1091 .unwrap(),
1092 UnionMode::Sparse,
1093 );
1094 let union_b = DataType::Union(
1095 UnionFields::try_new(
1096 vec![1, 2],
1097 vec![
1098 Field::new("ff1", DataType::Utf8, false),
1099 Field::new("ff2", DataType::UInt8, false),
1100 ],
1101 )
1102 .unwrap(),
1103 UnionMode::Sparse,
1104 );
1105 let union_c = DataType::Union(
1106 UnionFields::try_new(
1107 vec![2, 1],
1108 vec![
1109 Field::new("fff2", DataType::UInt8, false),
1110 Field::new("fff1", DataType::Utf8, false),
1111 ],
1112 )
1113 .unwrap(),
1114 UnionMode::Sparse,
1115 );
1116 let union_d = DataType::Union(
1117 UnionFields::try_new(
1118 vec![2, 1],
1119 vec![
1120 Field::new("fff1", DataType::Int8, false),
1121 Field::new("fff2", DataType::UInt8, false),
1122 ],
1123 )
1124 .unwrap(),
1125 UnionMode::Sparse,
1126 );
1127 let union_e = DataType::Union(
1128 UnionFields::try_new(
1129 vec![1, 2],
1130 vec![
1131 Field::new("f1", DataType::Utf8, true),
1132 Field::new("f2", DataType::UInt8, false),
1133 ],
1134 )
1135 .unwrap(),
1136 UnionMode::Sparse,
1137 );
1138
1139 assert!(union_a.equals_datatype(&union_b));
1140 assert!(union_a.equals_datatype(&union_c));
1141 assert!(!union_a.equals_datatype(&union_d));
1142 assert!(!union_a.equals_datatype(&union_e));
1143
1144 let list_w = DataType::RunEndEncoded(
1145 Arc::new(Field::new("f1", DataType::Int64, true)),
1146 Arc::new(Field::new("f2", DataType::Utf8, true)),
1147 );
1148 let list_x = DataType::RunEndEncoded(
1149 Arc::new(Field::new("ff1", DataType::Int64, true)),
1150 Arc::new(Field::new("ff2", DataType::Utf8, true)),
1151 );
1152 let list_y = DataType::RunEndEncoded(
1153 Arc::new(Field::new("ff1", DataType::UInt16, true)),
1154 Arc::new(Field::new("ff2", DataType::Utf8, true)),
1155 );
1156 let list_z = DataType::RunEndEncoded(
1157 Arc::new(Field::new("f1", DataType::Int64, false)),
1158 Arc::new(Field::new("f2", DataType::Utf8, true)),
1159 );
1160
1161 assert!(list_w.equals_datatype(&list_x));
1162 assert!(!list_w.equals_datatype(&list_y));
1163 assert!(!list_w.equals_datatype(&list_z));
1164 }
1165
1166 #[test]
1167 fn create_struct_type() {
1168 let _person = DataType::Struct(Fields::from(vec![
1169 Field::new("first_name", DataType::Utf8, false),
1170 Field::new("last_name", DataType::Utf8, false),
1171 Field::new(
1172 "address",
1173 DataType::Struct(Fields::from(vec![
1174 Field::new("street", DataType::Utf8, false),
1175 Field::new("zip", DataType::UInt16, false),
1176 ])),
1177 false,
1178 ),
1179 ]));
1180 }
1181
1182 #[test]
1183 fn test_nested() {
1184 let list = DataType::List(Arc::new(Field::new("foo", DataType::Utf8, true)));
1185 let list_view = DataType::ListView(Arc::new(Field::new("foo", DataType::Utf8, true)));
1186 let large_list_view =
1187 DataType::LargeListView(Arc::new(Field::new("foo", DataType::Utf8, true)));
1188
1189 assert!(!DataType::is_nested(&DataType::Boolean));
1190 assert!(!DataType::is_nested(&DataType::Int32));
1191 assert!(!DataType::is_nested(&DataType::Utf8));
1192 assert!(DataType::is_nested(&list));
1193 assert!(DataType::is_nested(&list_view));
1194 assert!(DataType::is_nested(&large_list_view));
1195
1196 assert!(!DataType::is_nested(&DataType::Dictionary(
1197 Box::new(DataType::Int32),
1198 Box::new(DataType::Boolean)
1199 )));
1200 assert!(!DataType::is_nested(&DataType::Dictionary(
1201 Box::new(DataType::Int32),
1202 Box::new(DataType::Int64)
1203 )));
1204 assert!(!DataType::is_nested(&DataType::Dictionary(
1205 Box::new(DataType::Int32),
1206 Box::new(DataType::LargeUtf8)
1207 )));
1208 assert!(DataType::is_nested(&DataType::Dictionary(
1209 Box::new(DataType::Int32),
1210 Box::new(list)
1211 )));
1212 }
1213
1214 #[test]
1215 fn test_integer() {
1216 // is_integer
1217 assert!(DataType::is_integer(&DataType::Int32));
1218 assert!(DataType::is_integer(&DataType::UInt64));
1219 assert!(!DataType::is_integer(&DataType::Float16));
1220
1221 // is_signed_integer
1222 assert!(DataType::is_signed_integer(&DataType::Int32));
1223 assert!(!DataType::is_signed_integer(&DataType::UInt64));
1224 assert!(!DataType::is_signed_integer(&DataType::Float16));
1225
1226 // is_unsigned_integer
1227 assert!(!DataType::is_unsigned_integer(&DataType::Int32));
1228 assert!(DataType::is_unsigned_integer(&DataType::UInt64));
1229 assert!(!DataType::is_unsigned_integer(&DataType::Float16));
1230
1231 // is_dictionary_key_type
1232 assert!(DataType::is_dictionary_key_type(&DataType::Int32));
1233 assert!(DataType::is_dictionary_key_type(&DataType::UInt64));
1234 assert!(!DataType::is_dictionary_key_type(&DataType::Float16));
1235 }
1236
1237 #[test]
1238 fn test_string() {
1239 assert!(DataType::is_string(&DataType::Utf8));
1240 assert!(DataType::is_string(&DataType::LargeUtf8));
1241 assert!(DataType::is_string(&DataType::Utf8View));
1242 assert!(!DataType::is_string(&DataType::Int32));
1243 }
1244
1245 #[test]
1246 fn test_floating() {
1247 assert!(DataType::is_floating(&DataType::Float16));
1248 assert!(!DataType::is_floating(&DataType::Int32));
1249 }
1250
1251 #[test]
1252 fn test_decimal() {
1253 assert!(DataType::is_decimal(&DataType::Decimal32(4, 2)));
1254 assert!(DataType::is_decimal(&DataType::Decimal64(4, 2)));
1255 assert!(DataType::is_decimal(&DataType::Decimal128(4, 2)));
1256 assert!(DataType::is_decimal(&DataType::Decimal256(4, 2)));
1257 assert!(!DataType::is_decimal(&DataType::Float16));
1258 }
1259
1260 #[test]
1261 fn test_datatype_is_null() {
1262 assert!(DataType::is_null(&DataType::Null));
1263 assert!(!DataType::is_null(&DataType::Int32));
1264 }
1265
1266 #[test]
1267 fn test_is_list() {
1268 assert!(DataType::is_list(&DataType::new_list(
1269 DataType::Int16,
1270 true
1271 )));
1272 assert!(DataType::is_list(&DataType::new_large_list(
1273 DataType::Int16,
1274 true
1275 )));
1276 assert!(DataType::is_list(&DataType::new_fixed_size_list(
1277 DataType::Int16,
1278 5,
1279 true
1280 )));
1281 assert!(DataType::is_list(&DataType::ListView(Arc::new(
1282 Field::new("f", DataType::Int16, true)
1283 ))));
1284 assert!(DataType::is_list(&DataType::LargeListView(Arc::new(
1285 Field::new("f", DataType::Int16, true)
1286 ))));
1287 assert!(!DataType::is_list(&DataType::Binary));
1288 }
1289
1290 #[test]
1291 fn test_is_binary() {
1292 assert!(DataType::is_binary(&DataType::Binary));
1293 assert!(DataType::is_binary(&DataType::LargeBinary));
1294 assert!(DataType::is_binary(&DataType::BinaryView));
1295 assert!(!DataType::is_list(&DataType::Utf8View));
1296 }
1297
1298 #[test]
1299 fn size_should_not_regress() {
1300 assert_eq!(std::mem::size_of::<DataType>(), 24);
1301 }
1302
1303 #[test]
1304 #[should_panic(expected = "duplicate type id: 1")]
1305 fn test_union_with_duplicated_type_id() {
1306 let type_ids = vec![1, 1];
1307 let _union = DataType::Union(
1308 UnionFields::try_new(
1309 type_ids,
1310 vec![
1311 Field::new("f1", DataType::Int32, false),
1312 Field::new("f2", DataType::Utf8, false),
1313 ],
1314 )
1315 .unwrap(),
1316 UnionMode::Dense,
1317 );
1318 }
1319
1320 #[test]
1321 fn test_try_from_str() {
1322 let data_type: DataType = "Int32".try_into().unwrap();
1323 assert_eq!(data_type, DataType::Int32);
1324 }
1325
1326 #[test]
1327 fn test_from_str() {
1328 let data_type: DataType = "UInt64".parse().unwrap();
1329 assert_eq!(data_type, DataType::UInt64);
1330 }
1331
1332 #[test]
1333 #[cfg_attr(miri, ignore)] // fork is not supported
1334 fn test_debug_format_field() {
1335 // Make sure the `Debug` formatting of `DataType` is readable and not too long
1336 insta::assert_debug_snapshot!(DataType::new_list(DataType::Int8, false), @r"
1337 List(
1338 Field {
1339 data_type: Int8,
1340 },
1341 )
1342 ");
1343 }
1344}