arrow_schema/datatype.rs
1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied. See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::str::FromStr;
19use std::sync::Arc;
20
21use crate::{ArrowError, Field, FieldRef, Fields, UnionFields};
22
23/// Datatypes supported by this implementation of Apache Arrow.
24///
25/// The variants of this enum include primitive fixed size types as well as
26/// parametric or nested types. See [`Schema.fbs`] for Arrow's specification.
27///
28/// # Examples
29///
30/// Primitive types
31/// ```
32/// # use arrow_schema::DataType;
33/// // create a new 32-bit signed integer
34/// let data_type = DataType::Int32;
35/// ```
36///
37/// Nested Types
38/// ```
39/// # use arrow_schema::{DataType, Field};
40/// # use std::sync::Arc;
41/// // create a new list of 32-bit signed integers directly
42/// let list_data_type = DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true)));
43/// // Create the same list type with constructor
44/// let list_data_type2 = DataType::new_list(DataType::Int32, true);
45/// assert_eq!(list_data_type, list_data_type2);
46/// ```
47///
48/// Dictionary Types
49/// ```
50/// # use arrow_schema::{DataType};
51/// // String Dictionary (key type Int32 and value type Utf8)
52/// let data_type = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
53/// ```
54///
55/// Timestamp Types
56/// ```
57/// # use arrow_schema::{DataType, TimeUnit};
58/// // timestamp with millisecond precision without timezone specified
59/// let data_type = DataType::Timestamp(TimeUnit::Millisecond, None);
60/// // timestamp with nanosecond precision in UTC timezone
61/// let data_type = DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into()));
62///```
63///
64/// # Display and FromStr
65///
66/// The `Display` and `FromStr` implementations for `DataType` are
67/// human-readable, parseable, and reversible.
68///
69/// ```
70/// # use arrow_schema::DataType;
71/// let data_type = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
72/// let data_type_string = data_type.to_string();
73/// assert_eq!(data_type_string, "Dictionary(Int32, Utf8)");
74/// // display can be parsed back into the original type
75/// let parsed_data_type: DataType = data_type.to_string().parse().unwrap();
76/// assert_eq!(data_type, parsed_data_type);
77/// ```
78///
79/// # Nested Support
80/// Currently, the Rust implementation supports the following nested types:
81/// - `List<T>`
82/// - `LargeList<T>`
83/// - `FixedSizeList<T>`
84/// - `Struct<T, U, V, ...>`
85/// - `Union<T, U, V, ...>`
86/// - `Map<K, V>`
87///
88/// Nested types can themselves be nested within other arrays.
89/// For more information on these types please see
90/// [the physical memory layout of Apache Arrow]
91///
92/// [`Schema.fbs`]: https://github.com/apache/arrow/blob/main/format/Schema.fbs
93/// [the physical memory layout of Apache Arrow]: https://arrow.apache.org/docs/format/Columnar.html#physical-memory-layout
94#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
95#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
96pub enum DataType {
97 /// Null type
98 Null,
99 /// A boolean datatype representing the values `true` and `false`.
100 Boolean,
101 /// A signed 8-bit integer.
102 Int8,
103 /// A signed 16-bit integer.
104 Int16,
105 /// A signed 32-bit integer.
106 Int32,
107 /// A signed 64-bit integer.
108 Int64,
109 /// An unsigned 8-bit integer.
110 UInt8,
111 /// An unsigned 16-bit integer.
112 UInt16,
113 /// An unsigned 32-bit integer.
114 UInt32,
115 /// An unsigned 64-bit integer.
116 UInt64,
117 /// A 16-bit floating point number.
118 Float16,
119 /// A 32-bit floating point number.
120 Float32,
121 /// A 64-bit floating point number.
122 Float64,
123 /// A timestamp with an optional timezone.
124 ///
125 /// Time is measured as a Unix epoch, counting the seconds from
126 /// 00:00:00.000 on 1 January 1970, excluding leap seconds,
127 /// as a signed 64-bit integer.
128 ///
129 /// The time zone is a string indicating the name of a time zone, one of:
130 ///
131 /// * As used in the Olson time zone database (the "tz database" or
132 /// "tzdata"), such as "America/New_York"
133 /// * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30
134 ///
135 /// Timestamps with a non-empty timezone
136 /// ------------------------------------
137 ///
138 /// If a Timestamp column has a non-empty timezone value, its epoch is
139 /// 1970-01-01 00:00:00 (January 1st 1970, midnight) in the *UTC* timezone
140 /// (the Unix epoch), regardless of the Timestamp's own timezone.
141 ///
142 /// Therefore, timestamp values with a non-empty timezone correspond to
143 /// physical points in time together with some additional information about
144 /// how the data was obtained and/or how to display it (the timezone).
145 ///
146 /// For example, the timestamp value 0 with the timezone string "Europe/Paris"
147 /// corresponds to "January 1st 1970, 00h00" in the UTC timezone, but the
148 /// application may prefer to display it as "January 1st 1970, 01h00" in
149 /// the Europe/Paris timezone (which is the same physical point in time).
150 ///
151 /// One consequence is that timestamp values with a non-empty timezone
152 /// can be compared and ordered directly, since they all share the same
153 /// well-known point of reference (the Unix epoch).
154 ///
155 /// Timestamps with an unset / empty timezone
156 /// -----------------------------------------
157 ///
158 /// If a Timestamp column has no timezone value, its epoch is
159 /// 1970-01-01 00:00:00 (January 1st 1970, midnight) in an *unknown* timezone.
160 ///
161 /// Therefore, timestamp values without a timezone cannot be meaningfully
162 /// interpreted as physical points in time, but only as calendar / clock
163 /// indications ("wall clock time") in an unspecified timezone.
164 ///
165 /// For example, the timestamp value 0 with an empty timezone string
166 /// corresponds to "January 1st 1970, 00h00" in an unknown timezone: there
167 /// is not enough information to interpret it as a well-defined physical
168 /// point in time.
169 ///
170 /// One consequence is that timestamp values without a timezone cannot
171 /// be reliably compared or ordered, since they may have different points of
172 /// reference. In particular, it is *not* possible to interpret an unset
173 /// or empty timezone as the same as "UTC".
174 ///
175 /// Conversion between timezones
176 /// ----------------------------
177 ///
178 /// If a Timestamp column has a non-empty timezone, changing the timezone
179 /// to a different non-empty value is a metadata-only operation:
180 /// the timestamp values need not change as their point of reference remains
181 /// the same (the Unix epoch).
182 ///
183 /// However, if a Timestamp column has no timezone value, changing it to a
184 /// non-empty value requires to think about the desired semantics.
185 /// One possibility is to assume that the original timestamp values are
186 /// relative to the epoch of the timezone being set; timestamp values should
187 /// then adjusted to the Unix epoch (for example, changing the timezone from
188 /// empty to "Europe/Paris" would require converting the timestamp values
189 /// from "Europe/Paris" to "UTC", which seems counter-intuitive but is
190 /// nevertheless correct).
191 ///
192 /// ```
193 /// # use arrow_schema::{DataType, TimeUnit};
194 /// DataType::Timestamp(TimeUnit::Second, None);
195 /// DataType::Timestamp(TimeUnit::Second, Some("literal".into()));
196 /// DataType::Timestamp(TimeUnit::Second, Some("string".to_string().into()));
197 /// ```
198 ///
199 /// # Timezone representation
200 /// ----------------------------
201 /// It is possible to use either the timezone string representation, such as "UTC", or the absolute time zone offset "+00:00".
202 /// For timezones with fixed offsets, such as "UTC" or "JST", the offset representation is recommended, as it is more explicit and less ambiguous.
203 ///
204 /// Most arrow-rs functionalities use the absolute offset representation,
205 /// such as [`PrimitiveArray::with_timezone_utc`] that applies a
206 /// UTC timezone to timestamp arrays.
207 ///
208 /// [`PrimitiveArray::with_timezone_utc`]: https://docs.rs/arrow/latest/arrow/array/struct.PrimitiveArray.html#method.with_timezone_utc
209 ///
210 /// Timezone string parsing
211 /// -----------------------
212 /// When feature `chrono-tz` is not enabled, allowed timezone strings are fixed offsets of the form "+09:00", "-09" or "+0930".
213 ///
214 /// When feature `chrono-tz` is enabled, additional strings supported by [chrono_tz](https://docs.rs/chrono-tz/latest/chrono_tz/)
215 /// are also allowed, which include [IANA database](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones)
216 /// timezones.
217 Timestamp(TimeUnit, Option<Arc<str>>),
218 /// A signed 32-bit date representing the elapsed time since UNIX epoch (1970-01-01)
219 /// in days.
220 Date32,
221 /// A signed 64-bit date representing the elapsed time since UNIX epoch (1970-01-01)
222 /// in milliseconds.
223 ///
224 /// # Valid Ranges
225 ///
226 /// According to the Arrow specification ([Schema.fbs]), values of Date64
227 /// are treated as the number of *days*, in milliseconds, since the UNIX
228 /// epoch. Therefore, values of this type must be evenly divisible by
229 /// `86_400_000`, the number of milliseconds in a standard day.
230 ///
231 /// It is not valid to store milliseconds that do not represent an exact
232 /// day. The reason for this restriction is compatibility with other
233 /// language's native libraries (specifically Java), which historically
234 /// lacked a dedicated date type and only supported timestamps.
235 ///
236 /// # Validation
237 ///
238 /// This library does not validate or enforce that Date64 values are evenly
239 /// divisible by `86_400_000` for performance and usability reasons. Date64
240 /// values are treated similarly to `Timestamp(TimeUnit::Millisecond,
241 /// None)`: values will be displayed with a time of day if the value does
242 /// not represent an exact day, and arithmetic will be done at the
243 /// millisecond granularity.
244 ///
245 /// # Recommendation
246 ///
247 /// Users should prefer [`Date32`] to cleanly represent the number
248 /// of days, or one of the Timestamp variants to include time as part of the
249 /// representation, depending on their use case.
250 ///
251 /// # Further Reading
252 ///
253 /// For more details, see [#5288](https://github.com/apache/arrow-rs/issues/5288).
254 ///
255 /// [`Date32`]: Self::Date32
256 /// [Schema.fbs]: https://github.com/apache/arrow/blob/main/format/Schema.fbs
257 Date64,
258 /// A signed 32-bit time representing the elapsed time since midnight in the unit of `TimeUnit`.
259 /// Must be either seconds or milliseconds.
260 Time32(TimeUnit),
261 /// A signed 64-bit time representing the elapsed time since midnight in the unit of `TimeUnit`.
262 /// Must be either microseconds or nanoseconds.
263 Time64(TimeUnit),
264 /// Measure of elapsed time in either seconds, milliseconds, microseconds or nanoseconds.
265 Duration(TimeUnit),
266 /// A "calendar" interval which models types that don't necessarily
267 /// have a precise duration without the context of a base timestamp (e.g.
268 /// days can differ in length during day light savings time transitions).
269 Interval(IntervalUnit),
270 /// Opaque binary data of variable length.
271 ///
272 /// A single Binary array can store up to [`i32::MAX`] bytes
273 /// of binary data in total.
274 Binary,
275 /// Opaque binary data of fixed size.
276 ///
277 /// Enum parameter specifies the number of bytes per value, defined by the
278 /// [`byteWidth` field] in the Arrow Spec
279 ///
280 /// [`byteWidth` field]: https://github.com/apache/arrow/blob/2a89d03bbefd620b42126b8e00f8ae57e99cd638/format/Schema.fbs#L211
281 FixedSizeBinary(i32),
282 /// Opaque binary data of variable length and 64-bit offsets.
283 ///
284 /// A single LargeBinary array can store up to [`i64::MAX`] bytes
285 /// of binary data in total.
286 LargeBinary,
287 /// Opaque binary data of variable length.
288 ///
289 /// Logically the same as [`Binary`], but the internal representation uses a view
290 /// struct that contains the string length and either the string's entire data
291 /// inline (for small strings) or an inlined prefix, an index of another buffer,
292 /// and an offset pointing to a slice in that buffer (for non-small strings).
293 ///
294 /// [`Binary`]: Self::Binary
295 BinaryView,
296 /// A variable-length string in Unicode with UTF-8 encoding.
297 ///
298 /// A single Utf8 array can store up to [`i32::MAX`] bytes
299 /// of string data in total.
300 Utf8,
301 /// A variable-length string in Unicode with UFT-8 encoding and 64-bit offsets.
302 ///
303 /// A single LargeUtf8 array can store up to [`i64::MAX`] bytes
304 /// of string data in total.
305 LargeUtf8,
306 /// A variable-length string in Unicode with UTF-8 encoding
307 ///
308 /// Logically the same as [`Utf8`], but the internal representation uses a view
309 /// struct that contains the string length and either the string's entire data
310 /// inline (for small strings) or an inlined prefix, an index of another buffer,
311 /// and an offset pointing to a slice in that buffer (for non-small strings).
312 ///
313 /// [`Utf8`]: Self::Utf8
314 Utf8View,
315 /// A list of some logical data type with variable length.
316 ///
317 /// A single List array can store up to [`i32::MAX`] elements in total.
318 List(FieldRef),
319 /// A list of some logical data type with variable length.
320 ///
321 /// Logically the same as [`List`], but the internal representation differs in how child
322 /// data is referenced, allowing flexibility in how data is laid out.
323 ///
324 /// [`List`]: Self::List
325 ListView(FieldRef),
326 /// A list of some logical data type with fixed length.
327 FixedSizeList(FieldRef, i32),
328 /// A list of some logical data type with variable length and 64-bit offsets.
329 ///
330 /// A single LargeList array can store up to [`i64::MAX`] elements in total.
331 LargeList(FieldRef),
332 /// A list of some logical data type with variable length and 64-bit offsets.
333 ///
334 /// Logically the same as [`LargeList`], but the internal representation differs in how child
335 /// data is referenced, allowing flexibility in how data is laid out.
336 ///
337 /// [`LargeList`]: Self::LargeList
338 LargeListView(FieldRef),
339 /// A nested datatype that contains a number of sub-fields.
340 Struct(Fields),
341 /// A nested datatype that can represent slots of differing types. Components:
342 ///
343 /// 1. [`UnionFields`]
344 /// 2. The type of union (Sparse or Dense)
345 Union(UnionFields, UnionMode),
346 /// A dictionary encoded array (`key_type`, `value_type`), where
347 /// each array element is an index of `key_type` into an
348 /// associated dictionary of `value_type`.
349 ///
350 /// Dictionary arrays are used to store columns of `value_type`
351 /// that contain many repeated values using less memory, but with
352 /// a higher CPU overhead for some operations.
353 ///
354 /// This type mostly used to represent low cardinality string
355 /// arrays or a limited set of primitive types as integers.
356 Dictionary(Box<DataType>, Box<DataType>),
357 /// Exact 32-bit width decimal value with precision and scale
358 ///
359 /// * precision is the maximum number of digits in the unscaled value
360 /// * scale controls the position of the decimal point
361 ///
362 /// The represented value is the unscaled integer multiplied by 10^{-scale}.
363 /// For example, the unscaled value 12345 with precision 5 and scale 2
364 /// represents 123.45.
365 ///
366 /// Scale can also be negative. For example, the unscaled value 12 with
367 /// precision 2 and scale -3 represents 12000.
368 Decimal32(u8, i8),
369 /// Exact 64-bit width decimal value with precision and scale
370 ///
371 /// * precision is the maximum number of digits in the unscaled value
372 /// * scale controls the position of the decimal point
373 ///
374 /// The represented value is the unscaled integer multiplied by 10^{-scale}.
375 /// For example, the unscaled value 12345 with precision 5 and scale 2
376 /// represents 123.45.
377 ///
378 /// Scale can also be negative. For example, the unscaled value 12 with
379 /// precision 2 and scale -3 represents 12000.
380 Decimal64(u8, i8),
381 /// Exact 128-bit width decimal value with precision and scale
382 ///
383 /// * precision is the maximum number of digits in the unscaled value
384 /// * scale controls the position of the decimal point
385 ///
386 /// The represented value is the unscaled integer multiplied by 10^{-scale}.
387 /// For example, the unscaled value 12345 with precision 5 and scale 2
388 /// represents 123.45.
389 ///
390 /// Scale can also be negative. For example, the unscaled value 12 with
391 /// precision 2 and scale -3 represents 12000.
392 Decimal128(u8, i8),
393 /// Exact 256-bit width decimal value with precision and scale
394 ///
395 /// * precision is the maximum number of digits in the unscaled value
396 /// * scale controls the position of the decimal point
397 ///
398 /// The represented value is the unscaled integer multiplied by 10^{-scale}.
399 /// For example, the unscaled value 12345 with precision 5 and scale 2
400 /// represents 123.45.
401 ///
402 /// Scale can also be negative. For example, the unscaled value 12 with
403 /// precision 2 and scale -3 represents 12000.
404 Decimal256(u8, i8),
405 /// A Map is a logical nested type that is represented as
406 ///
407 /// `List<entries: Struct<key: K, value: V>>`
408 ///
409 /// The keys and values are each respectively contiguous.
410 /// The key and value types are not constrained, but keys should be
411 /// hashable and unique.
412 /// Whether the keys are sorted can be set in the `bool` after the `Field`.
413 ///
414 /// In a field with Map type, the field has a child Struct field, which then
415 /// has two children: key type and the second the value type. The names of the
416 /// child fields may be respectively "entries", "key", and "value", but this is
417 /// not enforced.
418 ///
419 /// # Requirements
420 /// - The entries [`Field`] (first argument) must be non-nullable.
421 /// - The entries field must be a [`DataType::Struct`] with exactly 2 children.
422 /// - The first child (key) must be non-nullable.
423 Map(FieldRef, bool),
424 /// A run-end encoding (REE) is a variation of run-length encoding (RLE). These
425 /// encodings are well-suited for representing data containing sequences of the
426 /// same value, called runs. Each run is represented as a value and an integer giving
427 /// the index in the array where the run ends.
428 ///
429 /// A run-end encoded array has no buffers by itself, but has two child arrays. The
430 /// first child array, called the run ends array, holds either 16, 32, or 64-bit
431 /// signed integers. The actual values of each run are held in the second child array.
432 ///
433 /// These child arrays are prescribed the standard names of "run_ends" and "values"
434 /// respectively.
435 ///
436 /// # Requirements
437 /// - The run_ends [`Field`] (first argument) must be non-nullable and of type
438 /// [`DataType::Int16`], [`DataType::Int32`], or [`DataType::Int64`].
439 RunEndEncoded(FieldRef, FieldRef),
440}
441
442/// An absolute length of time in seconds, milliseconds, microseconds or nanoseconds.
443#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
444#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
445pub enum TimeUnit {
446 /// Time in seconds.
447 Second,
448 /// Time in milliseconds.
449 Millisecond,
450 /// Time in microseconds.
451 Microsecond,
452 /// Time in nanoseconds.
453 Nanosecond,
454}
455
456impl std::fmt::Display for TimeUnit {
457 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
458 match self {
459 TimeUnit::Second => write!(f, "s"),
460 TimeUnit::Millisecond => write!(f, "ms"),
461 TimeUnit::Microsecond => write!(f, "µs"),
462 TimeUnit::Nanosecond => write!(f, "ns"),
463 }
464 }
465}
466
467/// YEAR_MONTH, DAY_TIME, MONTH_DAY_NANO interval in SQL style.
468#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
469#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
470pub enum IntervalUnit {
471 /// Indicates the number of elapsed whole months, stored as 4-byte integers.
472 YearMonth,
473 /// Indicates the number of elapsed days and milliseconds,
474 /// stored as 2 contiguous 32-bit integers (days, milliseconds) (8-bytes in total).
475 DayTime,
476 /// A triple of the number of elapsed months, days, and nanoseconds.
477 /// The values are stored contiguously in 16 byte blocks. Months and
478 /// days are encoded as 32 bit integers and nanoseconds is encoded as a
479 /// 64 bit integer. All integers are signed. Each field is independent
480 /// (e.g. there is no constraint that nanoseconds have the same sign
481 /// as days or that the quantity of nanoseconds represents less
482 /// than a day's worth of time).
483 MonthDayNano,
484}
485
486/// Sparse or Dense union layouts
487#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Copy)]
488#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
489pub enum UnionMode {
490 /// Sparse union layout
491 Sparse,
492 /// Dense union layout
493 Dense,
494}
495
496/// Parses `str` into a `DataType`.
497///
498/// This is the reverse of [`DataType`]'s `Display`
499/// impl, and maintains the invariant that
500/// `DataType::try_from(&data_type.to_string()).unwrap() == data_type`
501///
502/// # Example
503/// ```
504/// use arrow_schema::DataType;
505///
506/// let data_type: DataType = "Int32".parse().unwrap();
507/// assert_eq!(data_type, DataType::Int32);
508/// ```
509impl FromStr for DataType {
510 type Err = ArrowError;
511
512 fn from_str(s: &str) -> Result<Self, Self::Err> {
513 crate::datatype_parse::parse_data_type(s)
514 }
515}
516
517impl TryFrom<&str> for DataType {
518 type Error = ArrowError;
519
520 fn try_from(value: &str) -> Result<Self, Self::Error> {
521 value.parse()
522 }
523}
524
525impl DataType {
526 /// Returns true if the type is primitive: (numeric, temporal).
527 #[inline]
528 pub fn is_primitive(&self) -> bool {
529 self.is_numeric() || self.is_temporal()
530 }
531
532 /// Returns true if this type is numeric: (UInt*, Int*, Float*, Decimal*).
533 #[inline]
534 pub fn is_numeric(&self) -> bool {
535 use DataType::*;
536 matches!(
537 self,
538 UInt8
539 | UInt16
540 | UInt32
541 | UInt64
542 | Int8
543 | Int16
544 | Int32
545 | Int64
546 | Float16
547 | Float32
548 | Float64
549 | Decimal32(_, _)
550 | Decimal64(_, _)
551 | Decimal128(_, _)
552 | Decimal256(_, _)
553 )
554 }
555
556 /// Returns true if this type is temporal: (Date*, Time*, Duration, or Interval).
557 #[inline]
558 pub fn is_temporal(&self) -> bool {
559 use DataType::*;
560 matches!(
561 self,
562 Date32 | Date64 | Timestamp(_, _) | Time32(_) | Time64(_) | Duration(_) | Interval(_)
563 )
564 }
565
566 /// Returns true if this type is floating: (Float*).
567 #[inline]
568 pub fn is_floating(&self) -> bool {
569 use DataType::*;
570 matches!(self, Float16 | Float32 | Float64)
571 }
572
573 /// Returns true if this type is integer: (Int*, UInt*).
574 #[inline]
575 pub fn is_integer(&self) -> bool {
576 self.is_signed_integer() || self.is_unsigned_integer()
577 }
578
579 /// Returns true if this type is signed integer: (Int*).
580 #[inline]
581 pub fn is_signed_integer(&self) -> bool {
582 use DataType::*;
583 matches!(self, Int8 | Int16 | Int32 | Int64)
584 }
585
586 /// Returns true if this type is unsigned integer: (UInt*).
587 #[inline]
588 pub fn is_unsigned_integer(&self) -> bool {
589 use DataType::*;
590 matches!(self, UInt8 | UInt16 | UInt32 | UInt64)
591 }
592
593 /// Returns true if this type is decimal: (Decimal*).
594 #[inline]
595 pub fn is_decimal(&self) -> bool {
596 use DataType::*;
597 matches!(
598 self,
599 Decimal32(..) | Decimal64(..) | Decimal128(..) | Decimal256(..)
600 )
601 }
602
603 /// Returns true if this type is valid as a dictionary key
604 #[inline]
605 pub fn is_dictionary_key_type(&self) -> bool {
606 self.is_integer()
607 }
608
609 /// Returns true if this type is valid for run-ends array in RunArray
610 #[inline]
611 pub fn is_run_ends_type(&self) -> bool {
612 use DataType::*;
613 matches!(self, Int16 | Int32 | Int64)
614 }
615
616 /// Returns true if this type is nested (List, FixedSizeList, LargeList, ListView. LargeListView, Struct, Union,
617 /// or Map), or a dictionary of a nested type
618 #[inline]
619 pub fn is_nested(&self) -> bool {
620 use DataType::*;
621 match self {
622 Dictionary(_, v) => DataType::is_nested(v.as_ref()),
623 RunEndEncoded(_, v) => DataType::is_nested(v.data_type()),
624 List(_)
625 | FixedSizeList(_, _)
626 | LargeList(_)
627 | ListView(_)
628 | LargeListView(_)
629 | Struct(_)
630 | Union(_, _)
631 | Map(_, _) => true,
632 _ => false,
633 }
634 }
635
636 /// Returns true if this type is DataType::Null.
637 #[inline]
638 pub fn is_null(&self) -> bool {
639 use DataType::*;
640 matches!(self, Null)
641 }
642
643 /// Returns true if this type is a String type
644 #[inline]
645 pub fn is_string(&self) -> bool {
646 use DataType::*;
647 matches!(self, Utf8 | LargeUtf8 | Utf8View)
648 }
649
650 /// Returns true if this type is a List type.
651 ///
652 /// List types include List, LargeList, FixedSizeList, ListView, and LargeListView.
653 #[inline]
654 pub fn is_list(&self) -> bool {
655 use DataType::*;
656 matches!(
657 self,
658 List(_) | LargeList(_) | FixedSizeList(_, _) | ListView(_) | LargeListView(_)
659 )
660 }
661
662 /// Returns true if this type is a Binary type.
663 ///
664 /// Binary types include Binary, LargeBinary, FixedSizeBinary and BinaryView.
665 #[inline]
666 pub fn is_binary(&self) -> bool {
667 use DataType::*;
668 matches!(self, Binary | LargeBinary | FixedSizeBinary(_) | BinaryView)
669 }
670
671 /// Compares the datatype with another, ignoring nested field names
672 /// and metadata.
673 pub fn equals_datatype(&self, other: &DataType) -> bool {
674 match (&self, other) {
675 (DataType::List(a), DataType::List(b))
676 | (DataType::LargeList(a), DataType::LargeList(b))
677 | (DataType::ListView(a), DataType::ListView(b))
678 | (DataType::LargeListView(a), DataType::LargeListView(b)) => {
679 a.is_nullable() == b.is_nullable() && a.data_type().equals_datatype(b.data_type())
680 }
681 (DataType::FixedSizeList(a, a_size), DataType::FixedSizeList(b, b_size)) => {
682 a_size == b_size
683 && a.is_nullable() == b.is_nullable()
684 && a.data_type().equals_datatype(b.data_type())
685 }
686 (DataType::Struct(a), DataType::Struct(b)) => {
687 a.len() == b.len()
688 && a.iter().zip(b).all(|(a, b)| {
689 a.is_nullable() == b.is_nullable()
690 && a.data_type().equals_datatype(b.data_type())
691 })
692 }
693 (DataType::Map(a_field, a_is_sorted), DataType::Map(b_field, b_is_sorted)) => {
694 a_field.is_nullable() == b_field.is_nullable()
695 && a_field.data_type().equals_datatype(b_field.data_type())
696 && a_is_sorted == b_is_sorted
697 }
698 (DataType::Dictionary(a_key, a_value), DataType::Dictionary(b_key, b_value)) => {
699 a_key.equals_datatype(b_key) && a_value.equals_datatype(b_value)
700 }
701 (
702 DataType::RunEndEncoded(a_run_ends, a_values),
703 DataType::RunEndEncoded(b_run_ends, b_values),
704 ) => {
705 a_run_ends.is_nullable() == b_run_ends.is_nullable()
706 && a_run_ends
707 .data_type()
708 .equals_datatype(b_run_ends.data_type())
709 && a_values.is_nullable() == b_values.is_nullable()
710 && a_values.data_type().equals_datatype(b_values.data_type())
711 }
712 (
713 DataType::Union(a_union_fields, a_union_mode),
714 DataType::Union(b_union_fields, b_union_mode),
715 ) => {
716 a_union_mode == b_union_mode
717 && a_union_fields.len() == b_union_fields.len()
718 && a_union_fields.iter().all(|a| {
719 b_union_fields.iter().any(|b| {
720 a.0 == b.0
721 && a.1.is_nullable() == b.1.is_nullable()
722 && a.1.data_type().equals_datatype(b.1.data_type())
723 })
724 })
725 }
726 _ => self == other,
727 }
728 }
729
730 /// Returns the byte width of this type if it is a primitive type
731 ///
732 /// Returns `None` if not a primitive type
733 #[inline]
734 pub fn primitive_width(&self) -> Option<usize> {
735 match self {
736 DataType::Null => None,
737 DataType::Boolean => None,
738 DataType::Int8 | DataType::UInt8 => Some(1),
739 DataType::Int16 | DataType::UInt16 | DataType::Float16 => Some(2),
740 DataType::Int32 | DataType::UInt32 | DataType::Float32 => Some(4),
741 DataType::Int64 | DataType::UInt64 | DataType::Float64 => Some(8),
742 DataType::Timestamp(_, _) => Some(8),
743 DataType::Date32 | DataType::Time32(_) => Some(4),
744 DataType::Date64 | DataType::Time64(_) => Some(8),
745 DataType::Duration(_) => Some(8),
746 DataType::Interval(IntervalUnit::YearMonth) => Some(4),
747 DataType::Interval(IntervalUnit::DayTime) => Some(8),
748 DataType::Interval(IntervalUnit::MonthDayNano) => Some(16),
749 DataType::Decimal32(_, _) => Some(4),
750 DataType::Decimal64(_, _) => Some(8),
751 DataType::Decimal128(_, _) => Some(16),
752 DataType::Decimal256(_, _) => Some(32),
753 DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => None,
754 DataType::Binary | DataType::LargeBinary | DataType::BinaryView => None,
755 DataType::FixedSizeBinary(_) => None,
756 DataType::List(_)
757 | DataType::ListView(_)
758 | DataType::LargeList(_)
759 | DataType::LargeListView(_)
760 | DataType::Map(_, _) => None,
761 DataType::FixedSizeList(_, _) => None,
762 DataType::Struct(_) => None,
763 DataType::Union(_, _) => None,
764 DataType::Dictionary(_, _) => None,
765 DataType::RunEndEncoded(_, _) => None,
766 }
767 }
768
769 /// Return size of this instance in bytes.
770 ///
771 /// Includes the size of `Self`.
772 pub fn size(&self) -> usize {
773 std::mem::size_of_val(self)
774 + match self {
775 DataType::Null
776 | DataType::Boolean
777 | DataType::Int8
778 | DataType::Int16
779 | DataType::Int32
780 | DataType::Int64
781 | DataType::UInt8
782 | DataType::UInt16
783 | DataType::UInt32
784 | DataType::UInt64
785 | DataType::Float16
786 | DataType::Float32
787 | DataType::Float64
788 | DataType::Date32
789 | DataType::Date64
790 | DataType::Time32(_)
791 | DataType::Time64(_)
792 | DataType::Duration(_)
793 | DataType::Interval(_)
794 | DataType::Binary
795 | DataType::FixedSizeBinary(_)
796 | DataType::LargeBinary
797 | DataType::BinaryView
798 | DataType::Utf8
799 | DataType::LargeUtf8
800 | DataType::Utf8View
801 | DataType::Decimal32(_, _)
802 | DataType::Decimal64(_, _)
803 | DataType::Decimal128(_, _)
804 | DataType::Decimal256(_, _) => 0,
805 DataType::Timestamp(_, s) => s.as_ref().map(|s| s.len()).unwrap_or_default(),
806 DataType::List(field)
807 | DataType::ListView(field)
808 | DataType::FixedSizeList(field, _)
809 | DataType::LargeList(field)
810 | DataType::LargeListView(field)
811 | DataType::Map(field, _) => field.size(),
812 DataType::Struct(fields) => fields.size(),
813 DataType::Union(fields, _) => fields.size(),
814 DataType::Dictionary(dt1, dt2) => dt1.size() + dt2.size(),
815 DataType::RunEndEncoded(run_ends, values) => {
816 run_ends.size() - std::mem::size_of_val(run_ends) + values.size()
817 - std::mem::size_of_val(values)
818 }
819 }
820 }
821
822 /// Check to see if `self` is a superset of `other`
823 ///
824 /// If DataType is a nested type, then it will check to see if the nested type is a superset of the other nested type
825 /// else it will check to see if the DataType is equal to the other DataType
826 pub fn contains(&self, other: &DataType) -> bool {
827 match (self, other) {
828 (DataType::List(f1), DataType::List(f2))
829 | (DataType::LargeList(f1), DataType::LargeList(f2))
830 | (DataType::ListView(f1), DataType::ListView(f2))
831 | (DataType::LargeListView(f1), DataType::LargeListView(f2)) => f1.contains(f2),
832 (DataType::FixedSizeList(f1, s1), DataType::FixedSizeList(f2, s2)) => {
833 s1 == s2 && f1.contains(f2)
834 }
835 (DataType::Map(f1, s1), DataType::Map(f2, s2)) => s1 == s2 && f1.contains(f2),
836 (DataType::Struct(f1), DataType::Struct(f2)) => f1.contains(f2),
837 (DataType::Union(f1, s1), DataType::Union(f2, s2)) => {
838 s1 == s2
839 && f1
840 .iter()
841 .all(|f1| f2.iter().any(|f2| f1.0 == f2.0 && f1.1.contains(f2.1)))
842 }
843 (DataType::Dictionary(k1, v1), DataType::Dictionary(k2, v2)) => {
844 k1.contains(k2) && v1.contains(v2)
845 }
846 _ => self == other,
847 }
848 }
849
850 /// Create a [`DataType::List`] with elements of the specified type
851 /// and nullability, and conventionally named inner [`Field`] (`"item"`).
852 ///
853 /// To specify field level metadata, construct the inner [`Field`]
854 /// directly via [`Field::new`] or [`Field::new_list_field`].
855 pub fn new_list(data_type: DataType, nullable: bool) -> Self {
856 DataType::List(Arc::new(Field::new_list_field(data_type, nullable)))
857 }
858
859 /// Create a [`DataType::LargeList`] with elements of the specified type
860 /// and nullability, and conventionally named inner [`Field`] (`"item"`).
861 ///
862 /// To specify field level metadata, construct the inner [`Field`]
863 /// directly via [`Field::new`] or [`Field::new_list_field`].
864 pub fn new_large_list(data_type: DataType, nullable: bool) -> Self {
865 DataType::LargeList(Arc::new(Field::new_list_field(data_type, nullable)))
866 }
867
868 /// Create a [`DataType::FixedSizeList`] with elements of the specified type, size
869 /// and nullability, and conventionally named inner [`Field`] (`"item"`).
870 ///
871 /// To specify field level metadata, construct the inner [`Field`]
872 /// directly via [`Field::new`] or [`Field::new_list_field`].
873 pub fn new_fixed_size_list(data_type: DataType, size: i32, nullable: bool) -> Self {
874 DataType::FixedSizeList(Arc::new(Field::new_list_field(data_type, nullable)), size)
875 }
876}
877
878/// The maximum precision for [DataType::Decimal32] values
879pub const DECIMAL32_MAX_PRECISION: u8 = 9;
880
881/// The maximum scale for [DataType::Decimal32] values
882pub const DECIMAL32_MAX_SCALE: i8 = 9;
883
884/// The maximum precision for [DataType::Decimal64] values
885pub const DECIMAL64_MAX_PRECISION: u8 = 18;
886
887/// The maximum scale for [DataType::Decimal64] values
888pub const DECIMAL64_MAX_SCALE: i8 = 18;
889
890/// The maximum precision for [DataType::Decimal128] values
891pub const DECIMAL128_MAX_PRECISION: u8 = 38;
892
893/// The maximum scale for [DataType::Decimal128] values
894pub const DECIMAL128_MAX_SCALE: i8 = 38;
895
896/// The maximum precision for [DataType::Decimal256] values
897pub const DECIMAL256_MAX_PRECISION: u8 = 76;
898
899/// The maximum scale for [DataType::Decimal256] values
900pub const DECIMAL256_MAX_SCALE: i8 = 76;
901
902/// The default scale for [DataType::Decimal32] values
903pub const DECIMAL32_DEFAULT_SCALE: i8 = 2;
904
905/// The default scale for [DataType::Decimal64] values
906pub const DECIMAL64_DEFAULT_SCALE: i8 = 6;
907
908/// The default scale for [DataType::Decimal128] and [DataType::Decimal256]
909/// values
910pub const DECIMAL_DEFAULT_SCALE: i8 = 10;
911
912#[cfg(test)]
913mod tests {
914 use super::*;
915
916 #[test]
917 #[cfg(feature = "serde")]
918 fn serde_struct_type() {
919 use std::collections::HashMap;
920
921 let kv_array = [("k".to_string(), "v".to_string())];
922 let field_metadata: HashMap<String, String> = kv_array.iter().cloned().collect();
923
924 // Non-empty map: should be converted as JSON obj { ... }
925 let first_name =
926 Field::new("first_name", DataType::Utf8, false).with_metadata(field_metadata);
927
928 // Empty map: should be omitted.
929 let last_name =
930 Field::new("last_name", DataType::Utf8, false).with_metadata(HashMap::default());
931
932 let person = DataType::Struct(Fields::from(vec![
933 first_name,
934 last_name,
935 Field::new(
936 "address",
937 DataType::Struct(Fields::from(vec![
938 Field::new("street", DataType::Utf8, false),
939 Field::new("zip", DataType::UInt16, false),
940 ])),
941 false,
942 ),
943 ]));
944
945 let serialized = serde_json::to_string(&person).unwrap();
946
947 // NOTE that this is testing the default (derived) serialization format, not the
948 // JSON format specified in metadata.md
949
950 assert_eq!(
951 "{\"Struct\":[\
952 {\"name\":\"first_name\",\"data_type\":\"Utf8\",\"nullable\":false,\"dict_id\":0,\"dict_is_ordered\":false,\"metadata\":{\"k\":\"v\"}},\
953 {\"name\":\"last_name\",\"data_type\":\"Utf8\",\"nullable\":false,\"dict_id\":0,\"dict_is_ordered\":false,\"metadata\":{}},\
954 {\"name\":\"address\",\"data_type\":{\"Struct\":\
955 [{\"name\":\"street\",\"data_type\":\"Utf8\",\"nullable\":false,\"dict_id\":0,\"dict_is_ordered\":false,\"metadata\":{}},\
956 {\"name\":\"zip\",\"data_type\":\"UInt16\",\"nullable\":false,\"dict_id\":0,\"dict_is_ordered\":false,\"metadata\":{}}\
957 ]},\"nullable\":false,\"dict_id\":0,\"dict_is_ordered\":false,\"metadata\":{}}]}",
958 serialized
959 );
960
961 let deserialized = serde_json::from_str(&serialized).unwrap();
962
963 assert_eq!(person, deserialized);
964 }
965
966 #[test]
967 fn test_list_datatype_equality() {
968 // tests that list type equality is checked while ignoring list names
969 let list_a = DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true)));
970 let list_b = DataType::List(Arc::new(Field::new("array", DataType::Int32, true)));
971 let list_c = DataType::List(Arc::new(Field::new_list_field(DataType::Int32, false)));
972 let list_d = DataType::List(Arc::new(Field::new_list_field(DataType::UInt32, true)));
973 assert!(list_a.equals_datatype(&list_b));
974 assert!(!list_a.equals_datatype(&list_c));
975 assert!(!list_b.equals_datatype(&list_c));
976 assert!(!list_a.equals_datatype(&list_d));
977
978 let list_e =
979 DataType::FixedSizeList(Arc::new(Field::new_list_field(list_a.clone(), false)), 3);
980 let list_f =
981 DataType::FixedSizeList(Arc::new(Field::new("array", list_b.clone(), false)), 3);
982 let list_g = DataType::FixedSizeList(
983 Arc::new(Field::new_list_field(DataType::FixedSizeBinary(3), true)),
984 3,
985 );
986 assert!(list_e.equals_datatype(&list_f));
987 assert!(!list_e.equals_datatype(&list_g));
988 assert!(!list_f.equals_datatype(&list_g));
989
990 let list_h = DataType::Struct(Fields::from(vec![Field::new("f1", list_e, true)]));
991 let list_i = DataType::Struct(Fields::from(vec![Field::new("f1", list_f.clone(), true)]));
992 let list_j = DataType::Struct(Fields::from(vec![Field::new("f1", list_f.clone(), false)]));
993 let list_k = DataType::Struct(Fields::from(vec![
994 Field::new("f1", list_f.clone(), false),
995 Field::new("f2", list_g.clone(), false),
996 Field::new("f3", DataType::Utf8, true),
997 ]));
998 let list_l = DataType::Struct(Fields::from(vec![
999 Field::new("ff1", list_f.clone(), false),
1000 Field::new("ff2", list_g.clone(), false),
1001 Field::new("ff3", DataType::LargeUtf8, true),
1002 ]));
1003 let list_m = DataType::Struct(Fields::from(vec![
1004 Field::new("ff1", list_f, false),
1005 Field::new("ff2", list_g, false),
1006 Field::new("ff3", DataType::Utf8, true),
1007 ]));
1008 assert!(list_h.equals_datatype(&list_i));
1009 assert!(!list_h.equals_datatype(&list_j));
1010 assert!(!list_k.equals_datatype(&list_l));
1011 assert!(list_k.equals_datatype(&list_m));
1012
1013 let list_n = DataType::Map(Arc::new(Field::new("f1", list_a.clone(), true)), true);
1014 let list_o = DataType::Map(Arc::new(Field::new("f2", list_b.clone(), true)), true);
1015 let list_p = DataType::Map(Arc::new(Field::new("f2", list_b.clone(), true)), false);
1016 let list_q = DataType::Map(Arc::new(Field::new("f2", list_c.clone(), true)), true);
1017 let list_r = DataType::Map(Arc::new(Field::new("f1", list_a.clone(), false)), true);
1018
1019 assert!(list_n.equals_datatype(&list_o));
1020 assert!(!list_n.equals_datatype(&list_p));
1021 assert!(!list_n.equals_datatype(&list_q));
1022 assert!(!list_n.equals_datatype(&list_r));
1023
1024 let list_s = DataType::Dictionary(Box::new(DataType::UInt8), Box::new(list_a));
1025 let list_t = DataType::Dictionary(Box::new(DataType::UInt8), Box::new(list_b.clone()));
1026 let list_u = DataType::Dictionary(Box::new(DataType::Int8), Box::new(list_b));
1027 let list_v = DataType::Dictionary(Box::new(DataType::UInt8), Box::new(list_c));
1028
1029 assert!(list_s.equals_datatype(&list_t));
1030 assert!(!list_s.equals_datatype(&list_u));
1031 assert!(!list_s.equals_datatype(&list_v));
1032
1033 let union_a = DataType::Union(
1034 UnionFields::try_new(
1035 vec![1, 2],
1036 vec![
1037 Field::new("f1", DataType::Utf8, false),
1038 Field::new("f2", DataType::UInt8, false),
1039 ],
1040 )
1041 .unwrap(),
1042 UnionMode::Sparse,
1043 );
1044 let union_b = DataType::Union(
1045 UnionFields::try_new(
1046 vec![1, 2],
1047 vec![
1048 Field::new("ff1", DataType::Utf8, false),
1049 Field::new("ff2", DataType::UInt8, false),
1050 ],
1051 )
1052 .unwrap(),
1053 UnionMode::Sparse,
1054 );
1055 let union_c = DataType::Union(
1056 UnionFields::try_new(
1057 vec![2, 1],
1058 vec![
1059 Field::new("fff2", DataType::UInt8, false),
1060 Field::new("fff1", DataType::Utf8, false),
1061 ],
1062 )
1063 .unwrap(),
1064 UnionMode::Sparse,
1065 );
1066 let union_d = DataType::Union(
1067 UnionFields::try_new(
1068 vec![2, 1],
1069 vec![
1070 Field::new("fff1", DataType::Int8, false),
1071 Field::new("fff2", DataType::UInt8, false),
1072 ],
1073 )
1074 .unwrap(),
1075 UnionMode::Sparse,
1076 );
1077 let union_e = DataType::Union(
1078 UnionFields::try_new(
1079 vec![1, 2],
1080 vec![
1081 Field::new("f1", DataType::Utf8, true),
1082 Field::new("f2", DataType::UInt8, false),
1083 ],
1084 )
1085 .unwrap(),
1086 UnionMode::Sparse,
1087 );
1088
1089 assert!(union_a.equals_datatype(&union_b));
1090 assert!(union_a.equals_datatype(&union_c));
1091 assert!(!union_a.equals_datatype(&union_d));
1092 assert!(!union_a.equals_datatype(&union_e));
1093
1094 let list_w = DataType::RunEndEncoded(
1095 Arc::new(Field::new("f1", DataType::Int64, true)),
1096 Arc::new(Field::new("f2", DataType::Utf8, true)),
1097 );
1098 let list_x = DataType::RunEndEncoded(
1099 Arc::new(Field::new("ff1", DataType::Int64, true)),
1100 Arc::new(Field::new("ff2", DataType::Utf8, true)),
1101 );
1102 let list_y = DataType::RunEndEncoded(
1103 Arc::new(Field::new("ff1", DataType::UInt16, true)),
1104 Arc::new(Field::new("ff2", DataType::Utf8, true)),
1105 );
1106 let list_z = DataType::RunEndEncoded(
1107 Arc::new(Field::new("f1", DataType::Int64, false)),
1108 Arc::new(Field::new("f2", DataType::Utf8, true)),
1109 );
1110
1111 assert!(list_w.equals_datatype(&list_x));
1112 assert!(!list_w.equals_datatype(&list_y));
1113 assert!(!list_w.equals_datatype(&list_z));
1114 }
1115
1116 #[test]
1117 fn create_struct_type() {
1118 let _person = DataType::Struct(Fields::from(vec![
1119 Field::new("first_name", DataType::Utf8, false),
1120 Field::new("last_name", DataType::Utf8, false),
1121 Field::new(
1122 "address",
1123 DataType::Struct(Fields::from(vec![
1124 Field::new("street", DataType::Utf8, false),
1125 Field::new("zip", DataType::UInt16, false),
1126 ])),
1127 false,
1128 ),
1129 ]));
1130 }
1131
1132 #[test]
1133 fn test_nested() {
1134 let list = DataType::List(Arc::new(Field::new("foo", DataType::Utf8, true)));
1135 let list_view = DataType::ListView(Arc::new(Field::new("foo", DataType::Utf8, true)));
1136 let large_list_view =
1137 DataType::LargeListView(Arc::new(Field::new("foo", DataType::Utf8, true)));
1138
1139 assert!(!DataType::is_nested(&DataType::Boolean));
1140 assert!(!DataType::is_nested(&DataType::Int32));
1141 assert!(!DataType::is_nested(&DataType::Utf8));
1142 assert!(DataType::is_nested(&list));
1143 assert!(DataType::is_nested(&list_view));
1144 assert!(DataType::is_nested(&large_list_view));
1145
1146 assert!(!DataType::is_nested(&DataType::Dictionary(
1147 Box::new(DataType::Int32),
1148 Box::new(DataType::Boolean)
1149 )));
1150 assert!(!DataType::is_nested(&DataType::Dictionary(
1151 Box::new(DataType::Int32),
1152 Box::new(DataType::Int64)
1153 )));
1154 assert!(!DataType::is_nested(&DataType::Dictionary(
1155 Box::new(DataType::Int32),
1156 Box::new(DataType::LargeUtf8)
1157 )));
1158 assert!(DataType::is_nested(&DataType::Dictionary(
1159 Box::new(DataType::Int32),
1160 Box::new(list)
1161 )));
1162 }
1163
1164 #[test]
1165 fn test_integer() {
1166 // is_integer
1167 assert!(DataType::is_integer(&DataType::Int32));
1168 assert!(DataType::is_integer(&DataType::UInt64));
1169 assert!(!DataType::is_integer(&DataType::Float16));
1170
1171 // is_signed_integer
1172 assert!(DataType::is_signed_integer(&DataType::Int32));
1173 assert!(!DataType::is_signed_integer(&DataType::UInt64));
1174 assert!(!DataType::is_signed_integer(&DataType::Float16));
1175
1176 // is_unsigned_integer
1177 assert!(!DataType::is_unsigned_integer(&DataType::Int32));
1178 assert!(DataType::is_unsigned_integer(&DataType::UInt64));
1179 assert!(!DataType::is_unsigned_integer(&DataType::Float16));
1180
1181 // is_dictionary_key_type
1182 assert!(DataType::is_dictionary_key_type(&DataType::Int32));
1183 assert!(DataType::is_dictionary_key_type(&DataType::UInt64));
1184 assert!(!DataType::is_dictionary_key_type(&DataType::Float16));
1185 }
1186
1187 #[test]
1188 fn test_string() {
1189 assert!(DataType::is_string(&DataType::Utf8));
1190 assert!(DataType::is_string(&DataType::LargeUtf8));
1191 assert!(DataType::is_string(&DataType::Utf8View));
1192 assert!(!DataType::is_string(&DataType::Int32));
1193 }
1194
1195 #[test]
1196 fn test_floating() {
1197 assert!(DataType::is_floating(&DataType::Float16));
1198 assert!(!DataType::is_floating(&DataType::Int32));
1199 }
1200
1201 #[test]
1202 fn test_decimal() {
1203 assert!(DataType::is_decimal(&DataType::Decimal32(4, 2)));
1204 assert!(DataType::is_decimal(&DataType::Decimal64(4, 2)));
1205 assert!(DataType::is_decimal(&DataType::Decimal128(4, 2)));
1206 assert!(DataType::is_decimal(&DataType::Decimal256(4, 2)));
1207 assert!(!DataType::is_decimal(&DataType::Float16));
1208 }
1209
1210 #[test]
1211 fn test_datatype_is_null() {
1212 assert!(DataType::is_null(&DataType::Null));
1213 assert!(!DataType::is_null(&DataType::Int32));
1214 }
1215
1216 #[test]
1217 fn test_is_list() {
1218 assert!(DataType::is_list(&DataType::new_list(
1219 DataType::Int16,
1220 true
1221 )));
1222 assert!(DataType::is_list(&DataType::new_large_list(
1223 DataType::Int16,
1224 true
1225 )));
1226 assert!(DataType::is_list(&DataType::new_fixed_size_list(
1227 DataType::Int16,
1228 5,
1229 true
1230 )));
1231 assert!(DataType::is_list(&DataType::ListView(Arc::new(
1232 Field::new("f", DataType::Int16, true)
1233 ))));
1234 assert!(DataType::is_list(&DataType::LargeListView(Arc::new(
1235 Field::new("f", DataType::Int16, true)
1236 ))));
1237 assert!(!DataType::is_list(&DataType::Binary));
1238 }
1239
1240 #[test]
1241 fn test_is_binary() {
1242 assert!(DataType::is_binary(&DataType::Binary));
1243 assert!(DataType::is_binary(&DataType::LargeBinary));
1244 assert!(DataType::is_binary(&DataType::BinaryView));
1245 assert!(!DataType::is_list(&DataType::Utf8View));
1246 }
1247
1248 #[test]
1249 fn size_should_not_regress() {
1250 assert_eq!(std::mem::size_of::<DataType>(), 24);
1251 }
1252
1253 #[test]
1254 #[should_panic(expected = "duplicate type id: 1")]
1255 fn test_union_with_duplicated_type_id() {
1256 let type_ids = vec![1, 1];
1257 let _union = DataType::Union(
1258 UnionFields::try_new(
1259 type_ids,
1260 vec![
1261 Field::new("f1", DataType::Int32, false),
1262 Field::new("f2", DataType::Utf8, false),
1263 ],
1264 )
1265 .unwrap(),
1266 UnionMode::Dense,
1267 );
1268 }
1269
1270 #[test]
1271 fn test_try_from_str() {
1272 let data_type: DataType = "Int32".try_into().unwrap();
1273 assert_eq!(data_type, DataType::Int32);
1274 }
1275
1276 #[test]
1277 fn test_from_str() {
1278 let data_type: DataType = "UInt64".parse().unwrap();
1279 assert_eq!(data_type, DataType::UInt64);
1280 }
1281
1282 #[test]
1283 #[cfg_attr(miri, ignore)] // fork is not supported
1284 fn test_debug_format_field() {
1285 // Make sure the `Debug` formatting of `DataType` is readable and not too long
1286 insta::assert_debug_snapshot!(DataType::new_list(DataType::Int8, false), @r"
1287 List(
1288 Field {
1289 data_type: Int8,
1290 },
1291 )
1292 ");
1293 }
1294}