parquet/
data_type.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Data types that connect Parquet physical types with their Rust-specific
19//! representations.
20use bytes::Bytes;
21use half::f16;
22use std::cmp::Ordering;
23use std::fmt;
24use std::mem;
25use std::ops::{Deref, DerefMut};
26use std::str::from_utf8;
27
28use crate::basic::Type;
29use crate::column::reader::{ColumnReader, ColumnReaderImpl};
30use crate::column::writer::{ColumnWriter, ColumnWriterImpl};
31use crate::errors::{ParquetError, Result};
32use crate::util::bit_util::FromBytes;
33
34/// Rust representation for logical type INT96, value is backed by an array of `u32`.
35/// The type only takes 12 bytes, without extra padding.
36#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
37pub struct Int96 {
38    value: [u32; 3],
39}
40
41const JULIAN_DAY_OF_EPOCH: i64 = 2_440_588;
42
43/// Number of seconds in a day
44const SECONDS_IN_DAY: i64 = 86_400;
45/// Number of milliseconds in a second
46const MILLISECONDS: i64 = 1_000;
47/// Number of microseconds in a second
48const MICROSECONDS: i64 = 1_000_000;
49/// Number of nanoseconds in a second
50const NANOSECONDS: i64 = 1_000_000_000;
51
52/// Number of milliseconds in a day
53const MILLISECONDS_IN_DAY: i64 = SECONDS_IN_DAY * MILLISECONDS;
54/// Number of microseconds in a day
55const MICROSECONDS_IN_DAY: i64 = SECONDS_IN_DAY * MICROSECONDS;
56/// Number of nanoseconds in a day
57const NANOSECONDS_IN_DAY: i64 = SECONDS_IN_DAY * NANOSECONDS;
58
59impl Int96 {
60    /// Creates new INT96 type struct with no data set.
61    pub fn new() -> Self {
62        Self { value: [0; 3] }
63    }
64
65    /// Returns underlying data as slice of [`u32`].
66    #[inline]
67    pub fn data(&self) -> &[u32] {
68        &self.value
69    }
70
71    /// Sets data for this INT96 type.
72    #[inline]
73    pub fn set_data(&mut self, elem0: u32, elem1: u32, elem2: u32) {
74        self.value = [elem0, elem1, elem2];
75    }
76
77    /// Converts this INT96 into an i64 representing the number of SECONDS since EPOCH
78    ///
79    /// Will wrap around on overflow
80    #[inline]
81    pub fn to_seconds(&self) -> i64 {
82        let (day, nanos) = self.data_as_days_and_nanos();
83        (day as i64 - JULIAN_DAY_OF_EPOCH)
84            .wrapping_mul(SECONDS_IN_DAY)
85            .wrapping_add(nanos / 1_000_000_000)
86    }
87
88    /// Converts this INT96 into an i64 representing the number of MILLISECONDS since EPOCH
89    ///
90    /// Will wrap around on overflow
91    #[inline]
92    pub fn to_millis(&self) -> i64 {
93        let (day, nanos) = self.data_as_days_and_nanos();
94        (day as i64 - JULIAN_DAY_OF_EPOCH)
95            .wrapping_mul(MILLISECONDS_IN_DAY)
96            .wrapping_add(nanos / 1_000_000)
97    }
98
99    /// Converts this INT96 into an i64 representing the number of MICROSECONDS since EPOCH
100    ///
101    /// Will wrap around on overflow
102    #[inline]
103    pub fn to_micros(&self) -> i64 {
104        let (day, nanos) = self.data_as_days_and_nanos();
105        (day as i64 - JULIAN_DAY_OF_EPOCH)
106            .wrapping_mul(MICROSECONDS_IN_DAY)
107            .wrapping_add(nanos / 1_000)
108    }
109
110    /// Converts this INT96 into an i64 representing the number of NANOSECONDS since EPOCH
111    ///
112    /// Will wrap around on overflow
113    #[inline]
114    pub fn to_nanos(&self) -> i64 {
115        let (day, nanos) = self.data_as_days_and_nanos();
116        (day as i64 - JULIAN_DAY_OF_EPOCH)
117            .wrapping_mul(NANOSECONDS_IN_DAY)
118            .wrapping_add(nanos)
119    }
120
121    #[inline]
122    fn get_days(&self) -> i32 {
123        self.data()[2] as i32
124    }
125
126    #[inline]
127    fn get_nanos(&self) -> i64 {
128        ((self.data()[1] as i64) << 32) + self.data()[0] as i64
129    }
130
131    #[inline]
132    fn data_as_days_and_nanos(&self) -> (i32, i64) {
133        (self.get_days(), self.get_nanos())
134    }
135}
136
137impl PartialOrd for Int96 {
138    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
139        Some(self.cmp(other))
140    }
141}
142
143impl Ord for Int96 {
144    /// Order `Int96` correctly for (deprecated) timestamp types.
145    ///
146    /// Note: this is done even though the Int96 type is deprecated.
147    /// Because some engines, notably Spark and Databricks Photon, still write
148    /// Int96 timestamps, a new `ColumnOrder` variant has been added to
149    /// the Parquet specification. See [parquet-format/#584].
150    ///
151    /// [parquet-format/#584]: https://github.com/apache/parquet-format/pull/584
152    fn cmp(&self, other: &Self) -> Ordering {
153        match self.get_days().cmp(&other.get_days()) {
154            Ordering::Equal => self.get_nanos().cmp(&other.get_nanos()),
155            ord => ord,
156        }
157    }
158}
159impl From<Vec<u32>> for Int96 {
160    fn from(buf: Vec<u32>) -> Self {
161        assert_eq!(buf.len(), 3);
162        let mut result = Self::new();
163        result.set_data(buf[0], buf[1], buf[2]);
164        result
165    }
166}
167
168impl fmt::Display for Int96 {
169    #[cold]
170    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
171        write!(f, "{:?}", self.data())
172    }
173}
174
175/// Rust representation for BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY Parquet physical types.
176/// Value is backed by a byte buffer.
177#[derive(Clone, Default)]
178pub struct ByteArray {
179    data: Option<Bytes>,
180}
181
182// Special case Debug that prints out byte arrays that are valid utf8 as &str's
183impl std::fmt::Debug for ByteArray {
184    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
185        let mut debug_struct = f.debug_struct("ByteArray");
186        match self.as_utf8() {
187            Ok(s) => debug_struct.field("data", &s),
188            Err(_) => debug_struct.field("data", &self.data),
189        };
190        debug_struct.finish()
191    }
192}
193
194impl PartialOrd for ByteArray {
195    fn partial_cmp(&self, other: &ByteArray) -> Option<Ordering> {
196        // sort nulls first (consistent with PartialCmp on Option)
197        //
198        // Since ByteBuffer doesn't implement PartialOrd, so can't
199        // derive an implementation
200        match (&self.data, &other.data) {
201            (None, None) => Some(Ordering::Equal),
202            (None, Some(_)) => Some(Ordering::Less),
203            (Some(_), None) => Some(Ordering::Greater),
204            (Some(self_data), Some(other_data)) => {
205                // compare slices directly
206                self_data.partial_cmp(&other_data)
207            }
208        }
209    }
210}
211
212impl ByteArray {
213    /// Creates new byte array with no data set.
214    #[inline]
215    pub fn new() -> Self {
216        ByteArray { data: None }
217    }
218
219    /// Gets length of the underlying byte buffer.
220    ///
221    /// # Panics
222    ///
223    /// Panics if no data has been set, e.g. on a [`ByteArray::new`] instance
224    #[inline]
225    pub fn len(&self) -> usize {
226        assert!(self.data.is_some());
227        self.data.as_ref().unwrap().len()
228    }
229
230    /// Checks if the underlying buffer is empty.
231    ///
232    /// # Panics
233    ///
234    /// Panics if no data has been set, see [`Self::len`]
235    #[inline]
236    pub fn is_empty(&self) -> bool {
237        self.len() == 0
238    }
239
240    /// Returns slice of data.
241    ///
242    /// # Panics
243    ///
244    /// Panics if no data has been set
245    #[inline]
246    pub fn data(&self) -> &[u8] {
247        self.data
248            .as_ref()
249            .expect("set_data should have been called")
250            .as_ref()
251    }
252
253    /// Set data from another byte buffer.
254    #[inline]
255    pub fn set_data(&mut self, data: Bytes) {
256        self.data = Some(data);
257    }
258
259    /// Returns `ByteArray` instance with slice of values for a data.
260    ///
261    /// # Panics
262    ///
263    /// Panics if no data has been set, or if `start + len` is out of bounds
264    #[inline]
265    pub fn slice(&self, start: usize, len: usize) -> Self {
266        Self::from(
267            self.data
268                .as_ref()
269                .expect("set_data should have been called")
270                .slice(start..start + len),
271        )
272    }
273
274    /// Try to convert the byte array to a utf8 slice
275    pub fn as_utf8(&self) -> Result<&str> {
276        let bytes = self
277            .data
278            .as_ref()
279            .map(|ptr| ptr.as_ref())
280            .ok_or_else(|| general_err!("Can't convert empty byte array to utf8"))?;
281        from_utf8(bytes).map_err(|e| e.into())
282    }
283}
284
285impl From<Vec<u8>> for ByteArray {
286    fn from(buf: Vec<u8>) -> ByteArray {
287        Self {
288            data: Some(buf.into()),
289        }
290    }
291}
292
293impl<'a> From<&'a [u8]> for ByteArray {
294    fn from(b: &'a [u8]) -> ByteArray {
295        let mut v = Vec::new();
296        v.extend_from_slice(b);
297        Self {
298            data: Some(v.into()),
299        }
300    }
301}
302
303impl<'a> From<&'a str> for ByteArray {
304    fn from(s: &'a str) -> ByteArray {
305        let mut v = Vec::new();
306        v.extend_from_slice(s.as_bytes());
307        Self {
308            data: Some(v.into()),
309        }
310    }
311}
312
313impl From<Bytes> for ByteArray {
314    fn from(value: Bytes) -> Self {
315        Self { data: Some(value) }
316    }
317}
318
319impl From<f16> for ByteArray {
320    fn from(value: f16) -> Self {
321        Self::from(value.to_le_bytes().as_slice())
322    }
323}
324
325impl PartialEq for ByteArray {
326    fn eq(&self, other: &ByteArray) -> bool {
327        match (&self.data, &other.data) {
328            (Some(d1), Some(d2)) => d1.as_ref() == d2.as_ref(),
329            (None, None) => true,
330            _ => false,
331        }
332    }
333}
334
335impl fmt::Display for ByteArray {
336    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
337        write!(f, "{:?}", self.data())
338    }
339}
340
341/// Wrapper type for performance reasons, this represents `FIXED_LEN_BYTE_ARRAY` but in all other
342/// considerations behaves the same as `ByteArray`
343///
344/// # Performance notes:
345/// This type is a little unfortunate, without it the compiler generates code that takes quite a
346/// big hit on the CPU pipeline. Essentially the previous version stalls awaiting the result of
347/// `T::get_physical_type() == Type::FIXED_LEN_BYTE_ARRAY`.
348///
349/// Its debatable if this is wanted, it is out of spec for what parquet documents as its base
350/// types, although there are code paths in the Rust (and potentially the C++) versions that
351/// warrant this.
352///
353/// With this wrapper type the compiler generates more targeted code paths matching the higher
354/// level logical types, removing the data-hazard from all decoding and encoding paths.
355#[repr(transparent)]
356#[derive(Clone, Debug, Default)]
357pub struct FixedLenByteArray(ByteArray);
358
359impl PartialEq for FixedLenByteArray {
360    fn eq(&self, other: &FixedLenByteArray) -> bool {
361        self.0.eq(&other.0)
362    }
363}
364
365impl PartialEq<ByteArray> for FixedLenByteArray {
366    fn eq(&self, other: &ByteArray) -> bool {
367        self.0.eq(other)
368    }
369}
370
371impl PartialEq<FixedLenByteArray> for ByteArray {
372    fn eq(&self, other: &FixedLenByteArray) -> bool {
373        self.eq(&other.0)
374    }
375}
376
377impl fmt::Display for FixedLenByteArray {
378    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
379        self.0.fmt(f)
380    }
381}
382
383impl PartialOrd for FixedLenByteArray {
384    fn partial_cmp(&self, other: &FixedLenByteArray) -> Option<Ordering> {
385        self.0.partial_cmp(&other.0)
386    }
387}
388
389impl PartialOrd<FixedLenByteArray> for ByteArray {
390    fn partial_cmp(&self, other: &FixedLenByteArray) -> Option<Ordering> {
391        self.partial_cmp(&other.0)
392    }
393}
394
395impl PartialOrd<ByteArray> for FixedLenByteArray {
396    fn partial_cmp(&self, other: &ByteArray) -> Option<Ordering> {
397        self.0.partial_cmp(other)
398    }
399}
400
401impl Deref for FixedLenByteArray {
402    type Target = ByteArray;
403
404    fn deref(&self) -> &Self::Target {
405        &self.0
406    }
407}
408
409impl DerefMut for FixedLenByteArray {
410    fn deref_mut(&mut self) -> &mut Self::Target {
411        &mut self.0
412    }
413}
414
415impl From<ByteArray> for FixedLenByteArray {
416    fn from(other: ByteArray) -> Self {
417        Self(other)
418    }
419}
420
421impl From<Vec<u8>> for FixedLenByteArray {
422    fn from(buf: Vec<u8>) -> FixedLenByteArray {
423        FixedLenByteArray(ByteArray::from(buf))
424    }
425}
426
427impl From<FixedLenByteArray> for ByteArray {
428    fn from(other: FixedLenByteArray) -> Self {
429        other.0
430    }
431}
432
433/// Rust representation for Decimal values.
434///
435/// This is not a representation of Parquet physical type, but rather a wrapper for
436/// DECIMAL logical type, and serves as container for raw parts of decimal values:
437/// unscaled value in bytes, precision and scale.
438#[derive(Clone, Debug)]
439pub enum Decimal {
440    /// Decimal backed by `i32`.
441    Int32 {
442        /// The underlying value
443        value: [u8; 4],
444        /// The total number of digits in the number
445        precision: i32,
446        /// The number of digits to the right of the decimal point
447        scale: i32,
448    },
449    /// Decimal backed by `i64`.
450    Int64 {
451        /// The underlying value
452        value: [u8; 8],
453        /// The total number of digits in the number
454        precision: i32,
455        /// The number of digits to the right of the decimal point
456        scale: i32,
457    },
458    /// Decimal backed by byte array.
459    Bytes {
460        /// The underlying value
461        value: ByteArray,
462        /// The total number of digits in the number
463        precision: i32,
464        /// The number of digits to the right of the decimal point
465        scale: i32,
466    },
467}
468
469impl Decimal {
470    /// Creates new decimal value from `i32`.
471    pub fn from_i32(value: i32, precision: i32, scale: i32) -> Self {
472        let bytes = value.to_be_bytes();
473        Decimal::Int32 {
474            value: bytes,
475            precision,
476            scale,
477        }
478    }
479
480    /// Creates new decimal value from `i64`.
481    pub fn from_i64(value: i64, precision: i32, scale: i32) -> Self {
482        let bytes = value.to_be_bytes();
483        Decimal::Int64 {
484            value: bytes,
485            precision,
486            scale,
487        }
488    }
489
490    /// Creates new decimal value from `ByteArray`.
491    pub fn from_bytes(value: ByteArray, precision: i32, scale: i32) -> Self {
492        Decimal::Bytes {
493            value,
494            precision,
495            scale,
496        }
497    }
498
499    /// Returns bytes of unscaled value.
500    pub fn data(&self) -> &[u8] {
501        match *self {
502            Decimal::Int32 { ref value, .. } => value,
503            Decimal::Int64 { ref value, .. } => value,
504            Decimal::Bytes { ref value, .. } => value.data(),
505        }
506    }
507
508    /// Returns decimal precision.
509    pub fn precision(&self) -> i32 {
510        match *self {
511            Decimal::Int32 { precision, .. } => precision,
512            Decimal::Int64 { precision, .. } => precision,
513            Decimal::Bytes { precision, .. } => precision,
514        }
515    }
516
517    /// Returns decimal scale.
518    pub fn scale(&self) -> i32 {
519        match *self {
520            Decimal::Int32 { scale, .. } => scale,
521            Decimal::Int64 { scale, .. } => scale,
522            Decimal::Bytes { scale, .. } => scale,
523        }
524    }
525}
526
527impl Default for Decimal {
528    fn default() -> Self {
529        Self::from_i32(0, 0, 0)
530    }
531}
532
533impl PartialEq for Decimal {
534    fn eq(&self, other: &Decimal) -> bool {
535        self.precision() == other.precision()
536            && self.scale() == other.scale()
537            && self.data() == other.data()
538    }
539}
540
541/// Converts an instance of data type to a slice of bytes as `u8`.
542pub trait AsBytes {
543    /// Returns slice of bytes for this data type.
544    fn as_bytes(&self) -> &[u8];
545}
546
547/// Converts an slice of a data type to a slice of bytes.
548pub trait SliceAsBytes: Sized {
549    /// Returns slice of bytes for a slice of this data type.
550    fn slice_as_bytes(self_: &[Self]) -> &[u8];
551    /// Return the internal representation as a mutable slice
552    ///
553    /// # Safety
554    /// If modified you are _required_ to ensure the internal representation
555    /// is valid and correct for the actual raw data
556    unsafe fn slice_as_bytes_mut(self_: &mut [Self]) -> &mut [u8];
557}
558
559impl AsBytes for [u8] {
560    fn as_bytes(&self) -> &[u8] {
561        self
562    }
563}
564
565macro_rules! gen_as_bytes {
566    ($source_ty:ident) => {
567        impl AsBytes for $source_ty {
568            fn as_bytes(&self) -> &[u8] {
569                // SAFETY: macro is only used with primitive types that have no padding, so the
570                // resulting slice always refers to initialized memory.
571                unsafe {
572                    std::slice::from_raw_parts(
573                        std::ptr::from_ref::<$source_ty>(self).cast::<u8>(),
574                        std::mem::size_of::<$source_ty>(),
575                    )
576                }
577            }
578        }
579
580        impl SliceAsBytes for $source_ty {
581            #[inline]
582            fn slice_as_bytes(self_: &[Self]) -> &[u8] {
583                // SAFETY: macro is only used with primitive types that have no padding, so the
584                // resulting slice always refers to initialized memory.
585                unsafe {
586                    std::slice::from_raw_parts(
587                        self_.as_ptr().cast::<u8>(),
588                        std::mem::size_of_val(self_),
589                    )
590                }
591            }
592
593            #[inline]
594            unsafe fn slice_as_bytes_mut(self_: &mut [Self]) -> &mut [u8] {
595                // SAFETY: macro is only used with primitive types that have no padding, so the
596                // resulting slice always refers to initialized memory. Moreover, self has no
597                // invalid bit patterns, so all writes to the resulting slice will be valid.
598                unsafe {
599                    std::slice::from_raw_parts_mut(
600                        self_.as_mut_ptr().cast::<u8>(),
601                        std::mem::size_of_val(self_),
602                    )
603                }
604            }
605        }
606    };
607}
608
609gen_as_bytes!(i8);
610gen_as_bytes!(i16);
611gen_as_bytes!(i32);
612gen_as_bytes!(i64);
613gen_as_bytes!(u8);
614gen_as_bytes!(u16);
615gen_as_bytes!(u32);
616gen_as_bytes!(u64);
617gen_as_bytes!(f32);
618gen_as_bytes!(f64);
619
620macro_rules! unimplemented_slice_as_bytes {
621    ($ty: ty) => {
622        impl SliceAsBytes for $ty {
623            fn slice_as_bytes(_self: &[Self]) -> &[u8] {
624                unimplemented!()
625            }
626
627            unsafe fn slice_as_bytes_mut(_self: &mut [Self]) -> &mut [u8] {
628                unimplemented!()
629            }
630        }
631    };
632}
633
634// TODO - Can Int96 and bool be implemented in these terms?
635unimplemented_slice_as_bytes!(Int96);
636unimplemented_slice_as_bytes!(bool);
637unimplemented_slice_as_bytes!(ByteArray);
638unimplemented_slice_as_bytes!(FixedLenByteArray);
639
640impl AsBytes for bool {
641    fn as_bytes(&self) -> &[u8] {
642        // SAFETY: a bool is guaranteed to be either 0x00 or 0x01 in memory, so the memory is
643        // valid.
644        unsafe { std::slice::from_raw_parts(std::ptr::from_ref::<bool>(self).cast::<u8>(), 1) }
645    }
646}
647
648impl AsBytes for Int96 {
649    fn as_bytes(&self) -> &[u8] {
650        // SAFETY: Int96::data is a &[u32; 3].
651        unsafe {
652            std::slice::from_raw_parts(std::ptr::from_ref::<[u32]>(self.data()).cast::<u8>(), 12)
653        }
654    }
655}
656
657impl AsBytes for ByteArray {
658    fn as_bytes(&self) -> &[u8] {
659        self.data()
660    }
661}
662
663impl AsBytes for FixedLenByteArray {
664    fn as_bytes(&self) -> &[u8] {
665        self.data()
666    }
667}
668
669impl AsBytes for Decimal {
670    fn as_bytes(&self) -> &[u8] {
671        self.data()
672    }
673}
674
675impl AsBytes for Vec<u8> {
676    fn as_bytes(&self) -> &[u8] {
677        self.as_slice()
678    }
679}
680
681impl AsBytes for &str {
682    fn as_bytes(&self) -> &[u8] {
683        (self as &str).as_bytes()
684    }
685}
686
687impl AsBytes for str {
688    fn as_bytes(&self) -> &[u8] {
689        (self as &str).as_bytes()
690    }
691}
692
693pub(crate) mod private {
694    use bytes::Bytes;
695
696    use crate::encodings::decoding::PlainDecoderDetails;
697    use crate::util::bit_util::{BitReader, BitWriter, read_num_bytes};
698
699    use super::{ParquetError, Result, SliceAsBytes};
700    use crate::basic::Type;
701    use crate::column::writer::min_max::{
702        fallback_min_max, min_max_f32, min_max_f64, min_max_i32, min_max_i64,
703    };
704    use crate::file::metadata::HeapSize;
705    use crate::schema::types::BasicTypeInfo;
706
707    /// Sealed trait to start to remove specialisation from implementations
708    ///
709    /// This is done to force the associated value type to be unimplementable outside of this
710    /// crate, and thus hint to the type system (and end user) traits are public for the contract
711    /// and not for extension.
712    pub trait ParquetValueType:
713        PartialEq
714        + std::fmt::Debug
715        + std::fmt::Display
716        + Default
717        + Clone
718        + super::AsBytes
719        + super::FromBytes
720        + SliceAsBytes
721        + PartialOrd
722        + Send
723        + HeapSize
724        + crate::encodings::decoding::private::GetDecoder
725        + crate::encodings::encoding::private::GetEncoder
726        + crate::file::statistics::private::MakeStatistics
727    {
728        const PHYSICAL_TYPE: Type;
729
730        /// Encode the value directly from a higher level encoder
731        fn encode<W: std::io::Write>(
732            values: &[Self],
733            writer: &mut W,
734            bit_writer: &mut BitWriter,
735        ) -> Result<()>;
736
737        /// Establish the data that will be decoded in a buffer
738        fn set_data(decoder: &mut PlainDecoderDetails, data: Bytes, num_values: usize);
739
740        /// Decode the value from a given buffer for a higher level decoder
741        fn decode(buffer: &mut [Self], decoder: &mut PlainDecoderDetails) -> Result<usize>;
742
743        fn skip(decoder: &mut PlainDecoderDetails, num_values: usize) -> Result<usize>;
744
745        /// Return the encoded size for a type
746        fn dict_encoding_size(&self) -> (usize, usize) {
747            (std::mem::size_of::<Self>(), 1)
748        }
749
750        /// Return the number of variable length bytes in a given slice of data
751        ///
752        /// Returns the sum of lengths for BYTE_ARRAY data, and None for all other data types
753        fn variable_length_bytes(_: &[Self]) -> Option<i64> {
754            None
755        }
756
757        /// Return the value as i64 if possible
758        ///
759        /// This is essentially the same as `std::convert::TryInto<i64>` but can't be
760        /// implemented for `f32` and `f64`, types that would fail orphan rules
761        fn as_i64(&self) -> Result<i64> {
762            Err(general_err!("Type cannot be converted to i64"))
763        }
764
765        /// Return the value as u64 if possible
766        ///
767        /// This is essentially the same as `std::convert::TryInto<u64>` but can't be
768        /// implemented for `f32` and `f64`, types that would fail orphan rules
769        fn as_u64(&self) -> Result<u64> {
770            self.as_i64()
771                .map_err(|_| general_err!("Type cannot be converted to u64"))
772                .map(|x| x as u64)
773        }
774
775        /// Return the min, max and NaN count of `values` for column statistics,
776        /// or `None` if `values` is empty
777        fn min_max(basic_type_info: &BasicTypeInfo, values: &[Self]) -> Option<(Self, Self, u64)> {
778            fallback_min_max(basic_type_info, values.iter())
779        }
780
781        /// Return the value as an Any to allow for downcasts without transmutation
782        fn as_any(&self) -> &dyn std::any::Any;
783
784        /// Return the value as an mutable Any to allow for downcasts without transmutation
785        fn as_mut_any(&mut self) -> &mut dyn std::any::Any;
786
787        /// Sets the value of this object from the provided [`Bytes`]
788        ///
789        /// Only implemented for `ByteArray` and `FixedLenByteArray`. Will panic for other types.
790        fn set_from_bytes(&mut self, _data: Bytes) {
791            unimplemented!();
792        }
793    }
794
795    impl ParquetValueType for bool {
796        const PHYSICAL_TYPE: Type = Type::BOOLEAN;
797
798        #[inline]
799        fn encode<W: std::io::Write>(
800            values: &[Self],
801            _: &mut W,
802            bit_writer: &mut BitWriter,
803        ) -> Result<()> {
804            bit_writer.put_batch(values, 1);
805            Ok(())
806        }
807
808        #[inline]
809        fn set_data(decoder: &mut PlainDecoderDetails, data: Bytes, num_values: usize) {
810            decoder.bit_reader.replace(BitReader::new(data));
811            decoder.num_values = num_values;
812        }
813
814        #[inline]
815        fn decode(buffer: &mut [Self], decoder: &mut PlainDecoderDetails) -> Result<usize> {
816            let bit_reader = decoder.bit_reader.as_mut().unwrap();
817            let num_values = std::cmp::min(buffer.len(), decoder.num_values);
818            let values_read = bit_reader.get_batch(&mut buffer[..num_values], 1);
819            decoder.num_values -= values_read;
820            Ok(values_read)
821        }
822
823        fn skip(decoder: &mut PlainDecoderDetails, num_values: usize) -> Result<usize> {
824            let bit_reader = decoder.bit_reader.as_mut().unwrap();
825            let num_values = std::cmp::min(num_values, decoder.num_values);
826            let values_read = bit_reader.skip(num_values, 1);
827            decoder.num_values -= values_read;
828            Ok(values_read)
829        }
830
831        #[inline]
832        fn as_i64(&self) -> Result<i64> {
833            Ok(*self as i64)
834        }
835
836        #[inline]
837        fn as_any(&self) -> &dyn std::any::Any {
838            self
839        }
840
841        #[inline]
842        fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
843            self
844        }
845    }
846
847    macro_rules! impl_from_raw {
848        ($ty: ty, $physical_ty: expr, $self: ident => $as_i64: block, min_max: ($info: ident, $values: ident) => $min_max: block) => {
849            impl ParquetValueType for $ty {
850                const PHYSICAL_TYPE: Type = $physical_ty;
851
852                #[inline]
853                fn encode<W: std::io::Write>(values: &[Self], writer: &mut W, _: &mut BitWriter) -> Result<()> {
854                    // SAFETY: Self is one of i32, i64, f32, f64, which have no padding.
855                    let raw = unsafe {
856                        std::slice::from_raw_parts(
857                            values.as_ptr().cast::<u8>(),
858                            std::mem::size_of_val(values),
859                        )
860                    };
861                    writer.write_all(raw)?;
862
863                    Ok(())
864                }
865
866                #[inline]
867                fn set_data(decoder: &mut PlainDecoderDetails, data: Bytes, num_values: usize) {
868                    decoder.data.replace(data);
869                    decoder.start = 0;
870                    decoder.num_values = num_values;
871                }
872
873                #[inline]
874                fn decode(buffer: &mut [Self], decoder: &mut PlainDecoderDetails) -> Result<usize> {
875                    let data = decoder.data.as_ref().expect("set_data should have been called");
876                    let num_values = std::cmp::min(buffer.len(), decoder.num_values);
877                    let bytes_left = data.len() - decoder.start;
878                    let bytes_to_decode = std::mem::size_of::<Self>() * num_values;
879
880                    if bytes_left < bytes_to_decode {
881                        return Err(eof_err!("Not enough bytes to decode"));
882                    }
883
884                    {
885                        // SAFETY: Self has no invalid bit patterns, so writing to the slice
886                        // obtained with slice_as_bytes_mut is always safe.
887                        let raw_buffer = &mut unsafe { Self::slice_as_bytes_mut(buffer) }[..bytes_to_decode];
888                        raw_buffer.copy_from_slice(data.slice(
889                            decoder.start..decoder.start + bytes_to_decode
890                        ).as_ref());
891                    };
892                    decoder.start += bytes_to_decode;
893                    decoder.num_values -= num_values;
894
895                    Ok(num_values)
896                }
897
898                #[inline]
899                fn skip(decoder: &mut PlainDecoderDetails, num_values: usize) -> Result<usize> {
900                    let data = decoder.data.as_ref().expect("set_data should have been called");
901                    let num_values = num_values.min(decoder.num_values);
902                    let bytes_left = data.len() - decoder.start;
903                    let bytes_to_skip = std::mem::size_of::<Self>() * num_values;
904
905                    if bytes_left < bytes_to_skip {
906                        return Err(eof_err!("Not enough bytes to skip"));
907                    }
908
909                    decoder.start += bytes_to_skip;
910                    decoder.num_values -= num_values;
911
912                    Ok(num_values)
913                }
914
915                #[inline]
916                fn as_i64(&$self) -> Result<i64> {
917                    $as_i64
918                }
919
920                #[inline]
921                fn min_max($info: &BasicTypeInfo, $values: &[Self]) -> Option<(Self, Self, u64)> {
922                    $min_max
923                }
924
925                #[inline]
926                fn as_any(&self) -> &dyn std::any::Any {
927                    self
928                }
929
930                #[inline]
931                fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
932                    self
933                }
934            }
935        }
936    }
937
938    impl_from_raw!(
939        i32, Type::INT32, self => { Ok(*self as i64) },
940        min_max: (info, values) => { min_max_i32(info, values).map(|(min, max)| (min, max, 0)) }
941    );
942    impl_from_raw!(
943        i64, Type::INT64, self => { Ok(*self) },
944        min_max: (info, values) => { min_max_i64(info, values).map(|(min, max)| (min, max, 0)) }
945    );
946    // The float kernels return `None` for all NaN input, which the scalar path handles
947    impl_from_raw!(
948        f32, Type::FLOAT, self => { Err(general_err!("Type cannot be converted to i64")) },
949        min_max: (info, values) => { min_max_f32(values).or_else(|| fallback_min_max(info, values.iter())) }
950    );
951    impl_from_raw!(
952        f64, Type::DOUBLE, self => { Err(general_err!("Type cannot be converted to i64")) },
953        min_max: (info, values) => { min_max_f64(values).or_else(|| fallback_min_max(info, values.iter())) }
954    );
955
956    impl ParquetValueType for super::Int96 {
957        const PHYSICAL_TYPE: Type = Type::INT96;
958
959        #[inline]
960        fn encode<W: std::io::Write>(
961            values: &[Self],
962            writer: &mut W,
963            _: &mut BitWriter,
964        ) -> Result<()> {
965            for value in values {
966                let raw = SliceAsBytes::slice_as_bytes(value.data());
967                writer.write_all(raw)?;
968            }
969            Ok(())
970        }
971
972        #[inline]
973        fn set_data(decoder: &mut PlainDecoderDetails, data: Bytes, num_values: usize) {
974            decoder.data.replace(data);
975            decoder.start = 0;
976            decoder.num_values = num_values;
977        }
978
979        #[inline]
980        fn decode(buffer: &mut [Self], decoder: &mut PlainDecoderDetails) -> Result<usize> {
981            // TODO - Remove the duplication between this and the general slice method
982            let data = decoder
983                .data
984                .as_ref()
985                .expect("set_data should have been called");
986            let num_values = std::cmp::min(buffer.len(), decoder.num_values);
987            let bytes_left = data.len() - decoder.start;
988            let bytes_to_decode = 12 * num_values;
989
990            if bytes_left < bytes_to_decode {
991                return Err(eof_err!("Not enough bytes to decode"));
992            }
993
994            let data_range = data.slice(decoder.start..decoder.start + bytes_to_decode);
995            let bytes: &[u8] = &data_range;
996            decoder.start += bytes_to_decode;
997
998            let mut pos = 0; // position in byte array
999            for item in buffer.iter_mut().take(num_values) {
1000                let elem0 = u32::from_le_bytes(bytes[pos..pos + 4].try_into().unwrap());
1001                let elem1 = u32::from_le_bytes(bytes[pos + 4..pos + 8].try_into().unwrap());
1002                let elem2 = u32::from_le_bytes(bytes[pos + 8..pos + 12].try_into().unwrap());
1003
1004                item.set_data(elem0, elem1, elem2);
1005                pos += 12;
1006            }
1007            decoder.num_values -= num_values;
1008
1009            Ok(num_values)
1010        }
1011
1012        fn skip(decoder: &mut PlainDecoderDetails, num_values: usize) -> Result<usize> {
1013            let data = decoder
1014                .data
1015                .as_ref()
1016                .expect("set_data should have been called");
1017            let num_values = std::cmp::min(num_values, decoder.num_values);
1018            let bytes_left = data.len() - decoder.start;
1019            let bytes_to_skip = 12 * num_values;
1020
1021            if bytes_left < bytes_to_skip {
1022                return Err(eof_err!("Not enough bytes to skip"));
1023            }
1024            decoder.start += bytes_to_skip;
1025            decoder.num_values -= num_values;
1026
1027            Ok(num_values)
1028        }
1029
1030        #[inline]
1031        fn as_any(&self) -> &dyn std::any::Any {
1032            self
1033        }
1034
1035        #[inline]
1036        fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
1037            self
1038        }
1039    }
1040
1041    impl HeapSize for super::Int96 {
1042        fn heap_size(&self) -> usize {
1043            0 // no heap allocations
1044        }
1045    }
1046
1047    impl ParquetValueType for super::ByteArray {
1048        const PHYSICAL_TYPE: Type = Type::BYTE_ARRAY;
1049
1050        #[inline]
1051        fn encode<W: std::io::Write>(
1052            values: &[Self],
1053            writer: &mut W,
1054            _: &mut BitWriter,
1055        ) -> Result<()> {
1056            for value in values {
1057                let len: u32 = value.len().try_into().unwrap();
1058                writer.write_all(&len.to_ne_bytes())?;
1059                let raw = value.data();
1060                writer.write_all(raw)?;
1061            }
1062            Ok(())
1063        }
1064
1065        #[inline]
1066        fn set_data(decoder: &mut PlainDecoderDetails, data: Bytes, num_values: usize) {
1067            decoder.data.replace(data);
1068            decoder.start = 0;
1069            decoder.num_values = num_values;
1070        }
1071
1072        #[inline]
1073        fn decode(buffer: &mut [Self], decoder: &mut PlainDecoderDetails) -> Result<usize> {
1074            let data = decoder
1075                .data
1076                .as_mut()
1077                .expect("set_data should have been called");
1078            let num_values = std::cmp::min(buffer.len(), decoder.num_values);
1079            for val_array in buffer.iter_mut().take(num_values) {
1080                let len: usize =
1081                    read_num_bytes::<u32>(4, data.slice(decoder.start..).as_ref()) as usize;
1082                decoder.start += std::mem::size_of::<u32>();
1083
1084                if data.len() < decoder.start + len {
1085                    return Err(eof_err!("Not enough bytes to decode"));
1086                }
1087
1088                val_array.set_data(data.slice(decoder.start..decoder.start + len));
1089                decoder.start += len;
1090            }
1091            decoder.num_values -= num_values;
1092
1093            Ok(num_values)
1094        }
1095
1096        fn variable_length_bytes(values: &[Self]) -> Option<i64> {
1097            Some(values.iter().map(|x| x.len() as i64).sum())
1098        }
1099
1100        fn skip(decoder: &mut PlainDecoderDetails, num_values: usize) -> Result<usize> {
1101            let data = decoder
1102                .data
1103                .as_mut()
1104                .expect("set_data should have been called");
1105            let num_values = num_values.min(decoder.num_values);
1106
1107            for _ in 0..num_values {
1108                let len: usize =
1109                    read_num_bytes::<u32>(4, data.slice(decoder.start..).as_ref()) as usize;
1110                decoder.start += std::mem::size_of::<u32>() + len;
1111            }
1112            decoder.num_values -= num_values;
1113
1114            Ok(num_values)
1115        }
1116
1117        #[inline]
1118        fn dict_encoding_size(&self) -> (usize, usize) {
1119            (std::mem::size_of::<u32>(), self.len())
1120        }
1121
1122        #[inline]
1123        fn as_any(&self) -> &dyn std::any::Any {
1124            self
1125        }
1126
1127        #[inline]
1128        fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
1129            self
1130        }
1131
1132        #[inline]
1133        fn set_from_bytes(&mut self, data: Bytes) {
1134            self.set_data(data);
1135        }
1136    }
1137
1138    impl HeapSize for super::ByteArray {
1139        fn heap_size(&self) -> usize {
1140            // note: this is an estimate, not exact, so just return the size
1141            // of the actual data used, don't try to handle the fact that it may
1142            // be shared.
1143            self.data.as_ref().map(|data| data.len()).unwrap_or(0)
1144        }
1145    }
1146
1147    impl ParquetValueType for super::FixedLenByteArray {
1148        const PHYSICAL_TYPE: Type = Type::FIXED_LEN_BYTE_ARRAY;
1149
1150        #[inline]
1151        fn encode<W: std::io::Write>(
1152            values: &[Self],
1153            writer: &mut W,
1154            _: &mut BitWriter,
1155        ) -> Result<()> {
1156            for value in values {
1157                let raw = value.data();
1158                writer.write_all(raw)?;
1159            }
1160            Ok(())
1161        }
1162
1163        #[inline]
1164        fn set_data(decoder: &mut PlainDecoderDetails, data: Bytes, num_values: usize) {
1165            decoder.data.replace(data);
1166            decoder.start = 0;
1167            decoder.num_values = num_values;
1168        }
1169
1170        #[inline]
1171        fn decode(buffer: &mut [Self], decoder: &mut PlainDecoderDetails) -> Result<usize> {
1172            assert!(decoder.type_length > 0);
1173
1174            let data = decoder
1175                .data
1176                .as_mut()
1177                .expect("set_data should have been called");
1178            let num_values = std::cmp::min(buffer.len(), decoder.num_values);
1179
1180            for item in buffer.iter_mut().take(num_values) {
1181                let len = decoder.type_length as usize;
1182
1183                if data.len() < decoder.start + len {
1184                    return Err(eof_err!("Not enough bytes to decode"));
1185                }
1186
1187                item.set_data(data.slice(decoder.start..decoder.start + len));
1188                decoder.start += len;
1189            }
1190            decoder.num_values -= num_values;
1191
1192            Ok(num_values)
1193        }
1194
1195        fn skip(decoder: &mut PlainDecoderDetails, num_values: usize) -> Result<usize> {
1196            assert!(decoder.type_length > 0);
1197
1198            let data = decoder
1199                .data
1200                .as_mut()
1201                .expect("set_data should have been called");
1202            let num_values = std::cmp::min(num_values, decoder.num_values);
1203            for _ in 0..num_values {
1204                let len = decoder.type_length as usize;
1205
1206                if data.len() < decoder.start + len {
1207                    return Err(eof_err!("Not enough bytes to skip"));
1208                }
1209
1210                decoder.start += len;
1211            }
1212            decoder.num_values -= num_values;
1213
1214            Ok(num_values)
1215        }
1216
1217        #[inline]
1218        fn dict_encoding_size(&self) -> (usize, usize) {
1219            (std::mem::size_of::<u32>(), self.len())
1220        }
1221
1222        #[inline]
1223        fn as_any(&self) -> &dyn std::any::Any {
1224            self
1225        }
1226
1227        #[inline]
1228        fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
1229            self
1230        }
1231
1232        #[inline]
1233        fn set_from_bytes(&mut self, data: Bytes) {
1234            self.set_data(data);
1235        }
1236    }
1237
1238    impl HeapSize for super::FixedLenByteArray {
1239        fn heap_size(&self) -> usize {
1240            self.0.heap_size()
1241        }
1242    }
1243}
1244
1245/// Contains the Parquet physical type information as well as the Rust primitive type
1246/// presentation.
1247pub trait DataType: 'static + Send {
1248    /// The physical type of the Parquet data type.
1249    type T: private::ParquetValueType;
1250
1251    /// Returns Parquet physical type.
1252    fn get_physical_type() -> Type {
1253        <Self::T as private::ParquetValueType>::PHYSICAL_TYPE
1254    }
1255
1256    /// Returns size in bytes for Rust representation of the physical type.
1257    fn get_type_size() -> usize;
1258
1259    /// Returns the underlying [`ColumnReaderImpl`] for the given [`ColumnReader`].
1260    fn get_column_reader(column_writer: ColumnReader) -> Option<ColumnReaderImpl<Self>>
1261    where
1262        Self: Sized;
1263
1264    /// Returns the underlying [`ColumnWriterImpl`] for the given [`ColumnWriter`].
1265    fn get_column_writer(column_writer: ColumnWriter<'_>) -> Option<ColumnWriterImpl<'_, Self>>
1266    where
1267        Self: Sized;
1268
1269    /// Returns a reference to the underlying [`ColumnWriterImpl`] for the given [`ColumnWriter`].
1270    fn get_column_writer_ref<'a, 'b: 'a>(
1271        column_writer: &'b ColumnWriter<'a>,
1272    ) -> Option<&'b ColumnWriterImpl<'a, Self>>
1273    where
1274        Self: Sized;
1275
1276    /// Returns a mutable reference to the underlying [`ColumnWriterImpl`] for the given
1277    fn get_column_writer_mut<'a, 'b: 'a>(
1278        column_writer: &'a mut ColumnWriter<'b>,
1279    ) -> Option<&'a mut ColumnWriterImpl<'b, Self>>
1280    where
1281        Self: Sized;
1282}
1283
1284macro_rules! make_type {
1285    ($name:ident, $reader_ident: ident, $writer_ident: ident, $native_ty:ty, $size:expr) => {
1286        #[doc = concat!("Parquet physical type: ", stringify!($name))]
1287        #[derive(Clone)]
1288        pub struct $name {}
1289
1290        impl DataType for $name {
1291            type T = $native_ty;
1292
1293            fn get_type_size() -> usize {
1294                $size
1295            }
1296
1297            fn get_column_reader(column_reader: ColumnReader) -> Option<ColumnReaderImpl<Self>> {
1298                match column_reader {
1299                    ColumnReader::$reader_ident(w) => Some(w),
1300                    _ => None,
1301                }
1302            }
1303
1304            fn get_column_writer(
1305                column_writer: ColumnWriter<'_>,
1306            ) -> Option<ColumnWriterImpl<'_, Self>> {
1307                match column_writer {
1308                    ColumnWriter::$writer_ident(w) => Some(w),
1309                    _ => None,
1310                }
1311            }
1312
1313            fn get_column_writer_ref<'a, 'b: 'a>(
1314                column_writer: &'a ColumnWriter<'b>,
1315            ) -> Option<&'a ColumnWriterImpl<'b, Self>> {
1316                match column_writer {
1317                    ColumnWriter::$writer_ident(w) => Some(w),
1318                    _ => None,
1319                }
1320            }
1321
1322            fn get_column_writer_mut<'a, 'b: 'a>(
1323                column_writer: &'a mut ColumnWriter<'b>,
1324            ) -> Option<&'a mut ColumnWriterImpl<'b, Self>> {
1325                match column_writer {
1326                    ColumnWriter::$writer_ident(w) => Some(w),
1327                    _ => None,
1328                }
1329            }
1330        }
1331    };
1332}
1333
1334// Generate struct definitions for all physical types
1335
1336make_type!(BoolType, BoolColumnReader, BoolColumnWriter, bool, 1);
1337make_type!(Int32Type, Int32ColumnReader, Int32ColumnWriter, i32, 4);
1338make_type!(Int64Type, Int64ColumnReader, Int64ColumnWriter, i64, 8);
1339make_type!(
1340    Int96Type,
1341    Int96ColumnReader,
1342    Int96ColumnWriter,
1343    Int96,
1344    mem::size_of::<Int96>()
1345);
1346make_type!(FloatType, FloatColumnReader, FloatColumnWriter, f32, 4);
1347make_type!(DoubleType, DoubleColumnReader, DoubleColumnWriter, f64, 8);
1348make_type!(
1349    ByteArrayType,
1350    ByteArrayColumnReader,
1351    ByteArrayColumnWriter,
1352    ByteArray,
1353    mem::size_of::<ByteArray>()
1354);
1355make_type!(
1356    FixedLenByteArrayType,
1357    FixedLenByteArrayColumnReader,
1358    FixedLenByteArrayColumnWriter,
1359    FixedLenByteArray,
1360    mem::size_of::<FixedLenByteArray>()
1361);
1362
1363impl AsRef<[u8]> for ByteArray {
1364    fn as_ref(&self) -> &[u8] {
1365        self.as_bytes()
1366    }
1367}
1368
1369impl AsRef<[u8]> for FixedLenByteArray {
1370    fn as_ref(&self) -> &[u8] {
1371        self.as_bytes()
1372    }
1373}
1374
1375/// Macro to reduce repetition in making type assertions on the physical type against `T`
1376macro_rules! ensure_phys_ty {
1377    ($($ty:pat_param)|+ , $($arg:tt)*) => {
1378        match T::get_physical_type() {
1379            $($ty => (),)*
1380            _ => panic!($($arg)*),
1381        };
1382    }
1383}
1384
1385#[cfg(test)]
1386mod tests {
1387    use super::*;
1388
1389    #[test]
1390    fn test_as_bytes() {
1391        // Test Int96
1392        let i96 = Int96::from(vec![1, 2, 3]);
1393        assert_eq!(i96.as_bytes(), &[1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0]);
1394
1395        // Test ByteArray
1396        let byte_arr = ByteArray::from(vec![1, 2, 3]);
1397        assert_eq!(byte_arr.as_bytes(), &[1, 2, 3]);
1398
1399        // Test Decimal
1400        let decimal = Decimal::from_i32(123, 5, 2);
1401        assert_eq!(decimal.as_bytes(), &[0, 0, 0, 123]);
1402        let decimal = Decimal::from_i64(123, 5, 2);
1403        assert_eq!(decimal.as_bytes(), &[0, 0, 0, 0, 0, 0, 0, 123]);
1404        let decimal = Decimal::from_bytes(ByteArray::from(vec![1, 2, 3]), 5, 2);
1405        assert_eq!(decimal.as_bytes(), &[1, 2, 3]);
1406    }
1407
1408    #[test]
1409    fn test_int96_from() {
1410        assert_eq!(
1411            Int96::from(vec![1, 12345, 1234567890]).data(),
1412            &[1, 12345, 1234567890]
1413        );
1414    }
1415
1416    #[test]
1417    fn test_byte_array_from() {
1418        assert_eq!(ByteArray::from(b"ABC".to_vec()).data(), b"ABC");
1419        assert_eq!(ByteArray::from("ABC").data(), b"ABC");
1420        assert_eq!(
1421            ByteArray::from(Bytes::from(vec![1u8, 2u8, 3u8, 4u8, 5u8])).data(),
1422            &[1u8, 2u8, 3u8, 4u8, 5u8]
1423        );
1424        let buf = vec![6u8, 7u8, 8u8, 9u8, 10u8];
1425        assert_eq!(ByteArray::from(buf).data(), &[6u8, 7u8, 8u8, 9u8, 10u8]);
1426    }
1427
1428    #[test]
1429    fn test_decimal_partial_eq() {
1430        assert_eq!(Decimal::default(), Decimal::from_i32(0, 0, 0));
1431        assert_eq!(Decimal::from_i32(222, 5, 2), Decimal::from_i32(222, 5, 2));
1432        assert_eq!(
1433            Decimal::from_bytes(ByteArray::from(vec![0, 0, 0, 3]), 5, 2),
1434            Decimal::from_i32(3, 5, 2)
1435        );
1436
1437        assert_ne!(Decimal::from_i32(222, 5, 2), Decimal::from_i32(111, 5, 2));
1438        assert_ne!(Decimal::from_i32(222, 5, 2), Decimal::from_i32(222, 6, 2));
1439        assert_ne!(Decimal::from_i32(222, 5, 2), Decimal::from_i32(222, 5, 3));
1440
1441        assert_ne!(Decimal::from_i64(222, 5, 2), Decimal::from_i32(222, 5, 2));
1442    }
1443
1444    #[test]
1445    fn test_byte_array_ord() {
1446        let byte_arr1 = ByteArray::from(vec![1, 2, 3]);
1447        let byte_arr11 = ByteArray::from(vec![1, 2, 3]);
1448        let byte_arr2 = ByteArray::from(vec![3, 4]);
1449        let byte_arr3 = ByteArray::from(vec![1, 2, 4]);
1450        let byte_arr4 = ByteArray::from(vec![]);
1451        let byte_arr5 = ByteArray::from(vec![2, 2, 3]);
1452
1453        assert!(byte_arr1 < byte_arr2);
1454        assert!(byte_arr3 > byte_arr1);
1455        assert!(byte_arr1 > byte_arr4);
1456        assert_eq!(byte_arr1, byte_arr11);
1457        assert!(byte_arr5 > byte_arr1);
1458    }
1459}
parquet/data_type.rs

parquet/
data_type.rs
