parquet_variant/
decoder.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17use crate::ShortString;
18use crate::utils::{
19    array_from_slice, overflow_error, slice_from_slice_at_offset, string_from_slice,
20};
21
22use arrow_schema::ArrowError;
23use chrono::{DateTime, Duration, NaiveDate, NaiveDateTime, NaiveTime, Utc};
24use uuid::Uuid;
25
26/// The basic type of a [`Variant`] value, encoded in the first two bits of the
27/// header byte.
28///
29/// See the [Variant Encoding specification] for details
30///
31/// [`Variant`]: crate::Variant
32/// [Variant Encoding specification]: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md#encoding-types
33#[derive(Debug, Clone, Copy, PartialEq)]
34pub enum VariantBasicType {
35    Primitive = 0,
36    ShortString = 1,
37    Object = 2,
38    Array = 3,
39}
40
41/// The type of [`VariantBasicType::Primitive`], for a primitive [`Variant`]
42/// value.
43///
44/// See the [Variant Encoding specification] for details
45///
46/// [`Variant`]: crate::Variant
47/// [Variant Encoding specification]: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md#encoding-types
48#[derive(Debug, Clone, Copy, PartialEq)]
49pub enum VariantPrimitiveType {
50    Null = 0,
51    BooleanTrue = 1,
52    BooleanFalse = 2,
53    Int8 = 3,
54    Int16 = 4,
55    Int32 = 5,
56    Int64 = 6,
57    Double = 7,
58    Decimal4 = 8,
59    Decimal8 = 9,
60    Decimal16 = 10,
61    Date = 11,
62    TimestampMicros = 12,
63    TimestampNtzMicros = 13,
64    Float = 14,
65    Binary = 15,
66    String = 16,
67    Time = 17,
68    TimestampNanos = 18,
69    TimestampNtzNanos = 19,
70    Uuid = 20,
71}
72
73/// Extracts the basic type from a header byte
74pub(crate) fn get_basic_type(header: u8) -> VariantBasicType {
75    // See https://github.com/apache/parquet-format/blob/master/VariantEncoding.md#value-encoding
76    let basic_type = header & 0x03; // Basic type is encoded in the first 2 bits
77    match basic_type {
78        0 => VariantBasicType::Primitive,
79        1 => VariantBasicType::ShortString,
80        2 => VariantBasicType::Object,
81        3 => VariantBasicType::Array,
82        _ => {
83            //NOTE:  A 2-bit value has a max of 4 different values (0-3), hence this is unreachable as we
84            // masked `basic_type` with 0x03 above.
85            unreachable!();
86        }
87    }
88}
89
90impl TryFrom<u8> for VariantPrimitiveType {
91    type Error = ArrowError;
92
93    fn try_from(value: u8) -> Result<Self, Self::Error> {
94        match value {
95            0 => Ok(VariantPrimitiveType::Null),
96            1 => Ok(VariantPrimitiveType::BooleanTrue),
97            2 => Ok(VariantPrimitiveType::BooleanFalse),
98            3 => Ok(VariantPrimitiveType::Int8),
99            4 => Ok(VariantPrimitiveType::Int16),
100            5 => Ok(VariantPrimitiveType::Int32),
101            6 => Ok(VariantPrimitiveType::Int64),
102            7 => Ok(VariantPrimitiveType::Double),
103            8 => Ok(VariantPrimitiveType::Decimal4),
104            9 => Ok(VariantPrimitiveType::Decimal8),
105            10 => Ok(VariantPrimitiveType::Decimal16),
106            11 => Ok(VariantPrimitiveType::Date),
107            12 => Ok(VariantPrimitiveType::TimestampMicros),
108            13 => Ok(VariantPrimitiveType::TimestampNtzMicros),
109            14 => Ok(VariantPrimitiveType::Float),
110            15 => Ok(VariantPrimitiveType::Binary),
111            16 => Ok(VariantPrimitiveType::String),
112            17 => Ok(VariantPrimitiveType::Time),
113            18 => Ok(VariantPrimitiveType::TimestampNanos),
114            19 => Ok(VariantPrimitiveType::TimestampNtzNanos),
115            20 => Ok(VariantPrimitiveType::Uuid),
116            _ => Err(ArrowError::InvalidArgumentError(format!(
117                "unknown primitive type: {value}",
118            ))),
119        }
120    }
121}
122
123/// Used to unpack offset array entries such as metadata dictionary offsets or object/array value
124/// offsets. Also used to unpack object field ids. These are always derived from a two-bit
125/// `XXX_size_minus_one` field in the corresponding header byte.
126#[derive(Debug, Clone, Copy, PartialEq)]
127pub(crate) enum OffsetSizeBytes {
128    One = 1,
129    Two = 2,
130    Three = 3,
131    Four = 4,
132}
133
134impl OffsetSizeBytes {
135    /// Build from the `offset_size_minus_one` bits (see spec).
136    pub(crate) fn try_new(offset_size_minus_one: u8) -> Result<Self, ArrowError> {
137        use OffsetSizeBytes::*;
138        let result = match offset_size_minus_one {
139            0 => One,
140            1 => Two,
141            2 => Three,
142            3 => Four,
143            _ => {
144                return Err(ArrowError::InvalidArgumentError(
145                    "offset_size_minus_one must be 0–3".to_string(),
146                ));
147            }
148        };
149        Ok(result)
150    }
151
152    /// Return one unsigned little-endian value from `bytes`.
153    ///
154    /// * `bytes` – the byte buffer to index
155    /// * `index` – 0-based index into the buffer
156    ///
157    /// Each value is `self as u32` bytes wide (1, 2, 3 or 4), zero-extended to 32 bits as needed.
158    pub(crate) fn unpack_u32(&self, bytes: &[u8], index: usize) -> Result<u32, ArrowError> {
159        self.unpack_u32_at_offset(bytes, 0, index)
160    }
161
162    /// Return one unsigned little-endian value from `bytes`.
163    ///
164    /// * `bytes` – the byte buffer to index
165    /// * `byte_offset` – number of bytes to skip **before** reading the first
166    ///   value (e.g. `1` to move past a header byte).
167    /// * `offset_index` – 0-based index **after** the skipped bytes
168    ///   (`0` is the first value, `1` the next, …).
169    ///
170    /// Each value is `self as u32` bytes wide (1, 2, 3 or 4), zero-extended to 32 bits as needed.
171    pub(crate) fn unpack_u32_at_offset(
172        &self,
173        bytes: &[u8],
174        byte_offset: usize,  // how many bytes to skip
175        offset_index: usize, // which offset in an array of offsets
176    ) -> Result<u32, ArrowError> {
177        use OffsetSizeBytes::*;
178
179        // Index into the byte array:
180        // byte_offset + (*self as usize) * offset_index
181        let offset = offset_index
182            .checked_mul(*self as usize)
183            .and_then(|n| n.checked_add(byte_offset))
184            .ok_or_else(|| overflow_error("unpacking offset array value"))?;
185        let value = match self {
186            One => u8::from_le_bytes(array_from_slice(bytes, offset)?).into(),
187            Two => u16::from_le_bytes(array_from_slice(bytes, offset)?).into(),
188            Three => {
189                // Let's grab the three byte le-chunk first
190                let b3_chunks: [u8; 3] = array_from_slice(bytes, offset)?;
191                // Let's pad it and construct a padded u32 from it.
192                let mut buf = [0u8; 4];
193                buf[..3].copy_from_slice(&b3_chunks);
194                u32::from_le_bytes(buf)
195            }
196            Four => u32::from_le_bytes(array_from_slice(bytes, offset)?),
197        };
198        Ok(value)
199    }
200}
201
202/// Converts a byte buffer to offset values based on the specific offset size
203pub(crate) fn map_bytes_to_offsets(
204    buffer: &[u8],
205    offset_size: OffsetSizeBytes,
206) -> impl Iterator<Item = usize> + use<'_> {
207    buffer
208        .chunks_exact(offset_size as usize)
209        .map(move |chunk| match offset_size {
210            OffsetSizeBytes::One => chunk[0] as usize,
211            OffsetSizeBytes::Two => u16::from_le_bytes([chunk[0], chunk[1]]) as usize,
212            OffsetSizeBytes::Three => {
213                u32::from_le_bytes([chunk[0], chunk[1], chunk[2], 0]) as usize
214            }
215            OffsetSizeBytes::Four => {
216                u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]) as usize
217            }
218        })
219}
220
221/// Extract the primitive type from a Variant value-metadata byte
222pub(crate) fn get_primitive_type(metadata: u8) -> Result<VariantPrimitiveType, ArrowError> {
223    // last 6 bits contain the primitive-type, see spec
224    VariantPrimitiveType::try_from(metadata >> 2)
225}
226
227/// Decodes an Int8 from the value section of a variant.
228pub(crate) fn decode_int8(data: &[u8]) -> Result<i8, ArrowError> {
229    Ok(i8::from_le_bytes(array_from_slice(data, 0)?))
230}
231
232/// Decodes an Int16 from the value section of a variant.
233pub(crate) fn decode_int16(data: &[u8]) -> Result<i16, ArrowError> {
234    Ok(i16::from_le_bytes(array_from_slice(data, 0)?))
235}
236
237/// Decodes an Int32 from the value section of a variant.
238pub(crate) fn decode_int32(data: &[u8]) -> Result<i32, ArrowError> {
239    Ok(i32::from_le_bytes(array_from_slice(data, 0)?))
240}
241
242/// Decodes an Int64 from the value section of a variant.
243pub(crate) fn decode_int64(data: &[u8]) -> Result<i64, ArrowError> {
244    Ok(i64::from_le_bytes(array_from_slice(data, 0)?))
245}
246
247/// Decodes a Decimal4 from the value section of a variant.
248pub(crate) fn decode_decimal4(data: &[u8]) -> Result<(i32, u8), ArrowError> {
249    let scale = u8::from_le_bytes(array_from_slice(data, 0)?);
250    let integer = i32::from_le_bytes(array_from_slice(data, 1)?);
251    Ok((integer, scale))
252}
253
254/// Decodes a Decimal8 from the value section of a variant.
255pub(crate) fn decode_decimal8(data: &[u8]) -> Result<(i64, u8), ArrowError> {
256    let scale = u8::from_le_bytes(array_from_slice(data, 0)?);
257    let integer = i64::from_le_bytes(array_from_slice(data, 1)?);
258    Ok((integer, scale))
259}
260
261/// Decodes a Decimal16 from the value section of a variant.
262pub(crate) fn decode_decimal16(data: &[u8]) -> Result<(i128, u8), ArrowError> {
263    let scale = u8::from_le_bytes(array_from_slice(data, 0)?);
264    let integer = i128::from_le_bytes(array_from_slice(data, 1)?);
265    Ok((integer, scale))
266}
267
268/// Decodes a Float from the value section of a variant.
269pub(crate) fn decode_float(data: &[u8]) -> Result<f32, ArrowError> {
270    Ok(f32::from_le_bytes(array_from_slice(data, 0)?))
271}
272
273/// Decodes a Double from the value section of a variant.
274pub(crate) fn decode_double(data: &[u8]) -> Result<f64, ArrowError> {
275    Ok(f64::from_le_bytes(array_from_slice(data, 0)?))
276}
277
278/// Decodes a Date from the value section of a variant.
279pub(crate) fn decode_date(data: &[u8]) -> Result<NaiveDate, ArrowError> {
280    let days_since_epoch = i32::from_le_bytes(array_from_slice(data, 0)?);
281    DateTime::UNIX_EPOCH
282        .checked_add_signed(Duration::days(i64::from(days_since_epoch)))
283        .map(|value| value.date_naive())
284        .ok_or_else(|| {
285            ArrowError::CastError(format!(
286                "Could not cast `{days_since_epoch}` days into a NaiveDate"
287            ))
288        })
289}
290
291/// Decodes a TimestampMicros from the value section of a variant.
292pub(crate) fn decode_timestamp_micros(data: &[u8]) -> Result<DateTime<Utc>, ArrowError> {
293    let micros_since_epoch = i64::from_le_bytes(array_from_slice(data, 0)?);
294    DateTime::from_timestamp_micros(micros_since_epoch).ok_or_else(|| {
295        ArrowError::CastError(format!(
296            "Could not cast `{micros_since_epoch}` microseconds into a DateTime<Utc>"
297        ))
298    })
299}
300
301/// Decodes a TimestampNtzMicros from the value section of a variant.
302pub(crate) fn decode_timestampntz_micros(data: &[u8]) -> Result<NaiveDateTime, ArrowError> {
303    let micros_since_epoch = i64::from_le_bytes(array_from_slice(data, 0)?);
304    DateTime::from_timestamp_micros(micros_since_epoch)
305        .ok_or_else(|| {
306            ArrowError::CastError(format!(
307                "Could not cast `{micros_since_epoch}` microseconds into a NaiveDateTime"
308            ))
309        })
310        .map(|v| v.naive_utc())
311}
312
313pub(crate) fn decode_time_ntz(data: &[u8]) -> Result<NaiveTime, ArrowError> {
314    let micros_since_epoch = u64::from_le_bytes(array_from_slice(data, 0)?);
315
316    let case_error = ArrowError::CastError(format!(
317        "Could not cast {micros_since_epoch} microseconds into a NaiveTime"
318    ));
319
320    if micros_since_epoch >= 86_400_000_000 {
321        return Err(case_error);
322    }
323
324    let nanos_since_midnight = micros_since_epoch * 1_000;
325    NaiveTime::from_num_seconds_from_midnight_opt(
326        (nanos_since_midnight / 1_000_000_000) as u32,
327        (nanos_since_midnight % 1_000_000_000) as u32,
328    )
329    .ok_or(case_error)
330}
331
332/// Decodes a TimestampNanos from the value section of a variant.
333pub(crate) fn decode_timestamp_nanos(data: &[u8]) -> Result<DateTime<Utc>, ArrowError> {
334    let nanos_since_epoch = i64::from_le_bytes(array_from_slice(data, 0)?);
335
336    // DateTime::from_timestamp_nanos would never fail
337    Ok(DateTime::from_timestamp_nanos(nanos_since_epoch))
338}
339
340/// Decodes a TimestampNtzNanos from the value section of a variant.
341pub(crate) fn decode_timestampntz_nanos(data: &[u8]) -> Result<NaiveDateTime, ArrowError> {
342    decode_timestamp_nanos(data).map(|v| v.naive_utc())
343}
344
345/// Decodes a UUID from the value section of a variant.
346pub(crate) fn decode_uuid(data: &[u8]) -> Result<Uuid, ArrowError> {
347    let bytes: [u8; 16] = array_from_slice(data, 0)?;
348    Ok(Uuid::from_bytes(bytes))
349}
350
351/// Decodes a Binary from the value section of a variant.
352pub(crate) fn decode_binary(data: &[u8]) -> Result<&[u8], ArrowError> {
353    let len = u32::from_le_bytes(array_from_slice(data, 0)?) as usize;
354    slice_from_slice_at_offset(data, 4, 0..len)
355}
356
357/// Decodes a long string from the value section of a variant.
358pub(crate) fn decode_long_string(data: &[u8]) -> Result<&str, ArrowError> {
359    let len = u32::from_le_bytes(array_from_slice(data, 0)?) as usize;
360    string_from_slice(data, 4, 0..len)
361}
362
363/// Decodes a short string from the value section of a variant.
364pub(crate) fn decode_short_string(
365    metadata: u8,
366    data: &[u8],
367) -> Result<ShortString<'_>, ArrowError> {
368    let len = (metadata >> 2) as usize;
369    let string = string_from_slice(data, 0, 0..len)?;
370    ShortString::try_new(string)
371}
372
373#[cfg(test)]
374mod tests {
375    use super::*;
376
377    macro_rules! test_decoder_bounds {
378        ($test_name:ident, $data:expr, $decode_fn:ident, $expected:expr) => {
379            mod $test_name {
380                use super::*;
381
382                #[test]
383                fn exact_length() {
384                    let result = $decode_fn(&$data).unwrap();
385                    assert_eq!(result, $expected);
386                }
387
388                #[test]
389                fn truncated_length() {
390                    // Remove the last byte of data so that there is not enough to decode
391                    let truncated_data = &$data[..$data.len() - 1];
392                    let result = $decode_fn(truncated_data);
393                    assert!(matches!(result, Err(ArrowError::InvalidArgumentError(_))));
394                }
395            }
396        };
397    }
398
399    mod integer {
400        use super::*;
401
402        test_decoder_bounds!(test_i8, [0x2a], decode_int8, 42);
403        test_decoder_bounds!(test_i16, [0xd2, 0x04], decode_int16, 1234);
404        test_decoder_bounds!(test_i32, [0x40, 0xe2, 0x01, 0x00], decode_int32, 123456);
405        test_decoder_bounds!(
406            test_i64,
407            [0x15, 0x81, 0xe9, 0x7d, 0xf4, 0x10, 0x22, 0x11],
408            decode_int64,
409            1234567890123456789
410        );
411    }
412
413    mod decimal {
414        use super::*;
415
416        test_decoder_bounds!(
417            test_decimal4,
418            [
419                0x02, // Scale
420                0xd2, 0x04, 0x00, 0x00, // Unscaled Value
421            ],
422            decode_decimal4,
423            (1234, 2)
424        );
425
426        test_decoder_bounds!(
427            test_decimal8,
428            [
429                0x02, // Scale
430                0xd2, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, // Unscaled Value
431            ],
432            decode_decimal8,
433            (1234567890, 2)
434        );
435
436        test_decoder_bounds!(
437            test_decimal16,
438            [
439                0x02, // Scale
440                0xd2, 0xb6, 0x23, 0xc0, 0xf4, 0x10, 0x22, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
441                0x00, 0x00, // Unscaled Value
442            ],
443            decode_decimal16,
444            (1234567891234567890, 2)
445        );
446    }
447
448    #[expect(clippy::float_cmp_const)]
449    mod float {
450        use super::*;
451
452        test_decoder_bounds!(
453            test_float,
454            [0x06, 0x2c, 0x93, 0x4e],
455            decode_float,
456            1_234_568_000.0
457        );
458
459        test_decoder_bounds!(
460            test_double,
461            [0xc9, 0xe5, 0x87, 0xb4, 0x80, 0x65, 0xd2, 0x41],
462            decode_double,
463            1234567890.1234
464        );
465    }
466
467    mod datetime {
468        use super::*;
469
470        test_decoder_bounds!(
471            test_date,
472            [0xe2, 0x4e, 0x0, 0x0],
473            decode_date,
474            NaiveDate::from_ymd_opt(2025, 4, 16).unwrap()
475        );
476
477        #[test]
478        fn test_date_out_of_range() {
479            // A day count that overflows chrono's supported date range must error, not panic
480            for days in [i32::MAX, i32::MIN] {
481                let result = decode_date(&days.to_le_bytes());
482                assert!(matches!(result, Err(ArrowError::CastError(_))));
483            }
484        }
485
486        test_decoder_bounds!(
487            test_timestamp_micros,
488            [0xe0, 0x52, 0x97, 0xdd, 0xe7, 0x32, 0x06, 0x00],
489            decode_timestamp_micros,
490            NaiveDate::from_ymd_opt(2025, 4, 16)
491                .unwrap()
492                .and_hms_milli_opt(16, 34, 56, 780)
493                .unwrap()
494                .and_utc()
495        );
496
497        test_decoder_bounds!(
498            test_timestampntz_micros,
499            [0xe0, 0x52, 0x97, 0xdd, 0xe7, 0x32, 0x06, 0x00],
500            decode_timestampntz_micros,
501            NaiveDate::from_ymd_opt(2025, 4, 16)
502                .unwrap()
503                .and_hms_milli_opt(16, 34, 56, 780)
504                .unwrap()
505        );
506
507        test_decoder_bounds!(
508            test_timestamp_nanos,
509            [0x15, 0x41, 0xa2, 0x5a, 0x36, 0xa2, 0x5b, 0x18],
510            decode_timestamp_nanos,
511            NaiveDate::from_ymd_opt(2025, 8, 14)
512                .unwrap()
513                .and_hms_nano_opt(12, 33, 54, 123456789)
514                .unwrap()
515                .and_utc()
516        );
517
518        test_decoder_bounds!(
519            test_timestamp_nanos_before_epoch,
520            [0x15, 0x41, 0x52, 0xd4, 0x94, 0xe5, 0xad, 0xfa],
521            decode_timestamp_nanos,
522            NaiveDate::from_ymd_opt(1957, 11, 7)
523                .unwrap()
524                .and_hms_nano_opt(12, 33, 54, 123456789)
525                .unwrap()
526                .and_utc()
527        );
528
529        test_decoder_bounds!(
530            test_timestampntz_nanos,
531            [0x15, 0x41, 0xa2, 0x5a, 0x36, 0xa2, 0x5b, 0x18],
532            decode_timestampntz_nanos,
533            NaiveDate::from_ymd_opt(2025, 8, 14)
534                .unwrap()
535                .and_hms_nano_opt(12, 33, 54, 123456789)
536                .unwrap()
537        );
538
539        test_decoder_bounds!(
540            test_timestampntz_nanos_before_epoch,
541            [0x15, 0x41, 0x52, 0xd4, 0x94, 0xe5, 0xad, 0xfa],
542            decode_timestampntz_nanos,
543            NaiveDate::from_ymd_opt(1957, 11, 7)
544                .unwrap()
545                .and_hms_nano_opt(12, 33, 54, 123456789)
546                .unwrap()
547        );
548    }
549
550    test_decoder_bounds!(
551        test_uuid,
552        [
553            0xf2, 0x4f, 0x9b, 0x64, 0x81, 0xfa, 0x49, 0xd1, 0xb7, 0x4e, 0x8c, 0x09, 0xa6, 0xe3,
554            0x1c, 0x56,
555        ],
556        decode_uuid,
557        Uuid::parse_str("f24f9b64-81fa-49d1-b74e-8c09a6e31c56").unwrap()
558    );
559
560    mod time {
561        use super::*;
562
563        test_decoder_bounds!(
564            test_timentz,
565            [0x53, 0x1f, 0x8e, 0xdf, 0x2, 0, 0, 0],
566            decode_time_ntz,
567            NaiveTime::from_num_seconds_from_midnight_opt(12340, 567_891_000).unwrap()
568        );
569
570        #[test]
571        fn test_decode_time_ntz_invalid() {
572            let invalid_second = u64::MAX;
573            let data = invalid_second.to_le_bytes();
574            let result = decode_time_ntz(&data);
575            assert!(matches!(result, Err(ArrowError::CastError(_))));
576        }
577    }
578
579    #[test]
580    fn test_binary_exact_length() {
581        let data = [
582            0x09, 0, 0, 0, // Length of binary data, 4-byte little-endian
583            0x03, 0x13, 0x37, 0xde, 0xad, 0xbe, 0xef, 0xca, 0xfe,
584        ];
585        let result = decode_binary(&data).unwrap();
586        assert_eq!(
587            result,
588            [0x03, 0x13, 0x37, 0xde, 0xad, 0xbe, 0xef, 0xca, 0xfe]
589        );
590    }
591
592    #[test]
593    fn test_binary_truncated_length() {
594        let data = [
595            0x09, 0, 0, 0, // Length of binary data, 4-byte little-endian
596            0x03, 0x13, 0x37, 0xde, 0xad, 0xbe, 0xef, 0xca,
597        ];
598        let result = decode_binary(&data);
599        assert!(matches!(result, Err(ArrowError::InvalidArgumentError(_))));
600    }
601
602    #[test]
603    fn test_short_string_exact_length() {
604        let data = b"Helloo";
605        let result = decode_short_string(1 | (5 << 2), data).unwrap();
606        assert_eq!(result.0, "Hello");
607    }
608
609    #[test]
610    fn test_short_string_truncated_length() {
611        let data = b"Abc";
612        let result = decode_short_string(1 | (5 << 2), data);
613        assert!(matches!(result, Err(ArrowError::InvalidArgumentError(_))));
614    }
615
616    #[test]
617    fn test_string_exact_length() {
618        let data = [
619            0x05, 0, 0, 0, // Length of string, 4-byte little-endian
620            b'H', b'e', b'l', b'l', b'o', b'o',
621        ];
622        let result = decode_long_string(&data).unwrap();
623        assert_eq!(result, "Hello");
624    }
625
626    #[test]
627    fn test_string_truncated_length() {
628        let data = [
629            0x05, 0, 0, 0, // Length of string, 4-byte little-endian
630            b'H', b'e', b'l',
631        ];
632        let result = decode_long_string(&data);
633        assert!(matches!(result, Err(ArrowError::InvalidArgumentError(_))));
634    }
635
636    #[test]
637    fn test_offset() {
638        assert_eq!(OffsetSizeBytes::try_new(0).unwrap(), OffsetSizeBytes::One);
639        assert_eq!(OffsetSizeBytes::try_new(1).unwrap(), OffsetSizeBytes::Two);
640        assert_eq!(OffsetSizeBytes::try_new(2).unwrap(), OffsetSizeBytes::Three);
641        assert_eq!(OffsetSizeBytes::try_new(3).unwrap(), OffsetSizeBytes::Four);
642
643        // everything outside 0-3 must error
644        assert!(OffsetSizeBytes::try_new(4).is_err());
645        assert!(OffsetSizeBytes::try_new(255).is_err());
646    }
647
648    #[test]
649    fn unpack_u32_all_widths() {
650        // One-byte offsets
651        let buf_one = [0x01u8, 0xAB, 0xCD];
652        assert_eq!(OffsetSizeBytes::One.unpack_u32(&buf_one, 0).unwrap(), 0x01);
653        assert_eq!(OffsetSizeBytes::One.unpack_u32(&buf_one, 2).unwrap(), 0xCD);
654
655        // Two-byte offsets (little-endian 0x1234, 0x5678)
656        let buf_two = [0x34, 0x12, 0x78, 0x56];
657        assert_eq!(
658            OffsetSizeBytes::Two.unpack_u32(&buf_two, 0).unwrap(),
659            0x1234
660        );
661        assert_eq!(
662            OffsetSizeBytes::Two.unpack_u32(&buf_two, 1).unwrap(),
663            0x5678
664        );
665
666        // Three-byte offsets (0x030201 and 0x0000FF)
667        let buf_three = [0x01, 0x02, 0x03, 0xFF, 0x00, 0x00];
668        assert_eq!(
669            OffsetSizeBytes::Three.unpack_u32(&buf_three, 0).unwrap(),
670            0x030201
671        );
672        assert_eq!(
673            OffsetSizeBytes::Three.unpack_u32(&buf_three, 1).unwrap(),
674            0x0000FF
675        );
676
677        // Four-byte offsets (0x12345678, 0x90ABCDEF)
678        let buf_four = [0x78, 0x56, 0x34, 0x12, 0xEF, 0xCD, 0xAB, 0x90];
679        assert_eq!(
680            OffsetSizeBytes::Four.unpack_u32(&buf_four, 0).unwrap(),
681            0x1234_5678
682        );
683        assert_eq!(
684            OffsetSizeBytes::Four.unpack_u32(&buf_four, 1).unwrap(),
685            0x90AB_CDEF
686        );
687    }
688
689    #[test]
690    fn unpack_u32_out_of_bounds() {
691        let tiny = [0x00u8]; // deliberately too short
692        assert!(OffsetSizeBytes::Two.unpack_u32(&tiny, 0).is_err());
693        assert!(OffsetSizeBytes::Three.unpack_u32(&tiny, 0).is_err());
694    }
695
696    #[test]
697    fn unpack_simple() {
698        let buf = [
699            0x41, // header
700            0x02, 0x00, // dictionary_size = 2
701            0x00, 0x00, // offset[0] = 0
702            0x05, 0x00, // offset[1] = 5
703            0x09, 0x00, // offset[2] = 9
704        ];
705
706        let width = OffsetSizeBytes::Two;
707
708        // dictionary_size starts immediately after the header byte
709        let dict_size = width.unpack_u32_at_offset(&buf, 1, 0).unwrap();
710        assert_eq!(dict_size, 2);
711
712        // offset array immediately follows the dictionary size
713        let first = width.unpack_u32_at_offset(&buf, 1, 1).unwrap();
714        assert_eq!(first, 0);
715
716        let second = width.unpack_u32_at_offset(&buf, 1, 2).unwrap();
717        assert_eq!(second, 5);
718
719        let third = width.unpack_u32_at_offset(&buf, 1, 3).unwrap();
720        assert_eq!(third, 9);
721
722        let err = width.unpack_u32_at_offset(&buf, 1, 4);
723        assert!(err.is_err())
724    }
725}
parquet_variant/decoder.rs

parquet_variant/
decoder.rs
