Skip to main content

parquet_variant/
decoder.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use crate::ShortString;
6use crate::utils::{
7    array_from_slice, overflow_error, slice_from_slice_at_offset, string_from_slice,
8};
9
10use arrow_schema::ArrowError;
11use chrono::{DateTime, Duration, NaiveDate, NaiveDateTime, NaiveTime, Utc};
12use uuid::Uuid;
13
14/// The basic type of a [`Variant`] value, encoded in the first two bits of the
15/// header byte.
16///
17/// See the [Variant Encoding specification] for details
18///
19/// [`Variant`]: crate::Variant
20/// [Variant Encoding specification]: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md#encoding-types
21#[derive(Debug, Clone, Copy, PartialEq)]
22pub enum VariantBasicType {
23    Primitive = 0,
24    ShortString = 1,
25    Object = 2,
26    Array = 3,
27}
28
29/// The type of [`VariantBasicType::Primitive`], for a primitive [`Variant`]
30/// value.
31///
32/// See the [Variant Encoding specification] for details
33///
34/// [`Variant`]: crate::Variant
35/// [Variant Encoding specification]: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md#encoding-types
36#[derive(Debug, Clone, Copy, PartialEq)]
37pub enum VariantPrimitiveType {
38    Null = 0,
39    BooleanTrue = 1,
40    BooleanFalse = 2,
41    Int8 = 3,
42    Int16 = 4,
43    Int32 = 5,
44    Int64 = 6,
45    Double = 7,
46    Decimal4 = 8,
47    Decimal8 = 9,
48    Decimal16 = 10,
49    Date = 11,
50    TimestampMicros = 12,
51    TimestampNtzMicros = 13,
52    Float = 14,
53    Binary = 15,
54    String = 16,
55    Time = 17,
56    TimestampNanos = 18,
57    TimestampNtzNanos = 19,
58    Uuid = 20,
59}
60
61/// Extracts the basic type from a header byte
62pub(crate) fn get_basic_type(header: u8) -> VariantBasicType {
63    // See https://github.com/apache/parquet-format/blob/master/VariantEncoding.md#value-encoding
64    let basic_type = header & 0x03; // Basic type is encoded in the first 2 bits
65    match basic_type {
66        0 => VariantBasicType::Primitive,
67        1 => VariantBasicType::ShortString,
68        2 => VariantBasicType::Object,
69        3 => VariantBasicType::Array,
70        _ => {
71            //NOTE:  A 2-bit value has a max of 4 different values (0-3), hence this is unreachable as we
72            // masked `basic_type` with 0x03 above.
73            unreachable!();
74        }
75    }
76}
77
78impl TryFrom<u8> for VariantPrimitiveType {
79    type Error = ArrowError;
80
81    fn try_from(value: u8) -> Result<Self, Self::Error> {
82        match value {
83            0 => Ok(VariantPrimitiveType::Null),
84            1 => Ok(VariantPrimitiveType::BooleanTrue),
85            2 => Ok(VariantPrimitiveType::BooleanFalse),
86            3 => Ok(VariantPrimitiveType::Int8),
87            4 => Ok(VariantPrimitiveType::Int16),
88            5 => Ok(VariantPrimitiveType::Int32),
89            6 => Ok(VariantPrimitiveType::Int64),
90            7 => Ok(VariantPrimitiveType::Double),
91            8 => Ok(VariantPrimitiveType::Decimal4),
92            9 => Ok(VariantPrimitiveType::Decimal8),
93            10 => Ok(VariantPrimitiveType::Decimal16),
94            11 => Ok(VariantPrimitiveType::Date),
95            12 => Ok(VariantPrimitiveType::TimestampMicros),
96            13 => Ok(VariantPrimitiveType::TimestampNtzMicros),
97            14 => Ok(VariantPrimitiveType::Float),
98            15 => Ok(VariantPrimitiveType::Binary),
99            16 => Ok(VariantPrimitiveType::String),
100            17 => Ok(VariantPrimitiveType::Time),
101            18 => Ok(VariantPrimitiveType::TimestampNanos),
102            19 => Ok(VariantPrimitiveType::TimestampNtzNanos),
103            20 => Ok(VariantPrimitiveType::Uuid),
104            _ => Err(ArrowError::InvalidArgumentError(format!(
105                "unknown primitive type: {value}",
106            ))),
107        }
108    }
109}
110
111/// Used to unpack offset array entries such as metadata dictionary offsets or object/array value
112/// offsets. Also used to unpack object field ids. These are always derived from a two-bit
113/// `XXX_size_minus_one` field in the corresponding header byte.
114#[derive(Debug, Clone, Copy, PartialEq)]
115pub(crate) enum OffsetSizeBytes {
116    One = 1,
117    Two = 2,
118    Three = 3,
119    Four = 4,
120}
121
122impl OffsetSizeBytes {
123    /// Build from the `offset_size_minus_one` bits (see spec).
124    pub(crate) fn try_new(offset_size_minus_one: u8) -> Result<Self, ArrowError> {
125        use OffsetSizeBytes::*;
126        let result = match offset_size_minus_one {
127            0 => One,
128            1 => Two,
129            2 => Three,
130            3 => Four,
131            _ => {
132                return Err(ArrowError::InvalidArgumentError(
133                    "offset_size_minus_one must be 0–3".to_string(),
134                ));
135            }
136        };
137        Ok(result)
138    }
139
140    /// Return one unsigned little-endian value from `bytes`.
141    ///
142    /// * `bytes` – the byte buffer to index
143    /// * `index` – 0-based index into the buffer
144    ///
145    /// Each value is `self as u32` bytes wide (1, 2, 3 or 4), zero-extended to 32 bits as needed.
146    pub(crate) fn unpack_u32(&self, bytes: &[u8], index: usize) -> Result<u32, ArrowError> {
147        self.unpack_u32_at_offset(bytes, 0, index)
148    }
149
150    /// Return one unsigned little-endian value from `bytes`.
151    ///
152    /// * `bytes` – the byte buffer to index
153    /// * `byte_offset` – number of bytes to skip **before** reading the first
154    ///   value (e.g. `1` to move past a header byte).
155    /// * `offset_index` – 0-based index **after** the skipped bytes
156    ///   (`0` is the first value, `1` the next, …).
157    ///
158    /// Each value is `self as u32` bytes wide (1, 2, 3 or 4), zero-extended to 32 bits as needed.
159    pub(crate) fn unpack_u32_at_offset(
160        &self,
161        bytes: &[u8],
162        byte_offset: usize,  // how many bytes to skip
163        offset_index: usize, // which offset in an array of offsets
164    ) -> Result<u32, ArrowError> {
165        use OffsetSizeBytes::*;
166
167        // Index into the byte array:
168        // byte_offset + (*self as usize) * offset_index
169        let offset = offset_index
170            .checked_mul(*self as usize)
171            .and_then(|n| n.checked_add(byte_offset))
172            .ok_or_else(|| overflow_error("unpacking offset array value"))?;
173        let value = match self {
174            One => u8::from_le_bytes(array_from_slice(bytes, offset)?).into(),
175            Two => u16::from_le_bytes(array_from_slice(bytes, offset)?).into(),
176            Three => {
177                // Let's grab the three byte le-chunk first
178                let b3_chunks: [u8; 3] = array_from_slice(bytes, offset)?;
179                // Let's pad it and construct a padded u32 from it.
180                let mut buf = [0u8; 4];
181                buf[..3].copy_from_slice(&b3_chunks);
182                u32::from_le_bytes(buf)
183            }
184            Four => u32::from_le_bytes(array_from_slice(bytes, offset)?),
185        };
186        Ok(value)
187    }
188}
189
190/// Converts a byte buffer to offset values based on the specific offset size
191pub(crate) fn map_bytes_to_offsets(
192    buffer: &[u8],
193    offset_size: OffsetSizeBytes,
194) -> impl Iterator<Item = usize> + use<'_> {
195    buffer
196        .chunks_exact(offset_size as usize)
197        .map(move |chunk| match offset_size {
198            OffsetSizeBytes::One => chunk[0] as usize,
199            OffsetSizeBytes::Two => u16::from_le_bytes([chunk[0], chunk[1]]) as usize,
200            OffsetSizeBytes::Three => {
201                u32::from_le_bytes([chunk[0], chunk[1], chunk[2], 0]) as usize
202            }
203            OffsetSizeBytes::Four => {
204                u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]) as usize
205            }
206        })
207}
208
209/// Extract the primitive type from a Variant value-metadata byte
210pub(crate) fn get_primitive_type(metadata: u8) -> Result<VariantPrimitiveType, ArrowError> {
211    // last 6 bits contain the primitive-type, see spec
212    VariantPrimitiveType::try_from(metadata >> 2)
213}
214
215/// Decodes an Int8 from the value section of a variant.
216pub(crate) fn decode_int8(data: &[u8]) -> Result<i8, ArrowError> {
217    Ok(i8::from_le_bytes(array_from_slice(data, 0)?))
218}
219
220/// Decodes an Int16 from the value section of a variant.
221pub(crate) fn decode_int16(data: &[u8]) -> Result<i16, ArrowError> {
222    Ok(i16::from_le_bytes(array_from_slice(data, 0)?))
223}
224
225/// Decodes an Int32 from the value section of a variant.
226pub(crate) fn decode_int32(data: &[u8]) -> Result<i32, ArrowError> {
227    Ok(i32::from_le_bytes(array_from_slice(data, 0)?))
228}
229
230/// Decodes an Int64 from the value section of a variant.
231pub(crate) fn decode_int64(data: &[u8]) -> Result<i64, ArrowError> {
232    Ok(i64::from_le_bytes(array_from_slice(data, 0)?))
233}
234
235/// Decodes a Decimal4 from the value section of a variant.
236pub(crate) fn decode_decimal4(data: &[u8]) -> Result<(i32, u8), ArrowError> {
237    let scale = u8::from_le_bytes(array_from_slice(data, 0)?);
238    let integer = i32::from_le_bytes(array_from_slice(data, 1)?);
239    Ok((integer, scale))
240}
241
242/// Decodes a Decimal8 from the value section of a variant.
243pub(crate) fn decode_decimal8(data: &[u8]) -> Result<(i64, u8), ArrowError> {
244    let scale = u8::from_le_bytes(array_from_slice(data, 0)?);
245    let integer = i64::from_le_bytes(array_from_slice(data, 1)?);
246    Ok((integer, scale))
247}
248
249/// Decodes a Decimal16 from the value section of a variant.
250pub(crate) fn decode_decimal16(data: &[u8]) -> Result<(i128, u8), ArrowError> {
251    let scale = u8::from_le_bytes(array_from_slice(data, 0)?);
252    let integer = i128::from_le_bytes(array_from_slice(data, 1)?);
253    Ok((integer, scale))
254}
255
256/// Decodes a Float from the value section of a variant.
257pub(crate) fn decode_float(data: &[u8]) -> Result<f32, ArrowError> {
258    Ok(f32::from_le_bytes(array_from_slice(data, 0)?))
259}
260
261/// Decodes a Double from the value section of a variant.
262pub(crate) fn decode_double(data: &[u8]) -> Result<f64, ArrowError> {
263    Ok(f64::from_le_bytes(array_from_slice(data, 0)?))
264}
265
266/// Decodes a Date from the value section of a variant.
267pub(crate) fn decode_date(data: &[u8]) -> Result<NaiveDate, ArrowError> {
268    let days_since_epoch = i32::from_le_bytes(array_from_slice(data, 0)?);
269    DateTime::UNIX_EPOCH
270        .checked_add_signed(Duration::days(i64::from(days_since_epoch)))
271        .map(|value| value.date_naive())
272        .ok_or_else(|| {
273            ArrowError::CastError(format!(
274                "Could not cast `{days_since_epoch}` days into a NaiveDate"
275            ))
276        })
277}
278
279/// Decodes a TimestampMicros from the value section of a variant.
280pub(crate) fn decode_timestamp_micros(data: &[u8]) -> Result<DateTime<Utc>, ArrowError> {
281    let micros_since_epoch = i64::from_le_bytes(array_from_slice(data, 0)?);
282    DateTime::from_timestamp_micros(micros_since_epoch).ok_or_else(|| {
283        ArrowError::CastError(format!(
284            "Could not cast `{micros_since_epoch}` microseconds into a DateTime<Utc>"
285        ))
286    })
287}
288
289/// Decodes a TimestampNtzMicros from the value section of a variant.
290pub(crate) fn decode_timestampntz_micros(data: &[u8]) -> Result<NaiveDateTime, ArrowError> {
291    let micros_since_epoch = i64::from_le_bytes(array_from_slice(data, 0)?);
292    DateTime::from_timestamp_micros(micros_since_epoch)
293        .ok_or_else(|| {
294            ArrowError::CastError(format!(
295                "Could not cast `{micros_since_epoch}` microseconds into a NaiveDateTime"
296            ))
297        })
298        .map(|v| v.naive_utc())
299}
300
301pub(crate) fn decode_time_ntz(data: &[u8]) -> Result<NaiveTime, ArrowError> {
302    let micros_since_epoch = u64::from_le_bytes(array_from_slice(data, 0)?);
303
304    let case_error = ArrowError::CastError(format!(
305        "Could not cast {micros_since_epoch} microseconds into a NaiveTime"
306    ));
307
308    if micros_since_epoch >= 86_400_000_000 {
309        return Err(case_error);
310    }
311
312    let nanos_since_midnight = micros_since_epoch * 1_000;
313    NaiveTime::from_num_seconds_from_midnight_opt(
314        (nanos_since_midnight / 1_000_000_000) as u32,
315        (nanos_since_midnight % 1_000_000_000) as u32,
316    )
317    .ok_or(case_error)
318}
319
320/// Decodes a TimestampNanos from the value section of a variant.
321pub(crate) fn decode_timestamp_nanos(data: &[u8]) -> Result<DateTime<Utc>, ArrowError> {
322    let nanos_since_epoch = i64::from_le_bytes(array_from_slice(data, 0)?);
323
324    // DateTime::from_timestamp_nanos would never fail
325    Ok(DateTime::from_timestamp_nanos(nanos_since_epoch))
326}
327
328/// Decodes a TimestampNtzNanos from the value section of a variant.
329pub(crate) fn decode_timestampntz_nanos(data: &[u8]) -> Result<NaiveDateTime, ArrowError> {
330    decode_timestamp_nanos(data).map(|v| v.naive_utc())
331}
332
333/// Decodes a UUID from the value section of a variant.
334pub(crate) fn decode_uuid(data: &[u8]) -> Result<Uuid, ArrowError> {
335    let bytes: [u8; 16] = array_from_slice(data, 0)?;
336    Ok(Uuid::from_bytes(bytes))
337}
338
339/// Decodes a Binary from the value section of a variant.
340pub(crate) fn decode_binary(data: &[u8]) -> Result<&[u8], ArrowError> {
341    let len = u32::from_le_bytes(array_from_slice(data, 0)?) as usize;
342    slice_from_slice_at_offset(data, 4, 0..len)
343}
344
345/// Decodes a long string from the value section of a variant.
346pub(crate) fn decode_long_string(data: &[u8]) -> Result<&str, ArrowError> {
347    let len = u32::from_le_bytes(array_from_slice(data, 0)?) as usize;
348    string_from_slice(data, 4, 0..len)
349}
350
351/// Decodes a short string from the value section of a variant.
352pub(crate) fn decode_short_string(
353    metadata: u8,
354    data: &[u8],
355) -> Result<ShortString<'_>, ArrowError> {
356    let len = (metadata >> 2) as usize;
357    let string = string_from_slice(data, 0, 0..len)?;
358    ShortString::try_new(string)
359}
360
361#[cfg(test)]
362mod tests {
363    use super::*;
364
365    macro_rules! test_decoder_bounds {
366        ($test_name:ident, $data:expr, $decode_fn:ident, $expected:expr) => {
367            mod $test_name {
368                use super::*;
369
370                #[test]
371                fn exact_length() {
372                    let result = $decode_fn(&$data).unwrap();
373                    assert_eq!(result, $expected);
374                }
375
376                #[test]
377                fn truncated_length() {
378                    // Remove the last byte of data so that there is not enough to decode
379                    let truncated_data = &$data[..$data.len() - 1];
380                    let result = $decode_fn(truncated_data);
381                    assert!(matches!(result, Err(ArrowError::InvalidArgumentError(_))));
382                }
383            }
384        };
385    }
386
387    mod integer {
388        use super::*;
389
390        test_decoder_bounds!(test_i8, [0x2a], decode_int8, 42);
391        test_decoder_bounds!(test_i16, [0xd2, 0x04], decode_int16, 1234);
392        test_decoder_bounds!(test_i32, [0x40, 0xe2, 0x01, 0x00], decode_int32, 123456);
393        test_decoder_bounds!(
394            test_i64,
395            [0x15, 0x81, 0xe9, 0x7d, 0xf4, 0x10, 0x22, 0x11],
396            decode_int64,
397            1234567890123456789
398        );
399    }
400
401    mod decimal {
402        use super::*;
403
404        test_decoder_bounds!(
405            test_decimal4,
406            [
407                0x02, // Scale
408                0xd2, 0x04, 0x00, 0x00, // Unscaled Value
409            ],
410            decode_decimal4,
411            (1234, 2)
412        );
413
414        test_decoder_bounds!(
415            test_decimal8,
416            [
417                0x02, // Scale
418                0xd2, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, // Unscaled Value
419            ],
420            decode_decimal8,
421            (1234567890, 2)
422        );
423
424        test_decoder_bounds!(
425            test_decimal16,
426            [
427                0x02, // Scale
428                0xd2, 0xb6, 0x23, 0xc0, 0xf4, 0x10, 0x22, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
429                0x00, 0x00, // Unscaled Value
430            ],
431            decode_decimal16,
432            (1234567891234567890, 2)
433        );
434    }
435
436    #[expect(clippy::float_cmp_const)]
437    mod float {
438        use super::*;
439
440        test_decoder_bounds!(
441            test_float,
442            [0x06, 0x2c, 0x93, 0x4e],
443            decode_float,
444            1_234_568_000.0
445        );
446
447        test_decoder_bounds!(
448            test_double,
449            [0xc9, 0xe5, 0x87, 0xb4, 0x80, 0x65, 0xd2, 0x41],
450            decode_double,
451            1234567890.1234
452        );
453    }
454
455    mod datetime {
456        use super::*;
457
458        test_decoder_bounds!(
459            test_date,
460            [0xe2, 0x4e, 0x0, 0x0],
461            decode_date,
462            NaiveDate::from_ymd_opt(2025, 4, 16).unwrap()
463        );
464
465        #[test]
466        fn test_date_out_of_range() {
467            // A day count that overflows chrono's supported date range must error, not panic
468            for days in [i32::MAX, i32::MIN] {
469                let result = decode_date(&days.to_le_bytes());
470                assert!(matches!(result, Err(ArrowError::CastError(_))));
471            }
472        }
473
474        test_decoder_bounds!(
475            test_timestamp_micros,
476            [0xe0, 0x52, 0x97, 0xdd, 0xe7, 0x32, 0x06, 0x00],
477            decode_timestamp_micros,
478            NaiveDate::from_ymd_opt(2025, 4, 16)
479                .unwrap()
480                .and_hms_milli_opt(16, 34, 56, 780)
481                .unwrap()
482                .and_utc()
483        );
484
485        test_decoder_bounds!(
486            test_timestampntz_micros,
487            [0xe0, 0x52, 0x97, 0xdd, 0xe7, 0x32, 0x06, 0x00],
488            decode_timestampntz_micros,
489            NaiveDate::from_ymd_opt(2025, 4, 16)
490                .unwrap()
491                .and_hms_milli_opt(16, 34, 56, 780)
492                .unwrap()
493        );
494
495        test_decoder_bounds!(
496            test_timestamp_nanos,
497            [0x15, 0x41, 0xa2, 0x5a, 0x36, 0xa2, 0x5b, 0x18],
498            decode_timestamp_nanos,
499            NaiveDate::from_ymd_opt(2025, 8, 14)
500                .unwrap()
501                .and_hms_nano_opt(12, 33, 54, 123456789)
502                .unwrap()
503                .and_utc()
504        );
505
506        test_decoder_bounds!(
507            test_timestamp_nanos_before_epoch,
508            [0x15, 0x41, 0x52, 0xd4, 0x94, 0xe5, 0xad, 0xfa],
509            decode_timestamp_nanos,
510            NaiveDate::from_ymd_opt(1957, 11, 7)
511                .unwrap()
512                .and_hms_nano_opt(12, 33, 54, 123456789)
513                .unwrap()
514                .and_utc()
515        );
516
517        test_decoder_bounds!(
518            test_timestampntz_nanos,
519            [0x15, 0x41, 0xa2, 0x5a, 0x36, 0xa2, 0x5b, 0x18],
520            decode_timestampntz_nanos,
521            NaiveDate::from_ymd_opt(2025, 8, 14)
522                .unwrap()
523                .and_hms_nano_opt(12, 33, 54, 123456789)
524                .unwrap()
525        );
526
527        test_decoder_bounds!(
528            test_timestampntz_nanos_before_epoch,
529            [0x15, 0x41, 0x52, 0xd4, 0x94, 0xe5, 0xad, 0xfa],
530            decode_timestampntz_nanos,
531            NaiveDate::from_ymd_opt(1957, 11, 7)
532                .unwrap()
533                .and_hms_nano_opt(12, 33, 54, 123456789)
534                .unwrap()
535        );
536    }
537
538    test_decoder_bounds!(
539        test_uuid,
540        [
541            0xf2, 0x4f, 0x9b, 0x64, 0x81, 0xfa, 0x49, 0xd1, 0xb7, 0x4e, 0x8c, 0x09, 0xa6, 0xe3,
542            0x1c, 0x56,
543        ],
544        decode_uuid,
545        Uuid::parse_str("f24f9b64-81fa-49d1-b74e-8c09a6e31c56").unwrap()
546    );
547
548    mod time {
549        use super::*;
550
551        test_decoder_bounds!(
552            test_timentz,
553            [0x53, 0x1f, 0x8e, 0xdf, 0x2, 0, 0, 0],
554            decode_time_ntz,
555            NaiveTime::from_num_seconds_from_midnight_opt(12340, 567_891_000).unwrap()
556        );
557
558        #[test]
559        fn test_decode_time_ntz_invalid() {
560            let invalid_second = u64::MAX;
561            let data = invalid_second.to_le_bytes();
562            let result = decode_time_ntz(&data);
563            assert!(matches!(result, Err(ArrowError::CastError(_))));
564        }
565    }
566
567    #[test]
568    fn test_binary_exact_length() {
569        let data = [
570            0x09, 0, 0, 0, // Length of binary data, 4-byte little-endian
571            0x03, 0x13, 0x37, 0xde, 0xad, 0xbe, 0xef, 0xca, 0xfe,
572        ];
573        let result = decode_binary(&data).unwrap();
574        assert_eq!(
575            result,
576            [0x03, 0x13, 0x37, 0xde, 0xad, 0xbe, 0xef, 0xca, 0xfe]
577        );
578    }
579
580    #[test]
581    fn test_binary_truncated_length() {
582        let data = [
583            0x09, 0, 0, 0, // Length of binary data, 4-byte little-endian
584            0x03, 0x13, 0x37, 0xde, 0xad, 0xbe, 0xef, 0xca,
585        ];
586        let result = decode_binary(&data);
587        assert!(matches!(result, Err(ArrowError::InvalidArgumentError(_))));
588    }
589
590    #[test]
591    fn test_short_string_exact_length() {
592        let data = b"Helloo";
593        let result = decode_short_string(1 | (5 << 2), data).unwrap();
594        assert_eq!(result.0, "Hello");
595    }
596
597    #[test]
598    fn test_short_string_truncated_length() {
599        let data = b"Abc";
600        let result = decode_short_string(1 | (5 << 2), data);
601        assert!(matches!(result, Err(ArrowError::InvalidArgumentError(_))));
602    }
603
604    #[test]
605    fn test_string_exact_length() {
606        let data = [
607            0x05, 0, 0, 0, // Length of string, 4-byte little-endian
608            b'H', b'e', b'l', b'l', b'o', b'o',
609        ];
610        let result = decode_long_string(&data).unwrap();
611        assert_eq!(result, "Hello");
612    }
613
614    #[test]
615    fn test_string_truncated_length() {
616        let data = [
617            0x05, 0, 0, 0, // Length of string, 4-byte little-endian
618            b'H', b'e', b'l',
619        ];
620        let result = decode_long_string(&data);
621        assert!(matches!(result, Err(ArrowError::InvalidArgumentError(_))));
622    }
623
624    #[test]
625    fn test_offset() {
626        assert_eq!(OffsetSizeBytes::try_new(0).unwrap(), OffsetSizeBytes::One);
627        assert_eq!(OffsetSizeBytes::try_new(1).unwrap(), OffsetSizeBytes::Two);
628        assert_eq!(OffsetSizeBytes::try_new(2).unwrap(), OffsetSizeBytes::Three);
629        assert_eq!(OffsetSizeBytes::try_new(3).unwrap(), OffsetSizeBytes::Four);
630
631        // everything outside 0-3 must error
632        assert!(OffsetSizeBytes::try_new(4).is_err());
633        assert!(OffsetSizeBytes::try_new(255).is_err());
634    }
635
636    #[test]
637    fn unpack_u32_all_widths() {
638        // One-byte offsets
639        let buf_one = [0x01u8, 0xAB, 0xCD];
640        assert_eq!(OffsetSizeBytes::One.unpack_u32(&buf_one, 0).unwrap(), 0x01);
641        assert_eq!(OffsetSizeBytes::One.unpack_u32(&buf_one, 2).unwrap(), 0xCD);
642
643        // Two-byte offsets (little-endian 0x1234, 0x5678)
644        let buf_two = [0x34, 0x12, 0x78, 0x56];
645        assert_eq!(
646            OffsetSizeBytes::Two.unpack_u32(&buf_two, 0).unwrap(),
647            0x1234
648        );
649        assert_eq!(
650            OffsetSizeBytes::Two.unpack_u32(&buf_two, 1).unwrap(),
651            0x5678
652        );
653
654        // Three-byte offsets (0x030201 and 0x0000FF)
655        let buf_three = [0x01, 0x02, 0x03, 0xFF, 0x00, 0x00];
656        assert_eq!(
657            OffsetSizeBytes::Three.unpack_u32(&buf_three, 0).unwrap(),
658            0x030201
659        );
660        assert_eq!(
661            OffsetSizeBytes::Three.unpack_u32(&buf_three, 1).unwrap(),
662            0x0000FF
663        );
664
665        // Four-byte offsets (0x12345678, 0x90ABCDEF)
666        let buf_four = [0x78, 0x56, 0x34, 0x12, 0xEF, 0xCD, 0xAB, 0x90];
667        assert_eq!(
668            OffsetSizeBytes::Four.unpack_u32(&buf_four, 0).unwrap(),
669            0x1234_5678
670        );
671        assert_eq!(
672            OffsetSizeBytes::Four.unpack_u32(&buf_four, 1).unwrap(),
673            0x90AB_CDEF
674        );
675    }
676
677    #[test]
678    fn unpack_u32_out_of_bounds() {
679        let tiny = [0x00u8]; // deliberately too short
680        assert!(OffsetSizeBytes::Two.unpack_u32(&tiny, 0).is_err());
681        assert!(OffsetSizeBytes::Three.unpack_u32(&tiny, 0).is_err());
682    }
683
684    #[test]
685    fn unpack_simple() {
686        let buf = [
687            0x41, // header
688            0x02, 0x00, // dictionary_size = 2
689            0x00, 0x00, // offset[0] = 0
690            0x05, 0x00, // offset[1] = 5
691            0x09, 0x00, // offset[2] = 9
692        ];
693
694        let width = OffsetSizeBytes::Two;
695
696        // dictionary_size starts immediately after the header byte
697        let dict_size = width.unpack_u32_at_offset(&buf, 1, 0).unwrap();
698        assert_eq!(dict_size, 2);
699
700        // offset array immediately follows the dictionary size
701        let first = width.unpack_u32_at_offset(&buf, 1, 1).unwrap();
702        assert_eq!(first, 0);
703
704        let second = width.unpack_u32_at_offset(&buf, 1, 2).unwrap();
705        assert_eq!(second, 5);
706
707        let third = width.unpack_u32_at_offset(&buf, 1, 3).unwrap();
708        assert_eq!(third, 9);
709
710        let err = width.unpack_u32_at_offset(&buf, 1, 4);
711        assert!(err.is_err())
712    }
713}