1use crate::ShortString;
6use crate::utils::{
7 array_from_slice, overflow_error, slice_from_slice_at_offset, string_from_slice,
8};
9
10use arrow_schema::ArrowError;
11use chrono::{DateTime, Duration, NaiveDate, NaiveDateTime, NaiveTime, Utc};
12use uuid::Uuid;
13
14#[derive(Debug, Clone, Copy, PartialEq)]
22pub enum VariantBasicType {
23 Primitive = 0,
24 ShortString = 1,
25 Object = 2,
26 Array = 3,
27}
28
29#[derive(Debug, Clone, Copy, PartialEq)]
37pub enum VariantPrimitiveType {
38 Null = 0,
39 BooleanTrue = 1,
40 BooleanFalse = 2,
41 Int8 = 3,
42 Int16 = 4,
43 Int32 = 5,
44 Int64 = 6,
45 Double = 7,
46 Decimal4 = 8,
47 Decimal8 = 9,
48 Decimal16 = 10,
49 Date = 11,
50 TimestampMicros = 12,
51 TimestampNtzMicros = 13,
52 Float = 14,
53 Binary = 15,
54 String = 16,
55 Time = 17,
56 TimestampNanos = 18,
57 TimestampNtzNanos = 19,
58 Uuid = 20,
59}
60
61pub(crate) fn get_basic_type(header: u8) -> VariantBasicType {
63 let basic_type = header & 0x03; match basic_type {
66 0 => VariantBasicType::Primitive,
67 1 => VariantBasicType::ShortString,
68 2 => VariantBasicType::Object,
69 3 => VariantBasicType::Array,
70 _ => {
71 unreachable!();
74 }
75 }
76}
77
78impl TryFrom<u8> for VariantPrimitiveType {
79 type Error = ArrowError;
80
81 fn try_from(value: u8) -> Result<Self, Self::Error> {
82 match value {
83 0 => Ok(VariantPrimitiveType::Null),
84 1 => Ok(VariantPrimitiveType::BooleanTrue),
85 2 => Ok(VariantPrimitiveType::BooleanFalse),
86 3 => Ok(VariantPrimitiveType::Int8),
87 4 => Ok(VariantPrimitiveType::Int16),
88 5 => Ok(VariantPrimitiveType::Int32),
89 6 => Ok(VariantPrimitiveType::Int64),
90 7 => Ok(VariantPrimitiveType::Double),
91 8 => Ok(VariantPrimitiveType::Decimal4),
92 9 => Ok(VariantPrimitiveType::Decimal8),
93 10 => Ok(VariantPrimitiveType::Decimal16),
94 11 => Ok(VariantPrimitiveType::Date),
95 12 => Ok(VariantPrimitiveType::TimestampMicros),
96 13 => Ok(VariantPrimitiveType::TimestampNtzMicros),
97 14 => Ok(VariantPrimitiveType::Float),
98 15 => Ok(VariantPrimitiveType::Binary),
99 16 => Ok(VariantPrimitiveType::String),
100 17 => Ok(VariantPrimitiveType::Time),
101 18 => Ok(VariantPrimitiveType::TimestampNanos),
102 19 => Ok(VariantPrimitiveType::TimestampNtzNanos),
103 20 => Ok(VariantPrimitiveType::Uuid),
104 _ => Err(ArrowError::InvalidArgumentError(format!(
105 "unknown primitive type: {value}",
106 ))),
107 }
108 }
109}
110
111#[derive(Debug, Clone, Copy, PartialEq)]
115pub(crate) enum OffsetSizeBytes {
116 One = 1,
117 Two = 2,
118 Three = 3,
119 Four = 4,
120}
121
122impl OffsetSizeBytes {
123 pub(crate) fn try_new(offset_size_minus_one: u8) -> Result<Self, ArrowError> {
125 use OffsetSizeBytes::*;
126 let result = match offset_size_minus_one {
127 0 => One,
128 1 => Two,
129 2 => Three,
130 3 => Four,
131 _ => {
132 return Err(ArrowError::InvalidArgumentError(
133 "offset_size_minus_one must be 0–3".to_string(),
134 ));
135 }
136 };
137 Ok(result)
138 }
139
140 pub(crate) fn unpack_u32(&self, bytes: &[u8], index: usize) -> Result<u32, ArrowError> {
147 self.unpack_u32_at_offset(bytes, 0, index)
148 }
149
150 pub(crate) fn unpack_u32_at_offset(
160 &self,
161 bytes: &[u8],
162 byte_offset: usize, offset_index: usize, ) -> Result<u32, ArrowError> {
165 use OffsetSizeBytes::*;
166
167 let offset = offset_index
170 .checked_mul(*self as usize)
171 .and_then(|n| n.checked_add(byte_offset))
172 .ok_or_else(|| overflow_error("unpacking offset array value"))?;
173 let value = match self {
174 One => u8::from_le_bytes(array_from_slice(bytes, offset)?).into(),
175 Two => u16::from_le_bytes(array_from_slice(bytes, offset)?).into(),
176 Three => {
177 let b3_chunks: [u8; 3] = array_from_slice(bytes, offset)?;
179 let mut buf = [0u8; 4];
181 buf[..3].copy_from_slice(&b3_chunks);
182 u32::from_le_bytes(buf)
183 }
184 Four => u32::from_le_bytes(array_from_slice(bytes, offset)?),
185 };
186 Ok(value)
187 }
188}
189
190pub(crate) fn map_bytes_to_offsets(
192 buffer: &[u8],
193 offset_size: OffsetSizeBytes,
194) -> impl Iterator<Item = usize> + use<'_> {
195 buffer
196 .chunks_exact(offset_size as usize)
197 .map(move |chunk| match offset_size {
198 OffsetSizeBytes::One => chunk[0] as usize,
199 OffsetSizeBytes::Two => u16::from_le_bytes([chunk[0], chunk[1]]) as usize,
200 OffsetSizeBytes::Three => {
201 u32::from_le_bytes([chunk[0], chunk[1], chunk[2], 0]) as usize
202 }
203 OffsetSizeBytes::Four => {
204 u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]) as usize
205 }
206 })
207}
208
209pub(crate) fn get_primitive_type(metadata: u8) -> Result<VariantPrimitiveType, ArrowError> {
211 VariantPrimitiveType::try_from(metadata >> 2)
213}
214
215pub(crate) fn decode_int8(data: &[u8]) -> Result<i8, ArrowError> {
217 Ok(i8::from_le_bytes(array_from_slice(data, 0)?))
218}
219
220pub(crate) fn decode_int16(data: &[u8]) -> Result<i16, ArrowError> {
222 Ok(i16::from_le_bytes(array_from_slice(data, 0)?))
223}
224
225pub(crate) fn decode_int32(data: &[u8]) -> Result<i32, ArrowError> {
227 Ok(i32::from_le_bytes(array_from_slice(data, 0)?))
228}
229
230pub(crate) fn decode_int64(data: &[u8]) -> Result<i64, ArrowError> {
232 Ok(i64::from_le_bytes(array_from_slice(data, 0)?))
233}
234
235pub(crate) fn decode_decimal4(data: &[u8]) -> Result<(i32, u8), ArrowError> {
237 let scale = u8::from_le_bytes(array_from_slice(data, 0)?);
238 let integer = i32::from_le_bytes(array_from_slice(data, 1)?);
239 Ok((integer, scale))
240}
241
242pub(crate) fn decode_decimal8(data: &[u8]) -> Result<(i64, u8), ArrowError> {
244 let scale = u8::from_le_bytes(array_from_slice(data, 0)?);
245 let integer = i64::from_le_bytes(array_from_slice(data, 1)?);
246 Ok((integer, scale))
247}
248
249pub(crate) fn decode_decimal16(data: &[u8]) -> Result<(i128, u8), ArrowError> {
251 let scale = u8::from_le_bytes(array_from_slice(data, 0)?);
252 let integer = i128::from_le_bytes(array_from_slice(data, 1)?);
253 Ok((integer, scale))
254}
255
256pub(crate) fn decode_float(data: &[u8]) -> Result<f32, ArrowError> {
258 Ok(f32::from_le_bytes(array_from_slice(data, 0)?))
259}
260
261pub(crate) fn decode_double(data: &[u8]) -> Result<f64, ArrowError> {
263 Ok(f64::from_le_bytes(array_from_slice(data, 0)?))
264}
265
266pub(crate) fn decode_date(data: &[u8]) -> Result<NaiveDate, ArrowError> {
268 let days_since_epoch = i32::from_le_bytes(array_from_slice(data, 0)?);
269 DateTime::UNIX_EPOCH
270 .checked_add_signed(Duration::days(i64::from(days_since_epoch)))
271 .map(|value| value.date_naive())
272 .ok_or_else(|| {
273 ArrowError::CastError(format!(
274 "Could not cast `{days_since_epoch}` days into a NaiveDate"
275 ))
276 })
277}
278
279pub(crate) fn decode_timestamp_micros(data: &[u8]) -> Result<DateTime<Utc>, ArrowError> {
281 let micros_since_epoch = i64::from_le_bytes(array_from_slice(data, 0)?);
282 DateTime::from_timestamp_micros(micros_since_epoch).ok_or_else(|| {
283 ArrowError::CastError(format!(
284 "Could not cast `{micros_since_epoch}` microseconds into a DateTime<Utc>"
285 ))
286 })
287}
288
289pub(crate) fn decode_timestampntz_micros(data: &[u8]) -> Result<NaiveDateTime, ArrowError> {
291 let micros_since_epoch = i64::from_le_bytes(array_from_slice(data, 0)?);
292 DateTime::from_timestamp_micros(micros_since_epoch)
293 .ok_or_else(|| {
294 ArrowError::CastError(format!(
295 "Could not cast `{micros_since_epoch}` microseconds into a NaiveDateTime"
296 ))
297 })
298 .map(|v| v.naive_utc())
299}
300
301pub(crate) fn decode_time_ntz(data: &[u8]) -> Result<NaiveTime, ArrowError> {
302 let micros_since_epoch = u64::from_le_bytes(array_from_slice(data, 0)?);
303
304 let case_error = ArrowError::CastError(format!(
305 "Could not cast {micros_since_epoch} microseconds into a NaiveTime"
306 ));
307
308 if micros_since_epoch >= 86_400_000_000 {
309 return Err(case_error);
310 }
311
312 let nanos_since_midnight = micros_since_epoch * 1_000;
313 NaiveTime::from_num_seconds_from_midnight_opt(
314 (nanos_since_midnight / 1_000_000_000) as u32,
315 (nanos_since_midnight % 1_000_000_000) as u32,
316 )
317 .ok_or(case_error)
318}
319
320pub(crate) fn decode_timestamp_nanos(data: &[u8]) -> Result<DateTime<Utc>, ArrowError> {
322 let nanos_since_epoch = i64::from_le_bytes(array_from_slice(data, 0)?);
323
324 Ok(DateTime::from_timestamp_nanos(nanos_since_epoch))
326}
327
328pub(crate) fn decode_timestampntz_nanos(data: &[u8]) -> Result<NaiveDateTime, ArrowError> {
330 decode_timestamp_nanos(data).map(|v| v.naive_utc())
331}
332
333pub(crate) fn decode_uuid(data: &[u8]) -> Result<Uuid, ArrowError> {
335 let bytes: [u8; 16] = array_from_slice(data, 0)?;
336 Ok(Uuid::from_bytes(bytes))
337}
338
339pub(crate) fn decode_binary(data: &[u8]) -> Result<&[u8], ArrowError> {
341 let len = u32::from_le_bytes(array_from_slice(data, 0)?) as usize;
342 slice_from_slice_at_offset(data, 4, 0..len)
343}
344
345pub(crate) fn decode_long_string(data: &[u8]) -> Result<&str, ArrowError> {
347 let len = u32::from_le_bytes(array_from_slice(data, 0)?) as usize;
348 string_from_slice(data, 4, 0..len)
349}
350
351pub(crate) fn decode_short_string(
353 metadata: u8,
354 data: &[u8],
355) -> Result<ShortString<'_>, ArrowError> {
356 let len = (metadata >> 2) as usize;
357 let string = string_from_slice(data, 0, 0..len)?;
358 ShortString::try_new(string)
359}
360
361#[cfg(test)]
362mod tests {
363 use super::*;
364
365 macro_rules! test_decoder_bounds {
366 ($test_name:ident, $data:expr, $decode_fn:ident, $expected:expr) => {
367 mod $test_name {
368 use super::*;
369
370 #[test]
371 fn exact_length() {
372 let result = $decode_fn(&$data).unwrap();
373 assert_eq!(result, $expected);
374 }
375
376 #[test]
377 fn truncated_length() {
378 let truncated_data = &$data[..$data.len() - 1];
380 let result = $decode_fn(truncated_data);
381 assert!(matches!(result, Err(ArrowError::InvalidArgumentError(_))));
382 }
383 }
384 };
385 }
386
387 mod integer {
388 use super::*;
389
390 test_decoder_bounds!(test_i8, [0x2a], decode_int8, 42);
391 test_decoder_bounds!(test_i16, [0xd2, 0x04], decode_int16, 1234);
392 test_decoder_bounds!(test_i32, [0x40, 0xe2, 0x01, 0x00], decode_int32, 123456);
393 test_decoder_bounds!(
394 test_i64,
395 [0x15, 0x81, 0xe9, 0x7d, 0xf4, 0x10, 0x22, 0x11],
396 decode_int64,
397 1234567890123456789
398 );
399 }
400
401 mod decimal {
402 use super::*;
403
404 test_decoder_bounds!(
405 test_decimal4,
406 [
407 0x02, 0xd2, 0x04, 0x00, 0x00, ],
410 decode_decimal4,
411 (1234, 2)
412 );
413
414 test_decoder_bounds!(
415 test_decimal8,
416 [
417 0x02, 0xd2, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, ],
420 decode_decimal8,
421 (1234567890, 2)
422 );
423
424 test_decoder_bounds!(
425 test_decimal16,
426 [
427 0x02, 0xd2, 0xb6, 0x23, 0xc0, 0xf4, 0x10, 0x22, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
429 0x00, 0x00, ],
431 decode_decimal16,
432 (1234567891234567890, 2)
433 );
434 }
435
436 #[expect(clippy::float_cmp_const)]
437 mod float {
438 use super::*;
439
440 test_decoder_bounds!(
441 test_float,
442 [0x06, 0x2c, 0x93, 0x4e],
443 decode_float,
444 1_234_568_000.0
445 );
446
447 test_decoder_bounds!(
448 test_double,
449 [0xc9, 0xe5, 0x87, 0xb4, 0x80, 0x65, 0xd2, 0x41],
450 decode_double,
451 1234567890.1234
452 );
453 }
454
455 mod datetime {
456 use super::*;
457
458 test_decoder_bounds!(
459 test_date,
460 [0xe2, 0x4e, 0x0, 0x0],
461 decode_date,
462 NaiveDate::from_ymd_opt(2025, 4, 16).unwrap()
463 );
464
465 #[test]
466 fn test_date_out_of_range() {
467 for days in [i32::MAX, i32::MIN] {
469 let result = decode_date(&days.to_le_bytes());
470 assert!(matches!(result, Err(ArrowError::CastError(_))));
471 }
472 }
473
474 test_decoder_bounds!(
475 test_timestamp_micros,
476 [0xe0, 0x52, 0x97, 0xdd, 0xe7, 0x32, 0x06, 0x00],
477 decode_timestamp_micros,
478 NaiveDate::from_ymd_opt(2025, 4, 16)
479 .unwrap()
480 .and_hms_milli_opt(16, 34, 56, 780)
481 .unwrap()
482 .and_utc()
483 );
484
485 test_decoder_bounds!(
486 test_timestampntz_micros,
487 [0xe0, 0x52, 0x97, 0xdd, 0xe7, 0x32, 0x06, 0x00],
488 decode_timestampntz_micros,
489 NaiveDate::from_ymd_opt(2025, 4, 16)
490 .unwrap()
491 .and_hms_milli_opt(16, 34, 56, 780)
492 .unwrap()
493 );
494
495 test_decoder_bounds!(
496 test_timestamp_nanos,
497 [0x15, 0x41, 0xa2, 0x5a, 0x36, 0xa2, 0x5b, 0x18],
498 decode_timestamp_nanos,
499 NaiveDate::from_ymd_opt(2025, 8, 14)
500 .unwrap()
501 .and_hms_nano_opt(12, 33, 54, 123456789)
502 .unwrap()
503 .and_utc()
504 );
505
506 test_decoder_bounds!(
507 test_timestamp_nanos_before_epoch,
508 [0x15, 0x41, 0x52, 0xd4, 0x94, 0xe5, 0xad, 0xfa],
509 decode_timestamp_nanos,
510 NaiveDate::from_ymd_opt(1957, 11, 7)
511 .unwrap()
512 .and_hms_nano_opt(12, 33, 54, 123456789)
513 .unwrap()
514 .and_utc()
515 );
516
517 test_decoder_bounds!(
518 test_timestampntz_nanos,
519 [0x15, 0x41, 0xa2, 0x5a, 0x36, 0xa2, 0x5b, 0x18],
520 decode_timestampntz_nanos,
521 NaiveDate::from_ymd_opt(2025, 8, 14)
522 .unwrap()
523 .and_hms_nano_opt(12, 33, 54, 123456789)
524 .unwrap()
525 );
526
527 test_decoder_bounds!(
528 test_timestampntz_nanos_before_epoch,
529 [0x15, 0x41, 0x52, 0xd4, 0x94, 0xe5, 0xad, 0xfa],
530 decode_timestampntz_nanos,
531 NaiveDate::from_ymd_opt(1957, 11, 7)
532 .unwrap()
533 .and_hms_nano_opt(12, 33, 54, 123456789)
534 .unwrap()
535 );
536 }
537
538 test_decoder_bounds!(
539 test_uuid,
540 [
541 0xf2, 0x4f, 0x9b, 0x64, 0x81, 0xfa, 0x49, 0xd1, 0xb7, 0x4e, 0x8c, 0x09, 0xa6, 0xe3,
542 0x1c, 0x56,
543 ],
544 decode_uuid,
545 Uuid::parse_str("f24f9b64-81fa-49d1-b74e-8c09a6e31c56").unwrap()
546 );
547
548 mod time {
549 use super::*;
550
551 test_decoder_bounds!(
552 test_timentz,
553 [0x53, 0x1f, 0x8e, 0xdf, 0x2, 0, 0, 0],
554 decode_time_ntz,
555 NaiveTime::from_num_seconds_from_midnight_opt(12340, 567_891_000).unwrap()
556 );
557
558 #[test]
559 fn test_decode_time_ntz_invalid() {
560 let invalid_second = u64::MAX;
561 let data = invalid_second.to_le_bytes();
562 let result = decode_time_ntz(&data);
563 assert!(matches!(result, Err(ArrowError::CastError(_))));
564 }
565 }
566
567 #[test]
568 fn test_binary_exact_length() {
569 let data = [
570 0x09, 0, 0, 0, 0x03, 0x13, 0x37, 0xde, 0xad, 0xbe, 0xef, 0xca, 0xfe,
572 ];
573 let result = decode_binary(&data).unwrap();
574 assert_eq!(
575 result,
576 [0x03, 0x13, 0x37, 0xde, 0xad, 0xbe, 0xef, 0xca, 0xfe]
577 );
578 }
579
580 #[test]
581 fn test_binary_truncated_length() {
582 let data = [
583 0x09, 0, 0, 0, 0x03, 0x13, 0x37, 0xde, 0xad, 0xbe, 0xef, 0xca,
585 ];
586 let result = decode_binary(&data);
587 assert!(matches!(result, Err(ArrowError::InvalidArgumentError(_))));
588 }
589
590 #[test]
591 fn test_short_string_exact_length() {
592 let data = b"Helloo";
593 let result = decode_short_string(1 | (5 << 2), data).unwrap();
594 assert_eq!(result.0, "Hello");
595 }
596
597 #[test]
598 fn test_short_string_truncated_length() {
599 let data = b"Abc";
600 let result = decode_short_string(1 | (5 << 2), data);
601 assert!(matches!(result, Err(ArrowError::InvalidArgumentError(_))));
602 }
603
604 #[test]
605 fn test_string_exact_length() {
606 let data = [
607 0x05, 0, 0, 0, b'H', b'e', b'l', b'l', b'o', b'o',
609 ];
610 let result = decode_long_string(&data).unwrap();
611 assert_eq!(result, "Hello");
612 }
613
614 #[test]
615 fn test_string_truncated_length() {
616 let data = [
617 0x05, 0, 0, 0, b'H', b'e', b'l',
619 ];
620 let result = decode_long_string(&data);
621 assert!(matches!(result, Err(ArrowError::InvalidArgumentError(_))));
622 }
623
624 #[test]
625 fn test_offset() {
626 assert_eq!(OffsetSizeBytes::try_new(0).unwrap(), OffsetSizeBytes::One);
627 assert_eq!(OffsetSizeBytes::try_new(1).unwrap(), OffsetSizeBytes::Two);
628 assert_eq!(OffsetSizeBytes::try_new(2).unwrap(), OffsetSizeBytes::Three);
629 assert_eq!(OffsetSizeBytes::try_new(3).unwrap(), OffsetSizeBytes::Four);
630
631 assert!(OffsetSizeBytes::try_new(4).is_err());
633 assert!(OffsetSizeBytes::try_new(255).is_err());
634 }
635
636 #[test]
637 fn unpack_u32_all_widths() {
638 let buf_one = [0x01u8, 0xAB, 0xCD];
640 assert_eq!(OffsetSizeBytes::One.unpack_u32(&buf_one, 0).unwrap(), 0x01);
641 assert_eq!(OffsetSizeBytes::One.unpack_u32(&buf_one, 2).unwrap(), 0xCD);
642
643 let buf_two = [0x34, 0x12, 0x78, 0x56];
645 assert_eq!(
646 OffsetSizeBytes::Two.unpack_u32(&buf_two, 0).unwrap(),
647 0x1234
648 );
649 assert_eq!(
650 OffsetSizeBytes::Two.unpack_u32(&buf_two, 1).unwrap(),
651 0x5678
652 );
653
654 let buf_three = [0x01, 0x02, 0x03, 0xFF, 0x00, 0x00];
656 assert_eq!(
657 OffsetSizeBytes::Three.unpack_u32(&buf_three, 0).unwrap(),
658 0x030201
659 );
660 assert_eq!(
661 OffsetSizeBytes::Three.unpack_u32(&buf_three, 1).unwrap(),
662 0x0000FF
663 );
664
665 let buf_four = [0x78, 0x56, 0x34, 0x12, 0xEF, 0xCD, 0xAB, 0x90];
667 assert_eq!(
668 OffsetSizeBytes::Four.unpack_u32(&buf_four, 0).unwrap(),
669 0x1234_5678
670 );
671 assert_eq!(
672 OffsetSizeBytes::Four.unpack_u32(&buf_four, 1).unwrap(),
673 0x90AB_CDEF
674 );
675 }
676
677 #[test]
678 fn unpack_u32_out_of_bounds() {
679 let tiny = [0x00u8]; assert!(OffsetSizeBytes::Two.unpack_u32(&tiny, 0).is_err());
681 assert!(OffsetSizeBytes::Three.unpack_u32(&tiny, 0).is_err());
682 }
683
684 #[test]
685 fn unpack_simple() {
686 let buf = [
687 0x41, 0x02, 0x00, 0x00, 0x00, 0x05, 0x00, 0x09, 0x00, ];
693
694 let width = OffsetSizeBytes::Two;
695
696 let dict_size = width.unpack_u32_at_offset(&buf, 1, 0).unwrap();
698 assert_eq!(dict_size, 2);
699
700 let first = width.unpack_u32_at_offset(&buf, 1, 1).unwrap();
702 assert_eq!(first, 0);
703
704 let second = width.unpack_u32_at_offset(&buf, 1, 2).unwrap();
705 assert_eq!(second, 5);
706
707 let third = width.unpack_u32_at_offset(&buf, 1, 3).unwrap();
708 assert_eq!(third, 9);
709
710 let err = width.unpack_u32_at_offset(&buf, 1, 4);
711 assert!(err.is_err())
712 }
713}