Skip to main content

parquet/
parquet_thrift.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Structs used for encoding and decoding Parquet Thrift objects.
6//!
7//! These include:
8//! * [`ThriftCompactInputProtocol`]: Trait implemented by Thrift decoders.
9//!     * [`ThriftSliceInputProtocol`]: Thrift decoder that takes a slice of bytes as input.
10//!     * [`ThriftReadInputProtocol`]: Thrift decoder that takes a [`Read`] as input.
11//! * [`ReadThrift`]: Trait implemented by serializable objects.
12//! * [`ThriftCompactOutputProtocol`]: Thrift encoder.
13//! * [`WriteThrift`]: Trait implemented by serializable objects.
14//! * [`WriteThriftField`]: Trait implemented by serializable objects that are fields in Thrift structs.
15
16use std::{
17    cmp::Ordering,
18    io::{Read, Write},
19};
20
21use crate::{
22    errors::{ParquetError, Result},
23    write_thrift_field,
24};
25use std::io::Error;
26use std::num::TryFromIntError;
27use std::str::Utf8Error;
28
29#[derive(Debug)]
30pub(crate) enum ThriftProtocolError {
31    Eof,
32    IO(Error),
33    InvalidFieldType(u8),
34    InvalidElementType(u8),
35    FieldDeltaOverflow { field_delta: u8, last_field_id: i16 },
36    InvalidBoolean(u8),
37    IntegerOverflow,
38    Utf8Error,
39    SkipDepth(FieldType),
40    SkipUnsupportedType(FieldType),
41}
42
43impl From<ThriftProtocolError> for ParquetError {
44    #[inline(never)]
45    fn from(e: ThriftProtocolError) -> Self {
46        match e {
47            ThriftProtocolError::Eof => eof_err!("Unexpected EOF"),
48            ThriftProtocolError::IO(e) => e.into(),
49            ThriftProtocolError::InvalidFieldType(value) => match FieldType::try_from(value) {
50                Ok(fld_type) => general_err!("Unexpected struct field type {:?}", fld_type),
51                Err(_) => general_err!("Unexpected struct field type {}", value),
52            },
53            ThriftProtocolError::InvalidElementType(value) => {
54                general_err!("Unexpected list/set element type {}", value)
55            }
56            ThriftProtocolError::FieldDeltaOverflow {
57                field_delta,
58                last_field_id,
59            } => general_err!("cannot add {} to {}", field_delta, last_field_id),
60            ThriftProtocolError::InvalidBoolean(value) => {
61                general_err!("cannot convert {} into bool", value)
62            }
63            ThriftProtocolError::IntegerOverflow => {
64                general_err!("integer overflow decoding thrift value")
65            }
66            ThriftProtocolError::Utf8Error => general_err!("invalid utf8"),
67            ThriftProtocolError::SkipDepth(field_type) => {
68                general_err!("cannot parse past {:?}", field_type)
69            }
70            ThriftProtocolError::SkipUnsupportedType(field_type) => {
71                general_err!("cannot skip field type {:?}", field_type)
72            }
73        }
74    }
75}
76
77impl From<Utf8Error> for ThriftProtocolError {
78    fn from(_: Utf8Error) -> Self {
79        // ignore error payload to reduce the size of ThriftProtocolError
80        Self::Utf8Error
81    }
82}
83
84impl From<Error> for ThriftProtocolError {
85    fn from(e: Error) -> Self {
86        Self::IO(e)
87    }
88}
89
90impl From<TryFromIntError> for ThriftProtocolError {
91    fn from(_: TryFromIntError) -> Self {
92        // ignore error payload to reduce the size of ThriftProtocolError
93        Self::IntegerOverflow
94    }
95}
96
97pub type ThriftProtocolResult<T> = Result<T, ThriftProtocolError>;
98
99/// Wrapper for thrift `double` fields. This is used to provide
100/// an implementation of `Eq` for floats. This implementation
101/// uses IEEE 754 total order.
102#[derive(Debug, Clone, Copy, PartialEq)]
103pub struct OrderedF64(f64);
104
105impl From<f64> for OrderedF64 {
106    fn from(value: f64) -> Self {
107        Self(value)
108    }
109}
110
111impl From<OrderedF64> for f64 {
112    fn from(value: OrderedF64) -> Self {
113        value.0
114    }
115}
116
117impl Eq for OrderedF64 {} // Marker trait, requires PartialEq
118
119impl Ord for OrderedF64 {
120    fn cmp(&self, other: &Self) -> Ordering {
121        self.0.total_cmp(&other.0)
122    }
123}
124
125impl PartialOrd for OrderedF64 {
126    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
127        Some(self.cmp(other))
128    }
129}
130
131// Thrift compact protocol types for struct fields.
132#[derive(Clone, Copy, Debug, Eq, PartialEq)]
133pub(crate) enum FieldType {
134    Stop = 0,
135    BooleanTrue = 1,
136    BooleanFalse = 2,
137    Byte = 3,
138    I16 = 4,
139    I32 = 5,
140    I64 = 6,
141    Double = 7,
142    Binary = 8,
143    List = 9,
144    Set = 10,
145    Map = 11,
146    Struct = 12,
147    Uuid = 13,
148}
149
150impl TryFrom<u8> for FieldType {
151    type Error = ThriftProtocolError;
152    fn try_from(value: u8) -> ThriftProtocolResult<Self> {
153        match value {
154            0 => Ok(Self::Stop),
155            1 => Ok(Self::BooleanTrue),
156            2 => Ok(Self::BooleanFalse),
157            3 => Ok(Self::Byte),
158            4 => Ok(Self::I16),
159            5 => Ok(Self::I32),
160            6 => Ok(Self::I64),
161            7 => Ok(Self::Double),
162            8 => Ok(Self::Binary),
163            9 => Ok(Self::List),
164            10 => Ok(Self::Set),
165            11 => Ok(Self::Map),
166            12 => Ok(Self::Struct),
167            13 => Ok(Self::Uuid),
168            _ => Err(ThriftProtocolError::InvalidFieldType(value)),
169        }
170    }
171}
172
173impl From<ElementType> for FieldType {
174    fn from(value: ElementType) -> Self {
175        match value {
176            ElementType::Bool => Self::BooleanTrue,
177            ElementType::Byte => Self::Byte,
178            ElementType::I16 => Self::I16,
179            ElementType::I32 => Self::I32,
180            ElementType::I64 => Self::I64,
181            ElementType::Double => Self::Double,
182            ElementType::Binary => Self::Binary,
183            ElementType::List => Self::List,
184            ElementType::Set => Self::Set,
185            ElementType::Map => Self::Map,
186            ElementType::Struct => Self::Struct,
187            ElementType::Uuid => Self::Uuid,
188        }
189    }
190}
191
192// Thrift compact protocol types for list elements
193#[derive(Clone, Copy, Debug, Eq, PartialEq)]
194pub(crate) enum ElementType {
195    Bool = 2,
196    Byte = 3,
197    I16 = 4,
198    I32 = 5,
199    I64 = 6,
200    Double = 7,
201    Binary = 8,
202    List = 9,
203    Set = 10,
204    Map = 11,
205    Struct = 12,
206    Uuid = 13,
207}
208
209impl TryFrom<u8> for ElementType {
210    type Error = ThriftProtocolError;
211    fn try_from(value: u8) -> ThriftProtocolResult<Self> {
212        match value {
213            // For historical and compatibility reasons, a reader should be capable to deal with both cases.
214            // The only valid value in the original spec was 2, but due to an widespread implementation bug
215            // the defacto standard across large parts of the library became 1 instead.
216            // As a result, both values are now allowed.
217            // https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md#list-and-set
218            1 | 2 => Ok(Self::Bool),
219            3 => Ok(Self::Byte),
220            4 => Ok(Self::I16),
221            5 => Ok(Self::I32),
222            6 => Ok(Self::I64),
223            7 => Ok(Self::Double),
224            8 => Ok(Self::Binary),
225            9 => Ok(Self::List),
226            10 => Ok(Self::Set),
227            11 => Ok(Self::Map),
228            12 => Ok(Self::Struct),
229            13 => Ok(Self::Uuid),
230            _ => Err(ThriftProtocolError::InvalidElementType(value)),
231        }
232    }
233}
234
235/// Struct used to describe a [thrift struct] field during decoding.
236///
237/// [thrift struct]: https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md#struct-encoding
238pub(crate) struct FieldIdentifier {
239    /// The type for the field.
240    pub(crate) field_type: FieldType,
241    /// The field's `id`. May be computed from delta or directly decoded.
242    pub(crate) id: i16,
243}
244
245impl FieldIdentifier {
246    pub(crate) fn bool_val(&self) -> ThriftProtocolResult<bool> {
247        match self.field_type {
248            FieldType::BooleanTrue => Ok(true),
249            FieldType::BooleanFalse => Ok(false),
250            _ => Err(ThriftProtocolError::InvalidFieldType(self.field_type as u8)),
251        }
252    }
253}
254
255/// Struct used to describe a [thrift list].
256///
257/// [thrift list]: https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md#list-and-set
258#[derive(Clone, Debug, Eq, PartialEq)]
259pub(crate) struct ListIdentifier {
260    /// The type for each element in the list.
261    pub(crate) element_type: ElementType,
262    /// Number of elements contained in the list.
263    pub(crate) size: i32,
264}
265
266/// Low-level object used to deserialize structs encoded with the Thrift [compact] protocol.
267///
268/// Implementation of this trait must provide the low-level functions `read_byte`, `read_bytes`,
269/// `skip_bytes`, and `read_double`. These primitives are used by the default functions provided
270/// here to perform deserialization.
271///
272/// [compact]: https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md
273pub(crate) trait ThriftCompactInputProtocol<'a> {
274    /// Read a single byte from the input.
275    fn read_byte(&mut self) -> ThriftProtocolResult<u8>;
276
277    /// Read a Thrift encoded [binary] from the input.
278    ///
279    /// [binary]: https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md#binary-encoding
280    fn read_bytes(&mut self) -> ThriftProtocolResult<&'a [u8]>;
281
282    fn read_bytes_owned(&mut self) -> ThriftProtocolResult<Vec<u8>>;
283
284    /// Skip the next `n` bytes of input.
285    fn skip_bytes(&mut self, n: usize) -> ThriftProtocolResult<()>;
286
287    /// Remaining unread bytes, if this protocol is backed by a finite buffer.
288    ///
289    /// Used to reject Thrift collection sizes that cannot fit in the remaining
290    /// input before allocating.
291    fn remaining_bytes(&self) -> Option<usize> {
292        None
293    }
294
295    /// Read a ULEB128 encoded unsigned varint from the input.
296    fn read_vlq(&mut self) -> ThriftProtocolResult<u64> {
297        // try the happy path first
298        let byte = self.read_byte()?;
299        if byte & 0x80 == 0 {
300            return Ok(byte as u64);
301        }
302        let mut in_progress = (byte & 0x7f) as u64;
303        let mut shift = 7;
304        loop {
305            let byte = self.read_byte()?;
306            in_progress |= ((byte & 0x7F) as u64).wrapping_shl(shift);
307            if byte & 0x80 == 0 {
308                return Ok(in_progress);
309            }
310            shift += 7;
311        }
312    }
313
314    /// Read a binary length, bounded to the signed 32-bit range used by Java and C++ Thrift.
315    fn read_binary_len(&mut self) -> ThriftProtocolResult<usize> {
316        Ok(i32::try_from(self.read_vlq()?)? as usize)
317    }
318
319    /// Read a zig-zag encoded signed varint from the input.
320    fn read_zig_zag(&mut self) -> ThriftProtocolResult<i64> {
321        let val = self.read_vlq()?;
322        Ok((val >> 1) as i64 ^ -((val & 1) as i64))
323    }
324
325    /// Read the [`ListIdentifier`] for a Thrift encoded list.
326    fn read_list_begin(&mut self) -> ThriftProtocolResult<ListIdentifier> {
327        let header = self.read_byte()?;
328        // some parquet writers will have an element_type of 0 for an empty list.
329        // account for that and return a bogus but valid element_type.
330        if header == 0 {
331            return Ok(ListIdentifier {
332                element_type: ElementType::Byte,
333                size: 0,
334            });
335        }
336        let element_type = ElementType::try_from(header & 0x0f)?;
337
338        let possible_element_count = (header & 0xF0) >> 4;
339        let element_count = if possible_element_count != 15 {
340            // high bits set high if count and type encoded separately
341            possible_element_count as i32
342        } else {
343            // The list size on the wire is an unsigned varint, but we represent
344            // it as `i32` (matching Java's `int` and the Thrift schema).
345            // A varint that decodes above `i32::MAX` is malformed input — reject
346            // it here at the protocol layer rather than letting the cast wrap
347            // into a negative size that downstream allocation code has to
348            // re-validate.
349            i32::try_from(self.read_vlq()?)?
350        };
351
352        Ok(ListIdentifier {
353            element_type,
354            size: element_count,
355        })
356    }
357
358    // Full field ids are uncommon.
359    // Not inlining this method reduces the code size of `read_field_begin`, which then ideally gets
360    // inlined everywhere.
361    #[cold]
362    fn read_full_field_id(&mut self) -> ThriftProtocolResult<i16> {
363        self.read_i16()
364    }
365
366    /// Read the [`FieldIdentifier`] for a field in a Thrift encoded struct.
367    fn read_field_begin(&mut self, last_field_id: i16) -> ThriftProtocolResult<FieldIdentifier> {
368        // we can read at least one byte, which is:
369        // - the type
370        // - the field delta and the type
371        let field_type = self.read_byte()?;
372        if field_type & 0xf == 0 {
373            return Ok(FieldIdentifier {
374                field_type: FieldType::Stop,
375                id: 0,
376            });
377        }
378
379        let field_delta = (field_type & 0xf0) >> 4;
380        let field_type = FieldType::try_from(field_type & 0xf)?;
381
382        let id = if field_delta != 0 {
383            last_field_id.checked_add(field_delta as i16).ok_or(
384                ThriftProtocolError::FieldDeltaOverflow {
385                    field_delta,
386                    last_field_id,
387                },
388            )?
389        } else {
390            self.read_full_field_id()?
391        };
392
393        Ok(FieldIdentifier { field_type, id })
394    }
395
396    /// This is a specialized version of [`Self::read_field_begin`], solely for use in parsing
397    /// simple structs. This function assumes that the delta field will always be less than 0xf,
398    /// fields will be in order, and no boolean fields will be read.
399    /// This also skips validation of the field type.
400    ///
401    /// Returns a tuple of `(field_type, field_delta)`.
402    fn read_field_header(&mut self) -> ThriftProtocolResult<(u8, u8)> {
403        let field_type = self.read_byte()?;
404        let field_delta = (field_type & 0xf0) >> 4;
405        let field_type = field_type & 0xf;
406        Ok((field_type, field_delta))
407    }
408
409    /// Read a boolean list element. This should not be used for struct fields. For the latter,
410    /// use the [`FieldIdentifier::bool_val`] field.
411    fn read_bool(&mut self) -> ThriftProtocolResult<bool> {
412        let b = self.read_byte()?;
413        // Previous versions of the thrift specification said to use 0 and 1 inside collections,
414        // but that differed from existing implementations.
415        // The specification was updated in https://github.com/apache/thrift/commit/2c29c5665bc442e703480bb0ee60fe925ffe02e8.
416        // At least the go implementation seems to have followed the previously documented values.
417        match b {
418            0x01 => Ok(true),
419            0x00 | 0x02 => Ok(false),
420            _ => Err(ThriftProtocolError::InvalidBoolean(b)),
421        }
422    }
423
424    /// Read a Thrift [binary] as a UTF-8 encoded string.
425    ///
426    /// [binary]: https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md#binary-encoding
427    fn read_string(&mut self) -> ThriftProtocolResult<&'a str> {
428        let slice = self.read_bytes()?;
429        Ok(std::str::from_utf8(slice)?)
430    }
431
432    /// Read an `i8`.
433    fn read_i8(&mut self) -> ThriftProtocolResult<i8> {
434        Ok(self.read_byte()? as _)
435    }
436
437    /// Read an `i16`.
438    fn read_i16(&mut self) -> ThriftProtocolResult<i16> {
439        Ok(self.read_zig_zag()? as _)
440    }
441
442    /// Read an `i32`.
443    fn read_i32(&mut self) -> ThriftProtocolResult<i32> {
444        Ok(self.read_zig_zag()? as _)
445    }
446
447    /// Read an `i64`.
448    fn read_i64(&mut self) -> ThriftProtocolResult<i64> {
449        self.read_zig_zag()
450    }
451
452    /// Read a Thrift `double` as `f64`.
453    fn read_double(&mut self) -> ThriftProtocolResult<f64>;
454
455    /// Skip a ULEB128 encoded varint.
456    fn skip_vlq(&mut self) -> ThriftProtocolResult<()> {
457        loop {
458            let byte = self.read_byte()?;
459            if byte & 0x80 == 0 {
460                return Ok(());
461            }
462        }
463    }
464
465    /// Skip a thrift [binary].
466    ///
467    /// [binary]: https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md#binary-encoding
468    fn skip_binary(&mut self) -> ThriftProtocolResult<()> {
469        let len = self.read_binary_len()?;
470        self.skip_bytes(len)
471    }
472
473    /// Skip a field with type `field_type` recursively until the default
474    /// maximum skip depth (currently 64) is reached.
475    fn skip(&mut self, field_type: FieldType) -> ThriftProtocolResult<()> {
476        const DEFAULT_SKIP_DEPTH: i8 = 64;
477        self.skip_till_depth(field_type, DEFAULT_SKIP_DEPTH)
478    }
479
480    /// Empty structs in unions consist of a single byte of 0 for the field stop record.
481    /// This skips that byte without encuring the cost of processing the [`FieldIdentifier`].
482    /// Will return an error if the struct is not actually empty.
483    fn skip_empty_struct(&mut self) -> Result<()> {
484        let b = self.read_byte()?;
485        if b != 0 {
486            Err(general_err!("Empty struct has fields"))
487        } else {
488            Ok(())
489        }
490    }
491
492    /// Skip a field with type `field_type` recursively up to `depth` levels.
493    fn skip_till_depth(&mut self, field_type: FieldType, depth: i8) -> ThriftProtocolResult<()> {
494        if depth == 0 {
495            return Err(ThriftProtocolError::SkipDepth(field_type));
496        }
497
498        match field_type {
499            // boolean field has no data
500            FieldType::BooleanFalse | FieldType::BooleanTrue => Ok(()),
501            FieldType::Byte => self.read_i8().map(|_| ()),
502            FieldType::I16 => self.skip_vlq(),
503            FieldType::I32 => self.skip_vlq(),
504            FieldType::I64 => self.skip_vlq(),
505            FieldType::Double => self.skip_bytes(8),
506            FieldType::Binary => self.skip_binary(),
507            // see https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md#struct
508            FieldType::Struct => {
509                loop {
510                    // we don't need field id for skipping, so always pass 0 for last id
511                    let field_ident = self.read_field_begin(0)?;
512                    if field_ident.field_type == FieldType::Stop {
513                        break;
514                    }
515                    self.skip_till_depth(field_ident.field_type, depth - 1)?;
516                }
517                Ok(())
518            }
519            // lists and sets are encoded the same
520            // see https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md#list-and-set
521            FieldType::List | FieldType::Set => {
522                let list_ident = self.read_list_begin()?;
523                if list_ident.element_type == ElementType::Bool {
524                    self.skip_bool_list(list_ident.size, depth - 1)
525                } else {
526                    let field_type = list_ident.element_type.into();
527                    for _ in 0..list_ident.size {
528                        self.skip_till_depth(field_type, depth - 1)?;
529                    }
530                    Ok(())
531                }
532            }
533            // see https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md#map
534            FieldType::Map => self.skip_map(depth),
535            // see https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md#universal-unique-identifier-encoding
536            FieldType::Uuid => self.skip_bytes(16),
537            FieldType::Stop => Err(ThriftProtocolError::SkipUnsupportedType(field_type)),
538        }
539    }
540
541    /// Skip boolean list or set payloads after reading their header.
542    #[cold]
543    fn skip_bool_list(&mut self, size: i32, depth: i8) -> ThriftProtocolResult<()> {
544        if size > 0 && depth == 0 {
545            return Err(ThriftProtocolError::SkipDepth(FieldType::BooleanTrue));
546        }
547        self.skip_bytes(size as usize)
548    }
549
550    /// Skip maps, which are not used by the current Parquet metadata schema.
551    #[cold]
552    fn skip_map(&mut self, depth: i8) -> ThriftProtocolResult<()> {
553        let size = i32::try_from(self.read_vlq()?)?;
554        if size > 0 {
555            let kv = self.read_byte()?;
556            let key_type = ElementType::try_from(kv >> 4)?;
557            let val_type = ElementType::try_from(kv & 0xf)?;
558            for _ in 0..size {
559                self.skip_element_till_depth(key_type, depth - 1)?;
560                self.skip_element_till_depth(val_type, depth - 1)?;
561            }
562        }
563        Ok(())
564    }
565
566    /// Collection booleans have a payload byte, unlike inline struct fields.
567    fn skip_element_till_depth(
568        &mut self,
569        element_type: ElementType,
570        depth: i8,
571    ) -> ThriftProtocolResult<()> {
572        if depth == 0 {
573            return Err(ThriftProtocolError::SkipDepth(element_type.into()));
574        }
575        match element_type {
576            ElementType::Bool => self.skip_bytes(1),
577            other => self.skip_till_depth(other.into(), depth),
578        }
579    }
580}
581
582/// A high performance Thrift reader that reads from a slice of bytes.
583pub(crate) struct ThriftSliceInputProtocol<'a> {
584    buf: &'a [u8],
585}
586
587impl<'a> ThriftSliceInputProtocol<'a> {
588    /// Create a new `ThriftSliceInputProtocol` using the bytes in `buf`.
589    pub fn new(buf: &'a [u8]) -> Self {
590        Self { buf }
591    }
592
593    /// Return the current buffer as a slice.
594    pub fn as_slice(&self) -> &'a [u8] {
595        self.buf
596    }
597}
598
599impl<'b, 'a: 'b> ThriftCompactInputProtocol<'b> for ThriftSliceInputProtocol<'a> {
600    #[inline]
601    fn read_byte(&mut self) -> ThriftProtocolResult<u8> {
602        let ret = *self.buf.first().ok_or(ThriftProtocolError::Eof)?;
603        self.buf = &self.buf[1..];
604        Ok(ret)
605    }
606
607    fn read_bytes(&mut self) -> ThriftProtocolResult<&'b [u8]> {
608        let len = self.read_binary_len()?;
609        let ret = self.buf.get(..len).ok_or(ThriftProtocolError::Eof)?;
610        self.buf = &self.buf[len..];
611        Ok(ret)
612    }
613
614    fn read_bytes_owned(&mut self) -> ThriftProtocolResult<Vec<u8>> {
615        Ok(self.read_bytes()?.to_vec())
616    }
617
618    #[inline]
619    fn skip_bytes(&mut self, n: usize) -> ThriftProtocolResult<()> {
620        self.buf.get(..n).ok_or(ThriftProtocolError::Eof)?;
621        self.buf = &self.buf[n..];
622        Ok(())
623    }
624
625    fn read_double(&mut self) -> ThriftProtocolResult<f64> {
626        let slice = self.buf.get(..8).ok_or(ThriftProtocolError::Eof)?;
627        self.buf = &self.buf[8..];
628        match slice.try_into() {
629            Ok(slice) => Ok(f64::from_le_bytes(slice)),
630            Err(_) => unreachable!(),
631        }
632    }
633
634    fn remaining_bytes(&self) -> Option<usize> {
635        Some(self.buf.len())
636    }
637}
638
639/// A Thrift input protocol that wraps a [`Read`] object.
640///
641/// Note that this is only intended for use in reading Parquet page headers. This will panic
642/// if Thrift `binary` data is encountered because a slice of that data cannot be returned.
643pub(crate) struct ThriftReadInputProtocol<R: Read> {
644    reader: R,
645}
646
647impl<R: Read> ThriftReadInputProtocol<R> {
648    pub(crate) fn new(reader: R) -> Self {
649        Self { reader }
650    }
651}
652
653impl<'a, R: Read> ThriftCompactInputProtocol<'a> for ThriftReadInputProtocol<R> {
654    #[inline]
655    fn read_byte(&mut self) -> ThriftProtocolResult<u8> {
656        let mut buf = [0_u8; 1];
657        self.reader.read_exact(&mut buf)?;
658        Ok(buf[0])
659    }
660
661    fn read_bytes(&mut self) -> ThriftProtocolResult<&'a [u8]> {
662        unimplemented!()
663    }
664
665    fn read_bytes_owned(&mut self) -> ThriftProtocolResult<Vec<u8>> {
666        let len = self.read_binary_len()?;
667        let mut v = Vec::with_capacity(len.min(8 * 1024));
668        let copied = std::io::copy(&mut self.reader.by_ref().take(len as u64), &mut v)?;
669        if copied != len as u64 {
670            return Err(ThriftProtocolError::Eof);
671        }
672        Ok(v)
673    }
674
675    fn skip_bytes(&mut self, n: usize) -> ThriftProtocolResult<()> {
676        let skipped = std::io::copy(
677            &mut self.reader.by_ref().take(n as u64),
678            &mut std::io::sink(),
679        )?;
680        if skipped != n as u64 {
681            return Err(ThriftProtocolError::Eof);
682        }
683        Ok(())
684    }
685
686    fn read_double(&mut self) -> ThriftProtocolResult<f64> {
687        let mut buf = [0_u8; 8];
688        self.reader.read_exact(&mut buf)?;
689        Ok(f64::from_le_bytes(buf))
690    }
691}
692
693/// Trait implemented for objects that can be deserialized from a Thrift input stream.
694/// Implementations are provided for Thrift primitive types.
695pub(crate) trait ReadThrift<'a, R: ThriftCompactInputProtocol<'a>> {
696    /// Read an object of type `Self` from the input protocol object.
697    fn read_thrift(prot: &mut R) -> Result<Self>
698    where
699        Self: Sized;
700}
701
702impl<'a, R: ThriftCompactInputProtocol<'a>> ReadThrift<'a, R> for bool {
703    fn read_thrift(prot: &mut R) -> Result<Self> {
704        Ok(prot.read_bool()?)
705    }
706}
707
708impl<'a, R: ThriftCompactInputProtocol<'a>> ReadThrift<'a, R> for i8 {
709    fn read_thrift(prot: &mut R) -> Result<Self> {
710        Ok(prot.read_i8()?)
711    }
712}
713
714impl<'a, R: ThriftCompactInputProtocol<'a>> ReadThrift<'a, R> for i16 {
715    fn read_thrift(prot: &mut R) -> Result<Self> {
716        Ok(prot.read_i16()?)
717    }
718}
719
720impl<'a, R: ThriftCompactInputProtocol<'a>> ReadThrift<'a, R> for i32 {
721    fn read_thrift(prot: &mut R) -> Result<Self> {
722        Ok(prot.read_i32()?)
723    }
724}
725
726impl<'a, R: ThriftCompactInputProtocol<'a>> ReadThrift<'a, R> for i64 {
727    fn read_thrift(prot: &mut R) -> Result<Self> {
728        Ok(prot.read_i64()?)
729    }
730}
731
732impl<'a, R: ThriftCompactInputProtocol<'a>> ReadThrift<'a, R> for OrderedF64 {
733    fn read_thrift(prot: &mut R) -> Result<Self> {
734        Ok(OrderedF64(prot.read_double()?))
735    }
736}
737
738impl<'a, R: ThriftCompactInputProtocol<'a>> ReadThrift<'a, R> for &'a str {
739    fn read_thrift(prot: &mut R) -> Result<Self> {
740        Ok(prot.read_string()?)
741    }
742}
743
744impl<'a, R: ThriftCompactInputProtocol<'a>> ReadThrift<'a, R> for String {
745    fn read_thrift(prot: &mut R) -> Result<Self> {
746        Ok(String::from_utf8(prot.read_bytes_owned()?)?)
747    }
748}
749
750impl<'a, R: ThriftCompactInputProtocol<'a>> ReadThrift<'a, R> for &'a [u8] {
751    fn read_thrift(prot: &mut R) -> Result<Self> {
752        Ok(prot.read_bytes()?)
753    }
754}
755
756/// Read a Thrift encoded [list] from the input protocol object.
757///
758/// [list]: https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md#list-and-set
759pub(crate) fn read_thrift_vec<'a, T, R>(prot: &mut R) -> Result<Vec<T>>
760where
761    R: ThriftCompactInputProtocol<'a>,
762    T: ReadThrift<'a, R> + WriteThrift,
763{
764    let list_ident = prot.read_list_begin()?;
765    validate_list_type(T::ELEMENT_TYPE, &list_ident)?;
766    let size = list_ident.size as usize;
767    check_list_size(prot.remaining_bytes(), size)?;
768    let mut res = Vec::with_capacity(size);
769    for _ in 0..list_ident.size {
770        let val = T::read_thrift(prot)?;
771        res.push(val);
772    }
773    Ok(res)
774}
775
776pub(crate) fn validate_list_type(expected: ElementType, got: &ListIdentifier) -> Result<()> {
777    // An empty list decodes no elements, and some writers (e.g. fastparquet)
778    // encode it as a bare 0x00 header with no meaningful element type.
779    if got.element_type != expected && got.size != 0 {
780        return Err(general_err!(
781            "Expected list element type of {:?} but got {:?}",
782            expected,
783            got.element_type
784        ));
785    }
786    Ok(())
787}
788
789/// Reject a declared list size that cannot fit the remaining input. Each list
790/// element is at least one byte on the wire, so a size past the remaining
791/// bytes is a malformed header; bounding it before reserving stops a huge
792/// allocation. `remaining` is `None` when the protocol is not backed by a
793/// finite buffer, in which case there is nothing to check against.
794pub(crate) fn check_list_size(remaining: Option<usize>, size: usize) -> Result<()> {
795    if let Some(remaining) = remaining
796        && size > remaining
797    {
798        return Err(general_err!(
799            "Thrift list size {size} exceeds remaining input length {remaining}"
800        ));
801    }
802    Ok(())
803}
804
805/////////////////////////
806// thrift compact output
807
808/// Low-level object used to serialize structs to the Thrift [compact output] protocol.
809///
810/// This struct serves as a wrapper around a [`Write`] object, to which thrift encoded data
811/// will written. The implementation provides functions to write Thrift primitive types, as well
812/// as functions used in the encoding of lists and structs. This should rarely be used directly,
813/// but is instead intended for use by implementers of [`WriteThrift`] and [`WriteThriftField`].
814///
815/// [compact output]: https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md
816pub(crate) struct ThriftCompactOutputProtocol<W: Write> {
817    writer: W,
818    write_path_in_schema: bool,
819    write_rg_ordinal: bool,
820}
821
822impl<W: Write> ThriftCompactOutputProtocol<W> {
823    /// Create a new `ThriftCompactOutputProtocol` wrapping the byte sink `writer`.
824    pub(crate) fn new(writer: W) -> Self {
825        Self {
826            writer,
827            write_path_in_schema: true,
828            write_rg_ordinal: true,
829        }
830    }
831
832    // TODO(ets): at some point there should probably be a properties object
833    // to control aspects of thrift output. But since this is the only option to date
834    // I'm choosing a simpler API.
835    /// Control the writing of the `path_in_schema` element of the `ColumnMetaData`
836    pub(crate) fn set_write_path_in_schema(&mut self, val: bool) {
837        self.write_path_in_schema = val;
838    }
839
840    /// Indicate whether or not to emit `path_in_schema`.
841    pub(crate) fn write_path_in_schema(&self) -> bool {
842        self.write_path_in_schema
843    }
844
845    /// Control the writing of the `ordinal` element of the `RowGroup` struct.
846    ///
847    /// The Thrift `ordinal` field on the `RowGroup` struct is `i16`, but the
848    /// Thrift compact protocol allows for up to 2^31 elements in a list. If
849    /// more than 2^15 row groups are to be written, this can be set to `false`
850    /// to prevent writing the ordinal for some row groups but not others.
851    pub(crate) fn set_write_row_group_ordinal(&mut self, val: bool) {
852        self.write_rg_ordinal = val;
853    }
854
855    /// Indicate whether or not to emit `ordinal`.
856    pub(crate) fn write_row_group_ordinal(&self) -> bool {
857        self.write_rg_ordinal
858    }
859
860    /// Write a single byte to the output stream.
861    fn write_byte(&mut self, b: u8) -> Result<()> {
862        self.writer.write_all(&[b])?;
863        Ok(())
864    }
865
866    /// Write the given `u64` as a ULEB128 encoded varint.
867    fn write_vlq(&mut self, val: u64) -> Result<()> {
868        let mut v = val;
869        while v > 0x7f {
870            self.write_byte(v as u8 | 0x80)?;
871            v >>= 7;
872        }
873        self.write_byte(v as u8)
874    }
875
876    /// Write the given `i64` as a zig-zag encoded varint.
877    fn write_zig_zag(&mut self, val: i64) -> Result<()> {
878        let s = (val < 0) as i64;
879        self.write_vlq((((val ^ -s) << 1) + s) as u64)
880    }
881
882    /// Used to mark the start of a Thrift struct field of type `field_type`. `last_field_id`
883    /// is used to compute a delta to the given `field_id` per the compact protocol [spec].
884    ///
885    /// [spec]: https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md#struct-encoding
886    pub(crate) fn write_field_begin(
887        &mut self,
888        field_type: FieldType,
889        field_id: i16,
890        last_field_id: i16,
891    ) -> Result<()> {
892        let delta = field_id.wrapping_sub(last_field_id);
893        if delta > 0 && delta <= 0xf {
894            self.write_byte(((delta as u8) << 4) | field_type as u8)
895        } else {
896            self.write_byte(field_type as u8)?;
897            self.write_i16(field_id)
898        }
899    }
900
901    /// Used to indicate the start of a list of `element_type` elements.
902    pub(crate) fn write_list_begin(&mut self, element_type: ElementType, len: usize) -> Result<()> {
903        if len < 15 {
904            self.write_byte(((len as u8) << 4) | element_type as u8)
905        } else {
906            self.write_byte(0xf0u8 | element_type as u8)?;
907            self.write_vlq(len as _)
908        }
909    }
910
911    /// Used to mark the end of a struct. This must be called after all fields of the struct have
912    /// been written.
913    pub(crate) fn write_struct_end(&mut self) -> Result<()> {
914        self.write_byte(0)
915    }
916
917    /// Serialize a slice of `u8`s. This will encode a length, and then write the bytes without
918    /// further encoding.
919    pub(crate) fn write_bytes(&mut self, val: &[u8]) -> Result<()> {
920        self.write_vlq(val.len() as u64)?;
921        self.writer.write_all(val)?;
922        Ok(())
923    }
924
925    /// Short-cut method used to encode structs that have no fields (often used in Thrift unions).
926    /// This simply encodes the field id and then immediately writes the end-of-struct marker.
927    pub(crate) fn write_empty_struct(&mut self, field_id: i16, last_field_id: i16) -> Result<i16> {
928        self.write_field_begin(FieldType::Struct, field_id, last_field_id)?;
929        self.write_struct_end()?;
930        Ok(last_field_id)
931    }
932
933    /// Write a boolean value.
934    pub(crate) fn write_bool(&mut self, val: bool) -> Result<()> {
935        match val {
936            true => self.write_byte(1),
937            false => self.write_byte(2),
938        }
939    }
940
941    /// Write a zig-zag encoded `i8` value.
942    pub(crate) fn write_i8(&mut self, val: i8) -> Result<()> {
943        self.write_byte(val as u8)
944    }
945
946    /// Write a zig-zag encoded `i16` value.
947    pub(crate) fn write_i16(&mut self, val: i16) -> Result<()> {
948        self.write_zig_zag(val as _)
949    }
950
951    /// Write a zig-zag encoded `i32` value.
952    pub(crate) fn write_i32(&mut self, val: i32) -> Result<()> {
953        self.write_zig_zag(val as _)
954    }
955
956    /// Write a zig-zag encoded `i64` value.
957    pub(crate) fn write_i64(&mut self, val: i64) -> Result<()> {
958        self.write_zig_zag(val)
959    }
960
961    /// Write a double value.
962    pub(crate) fn write_double(&mut self, val: f64) -> Result<()> {
963        self.writer.write_all(&val.to_le_bytes())?;
964        Ok(())
965    }
966}
967
968/// Trait implemented by objects that are to be serialized to a Thrift [compact output] protocol
969/// stream. Implementations are also provided for primitive Thrift types.
970///
971/// [compact output]: https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md
972pub(crate) trait WriteThrift {
973    /// The [`ElementType`] to use when a list of this object is written.
974    const ELEMENT_TYPE: ElementType;
975
976    /// Serialize this object to the given `writer`.
977    fn write_thrift<W: Write>(&self, writer: &mut ThriftCompactOutputProtocol<W>) -> Result<()>;
978}
979
980/// Implementation for a vector of thrift serializable objects that implement [`WriteThrift`].
981/// This will write the necessary list header and then serialize the elements one-at-a-time.
982impl<T> WriteThrift for Vec<T>
983where
984    T: WriteThrift,
985{
986    const ELEMENT_TYPE: ElementType = ElementType::List;
987
988    fn write_thrift<W: Write>(&self, writer: &mut ThriftCompactOutputProtocol<W>) -> Result<()> {
989        writer.write_list_begin(T::ELEMENT_TYPE, self.len())?;
990        for item in self {
991            item.write_thrift(writer)?;
992        }
993        Ok(())
994    }
995}
996
997impl WriteThrift for bool {
998    const ELEMENT_TYPE: ElementType = ElementType::Bool;
999
1000    fn write_thrift<W: Write>(&self, writer: &mut ThriftCompactOutputProtocol<W>) -> Result<()> {
1001        writer.write_bool(*self)
1002    }
1003}
1004
1005impl WriteThrift for i8 {
1006    const ELEMENT_TYPE: ElementType = ElementType::Byte;
1007
1008    fn write_thrift<W: Write>(&self, writer: &mut ThriftCompactOutputProtocol<W>) -> Result<()> {
1009        writer.write_i8(*self)
1010    }
1011}
1012
1013impl WriteThrift for i16 {
1014    const ELEMENT_TYPE: ElementType = ElementType::I16;
1015
1016    fn write_thrift<W: Write>(&self, writer: &mut ThriftCompactOutputProtocol<W>) -> Result<()> {
1017        writer.write_i16(*self)
1018    }
1019}
1020
1021impl WriteThrift for i32 {
1022    const ELEMENT_TYPE: ElementType = ElementType::I32;
1023
1024    fn write_thrift<W: Write>(&self, writer: &mut ThriftCompactOutputProtocol<W>) -> Result<()> {
1025        writer.write_i32(*self)
1026    }
1027}
1028
1029impl WriteThrift for i64 {
1030    const ELEMENT_TYPE: ElementType = ElementType::I64;
1031
1032    fn write_thrift<W: Write>(&self, writer: &mut ThriftCompactOutputProtocol<W>) -> Result<()> {
1033        writer.write_i64(*self)
1034    }
1035}
1036
1037impl WriteThrift for OrderedF64 {
1038    const ELEMENT_TYPE: ElementType = ElementType::Double;
1039
1040    fn write_thrift<W: Write>(&self, writer: &mut ThriftCompactOutputProtocol<W>) -> Result<()> {
1041        writer.write_double(self.0)
1042    }
1043}
1044
1045impl WriteThrift for f64 {
1046    const ELEMENT_TYPE: ElementType = ElementType::Double;
1047
1048    fn write_thrift<W: Write>(&self, writer: &mut ThriftCompactOutputProtocol<W>) -> Result<()> {
1049        writer.write_double(*self)
1050    }
1051}
1052
1053impl WriteThrift for &[u8] {
1054    const ELEMENT_TYPE: ElementType = ElementType::Binary;
1055
1056    fn write_thrift<W: Write>(&self, writer: &mut ThriftCompactOutputProtocol<W>) -> Result<()> {
1057        writer.write_bytes(self)
1058    }
1059}
1060
1061impl WriteThrift for &str {
1062    const ELEMENT_TYPE: ElementType = ElementType::Binary;
1063
1064    fn write_thrift<W: Write>(&self, writer: &mut ThriftCompactOutputProtocol<W>) -> Result<()> {
1065        writer.write_bytes(self.as_bytes())
1066    }
1067}
1068
1069impl WriteThrift for String {
1070    const ELEMENT_TYPE: ElementType = ElementType::Binary;
1071
1072    fn write_thrift<W: Write>(&self, writer: &mut ThriftCompactOutputProtocol<W>) -> Result<()> {
1073        writer.write_bytes(self.as_bytes())
1074    }
1075}
1076
1077/// Trait implemented by objects that are fields of Thrift structs.
1078///
1079/// For example, given the Thrift struct definition
1080/// ```ignore
1081/// struct MyStruct {
1082///   1: required i32 field1
1083///   2: optional bool field2
1084///   3: optional OtherStruct field3
1085/// }
1086/// ```
1087///
1088/// which becomes in Rust
1089/// ```ignore
1090/// # struct OtherStruct {}
1091/// struct MyStruct {
1092///   field1: i32,
1093///   field2: Option<bool>,
1094///   field3: Option<OtherStruct>,
1095/// }
1096/// ```
1097/// the impl of `WriteThrift` for `MyStruct` will use the `WriteThriftField` impls for `i32`,
1098/// `bool`, and `OtherStruct`.
1099///
1100/// ```ignore
1101/// impl WriteThrift for MyStruct {
1102///   fn write_thrift<W: Write>(&self, writer: &mut ThriftCompactOutputProtocol<W>) -> Result<()> {
1103///     let mut last_field_id = 0i16;
1104///     last_field_id = self.field1.write_thrift_field(writer, 1, last_field_id)?;
1105///     if self.field2.is_some() {
1106///       // if field2 is `None` then this assignment won't happen and last_field_id will remain
1107///       // `1` when writing `field3`
1108///       last_field_id = self.field2.write_thrift_field(writer, 2, last_field_id)?;
1109///     }
1110///     if self.field3.is_some() {
1111///       // no need to assign last_field_id since this is the final field.
1112///       self.field3.write_thrift_field(writer, 3, last_field_id)?;
1113///     }
1114///     writer.write_struct_end()
1115///   }
1116/// }
1117/// ```
1118///
1119pub(crate) trait WriteThriftField {
1120    /// Used to write struct fields (which may be primitive or IDL defined types). This will
1121    /// write the field marker for the given `field_id`, using `last_field_id` to compute the
1122    /// field delta used by the Thrift [compact protocol]. On success this will return `field_id`
1123    /// to be used in chaining.
1124    ///
1125    /// [compact protocol]: https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md#struct-encoding
1126    fn write_thrift_field<W: Write>(
1127        &self,
1128        writer: &mut ThriftCompactOutputProtocol<W>,
1129        field_id: i16,
1130        last_field_id: i16,
1131    ) -> Result<i16>;
1132}
1133
1134// bool struct fields are written differently to bool values
1135impl WriteThriftField for bool {
1136    fn write_thrift_field<W: Write>(
1137        &self,
1138        writer: &mut ThriftCompactOutputProtocol<W>,
1139        field_id: i16,
1140        last_field_id: i16,
1141    ) -> Result<i16> {
1142        // boolean only writes the field header
1143        match *self {
1144            true => writer.write_field_begin(FieldType::BooleanTrue, field_id, last_field_id)?,
1145            false => writer.write_field_begin(FieldType::BooleanFalse, field_id, last_field_id)?,
1146        }
1147        Ok(field_id)
1148    }
1149}
1150
1151write_thrift_field!(i8, FieldType::Byte);
1152write_thrift_field!(i16, FieldType::I16);
1153write_thrift_field!(i32, FieldType::I32);
1154write_thrift_field!(i64, FieldType::I64);
1155write_thrift_field!(OrderedF64, FieldType::Double);
1156write_thrift_field!(f64, FieldType::Double);
1157write_thrift_field!(String, FieldType::Binary);
1158
1159impl WriteThriftField for &[u8] {
1160    fn write_thrift_field<W: Write>(
1161        &self,
1162        writer: &mut ThriftCompactOutputProtocol<W>,
1163        field_id: i16,
1164        last_field_id: i16,
1165    ) -> Result<i16> {
1166        writer.write_field_begin(FieldType::Binary, field_id, last_field_id)?;
1167        writer.write_bytes(self)?;
1168        Ok(field_id)
1169    }
1170}
1171
1172impl WriteThriftField for &str {
1173    fn write_thrift_field<W: Write>(
1174        &self,
1175        writer: &mut ThriftCompactOutputProtocol<W>,
1176        field_id: i16,
1177        last_field_id: i16,
1178    ) -> Result<i16> {
1179        writer.write_field_begin(FieldType::Binary, field_id, last_field_id)?;
1180        writer.write_bytes(self.as_bytes())?;
1181        Ok(field_id)
1182    }
1183}
1184
1185impl<T> WriteThriftField for Vec<T>
1186where
1187    T: WriteThrift,
1188{
1189    fn write_thrift_field<W: Write>(
1190        &self,
1191        writer: &mut ThriftCompactOutputProtocol<W>,
1192        field_id: i16,
1193        last_field_id: i16,
1194    ) -> Result<i16> {
1195        writer.write_field_begin(FieldType::List, field_id, last_field_id)?;
1196        self.write_thrift(writer)?;
1197        Ok(field_id)
1198    }
1199}
1200
1201#[cfg(test)]
1202pub(crate) mod tests {
1203    use crate::basic::{TimeUnit, Type};
1204
1205    use super::*;
1206    use std::fmt::Debug;
1207    use std::io::{ErrorKind, Result as IoResult};
1208
1209    pub(crate) fn test_roundtrip<T>(val: T)
1210    where
1211        T: for<'a> ReadThrift<'a, ThriftSliceInputProtocol<'a>> + WriteThrift + PartialEq + Debug,
1212    {
1213        let mut buf = Vec::<u8>::new();
1214        {
1215            let mut writer = ThriftCompactOutputProtocol::new(&mut buf);
1216            val.write_thrift(&mut writer).unwrap();
1217        }
1218
1219        let mut prot = ThriftSliceInputProtocol::new(&buf);
1220        let read_val = T::read_thrift(&mut prot).unwrap();
1221        assert_eq!(val, read_val);
1222    }
1223
1224    #[test]
1225    fn test_binary_length_bound() {
1226        let oversized = [0x80, 0x80, 0x80, 0x80, 8]; // i32::MAX + 1
1227        let maximum = [0xff, 0xff, 0xff, 0xff, 7]; // i32::MAX, no payload
1228        for bytes in [&oversized, &maximum] {
1229            let check = |result: ThriftProtocolResult<()>| {
1230                if bytes == &oversized {
1231                    assert!(matches!(result, Err(ThriftProtocolError::IntegerOverflow)));
1232                } else {
1233                    assert!(matches!(result, Err(ThriftProtocolError::Eof)));
1234                }
1235            };
1236            check(
1237                ThriftSliceInputProtocol::new(bytes)
1238                    .read_bytes()
1239                    .map(|_| ()),
1240            );
1241            check(
1242                ThriftSliceInputProtocol::new(bytes)
1243                    .read_bytes_owned()
1244                    .map(|_| ()),
1245            );
1246            check(ThriftSliceInputProtocol::new(bytes).skip_binary());
1247            check(
1248                ThriftReadInputProtocol::new(bytes.as_slice())
1249                    .read_bytes_owned()
1250                    .map(|_| ()),
1251            );
1252            check(ThriftReadInputProtocol::new(bytes.as_slice()).skip_binary());
1253        }
1254    }
1255
1256    struct FragmentedReader<'a>(&'a [u8]);
1257
1258    impl Read for FragmentedReader<'_> {
1259        fn read(&mut self, buf: &mut [u8]) -> IoResult<usize> {
1260            let len = buf.len().min(2);
1261            self.0.read(&mut buf[..len])
1262        }
1263    }
1264
1265    #[test]
1266    fn test_read_bytes_owned_complete_and_fragmented() {
1267        let large_payload = vec![7; 16 * 1024];
1268        for payload in [b"".as_slice(), b"abcde".as_slice(), &large_payload] {
1269            let mut data = Vec::new();
1270            ThriftCompactOutputProtocol::new(&mut data)
1271                .write_bytes(payload)
1272                .unwrap();
1273            data.push(42);
1274            let mut slice = ThriftSliceInputProtocol::new(&data);
1275            let mut stream = ThriftReadInputProtocol::new(data.as_slice());
1276            let mut fragmented = ThriftReadInputProtocol::new(FragmentedReader(&data));
1277            assert_eq!(slice.read_bytes_owned().unwrap(), payload);
1278            assert_eq!(stream.read_bytes_owned().unwrap(), payload);
1279            assert_eq!(fragmented.read_bytes_owned().unwrap(), payload);
1280            assert_eq!(slice.read_byte().unwrap(), 42);
1281            assert_eq!(stream.read_byte().unwrap(), 42);
1282            assert_eq!(fragmented.read_byte().unwrap(), 42);
1283        }
1284    }
1285
1286    #[test]
1287    fn test_read_bytes_owned_truncated_payload() {
1288        for len in [5, 1024 * 1024, i32::MAX as u64] {
1289            let mut data = Vec::new();
1290            ThriftCompactOutputProtocol::new(&mut data)
1291                .write_vlq(len)
1292                .unwrap();
1293            data.extend_from_slice(b"abc");
1294            let mut stream = ThriftReadInputProtocol::new(FragmentedReader(&data));
1295            assert!(matches!(
1296                stream.read_bytes_owned(),
1297                Err(ThriftProtocolError::Eof)
1298            ));
1299        }
1300    }
1301
1302    #[test]
1303    fn test_read_bytes_owned_preserves_io_error() {
1304        struct FailingReader;
1305        impl Read for FailingReader {
1306            fn read(&mut self, _: &mut [u8]) -> IoResult<usize> {
1307                Err(Error::new(ErrorKind::PermissionDenied, "payload error"))
1308            }
1309        }
1310        let reader = b"\x05ab".as_slice().chain(FailingReader);
1311        let mut stream = ThriftReadInputProtocol::new(reader);
1312        assert!(
1313            matches!(stream.read_bytes_owned(), Err(ThriftProtocolError::IO(e)) if e.kind() == ErrorKind::PermissionDenied)
1314        );
1315    }
1316
1317    #[test]
1318    fn test_enum_roundtrip() {
1319        test_roundtrip(Type::BOOLEAN);
1320        test_roundtrip(Type::INT32);
1321        test_roundtrip(Type::INT64);
1322        test_roundtrip(Type::INT96);
1323        test_roundtrip(Type::FLOAT);
1324        test_roundtrip(Type::DOUBLE);
1325        test_roundtrip(Type::BYTE_ARRAY);
1326        test_roundtrip(Type::FIXED_LEN_BYTE_ARRAY);
1327    }
1328
1329    #[test]
1330    fn test_union_all_empty_roundtrip() {
1331        test_roundtrip(TimeUnit::MILLIS);
1332        test_roundtrip(TimeUnit::MICROS);
1333        test_roundtrip(TimeUnit::NANOS);
1334    }
1335
1336    #[test]
1337    fn test_decode_empty_list() {
1338        let data = vec![0u8; 1];
1339        let mut prot = ThriftSliceInputProtocol::new(&data);
1340        let header = prot.read_list_begin().expect("error reading list header");
1341        assert_eq!(header.size, 0);
1342        assert_eq!(header.element_type, ElementType::Byte);
1343    }
1344
1345    #[test]
1346    fn test_read_empty_list_with_zero_header() {
1347        let data = [0u8];
1348        let mut prot = ThriftSliceInputProtocol::new(&data);
1349        let result = read_thrift_vec::<i32, ThriftSliceInputProtocol>(&mut prot).unwrap();
1350        assert!(result.is_empty());
1351    }
1352
1353    /// A Thrift list header whose `size` varint decodes above `i32::MAX`
1354    /// must be rejected at the protocol layer rather than wrapping into a
1355    /// negative `i32` and being smuggled into downstream allocation code.
1356    #[test]
1357    fn test_read_list_begin_size_above_i32_max_returns_err() {
1358        // List header: element_type=8 (Binary), 0xF=follow-up varint.
1359        // Varint 80 80 80 80 08 decodes to 0x8000_0000 = i32::MAX + 1.
1360        let mut data: Vec<u8> = vec![0xF8];
1361        data.extend_from_slice(&[0x80, 0x80, 0x80, 0x80, 0x08]);
1362        let mut prot = ThriftSliceInputProtocol::new(&data);
1363        let result = prot.read_list_begin();
1364        assert!(result.is_err(), "expected error, got {result:?}");
1365    }
1366
1367    // Unknown fields must leave the following known field aligned.
1368    crate::thrift_struct! {
1369        struct KnownField {
1370            2: required i32 value;
1371        }
1372    }
1373
1374    #[test]
1375    fn test_skip_boolean_collections_before_known_field() {
1376        for collection_type in [FieldType::List, FieldType::Set] {
1377            // False then true must not be mistaken for inline field headers.
1378            // Known field 2 follows with i32(42).
1379            let data = [0x10 | collection_type as u8, 0x22, 2, 1, 0x15, 84, 0];
1380            let mut protocol = ThriftSliceInputProtocol::new(&data);
1381            assert_eq!(KnownField::read_thrift(&mut protocol).unwrap().value, 42);
1382            assert!(protocol.as_slice().is_empty());
1383        }
1384    }
1385
1386    #[test]
1387    fn test_skip_map_before_known_field() {
1388        // Unknown field 1: map<bool, i32> with two entries. Known field 2: i32(42).
1389        let data = [0x1b, 2, 0x25, 1, 14, 2, 16, 0x15, 84, 0];
1390        let mut protocol = ThriftSliceInputProtocol::new(&data);
1391        assert_eq!(KnownField::read_thrift(&mut protocol).unwrap().value, 42);
1392        assert!(protocol.as_slice().is_empty());
1393    }
1394
1395    #[test]
1396    fn test_read_list_wrong_type() {
1397        // list header: 4 elements of `Boolean`
1398        let data = [0x42, 0x01];
1399        let mut prot = ThriftSliceInputProtocol::new(&data);
1400        // try to read as list<i32>
1401        let result = read_thrift_vec::<i32, ThriftSliceInputProtocol>(&mut prot);
1402        println!("{result:?}");
1403        assert!(
1404            result
1405                .unwrap_err()
1406                .to_string()
1407                .contains("Expected list element type of I32 but got Bool")
1408        );
1409    }
1410
1411    #[test]
1412    fn test_read_thrift_vec_roundtrip_i32() {
1413        // 2-element list of i32: header 0x25 (count=2, type=I32), two zigzag zeros.
1414        let data = [0x25, 0x00, 0x00];
1415        let mut prot = ThriftSliceInputProtocol::new(&data);
1416        let result = read_thrift_vec::<i32, ThriftSliceInputProtocol>(&mut prot).unwrap();
1417        assert_eq!(result, vec![0, 0]);
1418    }
1419
1420    #[test]
1421    fn test_read_thrift_vec_size_exceeds_remaining_returns_err() {
1422        // Header 0xE5: 14 i32 elements, no payload. After reading the header
1423        // zero bytes remain, so this must error instead of reserving 14 slots
1424        // (and, for a larger declared size, gigabytes).
1425        let data = [0xE5];
1426        let mut prot = ThriftSliceInputProtocol::new(&data);
1427        let result = read_thrift_vec::<i32, ThriftSliceInputProtocol>(&mut prot);
1428        assert!(result.is_err(), "expected error, got {result:?}");
1429        assert!(
1430            result
1431                .unwrap_err()
1432                .to_string()
1433                .contains("Thrift list size 14 exceeds remaining input length 0")
1434        );
1435    }
1436
1437    #[test]
1438    fn test_read_thrift_vec_huge_declared_size_returns_err() {
1439        // Compact list header: 0xF5 = I32 elements, size follows as a varint.
1440        // 0xfc 0xfc 0xfc 0x33 decodes to 109_002_364 — the schema-list case
1441        // from #10920 — with only two leftover bytes.
1442        let data = [0xF5, 0xfc, 0xfc, 0xfc, 0x33, 0x00, 0x00];
1443        let mut prot = ThriftSliceInputProtocol::new(&data);
1444        let result = read_thrift_vec::<i32, ThriftSliceInputProtocol>(&mut prot);
1445        assert!(result.is_err(), "expected error, got {result:?}");
1446        let err = result.unwrap_err().to_string();
1447        assert!(
1448            err.contains("Thrift list size 109002364 exceeds remaining input length 2"),
1449            "{err}"
1450        );
1451    }
1452}