parquet_variant/variant/
list.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17use crate::decoder::{map_bytes_to_offsets, OffsetSizeBytes};
18use crate::utils::{
19    first_byte_from_slice, overflow_error, slice_from_slice, slice_from_slice_at_offset,
20};
21use crate::variant::{Variant, VariantMetadata};
22
23use arrow_schema::ArrowError;
24
25// The value header occupies one byte; use a named constant for readability
26const NUM_HEADER_BYTES: u32 = 1;
27
28/// A parsed version of the variant array value header byte.
29#[derive(Debug, Clone, PartialEq)]
30pub(crate) struct VariantListHeader {
31    num_elements_size: OffsetSizeBytes,
32    offset_size: OffsetSizeBytes,
33}
34
35impl VariantListHeader {
36    // Hide the ugly casting
37    const fn num_elements_size(&self) -> u32 {
38        self.num_elements_size as _
39    }
40    const fn offset_size(&self) -> u32 {
41        self.offset_size as _
42    }
43
44    // Avoid materializing this offset, since it's cheaply and safely computable
45    const fn first_offset_byte(&self) -> u32 {
46        NUM_HEADER_BYTES + self.num_elements_size()
47    }
48
49    pub(crate) fn try_new(header_byte: u8) -> Result<Self, ArrowError> {
50        // The 6 first bits to the left are the value_header and the 2 bits
51        // to the right are the basic type, so we shift to get only the value_header
52        let value_header = header_byte >> 2;
53        let is_large = (value_header & 0x04) != 0; // 3rd bit from the right
54        let field_offset_size_minus_one = value_header & 0x03; // Last two bits
55
56        // The size of the num_elements entry in the array value_data is 4 bytes if
57        // is_large is true, otherwise 1 byte.
58        let num_elements_size = match is_large {
59            true => OffsetSizeBytes::Four,
60            false => OffsetSizeBytes::One,
61        };
62        let offset_size = OffsetSizeBytes::try_new(field_offset_size_minus_one)?;
63
64        Ok(Self {
65            num_elements_size,
66            offset_size,
67        })
68    }
69}
70
71/// [`Variant`] Array.
72///
73/// See the [Variant spec] for details.
74///
75/// NOTE: The "list" naming differs from the variant spec -- which calls it "array" -- in order to be
76/// consistent with Parquet and Arrow type naming. Otherwise, the name would conflict with the
77/// `VariantArray : Array` we must eventually define for variant-typed arrow arrays.
78///
79/// # Validation
80///
81/// Every instance of variant list is either _valid_ or _invalid_. depending on whether the
82/// underlying bytes are a valid encoding of a variant array (see below).
83///
84/// Instances produced by [`Self::try_new`] or [`Self::with_full_validation`] are fully _validated_. They always
85/// contain _valid_ data, and infallible accesses such as iteration and indexing are panic-free. The
86/// validation cost is linear in the number of underlying bytes.
87///
88/// Instances produced by [`Self::new`] are _unvalidated_ and so they may contain either _valid_ or
89/// _invalid_ data. Infallible accesses such as iteration and indexing will panic if the underlying
90/// bytes are _invalid_, and fallible alternatives such as [`Self::iter_try`] and [`Self::get`] are
91/// provided as panic-free alternatives. [`Self::with_full_validation`] can also be used to _validate_ an
92/// _unvalidated_ instance, if desired.
93///
94/// _Unvalidated_ instances can be constructed in constant time. This can be useful if the caller
95/// knows the underlying bytes were already validated previously, or if the caller intends to
96/// perform a small number of (fallible) accesses to a large list.
97///
98/// A _validated_ variant list instance guarantees that:
99///
100/// - header byte is valid
101/// - num_elements is in bounds
102/// - offset array content is in-bounds
103/// - first offset is zero
104/// - last offset is in-bounds
105/// - all other offsets are in-bounds (*)
106/// - all offsets are monotonically increasing (*)
107/// - all values are (recursively) valid variant objects (*)
108/// - the associated variant metadata is [valid] (*)
109///
110/// NOTE: [`Self::new`] only skips expensive (non-constant cost) validation checks (marked by `(*)`
111/// in the list above); it panics any of the other checks fails.
112///
113/// # Safety
114///
115/// Even an _invalid_ variant list instance is still _safe_ to use in the Rust sense. Accessing
116/// it with infallible methods may cause panics but will never lead to undefined behavior.
117///
118/// [valid]: VariantMetadata#Validation
119/// [Variant spec]: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md#value-data-for-array-basic_type3
120#[derive(Debug, Clone)]
121pub struct VariantList<'m, 'v> {
122    pub metadata: VariantMetadata<'m>,
123    pub value: &'v [u8],
124    header: VariantListHeader,
125    num_elements: u32,
126    first_value_byte: u32,
127    validated: bool,
128}
129
130// We don't want this to grow because it could increase the size of `Variant` and hurt performance.
131const _: () = crate::utils::expect_size_of::<VariantList>(64);
132
133impl<'m, 'v> VariantList<'m, 'v> {
134    /// Attempts to interpret `value` as a variant array value.
135    ///
136    /// # Validation
137    ///
138    /// This constructor verifies that `value` points to a valid variant array value. In particular,
139    /// that all offsets are in-bounds and point to valid (recursively validated) objects.
140    pub fn try_new(metadata: VariantMetadata<'m>, value: &'v [u8]) -> Result<Self, ArrowError> {
141        Self::try_new_with_shallow_validation(metadata, value)?.with_full_validation()
142    }
143
144    pub fn new(metadata: VariantMetadata<'m>, value: &'v [u8]) -> Self {
145        Self::try_new_with_shallow_validation(metadata, value).expect("Invalid variant list value")
146    }
147
148    /// Attempts to interpet `metadata` and `value` as a variant array, performing only basic
149    /// (constant-cost) [validation].
150    ///
151    /// [validation]: Self#Validation
152    pub(crate) fn try_new_with_shallow_validation(
153        metadata: VariantMetadata<'m>,
154        value: &'v [u8],
155    ) -> Result<Self, ArrowError> {
156        let header_byte = first_byte_from_slice(value)?;
157        let header = VariantListHeader::try_new(header_byte)?;
158
159        // Skip the header byte to read the num_elements; the offset array immediately follows
160        let num_elements =
161            header
162                .num_elements_size
163                .unpack_u32_at_offset(value, NUM_HEADER_BYTES as _, 0)?;
164
165        // (num_elements + 1) * offset_size + first_offset_byte
166        let first_value_byte = num_elements
167            .checked_add(1)
168            .and_then(|n| n.checked_mul(header.offset_size()))
169            .and_then(|n| n.checked_add(header.first_offset_byte()))
170            .ok_or_else(|| overflow_error("offset of variant list values"))?;
171
172        let mut new_self = Self {
173            metadata,
174            value,
175            header,
176            num_elements,
177            first_value_byte,
178            validated: false,
179        };
180
181        // Validate just the first and last offset, ignoring the other offsets and all value bytes.
182        let first_offset = new_self.get_offset(0)?;
183        if first_offset != 0 {
184            return Err(ArrowError::InvalidArgumentError(format!(
185                "First offset is not zero: {first_offset}"
186            )));
187        }
188
189        // Use the last offset to upper-bound the value buffer
190        let last_offset = new_self
191            .get_offset(num_elements as _)?
192            .checked_add(first_value_byte)
193            .ok_or_else(|| overflow_error("variant array size"))?;
194        new_self.value = slice_from_slice(value, ..last_offset as _)?;
195        Ok(new_self)
196    }
197
198    /// True if this instance is fully [validated] for panic-free infallible accesses.
199    ///
200    /// [validated]: Self#Validation
201    pub fn is_fully_validated(&self) -> bool {
202        self.validated
203    }
204
205    /// Performs a full [validation] of this variant array and returns the result.
206    ///
207    /// [validation]: Self#Validation
208    pub fn with_full_validation(mut self) -> Result<Self, ArrowError> {
209        if !self.validated {
210            // Validate the metadata dictionary first, if not already validated, because we pass it
211            // by value to all the children (who would otherwise re-validate it repeatedly).
212            self.metadata = self.metadata.with_full_validation()?;
213
214            let offset_buffer = slice_from_slice(
215                self.value,
216                self.header.first_offset_byte() as _..self.first_value_byte as _,
217            )?;
218
219            let value_buffer = slice_from_slice(self.value, self.first_value_byte as _..)?;
220
221            // Validate whether values are valid variant objects
222            //
223            // Since we use offsets to slice into the value buffer, this also verifies all offsets are in-bounds
224            // and monotonically increasing
225            let mut offset_iter = map_bytes_to_offsets(offset_buffer, self.header.offset_size);
226            let mut current_offset = offset_iter.next().unwrap_or(0);
227
228            for next_offset in offset_iter {
229                let value_bytes = slice_from_slice(value_buffer, current_offset..next_offset)?;
230                Variant::try_new_with_metadata(self.metadata.clone(), value_bytes)?;
231                current_offset = next_offset;
232            }
233
234            self.validated = true;
235        }
236        Ok(self)
237    }
238
239    /// Return the length of this array
240    pub fn len(&self) -> usize {
241        self.num_elements as _
242    }
243
244    /// Is the array of zero length
245    pub fn is_empty(&self) -> bool {
246        self.len() == 0
247    }
248
249    /// Returns element by index in `0..self.len()`, if any. May panic if this list is [invalid].
250    ///
251    /// [invalid]: Self#Validation
252    pub fn get(&self, index: usize) -> Option<Variant<'m, 'v>> {
253        (index < self.len()).then(|| {
254            self.try_get_with_shallow_validation(index)
255                .expect("Invalid variant array element")
256        })
257    }
258
259    /// Fallible version of `get`. Returns element by index, capturing validation errors
260    pub fn try_get(&self, index: usize) -> Result<Variant<'m, 'v>, ArrowError> {
261        self.try_get_with_shallow_validation(index)?
262            .with_full_validation()
263    }
264
265    // Fallible version of `get`, performing only basic (constant-time) validation.
266    fn try_get_with_shallow_validation(&self, index: usize) -> Result<Variant<'m, 'v>, ArrowError> {
267        // Fetch the value bytes between the two offsets for this index, from the value array region
268        // of the byte buffer
269        let byte_range = self.get_offset(index)? as _..self.get_offset(index + 1)? as _;
270        let value_bytes =
271            slice_from_slice_at_offset(self.value, self.first_value_byte as _, byte_range)?;
272        Variant::try_new_with_metadata_and_shallow_validation(self.metadata.clone(), value_bytes)
273    }
274
275    /// Iterates over the values of this list. When working with [unvalidated] input, consider
276    /// [`Self::iter_try`] to avoid panics due to invalid data.
277    ///
278    /// [unvalidated]: Self#Validation
279    pub fn iter(&self) -> impl Iterator<Item = Variant<'m, 'v>> + '_ {
280        self.iter_try_with_shallow_validation()
281            .map(|result| result.expect("Invalid variant list entry"))
282    }
283
284    /// Fallible iteration over the elements of this list.
285    pub fn iter_try(&self) -> impl Iterator<Item = Result<Variant<'m, 'v>, ArrowError>> + '_ {
286        self.iter_try_with_shallow_validation()
287            .map(|result| result?.with_full_validation())
288    }
289
290    // Fallible iteration that only performs basic (constant-time) validation.
291    fn iter_try_with_shallow_validation(
292        &self,
293    ) -> impl Iterator<Item = Result<Variant<'m, 'v>, ArrowError>> + '_ {
294        (0..self.len()).map(|i| self.try_get_with_shallow_validation(i))
295    }
296
297    // Attempts to retrieve the ith offset from the offset array region of the byte buffer.
298    fn get_offset(&self, index: usize) -> Result<u32, ArrowError> {
299        let byte_range = self.header.first_offset_byte() as _..self.first_value_byte as _;
300        let offset_bytes = slice_from_slice(self.value, byte_range)?;
301        self.header.offset_size.unpack_u32(offset_bytes, index)
302    }
303}
304
305// Custom implementation of PartialEq for variant arrays
306//
307// Instead of comparing the raw bytes of 2 variant lists, this implementation recursively
308// checks whether their elements are equal.
309impl<'m, 'v> PartialEq for VariantList<'m, 'v> {
310    fn eq(&self, other: &Self) -> bool {
311        if self.num_elements != other.num_elements {
312            return false;
313        }
314
315        self.iter().zip(other.iter()).all(|(a, b)| a == b)
316    }
317}
318
319#[cfg(test)]
320mod tests {
321    use super::*;
322    use crate::VariantBuilder;
323    use std::iter::repeat_n;
324    use std::ops::Range;
325
326    #[test]
327    fn test_variant_list_simple() {
328        // Create simple metadata (empty dictionary for this test)
329        let metadata_bytes = vec![
330            0x01, // header: version=1, sorted=0, offset_size_minus_one=0
331            0,    // dictionary_size = 0
332            0,    // offset[0] = 0 (end of dictionary)
333        ];
334        let metadata = VariantMetadata::try_new(&metadata_bytes).unwrap();
335
336        // Create list value data for: [42, true, "hi"]
337        // Header: basic_type=3 (array), field_offset_size_minus_one=0, is_large=0
338        // value_header = 0000_0_0_00 = 0x00
339        // So header byte = (0x00 << 2) | 3 = 0x03
340        let list_value = vec![
341            0x03, // header: basic_type=3, value_header=0x00
342            3,    // num_elements = 3
343            // Offsets (1 byte each): 4 offsets total
344            0, // offset to first value (int8)
345            2, // offset to second value (boolean true)
346            3, // offset to third value (short string)
347            6, // end offset
348            // Values:
349            0x0C,
350            42,   // int8: primitive_header=3, basic_type=0 -> (3 << 2) | 0 = 0x0C, then value 42
351            0x04, // boolean true: primitive_header=1, basic_type=0 -> (1 << 2) | 0 = 0x04
352            0x09, b'h', b'i', // short string: length=2, basic_type=1 -> (2 << 2) | 1 = 0x09
353        ];
354
355        let variant_list = VariantList::try_new(metadata, &list_value).unwrap();
356
357        // Test basic properties
358        assert_eq!(variant_list.len(), 3);
359        assert!(!variant_list.is_empty());
360
361        // Test individual element access
362        let elem0 = variant_list.get(0).unwrap();
363        assert_eq!(elem0.as_int8(), Some(42));
364
365        let elem1 = variant_list.get(1).unwrap();
366        assert_eq!(elem1.as_boolean(), Some(true));
367
368        let elem2 = variant_list.get(2).unwrap();
369        assert_eq!(elem2.as_string(), Some("hi"));
370
371        // Test out of bounds access
372        let out_of_bounds = variant_list.get(3);
373        assert!(out_of_bounds.is_none());
374
375        // Test values iterator
376        let values: Vec<_> = variant_list.iter().collect();
377        assert_eq!(values.len(), 3);
378        assert_eq!(values[0].as_int8(), Some(42));
379        assert_eq!(values[1].as_boolean(), Some(true));
380        assert_eq!(values[2].as_string(), Some("hi"));
381    }
382
383    #[test]
384    fn test_variant_list_empty() {
385        // Create simple metadata (empty dictionary)
386        let metadata_bytes = vec![
387            0x01, // header: version=1, sorted=0, offset_size_minus_one=0
388            0,    // dictionary_size = 0
389            0,    // offset[0] = 0 (end of dictionary)
390        ];
391        let metadata = VariantMetadata::try_new(&metadata_bytes).unwrap();
392
393        // Create empty list value data: []
394        let list_value = vec![
395            0x03, // header: basic_type=3, value_header=0x00
396            0,    // num_elements = 0
397            0,    // single offset pointing to end
398                  // No values
399        ];
400
401        let variant_list = VariantList::try_new(metadata, &list_value).unwrap();
402
403        // Test basic properties
404        assert_eq!(variant_list.len(), 0);
405        assert!(variant_list.is_empty());
406
407        // Test out of bounds access on empty list
408        let out_of_bounds = variant_list.get(0);
409        assert!(out_of_bounds.is_none());
410
411        // Test values iterator on empty list
412        let values: Vec<_> = variant_list.iter().collect();
413        assert_eq!(values.len(), 0);
414    }
415
416    #[test]
417    fn test_variant_list_large() {
418        // Create simple metadata (empty dictionary)
419        let metadata_bytes = vec![
420            0x01, // header: version=1, sorted=0, offset_size_minus_one=0
421            0,    // dictionary_size = 0
422            0,    // offset[0] = 0 (end of dictionary)
423        ];
424        let metadata = VariantMetadata::try_new(&metadata_bytes).unwrap();
425
426        // Create large list value data with 2-byte offsets: [null, false]
427        // Header: is_large=1, field_offset_size_minus_one=1, basic_type=3 (array)
428        let list_bytes = vec![
429            0x17, // header = 000_1_01_11 = 0x17
430            2, 0, 0, 0, // num_elements = 2 (4 bytes because is_large=1)
431            // Offsets (2 bytes each): 3 offsets total
432            0x00, 0x00, 0x01, 0x00, // first value (null)
433            0x02, 0x00, // second value (boolean false)
434            // Values:
435            0x00, // null: primitive_header=0, basic_type=0 -> (0 << 2) | 0 = 0x00
436            0x08, // boolean false: primitive_header=2, basic_type=0 -> (2 << 2) | 0 = 0x08
437        ];
438
439        let variant_list = VariantList::try_new(metadata, &list_bytes).unwrap();
440
441        // Test basic properties
442        assert_eq!(variant_list.len(), 2);
443        assert!(!variant_list.is_empty());
444
445        // Test individual element access
446        let elem0 = variant_list.get(0).unwrap();
447        assert_eq!(elem0.as_null(), Some(()));
448
449        let elem1 = variant_list.get(1).unwrap();
450        assert_eq!(elem1.as_boolean(), Some(false));
451    }
452
453    #[test]
454    fn test_large_variant_list_with_total_child_length_between_2_pow_8_and_2_pow_16() {
455        // all the tests below will set the total child size to ~500,
456        // which is larger than 2^8 but less than 2^16.
457        // total child size = list_size * single_child_item_len
458
459        let mut list_size: usize = 1;
460        let mut single_child_item_len: usize = 500;
461
462        // offset size will be OffSizeBytes::Two as the total child length between 2^8 and 2^16
463        let expected_offset_size = OffsetSizeBytes::Two;
464
465        test_large_variant_list_with_child_length(
466            list_size,             // the elements in the list
467            single_child_item_len, // this will control the total child size in the list
468            OffsetSizeBytes::One, // will be OffsetSizeBytes::One as the size of the list is less than 256
469            expected_offset_size,
470        );
471
472        list_size = 255;
473        single_child_item_len = 2;
474        test_large_variant_list_with_child_length(
475            list_size,
476            single_child_item_len,
477            OffsetSizeBytes::One, // will be OffsetSizeBytes::One as the size of the list is less than 256
478            expected_offset_size,
479        );
480
481        list_size = 256;
482        single_child_item_len = 2;
483        test_large_variant_list_with_child_length(
484            list_size,
485            single_child_item_len,
486            OffsetSizeBytes::Four, // will be OffsetSizeBytes::Four as the size of the list is bigger than 255
487            expected_offset_size,
488        );
489
490        list_size = 300;
491        single_child_item_len = 2;
492        test_large_variant_list_with_child_length(
493            list_size,
494            single_child_item_len,
495            OffsetSizeBytes::Four, // will be OffsetSizeBytes::Four as the size of the list is bigger than 255
496            expected_offset_size,
497        );
498    }
499
500    #[test]
501    fn test_large_variant_list_with_total_child_length_between_2_pow_16_and_2_pow_24() {
502        // all the tests below will set the total child size to ~70,000,
503        // which is larger than 2^16 but less than 2^24.
504        // total child size = list_size * single_child_item_len
505
506        let mut list_size: usize = 1;
507        let mut single_child_item_len: usize = 70000;
508
509        // offset size will be OffSizeBytes::Two as the total child length between 2^16 and 2^24
510        let expected_offset_size = OffsetSizeBytes::Three;
511
512        test_large_variant_list_with_child_length(
513            list_size,
514            single_child_item_len,
515            OffsetSizeBytes::One, // will be OffsetSizeBytes::One as the size of the list is less than 256
516            expected_offset_size,
517        );
518
519        list_size = 255;
520        single_child_item_len = 275;
521        // total child size = 255 * 275 = 70,125
522        test_large_variant_list_with_child_length(
523            list_size,
524            single_child_item_len,
525            OffsetSizeBytes::One, // will be OffsetSizeBytes::One as the size of the list is less than 256
526            expected_offset_size,
527        );
528
529        list_size = 256;
530        single_child_item_len = 274;
531        // total child size = 256 * 274 = 70,144
532        test_large_variant_list_with_child_length(
533            list_size,
534            single_child_item_len,
535            OffsetSizeBytes::Four, // will be OffsetSizeBytes::Four as the size of the list is bigger than 255
536            expected_offset_size,
537        );
538
539        list_size = 300;
540        single_child_item_len = 234;
541        // total child size = 300 * 234 = 70,200
542        test_large_variant_list_with_child_length(
543            list_size,
544            single_child_item_len,
545            OffsetSizeBytes::Four, // will be OffsetSizeBytes::Four as the size of the list is bigger than 255
546            expected_offset_size,
547        );
548    }
549
550    #[test]
551    fn test_large_variant_list_with_total_child_length_between_2_pow_24_and_2_pow_32() {
552        // all the tests below will set the total child size to ~20,000,000,
553        // which is larger than 2^24 but less than 2^32.
554        // total child size = list_size * single_child_item_len
555
556        let mut list_size: usize = 1;
557        let mut single_child_item_len: usize = 20000000;
558
559        // offset size will be OffSizeBytes::Two as the total child length between 2^24 and 2^32
560        let expected_offset_size = OffsetSizeBytes::Four;
561
562        test_large_variant_list_with_child_length(
563            list_size,
564            single_child_item_len,
565            OffsetSizeBytes::One, // will be OffsetSizeBytes::One as the size of the list is less than 256
566            expected_offset_size,
567        );
568
569        list_size = 255;
570        single_child_item_len = 78432;
571        // total child size = 255 * 78,432 = 20,000,160
572        test_large_variant_list_with_child_length(
573            list_size,
574            single_child_item_len,
575            OffsetSizeBytes::One, // will be OffsetSizeBytes::One as the size of the list is less than 256
576            expected_offset_size,
577        );
578
579        list_size = 256;
580        single_child_item_len = 78125;
581        // total child size = 256 * 78,125 = 20,000,000
582        test_large_variant_list_with_child_length(
583            list_size,
584            single_child_item_len,
585            OffsetSizeBytes::Four, // will be OffsetSizeBytes::Four as the size of the list is bigger than 255
586            expected_offset_size,
587        );
588
589        list_size = 300;
590        single_child_item_len = 66667;
591        // total child size = 300 * 66,667 = 20,000,100
592        test_large_variant_list_with_child_length(
593            list_size,
594            single_child_item_len,
595            OffsetSizeBytes::Four, // will be OffsetSizeBytes::Four as the size of the list is bigger than 255
596            expected_offset_size,
597        );
598    }
599
600    // this function will create a large variant list from VariantBuilder
601    // with specified size and each child item with the given length.
602    // and verify the content and some meta for the variant list in the final.
603    fn test_large_variant_list_with_child_length(
604        list_size: usize,
605        single_child_item_len: usize,
606        expected_num_element_size: OffsetSizeBytes,
607        expected_offset_size_bytes: OffsetSizeBytes,
608    ) {
609        let mut builder = VariantBuilder::new();
610        let mut list_builder = builder.new_list();
611
612        let mut expected_list = vec![];
613        for i in 0..list_size {
614            let random_string: String =
615                repeat_n(char::from((i % 256) as u8), single_child_item_len).collect();
616
617            list_builder.append_value(Variant::String(random_string.as_str()));
618            expected_list.push(random_string);
619        }
620
621        list_builder.finish();
622        // Finish the builder to get the metadata and value
623        let (metadata, value) = builder.finish();
624        // use the Variant API to verify the result
625        let variant = Variant::try_new(&metadata, &value).unwrap();
626
627        let variant_list = variant.as_list().unwrap();
628
629        // verify that the head is expected
630        assert_eq!(expected_offset_size_bytes, variant_list.header.offset_size);
631        assert_eq!(
632            expected_num_element_size,
633            variant_list.header.num_elements_size
634        );
635        assert_eq!(list_size, variant_list.num_elements as usize);
636
637        // verify the data in the variant
638        assert_eq!(list_size, variant_list.len());
639        for i in 0..list_size {
640            let item = variant_list.get(i).unwrap();
641            let item_str = item.as_string().unwrap();
642            assert_eq!(expected_list.get(i).unwrap(), item_str);
643        }
644    }
645
646    #[test]
647    fn test_variant_list_equality() {
648        // Create two lists with the same values (0..10)
649        let (metadata1, value1) = make_listi32(0..10);
650        let list1 = Variant::new(&metadata1, &value1);
651        let (metadata2, value2) = make_listi32(0..10);
652        let list2 = Variant::new(&metadata2, &value2);
653        // They should be equal
654        assert_eq!(list1, list2);
655    }
656
657    #[test]
658    fn test_variant_list_equality_different_length() {
659        // Create two lists with different lengths
660        let (metadata1, value1) = make_listi32(0..10);
661        let list1 = Variant::new(&metadata1, &value1);
662        let (metadata2, value2) = make_listi32(0..5);
663        let list2 = Variant::new(&metadata2, &value2);
664        // They should not be equal
665        assert_ne!(list1, list2);
666    }
667
668    #[test]
669    fn test_variant_list_equality_different_values() {
670        // Create two lists with different values
671        let (metadata1, value1) = make_listi32(0..10);
672        let list1 = Variant::new(&metadata1, &value1);
673        let (metadata2, value2) = make_listi32(5..15);
674        let list2 = Variant::new(&metadata2, &value2);
675        // They should not be equal
676        assert_ne!(list1, list2);
677    }
678
679    #[test]
680    fn test_variant_list_equality_different_types() {
681        // Create two lists with different types
682        let (metadata1, value1) = make_listi32(0i32..10i32);
683        let list1 = Variant::new(&metadata1, &value1);
684        let (metadata2, value2) = make_listi64(0..10);
685        let list2 = Variant::new(&metadata2, &value2);
686        // They should not be equal due to type mismatch
687        assert_ne!(list1, list2);
688    }
689
690    #[test]
691    fn test_variant_list_equality_slices() {
692        // Make an object like this and make sure equality works
693        // when the lists are sub fields
694        //
695        // {
696        //   "list1": [0, 1, 2, ..., 9],
697        //   "list2": [0, 1, 2, ..., 9],
698        //   "list3": [10, 11, 12, ..., 19],
699        //  }
700        let (metadata, value) = {
701            let mut builder = VariantBuilder::new();
702            let mut object_builder = builder.new_object();
703            // list1 (0..10)
704            let (metadata1, value1) = make_listi32(0i32..10i32);
705            object_builder.insert("list1", Variant::new(&metadata1, &value1));
706
707            // list2 (0..10)
708            let (metadata2, value2) = make_listi32(0i32..10i32);
709            object_builder.insert("list2", Variant::new(&metadata2, &value2));
710
711            // list3 (10..20)
712            let (metadata3, value3) = make_listi32(10i32..20i32);
713            object_builder.insert("list3", Variant::new(&metadata3, &value3));
714            object_builder.finish();
715            builder.finish()
716        };
717
718        let variant = Variant::try_new(&metadata, &value).unwrap();
719        let object = variant.as_object().unwrap();
720        // Check that list1 and list2 are equal
721        assert_eq!(object.get("list1").unwrap(), object.get("list2").unwrap());
722        // Check that list1 and list3 are not equal
723        assert_ne!(object.get("list1").unwrap(), object.get("list3").unwrap());
724    }
725
726    /// return metadata/value for a simple variant list with values in a range
727    fn make_listi32(range: Range<i32>) -> (Vec<u8>, Vec<u8>) {
728        let mut variant_builder = VariantBuilder::new();
729        let mut list_builder = variant_builder.new_list();
730        for i in range {
731            list_builder.append_value(i);
732        }
733        list_builder.finish();
734        variant_builder.finish()
735    }
736
737    /// return metadata/value for a simple variant list with values in a range
738    fn make_listi64(range: Range<i64>) -> (Vec<u8>, Vec<u8>) {
739        let mut variant_builder = VariantBuilder::new();
740        let mut list_builder = variant_builder.new_list();
741        for i in range {
742            list_builder.append_value(i);
743        }
744        list_builder.finish();
745        variant_builder.finish()
746    }
747}