parquet_variant/variant/list.rs
1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied. See the License for the
15// specific language governing permissions and limitations
16// under the License.
17use crate::decoder::{map_bytes_to_offsets, OffsetSizeBytes};
18use crate::utils::{
19 first_byte_from_slice, overflow_error, slice_from_slice, slice_from_slice_at_offset,
20};
21use crate::variant::{Variant, VariantMetadata};
22
23use arrow_schema::ArrowError;
24
25// The value header occupies one byte; use a named constant for readability
26const NUM_HEADER_BYTES: u32 = 1;
27
28/// A parsed version of the variant array value header byte.
29#[derive(Debug, Clone, PartialEq)]
30pub(crate) struct VariantListHeader {
31 num_elements_size: OffsetSizeBytes,
32 offset_size: OffsetSizeBytes,
33}
34
35impl VariantListHeader {
36 // Hide the ugly casting
37 const fn num_elements_size(&self) -> u32 {
38 self.num_elements_size as _
39 }
40 const fn offset_size(&self) -> u32 {
41 self.offset_size as _
42 }
43
44 // Avoid materializing this offset, since it's cheaply and safely computable
45 const fn first_offset_byte(&self) -> u32 {
46 NUM_HEADER_BYTES + self.num_elements_size()
47 }
48
49 pub(crate) fn try_new(header_byte: u8) -> Result<Self, ArrowError> {
50 // The 6 first bits to the left are the value_header and the 2 bits
51 // to the right are the basic type, so we shift to get only the value_header
52 let value_header = header_byte >> 2;
53 let is_large = (value_header & 0x04) != 0; // 3rd bit from the right
54 let field_offset_size_minus_one = value_header & 0x03; // Last two bits
55
56 // The size of the num_elements entry in the array value_data is 4 bytes if
57 // is_large is true, otherwise 1 byte.
58 let num_elements_size = match is_large {
59 true => OffsetSizeBytes::Four,
60 false => OffsetSizeBytes::One,
61 };
62 let offset_size = OffsetSizeBytes::try_new(field_offset_size_minus_one)?;
63
64 Ok(Self {
65 num_elements_size,
66 offset_size,
67 })
68 }
69}
70
71/// [`Variant`] Array.
72///
73/// See the [Variant spec] for details.
74///
75/// NOTE: The "list" naming differs from the variant spec -- which calls it "array" -- in order to be
76/// consistent with Parquet and Arrow type naming. Otherwise, the name would conflict with the
77/// `VariantArray : Array` we must eventually define for variant-typed arrow arrays.
78///
79/// # Validation
80///
81/// Every instance of variant list is either _valid_ or _invalid_. depending on whether the
82/// underlying bytes are a valid encoding of a variant array (see below).
83///
84/// Instances produced by [`Self::try_new`] or [`Self::with_full_validation`] are fully _validated_. They always
85/// contain _valid_ data, and infallible accesses such as iteration and indexing are panic-free. The
86/// validation cost is linear in the number of underlying bytes.
87///
88/// Instances produced by [`Self::new`] are _unvalidated_ and so they may contain either _valid_ or
89/// _invalid_ data. Infallible accesses such as iteration and indexing will panic if the underlying
90/// bytes are _invalid_, and fallible alternatives such as [`Self::iter_try`] and [`Self::get`] are
91/// provided as panic-free alternatives. [`Self::with_full_validation`] can also be used to _validate_ an
92/// _unvalidated_ instance, if desired.
93///
94/// _Unvalidated_ instances can be constructed in constant time. This can be useful if the caller
95/// knows the underlying bytes were already validated previously, or if the caller intends to
96/// perform a small number of (fallible) accesses to a large list.
97///
98/// A _validated_ variant list instance guarantees that:
99///
100/// - header byte is valid
101/// - num_elements is in bounds
102/// - offset array content is in-bounds
103/// - first offset is zero
104/// - last offset is in-bounds
105/// - all other offsets are in-bounds (*)
106/// - all offsets are monotonically increasing (*)
107/// - all values are (recursively) valid variant objects (*)
108/// - the associated variant metadata is [valid] (*)
109///
110/// NOTE: [`Self::new`] only skips expensive (non-constant cost) validation checks (marked by `(*)`
111/// in the list above); it panics any of the other checks fails.
112///
113/// # Safety
114///
115/// Even an _invalid_ variant list instance is still _safe_ to use in the Rust sense. Accessing
116/// it with infallible methods may cause panics but will never lead to undefined behavior.
117///
118/// [valid]: VariantMetadata#Validation
119/// [Variant spec]: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md#value-data-for-array-basic_type3
120#[derive(Debug, Clone)]
121pub struct VariantList<'m, 'v> {
122 pub metadata: VariantMetadata<'m>,
123 pub value: &'v [u8],
124 header: VariantListHeader,
125 num_elements: u32,
126 first_value_byte: u32,
127 validated: bool,
128}
129
130// We don't want this to grow because it could increase the size of `Variant` and hurt performance.
131const _: () = crate::utils::expect_size_of::<VariantList>(64);
132
133impl<'m, 'v> VariantList<'m, 'v> {
134 /// Attempts to interpret `value` as a variant array value.
135 ///
136 /// # Validation
137 ///
138 /// This constructor verifies that `value` points to a valid variant array value. In particular,
139 /// that all offsets are in-bounds and point to valid (recursively validated) objects.
140 pub fn try_new(metadata: VariantMetadata<'m>, value: &'v [u8]) -> Result<Self, ArrowError> {
141 Self::try_new_with_shallow_validation(metadata, value)?.with_full_validation()
142 }
143
144 pub fn new(metadata: VariantMetadata<'m>, value: &'v [u8]) -> Self {
145 Self::try_new_with_shallow_validation(metadata, value).expect("Invalid variant list value")
146 }
147
148 /// Attempts to interpet `metadata` and `value` as a variant array, performing only basic
149 /// (constant-cost) [validation].
150 ///
151 /// [validation]: Self#Validation
152 pub(crate) fn try_new_with_shallow_validation(
153 metadata: VariantMetadata<'m>,
154 value: &'v [u8],
155 ) -> Result<Self, ArrowError> {
156 let header_byte = first_byte_from_slice(value)?;
157 let header = VariantListHeader::try_new(header_byte)?;
158
159 // Skip the header byte to read the num_elements; the offset array immediately follows
160 let num_elements =
161 header
162 .num_elements_size
163 .unpack_u32_at_offset(value, NUM_HEADER_BYTES as _, 0)?;
164
165 // (num_elements + 1) * offset_size + first_offset_byte
166 let first_value_byte = num_elements
167 .checked_add(1)
168 .and_then(|n| n.checked_mul(header.offset_size()))
169 .and_then(|n| n.checked_add(header.first_offset_byte()))
170 .ok_or_else(|| overflow_error("offset of variant list values"))?;
171
172 let mut new_self = Self {
173 metadata,
174 value,
175 header,
176 num_elements,
177 first_value_byte,
178 validated: false,
179 };
180
181 // Validate just the first and last offset, ignoring the other offsets and all value bytes.
182 let first_offset = new_self.get_offset(0)?;
183 if first_offset != 0 {
184 return Err(ArrowError::InvalidArgumentError(format!(
185 "First offset is not zero: {first_offset}"
186 )));
187 }
188
189 // Use the last offset to upper-bound the value buffer
190 let last_offset = new_self
191 .get_offset(num_elements as _)?
192 .checked_add(first_value_byte)
193 .ok_or_else(|| overflow_error("variant array size"))?;
194 new_self.value = slice_from_slice(value, ..last_offset as _)?;
195 Ok(new_self)
196 }
197
198 /// True if this instance is fully [validated] for panic-free infallible accesses.
199 ///
200 /// [validated]: Self#Validation
201 pub fn is_fully_validated(&self) -> bool {
202 self.validated
203 }
204
205 /// Performs a full [validation] of this variant array and returns the result.
206 ///
207 /// [validation]: Self#Validation
208 pub fn with_full_validation(mut self) -> Result<Self, ArrowError> {
209 if !self.validated {
210 // Validate the metadata dictionary first, if not already validated, because we pass it
211 // by value to all the children (who would otherwise re-validate it repeatedly).
212 self.metadata = self.metadata.with_full_validation()?;
213
214 let offset_buffer = slice_from_slice(
215 self.value,
216 self.header.first_offset_byte() as _..self.first_value_byte as _,
217 )?;
218
219 let value_buffer = slice_from_slice(self.value, self.first_value_byte as _..)?;
220
221 // Validate whether values are valid variant objects
222 //
223 // Since we use offsets to slice into the value buffer, this also verifies all offsets are in-bounds
224 // and monotonically increasing
225 let mut offset_iter = map_bytes_to_offsets(offset_buffer, self.header.offset_size);
226 let mut current_offset = offset_iter.next().unwrap_or(0);
227
228 for next_offset in offset_iter {
229 let value_bytes = slice_from_slice(value_buffer, current_offset..next_offset)?;
230 Variant::try_new_with_metadata(self.metadata.clone(), value_bytes)?;
231 current_offset = next_offset;
232 }
233
234 self.validated = true;
235 }
236 Ok(self)
237 }
238
239 /// Return the length of this array
240 pub fn len(&self) -> usize {
241 self.num_elements as _
242 }
243
244 /// Is the array of zero length
245 pub fn is_empty(&self) -> bool {
246 self.len() == 0
247 }
248
249 /// Returns element by index in `0..self.len()`, if any. May panic if this list is [invalid].
250 ///
251 /// [invalid]: Self#Validation
252 pub fn get(&self, index: usize) -> Option<Variant<'m, 'v>> {
253 (index < self.len()).then(|| {
254 self.try_get_with_shallow_validation(index)
255 .expect("Invalid variant array element")
256 })
257 }
258
259 /// Fallible version of `get`. Returns element by index, capturing validation errors
260 pub fn try_get(&self, index: usize) -> Result<Variant<'m, 'v>, ArrowError> {
261 self.try_get_with_shallow_validation(index)?
262 .with_full_validation()
263 }
264
265 // Fallible version of `get`, performing only basic (constant-time) validation.
266 fn try_get_with_shallow_validation(&self, index: usize) -> Result<Variant<'m, 'v>, ArrowError> {
267 // Fetch the value bytes between the two offsets for this index, from the value array region
268 // of the byte buffer
269 let byte_range = self.get_offset(index)? as _..self.get_offset(index + 1)? as _;
270 let value_bytes =
271 slice_from_slice_at_offset(self.value, self.first_value_byte as _, byte_range)?;
272 Variant::try_new_with_metadata_and_shallow_validation(self.metadata.clone(), value_bytes)
273 }
274
275 /// Iterates over the values of this list. When working with [unvalidated] input, consider
276 /// [`Self::iter_try`] to avoid panics due to invalid data.
277 ///
278 /// [unvalidated]: Self#Validation
279 pub fn iter(&self) -> impl Iterator<Item = Variant<'m, 'v>> + '_ {
280 self.iter_try_with_shallow_validation()
281 .map(|result| result.expect("Invalid variant list entry"))
282 }
283
284 /// Fallible iteration over the elements of this list.
285 pub fn iter_try(&self) -> impl Iterator<Item = Result<Variant<'m, 'v>, ArrowError>> + '_ {
286 self.iter_try_with_shallow_validation()
287 .map(|result| result?.with_full_validation())
288 }
289
290 // Fallible iteration that only performs basic (constant-time) validation.
291 fn iter_try_with_shallow_validation(
292 &self,
293 ) -> impl Iterator<Item = Result<Variant<'m, 'v>, ArrowError>> + '_ {
294 (0..self.len()).map(|i| self.try_get_with_shallow_validation(i))
295 }
296
297 // Attempts to retrieve the ith offset from the offset array region of the byte buffer.
298 fn get_offset(&self, index: usize) -> Result<u32, ArrowError> {
299 let byte_range = self.header.first_offset_byte() as _..self.first_value_byte as _;
300 let offset_bytes = slice_from_slice(self.value, byte_range)?;
301 self.header.offset_size.unpack_u32(offset_bytes, index)
302 }
303}
304
305// Custom implementation of PartialEq for variant arrays
306//
307// Instead of comparing the raw bytes of 2 variant lists, this implementation recursively
308// checks whether their elements are equal.
309impl<'m, 'v> PartialEq for VariantList<'m, 'v> {
310 fn eq(&self, other: &Self) -> bool {
311 if self.num_elements != other.num_elements {
312 return false;
313 }
314
315 self.iter().zip(other.iter()).all(|(a, b)| a == b)
316 }
317}
318
319#[cfg(test)]
320mod tests {
321 use super::*;
322 use crate::VariantBuilder;
323 use std::iter::repeat_n;
324 use std::ops::Range;
325
326 #[test]
327 fn test_variant_list_simple() {
328 // Create simple metadata (empty dictionary for this test)
329 let metadata_bytes = vec![
330 0x01, // header: version=1, sorted=0, offset_size_minus_one=0
331 0, // dictionary_size = 0
332 0, // offset[0] = 0 (end of dictionary)
333 ];
334 let metadata = VariantMetadata::try_new(&metadata_bytes).unwrap();
335
336 // Create list value data for: [42, true, "hi"]
337 // Header: basic_type=3 (array), field_offset_size_minus_one=0, is_large=0
338 // value_header = 0000_0_0_00 = 0x00
339 // So header byte = (0x00 << 2) | 3 = 0x03
340 let list_value = vec![
341 0x03, // header: basic_type=3, value_header=0x00
342 3, // num_elements = 3
343 // Offsets (1 byte each): 4 offsets total
344 0, // offset to first value (int8)
345 2, // offset to second value (boolean true)
346 3, // offset to third value (short string)
347 6, // end offset
348 // Values:
349 0x0C,
350 42, // int8: primitive_header=3, basic_type=0 -> (3 << 2) | 0 = 0x0C, then value 42
351 0x04, // boolean true: primitive_header=1, basic_type=0 -> (1 << 2) | 0 = 0x04
352 0x09, b'h', b'i', // short string: length=2, basic_type=1 -> (2 << 2) | 1 = 0x09
353 ];
354
355 let variant_list = VariantList::try_new(metadata, &list_value).unwrap();
356
357 // Test basic properties
358 assert_eq!(variant_list.len(), 3);
359 assert!(!variant_list.is_empty());
360
361 // Test individual element access
362 let elem0 = variant_list.get(0).unwrap();
363 assert_eq!(elem0.as_int8(), Some(42));
364
365 let elem1 = variant_list.get(1).unwrap();
366 assert_eq!(elem1.as_boolean(), Some(true));
367
368 let elem2 = variant_list.get(2).unwrap();
369 assert_eq!(elem2.as_string(), Some("hi"));
370
371 // Test out of bounds access
372 let out_of_bounds = variant_list.get(3);
373 assert!(out_of_bounds.is_none());
374
375 // Test values iterator
376 let values: Vec<_> = variant_list.iter().collect();
377 assert_eq!(values.len(), 3);
378 assert_eq!(values[0].as_int8(), Some(42));
379 assert_eq!(values[1].as_boolean(), Some(true));
380 assert_eq!(values[2].as_string(), Some("hi"));
381 }
382
383 #[test]
384 fn test_variant_list_empty() {
385 // Create simple metadata (empty dictionary)
386 let metadata_bytes = vec![
387 0x01, // header: version=1, sorted=0, offset_size_minus_one=0
388 0, // dictionary_size = 0
389 0, // offset[0] = 0 (end of dictionary)
390 ];
391 let metadata = VariantMetadata::try_new(&metadata_bytes).unwrap();
392
393 // Create empty list value data: []
394 let list_value = vec![
395 0x03, // header: basic_type=3, value_header=0x00
396 0, // num_elements = 0
397 0, // single offset pointing to end
398 // No values
399 ];
400
401 let variant_list = VariantList::try_new(metadata, &list_value).unwrap();
402
403 // Test basic properties
404 assert_eq!(variant_list.len(), 0);
405 assert!(variant_list.is_empty());
406
407 // Test out of bounds access on empty list
408 let out_of_bounds = variant_list.get(0);
409 assert!(out_of_bounds.is_none());
410
411 // Test values iterator on empty list
412 let values: Vec<_> = variant_list.iter().collect();
413 assert_eq!(values.len(), 0);
414 }
415
416 #[test]
417 fn test_variant_list_large() {
418 // Create simple metadata (empty dictionary)
419 let metadata_bytes = vec![
420 0x01, // header: version=1, sorted=0, offset_size_minus_one=0
421 0, // dictionary_size = 0
422 0, // offset[0] = 0 (end of dictionary)
423 ];
424 let metadata = VariantMetadata::try_new(&metadata_bytes).unwrap();
425
426 // Create large list value data with 2-byte offsets: [null, false]
427 // Header: is_large=1, field_offset_size_minus_one=1, basic_type=3 (array)
428 let list_bytes = vec![
429 0x17, // header = 000_1_01_11 = 0x17
430 2, 0, 0, 0, // num_elements = 2 (4 bytes because is_large=1)
431 // Offsets (2 bytes each): 3 offsets total
432 0x00, 0x00, 0x01, 0x00, // first value (null)
433 0x02, 0x00, // second value (boolean false)
434 // Values:
435 0x00, // null: primitive_header=0, basic_type=0 -> (0 << 2) | 0 = 0x00
436 0x08, // boolean false: primitive_header=2, basic_type=0 -> (2 << 2) | 0 = 0x08
437 ];
438
439 let variant_list = VariantList::try_new(metadata, &list_bytes).unwrap();
440
441 // Test basic properties
442 assert_eq!(variant_list.len(), 2);
443 assert!(!variant_list.is_empty());
444
445 // Test individual element access
446 let elem0 = variant_list.get(0).unwrap();
447 assert_eq!(elem0.as_null(), Some(()));
448
449 let elem1 = variant_list.get(1).unwrap();
450 assert_eq!(elem1.as_boolean(), Some(false));
451 }
452
453 #[test]
454 fn test_large_variant_list_with_total_child_length_between_2_pow_8_and_2_pow_16() {
455 // all the tests below will set the total child size to ~500,
456 // which is larger than 2^8 but less than 2^16.
457 // total child size = list_size * single_child_item_len
458
459 let mut list_size: usize = 1;
460 let mut single_child_item_len: usize = 500;
461
462 // offset size will be OffSizeBytes::Two as the total child length between 2^8 and 2^16
463 let expected_offset_size = OffsetSizeBytes::Two;
464
465 test_large_variant_list_with_child_length(
466 list_size, // the elements in the list
467 single_child_item_len, // this will control the total child size in the list
468 OffsetSizeBytes::One, // will be OffsetSizeBytes::One as the size of the list is less than 256
469 expected_offset_size,
470 );
471
472 list_size = 255;
473 single_child_item_len = 2;
474 test_large_variant_list_with_child_length(
475 list_size,
476 single_child_item_len,
477 OffsetSizeBytes::One, // will be OffsetSizeBytes::One as the size of the list is less than 256
478 expected_offset_size,
479 );
480
481 list_size = 256;
482 single_child_item_len = 2;
483 test_large_variant_list_with_child_length(
484 list_size,
485 single_child_item_len,
486 OffsetSizeBytes::Four, // will be OffsetSizeBytes::Four as the size of the list is bigger than 255
487 expected_offset_size,
488 );
489
490 list_size = 300;
491 single_child_item_len = 2;
492 test_large_variant_list_with_child_length(
493 list_size,
494 single_child_item_len,
495 OffsetSizeBytes::Four, // will be OffsetSizeBytes::Four as the size of the list is bigger than 255
496 expected_offset_size,
497 );
498 }
499
500 #[test]
501 fn test_large_variant_list_with_total_child_length_between_2_pow_16_and_2_pow_24() {
502 // all the tests below will set the total child size to ~70,000,
503 // which is larger than 2^16 but less than 2^24.
504 // total child size = list_size * single_child_item_len
505
506 let mut list_size: usize = 1;
507 let mut single_child_item_len: usize = 70000;
508
509 // offset size will be OffSizeBytes::Two as the total child length between 2^16 and 2^24
510 let expected_offset_size = OffsetSizeBytes::Three;
511
512 test_large_variant_list_with_child_length(
513 list_size,
514 single_child_item_len,
515 OffsetSizeBytes::One, // will be OffsetSizeBytes::One as the size of the list is less than 256
516 expected_offset_size,
517 );
518
519 list_size = 255;
520 single_child_item_len = 275;
521 // total child size = 255 * 275 = 70,125
522 test_large_variant_list_with_child_length(
523 list_size,
524 single_child_item_len,
525 OffsetSizeBytes::One, // will be OffsetSizeBytes::One as the size of the list is less than 256
526 expected_offset_size,
527 );
528
529 list_size = 256;
530 single_child_item_len = 274;
531 // total child size = 256 * 274 = 70,144
532 test_large_variant_list_with_child_length(
533 list_size,
534 single_child_item_len,
535 OffsetSizeBytes::Four, // will be OffsetSizeBytes::Four as the size of the list is bigger than 255
536 expected_offset_size,
537 );
538
539 list_size = 300;
540 single_child_item_len = 234;
541 // total child size = 300 * 234 = 70,200
542 test_large_variant_list_with_child_length(
543 list_size,
544 single_child_item_len,
545 OffsetSizeBytes::Four, // will be OffsetSizeBytes::Four as the size of the list is bigger than 255
546 expected_offset_size,
547 );
548 }
549
550 #[test]
551 fn test_large_variant_list_with_total_child_length_between_2_pow_24_and_2_pow_32() {
552 // all the tests below will set the total child size to ~20,000,000,
553 // which is larger than 2^24 but less than 2^32.
554 // total child size = list_size * single_child_item_len
555
556 let mut list_size: usize = 1;
557 let mut single_child_item_len: usize = 20000000;
558
559 // offset size will be OffSizeBytes::Two as the total child length between 2^24 and 2^32
560 let expected_offset_size = OffsetSizeBytes::Four;
561
562 test_large_variant_list_with_child_length(
563 list_size,
564 single_child_item_len,
565 OffsetSizeBytes::One, // will be OffsetSizeBytes::One as the size of the list is less than 256
566 expected_offset_size,
567 );
568
569 list_size = 255;
570 single_child_item_len = 78432;
571 // total child size = 255 * 78,432 = 20,000,160
572 test_large_variant_list_with_child_length(
573 list_size,
574 single_child_item_len,
575 OffsetSizeBytes::One, // will be OffsetSizeBytes::One as the size of the list is less than 256
576 expected_offset_size,
577 );
578
579 list_size = 256;
580 single_child_item_len = 78125;
581 // total child size = 256 * 78,125 = 20,000,000
582 test_large_variant_list_with_child_length(
583 list_size,
584 single_child_item_len,
585 OffsetSizeBytes::Four, // will be OffsetSizeBytes::Four as the size of the list is bigger than 255
586 expected_offset_size,
587 );
588
589 list_size = 300;
590 single_child_item_len = 66667;
591 // total child size = 300 * 66,667 = 20,000,100
592 test_large_variant_list_with_child_length(
593 list_size,
594 single_child_item_len,
595 OffsetSizeBytes::Four, // will be OffsetSizeBytes::Four as the size of the list is bigger than 255
596 expected_offset_size,
597 );
598 }
599
600 // this function will create a large variant list from VariantBuilder
601 // with specified size and each child item with the given length.
602 // and verify the content and some meta for the variant list in the final.
603 fn test_large_variant_list_with_child_length(
604 list_size: usize,
605 single_child_item_len: usize,
606 expected_num_element_size: OffsetSizeBytes,
607 expected_offset_size_bytes: OffsetSizeBytes,
608 ) {
609 let mut builder = VariantBuilder::new();
610 let mut list_builder = builder.new_list();
611
612 let mut expected_list = vec![];
613 for i in 0..list_size {
614 let random_string: String =
615 repeat_n(char::from((i % 256) as u8), single_child_item_len).collect();
616
617 list_builder.append_value(Variant::String(random_string.as_str()));
618 expected_list.push(random_string);
619 }
620
621 list_builder.finish();
622 // Finish the builder to get the metadata and value
623 let (metadata, value) = builder.finish();
624 // use the Variant API to verify the result
625 let variant = Variant::try_new(&metadata, &value).unwrap();
626
627 let variant_list = variant.as_list().unwrap();
628
629 // verify that the head is expected
630 assert_eq!(expected_offset_size_bytes, variant_list.header.offset_size);
631 assert_eq!(
632 expected_num_element_size,
633 variant_list.header.num_elements_size
634 );
635 assert_eq!(list_size, variant_list.num_elements as usize);
636
637 // verify the data in the variant
638 assert_eq!(list_size, variant_list.len());
639 for i in 0..list_size {
640 let item = variant_list.get(i).unwrap();
641 let item_str = item.as_string().unwrap();
642 assert_eq!(expected_list.get(i).unwrap(), item_str);
643 }
644 }
645
646 #[test]
647 fn test_variant_list_equality() {
648 // Create two lists with the same values (0..10)
649 let (metadata1, value1) = make_listi32(0..10);
650 let list1 = Variant::new(&metadata1, &value1);
651 let (metadata2, value2) = make_listi32(0..10);
652 let list2 = Variant::new(&metadata2, &value2);
653 // They should be equal
654 assert_eq!(list1, list2);
655 }
656
657 #[test]
658 fn test_variant_list_equality_different_length() {
659 // Create two lists with different lengths
660 let (metadata1, value1) = make_listi32(0..10);
661 let list1 = Variant::new(&metadata1, &value1);
662 let (metadata2, value2) = make_listi32(0..5);
663 let list2 = Variant::new(&metadata2, &value2);
664 // They should not be equal
665 assert_ne!(list1, list2);
666 }
667
668 #[test]
669 fn test_variant_list_equality_different_values() {
670 // Create two lists with different values
671 let (metadata1, value1) = make_listi32(0..10);
672 let list1 = Variant::new(&metadata1, &value1);
673 let (metadata2, value2) = make_listi32(5..15);
674 let list2 = Variant::new(&metadata2, &value2);
675 // They should not be equal
676 assert_ne!(list1, list2);
677 }
678
679 #[test]
680 fn test_variant_list_equality_different_types() {
681 // Create two lists with different types
682 let (metadata1, value1) = make_listi32(0i32..10i32);
683 let list1 = Variant::new(&metadata1, &value1);
684 let (metadata2, value2) = make_listi64(0..10);
685 let list2 = Variant::new(&metadata2, &value2);
686 // They should not be equal due to type mismatch
687 assert_ne!(list1, list2);
688 }
689
690 #[test]
691 fn test_variant_list_equality_slices() {
692 // Make an object like this and make sure equality works
693 // when the lists are sub fields
694 //
695 // {
696 // "list1": [0, 1, 2, ..., 9],
697 // "list2": [0, 1, 2, ..., 9],
698 // "list3": [10, 11, 12, ..., 19],
699 // }
700 let (metadata, value) = {
701 let mut builder = VariantBuilder::new();
702 let mut object_builder = builder.new_object();
703 // list1 (0..10)
704 let (metadata1, value1) = make_listi32(0i32..10i32);
705 object_builder.insert("list1", Variant::new(&metadata1, &value1));
706
707 // list2 (0..10)
708 let (metadata2, value2) = make_listi32(0i32..10i32);
709 object_builder.insert("list2", Variant::new(&metadata2, &value2));
710
711 // list3 (10..20)
712 let (metadata3, value3) = make_listi32(10i32..20i32);
713 object_builder.insert("list3", Variant::new(&metadata3, &value3));
714 object_builder.finish();
715 builder.finish()
716 };
717
718 let variant = Variant::try_new(&metadata, &value).unwrap();
719 let object = variant.as_object().unwrap();
720 // Check that list1 and list2 are equal
721 assert_eq!(object.get("list1").unwrap(), object.get("list2").unwrap());
722 // Check that list1 and list3 are not equal
723 assert_ne!(object.get("list1").unwrap(), object.get("list3").unwrap());
724 }
725
726 /// return metadata/value for a simple variant list with values in a range
727 fn make_listi32(range: Range<i32>) -> (Vec<u8>, Vec<u8>) {
728 let mut variant_builder = VariantBuilder::new();
729 let mut list_builder = variant_builder.new_list();
730 for i in range {
731 list_builder.append_value(i);
732 }
733 list_builder.finish();
734 variant_builder.finish()
735 }
736
737 /// return metadata/value for a simple variant list with values in a range
738 fn make_listi64(range: Range<i64>) -> (Vec<u8>, Vec<u8>) {
739 let mut variant_builder = VariantBuilder::new();
740 let mut list_builder = variant_builder.new_list();
741 for i in range {
742 list_builder.append_value(i);
743 }
744 list_builder.finish();
745 variant_builder.finish()
746 }
747}