arrow_array/array/
list_view_array.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use arrow_buffer::{Buffer, NullBuffer, ScalarBuffer};
6use arrow_data::{ArrayData, ArrayDataBuilder};
7use arrow_schema::{ArrowError, DataType, FieldRef};
8use std::any::Any;
9use std::ops::Add;
10use std::sync::Arc;
11
12use crate::array::{make_array, print_long_array};
13use crate::builder::{GenericListViewBuilder, PrimitiveBuilder};
14use crate::iterator::GenericListViewArrayIter;
15use crate::{
16    Array, ArrayAccessor, ArrayRef, ArrowPrimitiveType, FixedSizeListArray, GenericListArray,
17    OffsetSizeTrait, new_empty_array,
18};
19
20/// A [`GenericListViewArray`] of variable size lists, storing offsets as `i32`.
21pub type ListViewArray = GenericListViewArray<i32>;
22
23/// A [`GenericListViewArray`] of variable size lists, storing offsets as `i64`.
24pub type LargeListViewArray = GenericListViewArray<i64>;
25
26/// An array of [variable length lists], specifically in the [list-view layout].
27///
28/// Differs from [`GenericListArray`] (which represents the [list layout]) in that
29/// the sizes of the child arrays are explicitly encoded in a separate buffer, instead
30/// of being derived from the difference between subsequent offsets in the offset buffer.
31///
32/// This allows the offsets (and subsequently child data) to be out of order. It also
33/// allows take / filter operations to be implemented without copying the underlying data.
34///
35/// # Representation
36///
37/// Given the same example array from [`GenericListArray`], it would be represented
38/// as such via a list-view layout array:
39///
40/// ```text
41///                                         ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
42///                                                                         ┌ ─ ─ ─ ─ ─ ─ ┐    │
43///  ┌─────────────┐  ┌───────┐             │     ┌───┐   ┌───┐   ┌───┐       ┌───┐ ┌───┐
44///  │   [A,B,C]   │  │ (0,3) │                   │ 1 │   │ 0 │   │ 3 │     │ │ 1 │ │ A │ │ 0  │
45///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
46///  │      []     │  │ (3,0) │                   │ 1 │   │ 3 │   │ 0 │     │ │ 1 │ │ B │ │ 1  │
47///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
48///  │    NULL     │  │ (?,?) │                   │ 0 │   │ ? │   │ ? │     │ │ 1 │ │ C │ │ 2  │
49///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
50///  │     [D]     │  │ (4,1) │                   │ 1 │   │ 4 │   │ 1 │     │ │ ? │ │ ? │ │ 3  │
51///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
52///  │  [NULL, F]  │  │ (5,2) │                   │ 1 │   │ 5 │   │ 2 │     │ │ 1 │ │ D │ │ 4  │
53///  └─────────────┘  └───────┘             │     └───┘   └───┘   └───┘       ├───┤ ├───┤
54///                                                                         │ │ 0 │ │ ? │ │ 5  │
55///     Logical       Logical               │  Validity  Offsets  Sizes       ├───┤ ├───┤
56///      Values       Offset                   (nulls)                      │ │ 1 │ │ F │ │ 6  │
57///                   & Size                │                                 └───┘ └───┘
58///                                                                         │    Values   │    │
59///                 (offsets[i],            │   ListViewArray                   (Array)
60///                  sizes[i])                                              └ ─ ─ ─ ─ ─ ─ ┘    │
61///                                         └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
62/// ```
63///
64/// Another way of representing the same array but taking advantage of the offsets being out of order:
65///
66/// ```text
67///                                         ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
68///                                                                         ┌ ─ ─ ─ ─ ─ ─ ┐    │
69///  ┌─────────────┐  ┌───────┐             │     ┌───┐   ┌───┐   ┌───┐       ┌───┐ ┌───┐
70///  │   [A,B,C]   │  │ (2,3) │                   │ 1 │   │ 2 │   │ 3 │     │ │ 0 │ │ ? │ │ 0  │
71///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
72///  │      []     │  │ (0,0) │                   │ 1 │   │ 0 │   │ 0 │     │ │ 1 │ │ F │ │ 1  │
73///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
74///  │    NULL     │  │ (?,?) │                   │ 0 │   │ ? │   │ ? │     │ │ 1 │ │ A │ │ 2  │
75///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
76///  │     [D]     │  │ (5,1) │                   │ 1 │   │ 5 │   │ 1 │     │ │ 1 │ │ B │ │ 3  │
77///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
78///  │  [NULL, F]  │  │ (0,2) │                   │ 1 │   │ 0 │   │ 2 │     │ │ 1 │ │ C │ │ 4  │
79///  └─────────────┘  └───────┘             │     └───┘   └───┘   └───┘       ├───┤ ├───┤
80///                                                                         │ │ 1 │ │ D │ │ 5  │
81///     Logical       Logical               │  Validity  Offsets  Sizes       └───┘ └───┘
82///      Values       Offset                   (nulls)                      │    Values   │    │
83///                   & Size                │                                   (Array)
84///                                                                         └ ─ ─ ─ ─ ─ ─ ┘    │
85///                 (offsets[i],            │   ListViewArray
86///                  sizes[i])                                                                 │
87///                                         └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
88/// ```
89///
90/// [`GenericListArray`]: crate::array::GenericListArray
91/// [variable length lists]: https://arrow.apache.org/docs/format/Columnar.html#variable-size-list-layout
92/// [list layout]: https://arrow.apache.org/docs/format/Columnar.html#list-layout
93/// [list-view layout]: https://arrow.apache.org/docs/format/Columnar.html#listview-layout
94#[derive(Clone)]
95pub struct GenericListViewArray<OffsetSize: OffsetSizeTrait> {
96    data_type: DataType,
97    nulls: Option<NullBuffer>,
98    values: ArrayRef,
99    // Unlike GenericListArray, we do not use OffsetBuffer here as offsets are not
100    // guaranteed to be monotonically increasing.
101    value_offsets: ScalarBuffer<OffsetSize>,
102    value_sizes: ScalarBuffer<OffsetSize>,
103}
104
105impl<OffsetSize: OffsetSizeTrait> GenericListViewArray<OffsetSize> {
106    /// The data type constructor of listview array.
107    /// The input is the schema of the child array and
108    /// the output is the [`DataType`], ListView or LargeListView.
109    pub const DATA_TYPE_CONSTRUCTOR: fn(FieldRef) -> DataType = if OffsetSize::IS_LARGE {
110        DataType::LargeListView
111    } else {
112        DataType::ListView
113    };
114
115    /// Create a new [`GenericListViewArray`] from the provided parts
116    ///
117    /// # Errors
118    ///
119    /// Errors if
120    ///
121    /// * `offsets.len() != sizes.len()`
122    /// * `offsets.len() != nulls.len()`
123    /// * `offsets[i] > values.len()`
124    /// * `!field.is_nullable() && values.logical_null_count() != 0`
125    /// * `field.data_type() != values.data_type()`
126    /// * `0 <= offsets[i] <= length of the child array`
127    /// * `0 <= offsets[i] + size[i] <= length of the child array`
128    pub fn try_new(
129        field: FieldRef,
130        offsets: ScalarBuffer<OffsetSize>,
131        sizes: ScalarBuffer<OffsetSize>,
132        values: ArrayRef,
133        nulls: Option<NullBuffer>,
134    ) -> Result<Self, ArrowError> {
135        let len = offsets.len();
136        if let Some(n) = nulls.as_ref()
137            && n.len() != len
138        {
139            return Err(ArrowError::InvalidArgumentError(format!(
140                "Incorrect length of null buffer for {}ListViewArray, expected {len} got {}",
141                OffsetSize::PREFIX,
142                n.len(),
143            )));
144        }
145        if len != sizes.len() {
146            return Err(ArrowError::InvalidArgumentError(format!(
147                "Length of offsets buffer and sizes buffer must be equal for {}ListViewArray, got {len} and {}",
148                OffsetSize::PREFIX,
149                sizes.len()
150            )));
151        }
152
153        for (offset, size) in offsets.iter().zip(sizes.iter()) {
154            let offset = offset.as_usize();
155            let size = size.as_usize();
156            if offset.checked_add(size).ok_or_else(|| {
157                ArrowError::InvalidArgumentError(format!(
158                    "Overflow in offset + size for {}ListViewArray",
159                    OffsetSize::PREFIX
160                ))
161            })? > values.len()
162            {
163                return Err(ArrowError::InvalidArgumentError(format!(
164                    "Offset + size for {}ListViewArray must be within the bounds of the child array, got offset: {offset}, size: {size}, child array length: {}",
165                    OffsetSize::PREFIX,
166                    values.len()
167                )));
168            }
169        }
170
171        if !field.is_nullable() && values.logical_null_count() != 0 {
172            return Err(ArrowError::InvalidArgumentError(format!(
173                "Non-nullable field of {}ListViewArray {:?} cannot contain nulls",
174                OffsetSize::PREFIX,
175                field.name()
176            )));
177        }
178
179        if field.data_type() != values.data_type() {
180            return Err(ArrowError::InvalidArgumentError(format!(
181                "{}ListViewArray expected data type {} got {} for {:?}",
182                OffsetSize::PREFIX,
183                field.data_type(),
184                values.data_type(),
185                field.name()
186            )));
187        }
188
189        Ok(Self {
190            data_type: Self::DATA_TYPE_CONSTRUCTOR(field),
191            nulls,
192            values,
193            value_offsets: offsets,
194            value_sizes: sizes,
195        })
196    }
197
198    /// Create a new [`GenericListViewArray`] from the provided parts
199    ///
200    /// # Panics
201    ///
202    /// Panics if [`Self::try_new`] returns an error
203    pub fn new(
204        field: FieldRef,
205        offsets: ScalarBuffer<OffsetSize>,
206        sizes: ScalarBuffer<OffsetSize>,
207        values: ArrayRef,
208        nulls: Option<NullBuffer>,
209    ) -> Self {
210        Self::try_new(field, offsets, sizes, values, nulls).unwrap()
211    }
212
213    /// Create a new [`GenericListViewArray`] from the provided parts without validation
214    ///
215    /// See [`Self::try_new`] for the checked version of this function, and the
216    /// documentation of that function for the invariants that must be upheld.
217    ///
218    /// # Safety
219    ///
220    /// The parts must form a valid [`ListViewArray`] or [`LargeListViewArray`] according
221    /// to the Arrow spec.
222    pub unsafe fn new_unchecked(
223        field: FieldRef,
224        offsets: ScalarBuffer<OffsetSize>,
225        sizes: ScalarBuffer<OffsetSize>,
226        values: ArrayRef,
227        nulls: Option<NullBuffer>,
228    ) -> Self {
229        if cfg!(feature = "force_validate") {
230            return Self::new(field, offsets, sizes, values, nulls);
231        }
232
233        Self {
234            data_type: Self::DATA_TYPE_CONSTRUCTOR(field),
235            nulls,
236            values,
237            value_offsets: offsets,
238            value_sizes: sizes,
239        }
240    }
241
242    /// Create a new [`GenericListViewArray`] of length `len` where all values are null
243    pub fn new_null(field: FieldRef, len: usize) -> Self {
244        let values = new_empty_array(field.data_type());
245        Self {
246            data_type: Self::DATA_TYPE_CONSTRUCTOR(field),
247            nulls: Some(NullBuffer::new_null(len)),
248            value_offsets: ScalarBuffer::from(vec![OffsetSize::usize_as(0); len]),
249            value_sizes: ScalarBuffer::from(vec![OffsetSize::usize_as(0); len]),
250            values,
251        }
252    }
253
254    /// Deconstruct this array into its constituent parts
255    pub fn into_parts(
256        self,
257    ) -> (
258        FieldRef,
259        ScalarBuffer<OffsetSize>,
260        ScalarBuffer<OffsetSize>,
261        ArrayRef,
262        Option<NullBuffer>,
263    ) {
264        let (DataType::ListView(f) | DataType::LargeListView(f)) = self.data_type else {
265            unreachable!()
266        };
267        (
268            f,
269            self.value_offsets,
270            self.value_sizes,
271            self.values,
272            self.nulls,
273        )
274    }
275
276    /// The field that describes the values of this list.
277    pub fn value_field(&self) -> &FieldRef {
278        match &self.data_type {
279            DataType::ListView(f) | DataType::LargeListView(f) => f,
280            _ => unreachable!(),
281        }
282    }
283
284    /// Returns a reference to the offsets of this list
285    ///
286    /// Unlike [`Self::value_offsets`] this returns the [`ScalarBuffer`]
287    /// allowing for zero-copy cloning
288    #[inline]
289    pub fn offsets(&self) -> &ScalarBuffer<OffsetSize> {
290        &self.value_offsets
291    }
292
293    /// Returns a reference to the values of this list
294    #[inline]
295    pub fn values(&self) -> &ArrayRef {
296        &self.values
297    }
298
299    /// Returns a reference to the sizes of this list
300    ///
301    /// Unlike [`Self::value_sizes`] this returns the [`ScalarBuffer`]
302    /// allowing for zero-copy cloning
303    #[inline]
304    pub fn sizes(&self) -> &ScalarBuffer<OffsetSize> {
305        &self.value_sizes
306    }
307
308    /// Returns a clone of the value type of this list.
309    pub fn value_type(&self) -> DataType {
310        self.values.data_type().clone()
311    }
312
313    /// Returns ith value of this list view array.
314    ///
315    /// Note: This method does not check for nulls and the value is arbitrary
316    /// if [`is_null`](Self::is_null) returns true for the index.
317    ///
318    /// # Safety
319    /// Caller must ensure that the index is within the array bounds
320    pub unsafe fn value_unchecked(&self, i: usize) -> ArrayRef {
321        let offset = unsafe { self.value_offsets().get_unchecked(i).as_usize() };
322        let length = unsafe { self.value_sizes().get_unchecked(i).as_usize() };
323        self.values.slice(offset, length)
324    }
325
326    /// Returns ith value of this list view array.
327    ///
328    /// Note: This method does not check for nulls and the value is arbitrary
329    /// (but still well-defined) if [`is_null`](Self::is_null) returns true for the index.
330    ///
331    /// # Panics
332    /// Panics if `i >= self.len()`
333    pub fn value(&self, i: usize) -> ArrayRef {
334        let offset = self.value_offsets()[i].as_usize();
335        let length = self.value_sizes()[i].as_usize();
336        self.values.slice(offset, length)
337    }
338
339    /// Returns the offset values in the offsets buffer
340    #[inline]
341    pub fn value_offsets(&self) -> &[OffsetSize] {
342        &self.value_offsets
343    }
344
345    /// Returns the sizes values in the offsets buffer
346    #[inline]
347    pub fn value_sizes(&self) -> &[OffsetSize] {
348        &self.value_sizes
349    }
350
351    /// Returns the size for value at index `i`.
352    ///
353    /// # Panics
354    /// Panics if `i >= self.len()`
355    #[inline]
356    pub fn value_size(&self, i: usize) -> OffsetSize {
357        self.value_sizes[i]
358    }
359
360    /// Returns the offset for value at index `i`.
361    ///
362    /// # Panics
363    /// Panics if `i >= self.len()`
364    pub fn value_offset(&self, i: usize) -> OffsetSize {
365        self.value_offsets[i]
366    }
367
368    /// Constructs a new iterator
369    pub fn iter(&self) -> GenericListViewArrayIter<'_, OffsetSize> {
370        GenericListViewArrayIter::<'_, OffsetSize>::new(self)
371    }
372
373    #[inline]
374    fn get_type(data_type: &DataType) -> Option<&DataType> {
375        match (OffsetSize::IS_LARGE, data_type) {
376            (true, DataType::LargeListView(child)) | (false, DataType::ListView(child)) => {
377                Some(child.data_type())
378            }
379            _ => None,
380        }
381    }
382
383    /// Returns a zero-copy slice of this array with the indicated offset and length.
384    ///
385    /// # Panics
386    /// Panics if `offset + length > self.len()`
387    pub fn slice(&self, offset: usize, length: usize) -> Self {
388        Self {
389            data_type: self.data_type.clone(),
390            nulls: self.nulls.as_ref().map(|n| n.slice(offset, length)),
391            values: self.values.clone(),
392            value_offsets: self.value_offsets.slice(offset, length),
393            value_sizes: self.value_sizes.slice(offset, length),
394        }
395    }
396
397    /// Creates a [`GenericListViewArray`] from an iterator of primitive values
398    /// # Example
399    /// ```
400    /// # use arrow_array::ListViewArray;
401    /// # use arrow_array::types::Int32Type;
402    ///
403    /// let data = vec![
404    ///    Some(vec![Some(0), Some(1), Some(2)]),
405    ///    None,
406    ///    Some(vec![Some(3), None, Some(5)]),
407    ///    Some(vec![Some(6), Some(7)]),
408    /// ];
409    /// let list_array = ListViewArray::from_iter_primitive::<Int32Type, _, _>(data);
410    /// println!("{:?}", list_array);
411    /// ```
412    pub fn from_iter_primitive<T, P, I>(iter: I) -> Self
413    where
414        T: ArrowPrimitiveType,
415        P: IntoIterator<Item = Option<<T as ArrowPrimitiveType>::Native>>,
416        I: IntoIterator<Item = Option<P>>,
417    {
418        let iter = iter.into_iter();
419        let size_hint = iter.size_hint().0;
420        let mut builder =
421            GenericListViewBuilder::with_capacity(PrimitiveBuilder::<T>::new(), size_hint);
422
423        for i in iter {
424            match i {
425                Some(p) => {
426                    for t in p {
427                        builder.values().append_option(t);
428                    }
429                    builder.append(true);
430                }
431                None => builder.append(false),
432            }
433        }
434        builder.finish()
435    }
436}
437
438impl<'a, OffsetSize: OffsetSizeTrait> IntoIterator for &'a GenericListViewArray<OffsetSize> {
439    type Item = Option<ArrayRef>;
440    type IntoIter = GenericListViewArrayIter<'a, OffsetSize>;
441
442    fn into_iter(self) -> Self::IntoIter {
443        GenericListViewArrayIter::<'a, OffsetSize>::new(self)
444    }
445}
446
447impl<OffsetSize: OffsetSizeTrait> ArrayAccessor for &GenericListViewArray<OffsetSize> {
448    type Item = ArrayRef;
449
450    fn value(&self, index: usize) -> Self::Item {
451        GenericListViewArray::value(self, index)
452    }
453
454    unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
455        unsafe { GenericListViewArray::value_unchecked(self, index) }
456    }
457}
458
459/// SAFETY: Correctly implements the contract of Arrow Arrays
460unsafe impl<OffsetSize: OffsetSizeTrait> Array for GenericListViewArray<OffsetSize> {
461    fn as_any(&self) -> &dyn Any {
462        self
463    }
464
465    fn to_data(&self) -> ArrayData {
466        self.clone().into()
467    }
468
469    fn into_data(self) -> ArrayData {
470        self.into()
471    }
472
473    fn data_type(&self) -> &DataType {
474        &self.data_type
475    }
476
477    fn slice(&self, offset: usize, length: usize) -> ArrayRef {
478        Arc::new(self.slice(offset, length))
479    }
480
481    fn len(&self) -> usize {
482        self.sizes().len()
483    }
484
485    fn is_empty(&self) -> bool {
486        self.value_sizes.is_empty()
487    }
488
489    fn shrink_to_fit(&mut self) {
490        if let Some(nulls) = &mut self.nulls {
491            nulls.shrink_to_fit();
492        }
493        self.values.shrink_to_fit();
494        self.value_offsets.shrink_to_fit();
495        self.value_sizes.shrink_to_fit();
496    }
497
498    fn offset(&self) -> usize {
499        0
500    }
501
502    fn nulls(&self) -> Option<&NullBuffer> {
503        self.nulls.as_ref()
504    }
505
506    fn logical_null_count(&self) -> usize {
507        // More efficient that the default implementation
508        self.null_count()
509    }
510
511    fn get_buffer_memory_size(&self) -> usize {
512        let mut size = self.values.get_buffer_memory_size();
513        size += self.value_offsets.inner().capacity();
514        size += self.value_sizes.inner().capacity();
515        if let Some(n) = self.nulls.as_ref() {
516            size += n.buffer().capacity();
517        }
518        size
519    }
520
521    fn get_array_memory_size(&self) -> usize {
522        let mut size = std::mem::size_of::<Self>() + self.values.get_array_memory_size();
523        size += self.value_offsets.inner().capacity();
524        size += self.value_sizes.inner().capacity();
525        if let Some(n) = self.nulls.as_ref() {
526            size += n.buffer().capacity();
527        }
528        size
529    }
530
531    #[cfg(feature = "pool")]
532    fn claim(&self, pool: &dyn arrow_buffer::MemoryPool) {
533        self.value_offsets.claim(pool);
534        self.value_sizes.claim(pool);
535        self.values.claim(pool);
536        if let Some(nulls) = &self.nulls {
537            nulls.claim(pool);
538        }
539    }
540}
541
542impl<OffsetSize: OffsetSizeTrait> super::ListLikeArray for GenericListViewArray<OffsetSize> {
543    fn values(&self) -> &ArrayRef {
544        self.values()
545    }
546
547    fn element_range(&self, index: usize) -> std::ops::Range<usize> {
548        let offset = self.value_offsets()[index].as_usize();
549        let size = self.value_sizes()[index].as_usize();
550        offset..(offset + size)
551    }
552}
553
554impl<OffsetSize: OffsetSizeTrait> std::fmt::Debug for GenericListViewArray<OffsetSize> {
555    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
556        let prefix = OffsetSize::PREFIX;
557        write!(f, "{prefix}ListViewArray\n[\n")?;
558        print_long_array(self, f, &mut |index, f| {
559            std::fmt::Debug::fmt(&self.value(index), f)
560        })?;
561        write!(f, "]")
562    }
563}
564
565impl<OffsetSize: OffsetSizeTrait> From<GenericListArray<OffsetSize>>
566    for GenericListViewArray<OffsetSize>
567{
568    fn from(value: GenericListArray<OffsetSize>) -> Self {
569        let (field, offsets, values, nulls) = value.into_parts();
570        let len = offsets.len() - 1;
571        let mut sizes = Vec::with_capacity(len);
572        let mut view_offsets = Vec::with_capacity(len);
573        for (i, offset) in offsets.iter().enumerate().take(len) {
574            view_offsets.push(*offset);
575            sizes.push(offsets[i + 1] - offsets[i]);
576        }
577
578        Self::new(
579            field,
580            ScalarBuffer::from(view_offsets),
581            ScalarBuffer::from(sizes),
582            values,
583            nulls,
584        )
585    }
586}
587
588impl<OffsetSize: OffsetSizeTrait> From<GenericListViewArray<OffsetSize>> for ArrayData {
589    fn from(array: GenericListViewArray<OffsetSize>) -> Self {
590        let len = array.len();
591        let builder = ArrayDataBuilder::new(array.data_type)
592            .len(len)
593            .nulls(array.nulls)
594            .buffers(vec![
595                array.value_offsets.into_inner(),
596                array.value_sizes.into_inner(),
597            ])
598            .child_data(vec![array.values.to_data()]);
599
600        unsafe { builder.build_unchecked() }
601    }
602}
603
604impl<OffsetSize: OffsetSizeTrait> From<ArrayData> for GenericListViewArray<OffsetSize> {
605    fn from(data: ArrayData) -> Self {
606        Self::try_new_from_array_data(data)
607            .expect("Expected infallible creation of GenericListViewArray from ArrayDataRef failed")
608    }
609}
610
611impl<OffsetSize: OffsetSizeTrait> From<FixedSizeListArray> for GenericListViewArray<OffsetSize> {
612    fn from(value: FixedSizeListArray) -> Self {
613        let (field, size) = match value.data_type() {
614            DataType::FixedSizeList(f, size) => (f, *size as usize),
615            _ => unreachable!(),
616        };
617        let mut acc = 0_usize;
618        let iter = std::iter::repeat_n(size, value.len());
619        let mut sizes = Vec::with_capacity(iter.size_hint().0);
620        let mut offsets = Vec::with_capacity(iter.size_hint().0);
621
622        for size in iter {
623            offsets.push(OffsetSize::usize_as(acc));
624            acc = acc.add(size);
625            sizes.push(OffsetSize::usize_as(size));
626        }
627        let sizes = ScalarBuffer::from(sizes);
628        let offsets = ScalarBuffer::from(offsets);
629        Self {
630            data_type: Self::DATA_TYPE_CONSTRUCTOR(field.clone()),
631            nulls: value.nulls().cloned(),
632            values: value.values().clone(),
633            value_offsets: offsets,
634            value_sizes: sizes,
635        }
636    }
637}
638
639impl<OffsetSize: OffsetSizeTrait> GenericListViewArray<OffsetSize> {
640    fn try_new_from_array_data(data: ArrayData) -> Result<Self, ArrowError> {
641        let (data_type, len, nulls, offset, buffers, child_data) = data.into_parts();
642
643        // ArrayData is valid, and verified type above
644        // buffer[0] is offsets, buffer[1] is sizes
645        let num_buffers = buffers.len();
646        let [offsets_buffer, sizes_buffer] : [Buffer; 2] = buffers.try_into().map_err(|_| {
647             ArrowError::InvalidArgumentError(format!(
648                "ListViewArray data should contain two buffers (value offsets & value sizes), had {num_buffers}",
649            ))
650        })?;
651
652        let num_child = child_data.len();
653        let [values]: [ArrayData; 1] = child_data.try_into().map_err(|_| {
654            ArrowError::InvalidArgumentError(format!(
655                "ListViewArray should contain a single child array (values array), had {num_child}",
656            ))
657        })?;
658
659        if let Some(child_data_type) = Self::get_type(&data_type) {
660            if values.data_type() != child_data_type {
661                return Err(ArrowError::InvalidArgumentError(format!(
662                    "{}ListViewArray's child datatype {:?} does not \
663                             correspond to the List's datatype {:?}",
664                    OffsetSize::PREFIX,
665                    values.data_type(),
666                    child_data_type
667                )));
668            }
669        } else {
670            return Err(ArrowError::InvalidArgumentError(format!(
671                "{}ListViewArray's datatype must be {}ListViewArray(). It is {:?}",
672                OffsetSize::PREFIX,
673                OffsetSize::PREFIX,
674                data_type
675            )));
676        }
677
678        let values = make_array(values);
679        let value_offsets = ScalarBuffer::new(offsets_buffer, offset, len);
680        let value_sizes = ScalarBuffer::new(sizes_buffer, offset, len);
681
682        Ok(Self {
683            data_type,
684            nulls,
685            values,
686            value_offsets,
687            value_sizes,
688        })
689    }
690}
691
692#[cfg(test)]
693mod tests {
694    use arrow_buffer::{BooleanBuffer, Buffer, NullBufferBuilder, ScalarBuffer, bit_util};
695    use arrow_schema::Field;
696
697    use crate::builder::{FixedSizeListBuilder, Int32Builder};
698    use crate::cast::AsArray;
699    use crate::types::Int32Type;
700    use crate::{Int8Array, Int8DictionaryArray, Int32Array, Int64Array, StringArray};
701
702    use super::*;
703
704    #[test]
705    fn test_empty_list_view_array() {
706        // Construct an empty value array
707        let vec: Vec<i32> = vec![];
708        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
709        let sizes = ScalarBuffer::from(vec![]);
710        let offsets = ScalarBuffer::from(vec![]);
711        let values = Int32Array::from(vec);
712        let list_array = LargeListViewArray::new(field, offsets, sizes, Arc::new(values), None);
713
714        assert_eq!(list_array.len(), 0)
715    }
716
717    #[test]
718    fn test_list_view_array() {
719        // Construct a value array
720        let value_data = ArrayData::builder(DataType::Int32)
721            .len(8)
722            .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7]))
723            .build()
724            .unwrap();
725
726        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
727        let sizes = ScalarBuffer::from(vec![3i32, 3, 2]);
728        let offsets = ScalarBuffer::from(vec![0i32, 3, 6]);
729        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7]);
730        let list_array = ListViewArray::new(field, offsets, sizes, Arc::new(values), None);
731
732        let values = list_array.values();
733        assert_eq!(value_data, values.to_data());
734        assert_eq!(DataType::Int32, list_array.value_type());
735        assert_eq!(3, list_array.len());
736        assert_eq!(0, list_array.null_count());
737        assert_eq!(6, list_array.value_offsets()[2]);
738        assert_eq!(2, list_array.value_sizes()[2]);
739        assert_eq!(2, list_array.value_size(2));
740        assert_eq!(0, list_array.value(0).as_primitive::<Int32Type>().value(0));
741        assert_eq!(
742            0,
743            unsafe { list_array.value_unchecked(0) }
744                .as_primitive::<Int32Type>()
745                .value(0)
746        );
747        for i in 0..3 {
748            assert!(list_array.is_valid(i));
749            assert!(!list_array.is_null(i));
750        }
751    }
752
753    #[test]
754    fn test_large_list_view_array() {
755        // Construct a value array
756        let value_data = ArrayData::builder(DataType::Int32)
757            .len(8)
758            .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7]))
759            .build()
760            .unwrap();
761
762        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
763        let sizes = ScalarBuffer::from(vec![3i64, 3, 2]);
764        let offsets = ScalarBuffer::from(vec![0i64, 3, 6]);
765        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7]);
766        let list_array = LargeListViewArray::new(field, offsets, sizes, Arc::new(values), None);
767
768        let values = list_array.values();
769        assert_eq!(value_data, values.to_data());
770        assert_eq!(DataType::Int32, list_array.value_type());
771        assert_eq!(3, list_array.len());
772        assert_eq!(0, list_array.null_count());
773        assert_eq!(6, list_array.value_offsets()[2]);
774        assert_eq!(2, list_array.value_sizes()[2]);
775        assert_eq!(2, list_array.value_size(2));
776        assert_eq!(0, list_array.value(0).as_primitive::<Int32Type>().value(0));
777        assert_eq!(
778            0,
779            unsafe { list_array.value_unchecked(0) }
780                .as_primitive::<Int32Type>()
781                .value(0)
782        );
783        for i in 0..3 {
784            assert!(list_array.is_valid(i));
785            assert!(!list_array.is_null(i));
786        }
787    }
788
789    #[test]
790    fn test_list_view_array_slice() {
791        // Construct a value array
792        let value_data = ArrayData::builder(DataType::Int32)
793            .len(10)
794            .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))
795            .build()
796            .unwrap();
797
798        // 01011001 00000001
799        let mut null_bits: [u8; 2] = [0; 2];
800        bit_util::set_bit(&mut null_bits, 0);
801        bit_util::set_bit(&mut null_bits, 3);
802        bit_util::set_bit(&mut null_bits, 4);
803        bit_util::set_bit(&mut null_bits, 6);
804        bit_util::set_bit(&mut null_bits, 8);
805        let buffer = BooleanBuffer::new(Buffer::from(null_bits), 0, 9);
806        let null_buffer = NullBuffer::new(buffer);
807
808        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
809        let sizes = ScalarBuffer::from(vec![2, 0, 0, 2, 2, 0, 3, 0, 1]);
810        let offsets = ScalarBuffer::from(vec![0, 2, 2, 2, 4, 6, 6, 9, 9]);
811        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
812        let list_array =
813            ListViewArray::new(field, offsets, sizes, Arc::new(values), Some(null_buffer));
814
815        let values = list_array.values();
816        assert_eq!(value_data, values.to_data());
817        assert_eq!(DataType::Int32, list_array.value_type());
818        assert_eq!(9, list_array.len());
819        assert_eq!(4, list_array.null_count());
820        assert_eq!(2, list_array.value_offsets()[3]);
821        assert_eq!(2, list_array.value_sizes()[3]);
822        assert_eq!(2, list_array.value_size(3));
823
824        let sliced_array = list_array.slice(1, 6);
825        assert_eq!(6, sliced_array.len());
826        assert_eq!(3, sliced_array.null_count());
827
828        for i in 0..sliced_array.len() {
829            if bit_util::get_bit(&null_bits, 1 + i) {
830                assert!(sliced_array.is_valid(i));
831            } else {
832                assert!(sliced_array.is_null(i));
833            }
834        }
835
836        // Check offset and length for each non-null value.
837        let sliced_list_array = sliced_array
838            .as_any()
839            .downcast_ref::<ListViewArray>()
840            .unwrap();
841        assert_eq!(2, sliced_list_array.value_offsets()[2]);
842        assert_eq!(2, sliced_list_array.value_sizes()[2]);
843        assert_eq!(2, sliced_list_array.value_size(2));
844
845        assert_eq!(4, sliced_list_array.value_offsets()[3]);
846        assert_eq!(2, sliced_list_array.value_sizes()[3]);
847        assert_eq!(2, sliced_list_array.value_size(3));
848
849        assert_eq!(6, sliced_list_array.value_offsets()[5]);
850        assert_eq!(3, sliced_list_array.value_sizes()[5]);
851        assert_eq!(3, sliced_list_array.value_size(5));
852    }
853
854    #[test]
855    fn test_large_list_view_array_slice() {
856        // Construct a value array
857        let value_data = ArrayData::builder(DataType::Int32)
858            .len(10)
859            .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))
860            .build()
861            .unwrap();
862
863        // 01011001 00000001
864        let mut null_bits: [u8; 2] = [0; 2];
865        bit_util::set_bit(&mut null_bits, 0);
866        bit_util::set_bit(&mut null_bits, 3);
867        bit_util::set_bit(&mut null_bits, 4);
868        bit_util::set_bit(&mut null_bits, 6);
869        bit_util::set_bit(&mut null_bits, 8);
870        let buffer = BooleanBuffer::new(Buffer::from(null_bits), 0, 9);
871        let null_buffer = NullBuffer::new(buffer);
872
873        // Construct a large list view array from the above two
874        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
875        let sizes = ScalarBuffer::from(vec![2i64, 0, 0, 2, 2, 0, 3, 0, 1]);
876        let offsets = ScalarBuffer::from(vec![0i64, 2, 2, 2, 4, 6, 6, 9, 9]);
877        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
878        let list_array =
879            LargeListViewArray::new(field, offsets, sizes, Arc::new(values), Some(null_buffer));
880
881        let values = list_array.values();
882        assert_eq!(value_data, values.to_data());
883        assert_eq!(DataType::Int32, list_array.value_type());
884        assert_eq!(9, list_array.len());
885        assert_eq!(4, list_array.null_count());
886        assert_eq!(2, list_array.value_offsets()[3]);
887        assert_eq!(2, list_array.value_sizes()[3]);
888        assert_eq!(2, list_array.value_size(3));
889
890        let sliced_array = list_array.slice(1, 6);
891        assert_eq!(6, sliced_array.len());
892        assert_eq!(3, sliced_array.null_count());
893
894        for i in 0..sliced_array.len() {
895            if bit_util::get_bit(&null_bits, 1 + i) {
896                assert!(sliced_array.is_valid(i));
897            } else {
898                assert!(sliced_array.is_null(i));
899            }
900        }
901
902        // Check offset and length for each non-null value.
903        let sliced_list_array = sliced_array
904            .as_any()
905            .downcast_ref::<LargeListViewArray>()
906            .unwrap();
907        assert_eq!(2, sliced_list_array.value_offsets()[2]);
908        assert_eq!(2, sliced_list_array.value_size(2));
909        assert_eq!(2, sliced_list_array.value_sizes()[2]);
910
911        assert_eq!(4, sliced_list_array.value_offsets()[3]);
912        assert_eq!(2, sliced_list_array.value_size(3));
913        assert_eq!(2, sliced_list_array.value_sizes()[3]);
914
915        assert_eq!(6, sliced_list_array.value_offsets()[5]);
916        assert_eq!(3, sliced_list_array.value_size(5));
917        assert_eq!(2, sliced_list_array.value_sizes()[3]);
918    }
919
920    #[test]
921    #[should_panic(expected = "index out of bounds: the len is 9 but the index is 10")]
922    fn test_list_view_array_index_out_of_bound() {
923        // 01011001 00000001
924        let mut null_bits: [u8; 2] = [0; 2];
925        bit_util::set_bit(&mut null_bits, 0);
926        bit_util::set_bit(&mut null_bits, 3);
927        bit_util::set_bit(&mut null_bits, 4);
928        bit_util::set_bit(&mut null_bits, 6);
929        bit_util::set_bit(&mut null_bits, 8);
930        let buffer = BooleanBuffer::new(Buffer::from(null_bits), 0, 9);
931        let null_buffer = NullBuffer::new(buffer);
932
933        // Construct a buffer for value offsets, for the nested array:
934        //  [[0, 1], null, null, [2, 3], [4, 5], null, [6, 7, 8], null, [9]]
935        // Construct a list array from the above two
936        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
937        let sizes = ScalarBuffer::from(vec![2i32, 0, 0, 2, 2, 0, 3, 0, 1]);
938        let offsets = ScalarBuffer::from(vec![0i32, 2, 2, 2, 4, 6, 6, 9, 9]);
939        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
940        let list_array =
941            ListViewArray::new(field, offsets, sizes, Arc::new(values), Some(null_buffer));
942
943        assert_eq!(9, list_array.len());
944        list_array.value(10);
945    }
946    #[test]
947    #[should_panic(
948        expected = "ListViewArray data should contain two buffers (value offsets & value sizes), had 0"
949    )]
950    #[cfg(not(feature = "force_validate"))]
951    fn test_list_view_array_invalid_buffer_len() {
952        let value_data = unsafe {
953            ArrayData::builder(DataType::Int32)
954                .len(8)
955                .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7]))
956                .build_unchecked()
957        };
958        let list_data_type =
959            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, false)));
960        let list_data = unsafe {
961            ArrayData::builder(list_data_type)
962                .len(3)
963                .add_child_data(value_data)
964                .build_unchecked()
965        };
966        drop(ListViewArray::from(list_data));
967    }
968
969    #[test]
970    #[should_panic(
971        expected = "ListViewArray data should contain two buffers (value offsets & value sizes), had 1"
972    )]
973    #[cfg(not(feature = "force_validate"))]
974    fn test_list_view_array_invalid_child_array_len() {
975        let value_offsets = Buffer::from_slice_ref([0, 2, 5, 7]);
976        let list_data_type =
977            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, false)));
978        let list_data = unsafe {
979            ArrayData::builder(list_data_type)
980                .len(3)
981                .add_buffer(value_offsets)
982                .build_unchecked()
983        };
984        drop(ListViewArray::from(list_data));
985    }
986
987    #[test]
988    fn test_list_view_array_offsets_need_not_start_at_zero() {
989        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
990        let sizes = ScalarBuffer::from(vec![0i32, 0, 3]);
991        let offsets = ScalarBuffer::from(vec![2i32, 2, 5]);
992        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7]);
993        let list_array = ListViewArray::new(field, offsets, sizes, Arc::new(values), None);
994
995        assert_eq!(list_array.value_size(0), 0);
996        assert_eq!(list_array.value_size(1), 0);
997        assert_eq!(list_array.value_size(2), 3);
998    }
999
1000    #[test]
1001    #[should_panic(expected = "Memory pointer is not aligned with the specified scalar type")]
1002    #[cfg(not(feature = "force_validate"))]
1003    fn test_list_view_array_alignment() {
1004        let offset_buf = Buffer::from_slice_ref([0_u64]);
1005        let offset_buf2 = offset_buf.slice(1);
1006
1007        let size_buf = Buffer::from_slice_ref([0_u64]);
1008        let size_buf2 = size_buf.slice(1);
1009
1010        let values: [i32; 8] = [0; 8];
1011        let value_data = unsafe {
1012            ArrayData::builder(DataType::Int32)
1013                .add_buffer(Buffer::from_slice_ref(values))
1014                .build_unchecked()
1015        };
1016
1017        let list_data_type =
1018            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, false)));
1019        let list_data = unsafe {
1020            ArrayData::builder(list_data_type)
1021                .add_buffer(offset_buf2)
1022                .add_buffer(size_buf2)
1023                .add_child_data(value_data)
1024                .build_unchecked()
1025        };
1026        drop(ListViewArray::from(list_data));
1027    }
1028
1029    #[test]
1030    fn test_empty_offsets() {
1031        let f = Arc::new(Field::new("element", DataType::Int32, true));
1032        let string = ListViewArray::from(
1033            ArrayData::builder(DataType::ListView(f.clone()))
1034                .buffers(vec![Buffer::from(&[]), Buffer::from(&[])])
1035                .add_child_data(ArrayData::new_empty(&DataType::Int32))
1036                .build()
1037                .unwrap(),
1038        );
1039        assert_eq!(string.value_offsets(), &[] as &[i32; 0]);
1040        assert_eq!(string.value_sizes(), &[] as &[i32; 0]);
1041
1042        let string = LargeListViewArray::from(
1043            ArrayData::builder(DataType::LargeListView(f))
1044                .buffers(vec![Buffer::from(&[]), Buffer::from(&[])])
1045                .add_child_data(ArrayData::new_empty(&DataType::Int32))
1046                .build()
1047                .unwrap(),
1048        );
1049        assert_eq!(string.len(), 0);
1050        assert_eq!(string.value_offsets(), &[] as &[i64; 0]);
1051        assert_eq!(string.value_sizes(), &[] as &[i64; 0]);
1052    }
1053
1054    #[test]
1055    fn test_try_new() {
1056        let offsets = ScalarBuffer::from(vec![0, 1, 4, 5]);
1057        let sizes = ScalarBuffer::from(vec![1, 3, 1, 0]);
1058        let values = Int32Array::new(vec![1, 2, 3, 4, 5].into(), None);
1059        let values = Arc::new(values) as ArrayRef;
1060
1061        let field = Arc::new(Field::new("element", DataType::Int32, false));
1062        ListViewArray::new(
1063            field.clone(),
1064            offsets.clone(),
1065            sizes.clone(),
1066            values.clone(),
1067            None,
1068        );
1069
1070        let nulls = NullBuffer::new_null(4);
1071        ListViewArray::new(
1072            field.clone(),
1073            offsets,
1074            sizes.clone(),
1075            values.clone(),
1076            Some(nulls),
1077        );
1078
1079        let nulls = NullBuffer::new_null(4);
1080        let offsets = ScalarBuffer::from(vec![0, 1, 2, 3, 4]);
1081        let sizes = ScalarBuffer::from(vec![1, 1, 1, 1, 0]);
1082        let err = LargeListViewArray::try_new(
1083            field,
1084            offsets.clone(),
1085            sizes.clone(),
1086            values.clone(),
1087            Some(nulls),
1088        )
1089        .unwrap_err();
1090
1091        assert_eq!(
1092            err.to_string(),
1093            "Invalid argument error: Incorrect length of null buffer for LargeListViewArray, expected 5 got 4"
1094        );
1095
1096        let field = Arc::new(Field::new("element", DataType::Int64, false));
1097        let err = LargeListViewArray::try_new(
1098            field.clone(),
1099            offsets.clone(),
1100            sizes.clone(),
1101            values.clone(),
1102            None,
1103        )
1104        .unwrap_err();
1105
1106        assert_eq!(
1107            err.to_string(),
1108            "Invalid argument error: LargeListViewArray expected data type Int64 got Int32 for \"element\""
1109        );
1110
1111        let nulls = NullBuffer::new_null(7);
1112        let values = Int64Array::new(vec![0; 7].into(), Some(nulls));
1113        let values = Arc::new(values);
1114
1115        let err = LargeListViewArray::try_new(
1116            field,
1117            offsets.clone(),
1118            sizes.clone(),
1119            values.clone(),
1120            None,
1121        )
1122        .unwrap_err();
1123
1124        assert_eq!(
1125            err.to_string(),
1126            "Invalid argument error: Non-nullable field of LargeListViewArray \"element\" cannot contain nulls"
1127        );
1128    }
1129
1130    #[test]
1131    fn test_try_new_non_nullable_field_dictionary_values() {
1132        let keys = Int8Array::new(vec![0i8, 1].into(), Some(NullBuffer::new_valid(2)));
1133        let values = StringArray::from(vec!["x", "y"]);
1134        let dict = Int8DictionaryArray::try_new(keys, Arc::new(values)).unwrap();
1135        let field = Arc::new(Field::new("element", dict.data_type().clone(), false));
1136        ListViewArray::try_new(
1137            field,
1138            ScalarBuffer::from(vec![0]),
1139            ScalarBuffer::from(vec![2]),
1140            Arc::new(dict),
1141            None,
1142        )
1143        .unwrap();
1144
1145        let keys = Int8Array::from(vec![Some(0i8), None]);
1146        let values = StringArray::from(vec!["x", "y"]);
1147        let dict = Int8DictionaryArray::try_new(keys, Arc::new(values)).unwrap();
1148        let field = Arc::new(Field::new("element", dict.data_type().clone(), false));
1149        let err = ListViewArray::try_new(
1150            field,
1151            ScalarBuffer::from(vec![0]),
1152            ScalarBuffer::from(vec![2]),
1153            Arc::new(dict),
1154            None,
1155        )
1156        .unwrap_err();
1157
1158        assert!(err.to_string().contains("cannot contain nulls"));
1159    }
1160
1161    #[test]
1162    fn test_from_fixed_size_list() {
1163        let mut builder = FixedSizeListBuilder::new(Int32Builder::new(), 3);
1164        builder.values().append_slice(&[1, 2, 3]);
1165        builder.append(true);
1166        builder.values().append_slice(&[0, 0, 0]);
1167        builder.append(false);
1168        builder.values().append_slice(&[4, 5, 6]);
1169        builder.append(true);
1170        let list: ListViewArray = builder.finish().into();
1171        let values: Vec<_> = list
1172            .iter()
1173            .map(|x| x.map(|x| x.as_primitive::<Int32Type>().values().to_vec()))
1174            .collect();
1175        assert_eq!(values, vec![Some(vec![1, 2, 3]), None, Some(vec![4, 5, 6])]);
1176        let offsets = list.value_offsets();
1177        assert_eq!(offsets, &[0, 3, 6]);
1178        let sizes = list.value_sizes();
1179        assert_eq!(sizes, &[3, 3, 3]);
1180    }
1181
1182    #[test]
1183    fn test_list_view_array_overlap_lists() {
1184        let value_data = unsafe {
1185            ArrayData::builder(DataType::Int32)
1186                .len(8)
1187                .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7]))
1188                .build_unchecked()
1189        };
1190        let list_data_type =
1191            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, false)));
1192        let list_data = unsafe {
1193            ArrayData::builder(list_data_type)
1194                .len(2)
1195                .add_buffer(Buffer::from_slice_ref([0, 3])) // offsets
1196                .add_buffer(Buffer::from_slice_ref([5, 5])) // sizes
1197                .add_child_data(value_data)
1198                .build_unchecked()
1199        };
1200        let array = ListViewArray::from(list_data);
1201
1202        assert_eq!(array.len(), 2);
1203        assert_eq!(array.value_size(0), 5);
1204        assert_eq!(array.value_size(1), 5);
1205
1206        let values: Vec<_> = array
1207            .iter()
1208            .map(|x| x.map(|x| x.as_primitive::<Int32Type>().values().to_vec()))
1209            .collect();
1210        assert_eq!(
1211            values,
1212            vec![Some(vec![0, 1, 2, 3, 4]), Some(vec![3, 4, 5, 6, 7])]
1213        );
1214    }
1215
1216    #[test]
1217    fn test_list_view_array_incomplete_offsets() {
1218        let value_data = unsafe {
1219            ArrayData::builder(DataType::Int32)
1220                .len(50)
1221                .add_buffer(Buffer::from_slice_ref((0..50).collect::<Vec<i32>>()))
1222                .build_unchecked()
1223        };
1224        let list_data_type =
1225            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, false)));
1226        let list_data = unsafe {
1227            ArrayData::builder(list_data_type)
1228                .len(3)
1229                .add_buffer(Buffer::from_slice_ref([0, 5, 10])) // offsets
1230                .add_buffer(Buffer::from_slice_ref([0, 5, 10])) // sizes
1231                .add_child_data(value_data)
1232                .build_unchecked()
1233        };
1234        let array = ListViewArray::from(list_data);
1235
1236        assert_eq!(array.len(), 3);
1237        assert_eq!(array.value_size(0), 0);
1238        assert_eq!(array.value_size(1), 5);
1239        assert_eq!(array.value_size(2), 10);
1240
1241        let values: Vec<_> = array
1242            .iter()
1243            .map(|x| x.map(|x| x.as_primitive::<Int32Type>().values().to_vec()))
1244            .collect();
1245        assert_eq!(
1246            values,
1247            vec![
1248                Some(vec![]),
1249                Some(vec![5, 6, 7, 8, 9]),
1250                Some(vec![10, 11, 12, 13, 14, 15, 16, 17, 18, 19])
1251            ]
1252        );
1253    }
1254
1255    #[test]
1256    fn test_list_view_array_empty_lists() {
1257        let value_data = unsafe {
1258            ArrayData::builder(DataType::Int32)
1259                .len(0)
1260                .add_buffer(Buffer::from_slice_ref::<i32, &[_; 0]>(&[]))
1261                .build_unchecked()
1262        };
1263        let list_data_type =
1264            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, false)));
1265        let list_data = unsafe {
1266            ArrayData::builder(list_data_type)
1267                .len(3)
1268                .add_buffer(Buffer::from_slice_ref([0, 0, 0])) // offsets
1269                .add_buffer(Buffer::from_slice_ref([0, 0, 0])) // sizes
1270                .add_child_data(value_data)
1271                .build_unchecked()
1272        };
1273        let array = ListViewArray::from(list_data);
1274
1275        assert_eq!(array.len(), 3);
1276        assert_eq!(array.value_size(0), 0);
1277        assert_eq!(array.value_size(1), 0);
1278        assert_eq!(array.value_size(2), 0);
1279
1280        let values: Vec<_> = array
1281            .iter()
1282            .map(|x| x.map(|x| x.as_primitive::<Int32Type>().values().to_vec()))
1283            .collect();
1284        assert_eq!(values, vec![Some(vec![]), Some(vec![]), Some(vec![])]);
1285    }
1286
1287    #[test]
1288    fn test_list_view_new_null_len() {
1289        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
1290        let array = ListViewArray::new_null(field, 5);
1291        assert_eq!(array.len(), 5);
1292    }
1293
1294    #[test]
1295    fn test_from_iter_primitive() {
1296        let data = vec![
1297            Some(vec![Some(0), Some(1), Some(2)]),
1298            None,
1299            Some(vec![Some(3), Some(4), Some(5)]),
1300            Some(vec![Some(6), Some(7)]),
1301        ];
1302        let list_array = ListViewArray::from_iter_primitive::<Int32Type, _, _>(data);
1303
1304        //  [[0, 1, 2], NULL, [3, 4, 5], [6, 7]]
1305        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7]);
1306        let offsets = ScalarBuffer::from(vec![0, 3, 3, 6]);
1307        let sizes = ScalarBuffer::from(vec![3, 0, 3, 2]);
1308        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
1309
1310        let mut nulls = NullBufferBuilder::new(4);
1311        nulls.append(true);
1312        nulls.append(false);
1313        nulls.append_n_non_nulls(2);
1314        let another = ListViewArray::new(field, offsets, sizes, Arc::new(values), nulls.finish());
1315
1316        assert_eq!(list_array, another)
1317    }
1318}
arrow_array/array/list_view_array.rs

arrow_array/array/
list_view_array.rs
