arrow_array/array/
list_view_array.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use arrow_buffer::{Buffer, NullBuffer, ScalarBuffer};
19use arrow_data::{ArrayData, ArrayDataBuilder};
20use arrow_schema::{ArrowError, DataType, FieldRef};
21use std::any::Any;
22use std::ops::Add;
23use std::sync::Arc;
24
25use crate::array::{make_array, print_long_array};
26use crate::builder::{GenericListViewBuilder, PrimitiveBuilder};
27use crate::iterator::GenericListViewArrayIter;
28use crate::{
29    Array, ArrayAccessor, ArrayRef, ArrowPrimitiveType, FixedSizeListArray, GenericListArray,
30    OffsetSizeTrait, new_empty_array,
31};
32
33/// A [`GenericListViewArray`] of variable size lists, storing offsets as `i32`.
34pub type ListViewArray = GenericListViewArray<i32>;
35
36/// A [`GenericListViewArray`] of variable size lists, storing offsets as `i64`.
37pub type LargeListViewArray = GenericListViewArray<i64>;
38
39/// An array of [variable length lists], specifically in the [list-view layout].
40///
41/// Differs from [`GenericListArray`] (which represents the [list layout]) in that
42/// the sizes of the child arrays are explicitly encoded in a separate buffer, instead
43/// of being derived from the difference between subsequent offsets in the offset buffer.
44///
45/// This allows the offsets (and subsequently child data) to be out of order. It also
46/// allows take / filter operations to be implemented without copying the underlying data.
47///
48/// # Representation
49///
50/// Given the same example array from [`GenericListArray`], it would be represented
51/// as such via a list-view layout array:
52///
53/// ```text
54///                                         ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
55///                                                                         ┌ ─ ─ ─ ─ ─ ─ ┐    │
56///  ┌─────────────┐  ┌───────┐             │     ┌───┐   ┌───┐   ┌───┐       ┌───┐ ┌───┐
57///  │   [A,B,C]   │  │ (0,3) │                   │ 1 │   │ 0 │   │ 3 │     │ │ 1 │ │ A │ │ 0  │
58///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
59///  │      []     │  │ (3,0) │                   │ 1 │   │ 3 │   │ 0 │     │ │ 1 │ │ B │ │ 1  │
60///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
61///  │    NULL     │  │ (?,?) │                   │ 0 │   │ ? │   │ ? │     │ │ 1 │ │ C │ │ 2  │
62///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
63///  │     [D]     │  │ (4,1) │                   │ 1 │   │ 4 │   │ 1 │     │ │ ? │ │ ? │ │ 3  │
64///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
65///  │  [NULL, F]  │  │ (5,2) │                   │ 1 │   │ 5 │   │ 2 │     │ │ 1 │ │ D │ │ 4  │
66///  └─────────────┘  └───────┘             │     └───┘   └───┘   └───┘       ├───┤ ├───┤
67///                                                                         │ │ 0 │ │ ? │ │ 5  │
68///     Logical       Logical               │  Validity  Offsets  Sizes       ├───┤ ├───┤
69///      Values       Offset                   (nulls)                      │ │ 1 │ │ F │ │ 6  │
70///                   & Size                │                                 └───┘ └───┘
71///                                                                         │    Values   │    │
72///                 (offsets[i],            │   ListViewArray                   (Array)
73///                  sizes[i])                                              └ ─ ─ ─ ─ ─ ─ ┘    │
74///                                         └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
75/// ```
76///
77/// Another way of representing the same array but taking advantage of the offsets being out of order:
78///
79/// ```text
80///                                         ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
81///                                                                         ┌ ─ ─ ─ ─ ─ ─ ┐    │
82///  ┌─────────────┐  ┌───────┐             │     ┌───┐   ┌───┐   ┌───┐       ┌───┐ ┌───┐
83///  │   [A,B,C]   │  │ (2,3) │                   │ 1 │   │ 2 │   │ 3 │     │ │ 0 │ │ ? │ │ 0  │
84///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
85///  │      []     │  │ (0,0) │                   │ 1 │   │ 0 │   │ 0 │     │ │ 1 │ │ F │ │ 1  │
86///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
87///  │    NULL     │  │ (?,?) │                   │ 0 │   │ ? │   │ ? │     │ │ 1 │ │ A │ │ 2  │
88///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
89///  │     [D]     │  │ (5,1) │                   │ 1 │   │ 5 │   │ 1 │     │ │ 1 │ │ B │ │ 3  │
90///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
91///  │  [NULL, F]  │  │ (0,2) │                   │ 1 │   │ 0 │   │ 2 │     │ │ 1 │ │ C │ │ 4  │
92///  └─────────────┘  └───────┘             │     └───┘   └───┘   └───┘       ├───┤ ├───┤
93///                                                                         │ │ 1 │ │ D │ │ 5  │
94///     Logical       Logical               │  Validity  Offsets  Sizes       └───┘ └───┘
95///      Values       Offset                   (nulls)                      │    Values   │    │
96///                   & Size                │                                   (Array)
97///                                                                         └ ─ ─ ─ ─ ─ ─ ┘    │
98///                 (offsets[i],            │   ListViewArray
99///                  sizes[i])                                                                 │
100///                                         └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
101/// ```
102///
103/// [`GenericListArray`]: crate::array::GenericListArray
104/// [variable length lists]: https://arrow.apache.org/docs/format/Columnar.html#variable-size-list-layout
105/// [list layout]: https://arrow.apache.org/docs/format/Columnar.html#list-layout
106/// [list-view layout]: https://arrow.apache.org/docs/format/Columnar.html#listview-layout
107#[derive(Clone)]
108pub struct GenericListViewArray<OffsetSize: OffsetSizeTrait> {
109    data_type: DataType,
110    nulls: Option<NullBuffer>,
111    values: ArrayRef,
112    // Unlike GenericListArray, we do not use OffsetBuffer here as offsets are not
113    // guaranteed to be monotonically increasing.
114    value_offsets: ScalarBuffer<OffsetSize>,
115    value_sizes: ScalarBuffer<OffsetSize>,
116}
117
118impl<OffsetSize: OffsetSizeTrait> GenericListViewArray<OffsetSize> {
119    /// The data type constructor of listview array.
120    /// The input is the schema of the child array and
121    /// the output is the [`DataType`], ListView or LargeListView.
122    pub const DATA_TYPE_CONSTRUCTOR: fn(FieldRef) -> DataType = if OffsetSize::IS_LARGE {
123        DataType::LargeListView
124    } else {
125        DataType::ListView
126    };
127
128    /// Create a new [`GenericListViewArray`] from the provided parts
129    ///
130    /// # Errors
131    ///
132    /// Errors if
133    ///
134    /// * `offsets.len() != sizes.len()`
135    /// * `offsets.len() != nulls.len()`
136    /// * `offsets[i] > values.len()`
137    /// * `!field.is_nullable() && values.is_nullable()`
138    /// * `field.data_type() != values.data_type()`
139    /// * `0 <= offsets[i] <= length of the child array`
140    /// * `0 <= offsets[i] + size[i] <= length of the child array`
141    pub fn try_new(
142        field: FieldRef,
143        offsets: ScalarBuffer<OffsetSize>,
144        sizes: ScalarBuffer<OffsetSize>,
145        values: ArrayRef,
146        nulls: Option<NullBuffer>,
147    ) -> Result<Self, ArrowError> {
148        let len = offsets.len();
149        if let Some(n) = nulls.as_ref() {
150            if n.len() != len {
151                return Err(ArrowError::InvalidArgumentError(format!(
152                    "Incorrect length of null buffer for {}ListViewArray, expected {len} got {}",
153                    OffsetSize::PREFIX,
154                    n.len(),
155                )));
156            }
157        }
158        if len != sizes.len() {
159            return Err(ArrowError::InvalidArgumentError(format!(
160                "Length of offsets buffer and sizes buffer must be equal for {}ListViewArray, got {len} and {}",
161                OffsetSize::PREFIX,
162                sizes.len()
163            )));
164        }
165
166        for (offset, size) in offsets.iter().zip(sizes.iter()) {
167            let offset = offset.as_usize();
168            let size = size.as_usize();
169            if offset.checked_add(size).ok_or_else(|| {
170                ArrowError::InvalidArgumentError(format!(
171                    "Overflow in offset + size for {}ListViewArray",
172                    OffsetSize::PREFIX
173                ))
174            })? > values.len()
175            {
176                return Err(ArrowError::InvalidArgumentError(format!(
177                    "Offset + size for {}ListViewArray must be within the bounds of the child array, got offset: {offset}, size: {size}, child array length: {}",
178                    OffsetSize::PREFIX,
179                    values.len()
180                )));
181            }
182        }
183
184        if !field.is_nullable() && values.is_nullable() {
185            return Err(ArrowError::InvalidArgumentError(format!(
186                "Non-nullable field of {}ListViewArray {:?} cannot contain nulls",
187                OffsetSize::PREFIX,
188                field.name()
189            )));
190        }
191
192        if field.data_type() != values.data_type() {
193            return Err(ArrowError::InvalidArgumentError(format!(
194                "{}ListViewArray expected data type {} got {} for {:?}",
195                OffsetSize::PREFIX,
196                field.data_type(),
197                values.data_type(),
198                field.name()
199            )));
200        }
201
202        Ok(Self {
203            data_type: Self::DATA_TYPE_CONSTRUCTOR(field),
204            nulls,
205            values,
206            value_offsets: offsets,
207            value_sizes: sizes,
208        })
209    }
210
211    /// Create a new [`GenericListViewArray`] from the provided parts
212    ///
213    /// # Panics
214    ///
215    /// Panics if [`Self::try_new`] returns an error
216    pub fn new(
217        field: FieldRef,
218        offsets: ScalarBuffer<OffsetSize>,
219        sizes: ScalarBuffer<OffsetSize>,
220        values: ArrayRef,
221        nulls: Option<NullBuffer>,
222    ) -> Self {
223        Self::try_new(field, offsets, sizes, values, nulls).unwrap()
224    }
225
226    /// Create a new [`GenericListViewArray`] from the provided parts without validation
227    ///
228    /// See [`Self::try_new`] for the checked version of this function, and the
229    /// documentation of that function for the invariants that must be upheld.
230    ///
231    /// # Safety
232    ///
233    /// The parts must form a valid [`ListViewArray`] or [`LargeListViewArray`] according
234    /// to the Arrow spec.
235    pub unsafe fn new_unchecked(
236        field: FieldRef,
237        offsets: ScalarBuffer<OffsetSize>,
238        sizes: ScalarBuffer<OffsetSize>,
239        values: ArrayRef,
240        nulls: Option<NullBuffer>,
241    ) -> Self {
242        if cfg!(feature = "force_validate") {
243            return Self::new(field, offsets, sizes, values, nulls);
244        }
245
246        Self {
247            data_type: Self::DATA_TYPE_CONSTRUCTOR(field),
248            nulls,
249            values,
250            value_offsets: offsets,
251            value_sizes: sizes,
252        }
253    }
254
255    /// Create a new [`GenericListViewArray`] of length `len` where all values are null
256    pub fn new_null(field: FieldRef, len: usize) -> Self {
257        let values = new_empty_array(field.data_type());
258        Self {
259            data_type: Self::DATA_TYPE_CONSTRUCTOR(field),
260            nulls: Some(NullBuffer::new_null(len)),
261            value_offsets: ScalarBuffer::from(vec![OffsetSize::usize_as(0); len]),
262            value_sizes: ScalarBuffer::from(vec![OffsetSize::usize_as(0); len]),
263            values,
264        }
265    }
266
267    /// Deconstruct this array into its constituent parts
268    pub fn into_parts(
269        self,
270    ) -> (
271        FieldRef,
272        ScalarBuffer<OffsetSize>,
273        ScalarBuffer<OffsetSize>,
274        ArrayRef,
275        Option<NullBuffer>,
276    ) {
277        let f = match self.data_type {
278            DataType::ListView(f) | DataType::LargeListView(f) => f,
279            _ => unreachable!(),
280        };
281        (
282            f,
283            self.value_offsets,
284            self.value_sizes,
285            self.values,
286            self.nulls,
287        )
288    }
289
290    /// Returns a reference to the offsets of this list
291    ///
292    /// Unlike [`Self::value_offsets`] this returns the [`ScalarBuffer`]
293    /// allowing for zero-copy cloning
294    #[inline]
295    pub fn offsets(&self) -> &ScalarBuffer<OffsetSize> {
296        &self.value_offsets
297    }
298
299    /// Returns a reference to the values of this list
300    #[inline]
301    pub fn values(&self) -> &ArrayRef {
302        &self.values
303    }
304
305    /// Returns a reference to the sizes of this list
306    ///
307    /// Unlike [`Self::value_sizes`] this returns the [`ScalarBuffer`]
308    /// allowing for zero-copy cloning
309    #[inline]
310    pub fn sizes(&self) -> &ScalarBuffer<OffsetSize> {
311        &self.value_sizes
312    }
313
314    /// Returns a clone of the value type of this list.
315    pub fn value_type(&self) -> DataType {
316        self.values.data_type().clone()
317    }
318
319    /// Returns ith value of this list view array.
320    ///
321    /// Note: This method does not check for nulls and the value is arbitrary
322    /// if [`is_null`](Self::is_null) returns true for the index.
323    ///
324    /// # Safety
325    /// Caller must ensure that the index is within the array bounds
326    pub unsafe fn value_unchecked(&self, i: usize) -> ArrayRef {
327        let offset = unsafe { self.value_offsets().get_unchecked(i).as_usize() };
328        let length = unsafe { self.value_sizes().get_unchecked(i).as_usize() };
329        self.values.slice(offset, length)
330    }
331
332    /// Returns ith value of this list view array.
333    ///
334    /// Note: This method does not check for nulls and the value is arbitrary
335    /// (but still well-defined) if [`is_null`](Self::is_null) returns true for the index.
336    ///
337    /// # Panics
338    /// Panics if the index is out of bounds
339    pub fn value(&self, i: usize) -> ArrayRef {
340        let offset = self.value_offsets()[i].as_usize();
341        let length = self.value_sizes()[i].as_usize();
342        self.values.slice(offset, length)
343    }
344
345    /// Returns the offset values in the offsets buffer
346    #[inline]
347    pub fn value_offsets(&self) -> &[OffsetSize] {
348        &self.value_offsets
349    }
350
351    /// Returns the sizes values in the offsets buffer
352    #[inline]
353    pub fn value_sizes(&self) -> &[OffsetSize] {
354        &self.value_sizes
355    }
356
357    /// Returns the size for value at index `i`.
358    #[inline]
359    pub fn value_size(&self, i: usize) -> OffsetSize {
360        self.value_sizes[i]
361    }
362
363    /// Returns the offset for value at index `i`.
364    pub fn value_offset(&self, i: usize) -> OffsetSize {
365        self.value_offsets[i]
366    }
367
368    /// Constructs a new iterator
369    pub fn iter(&self) -> GenericListViewArrayIter<'_, OffsetSize> {
370        GenericListViewArrayIter::<'_, OffsetSize>::new(self)
371    }
372
373    #[inline]
374    fn get_type(data_type: &DataType) -> Option<&DataType> {
375        match (OffsetSize::IS_LARGE, data_type) {
376            (true, DataType::LargeListView(child)) | (false, DataType::ListView(child)) => {
377                Some(child.data_type())
378            }
379            _ => None,
380        }
381    }
382
383    /// Returns a zero-copy slice of this array with the indicated offset and length.
384    pub fn slice(&self, offset: usize, length: usize) -> Self {
385        Self {
386            data_type: self.data_type.clone(),
387            nulls: self.nulls.as_ref().map(|n| n.slice(offset, length)),
388            values: self.values.clone(),
389            value_offsets: self.value_offsets.slice(offset, length),
390            value_sizes: self.value_sizes.slice(offset, length),
391        }
392    }
393
394    /// Creates a [`GenericListViewArray`] from an iterator of primitive values
395    /// # Example
396    /// ```
397    /// # use arrow_array::ListViewArray;
398    /// # use arrow_array::types::Int32Type;
399    ///
400    /// let data = vec![
401    ///    Some(vec![Some(0), Some(1), Some(2)]),
402    ///    None,
403    ///    Some(vec![Some(3), None, Some(5)]),
404    ///    Some(vec![Some(6), Some(7)]),
405    /// ];
406    /// let list_array = ListViewArray::from_iter_primitive::<Int32Type, _, _>(data);
407    /// println!("{:?}", list_array);
408    /// ```
409    pub fn from_iter_primitive<T, P, I>(iter: I) -> Self
410    where
411        T: ArrowPrimitiveType,
412        P: IntoIterator<Item = Option<<T as ArrowPrimitiveType>::Native>>,
413        I: IntoIterator<Item = Option<P>>,
414    {
415        let iter = iter.into_iter();
416        let size_hint = iter.size_hint().0;
417        let mut builder =
418            GenericListViewBuilder::with_capacity(PrimitiveBuilder::<T>::new(), size_hint);
419
420        for i in iter {
421            match i {
422                Some(p) => {
423                    for t in p {
424                        builder.values().append_option(t);
425                    }
426                    builder.append(true);
427                }
428                None => builder.append(false),
429            }
430        }
431        builder.finish()
432    }
433}
434
435impl<OffsetSize: OffsetSizeTrait> ArrayAccessor for &GenericListViewArray<OffsetSize> {
436    type Item = ArrayRef;
437
438    fn value(&self, index: usize) -> Self::Item {
439        GenericListViewArray::value(self, index)
440    }
441
442    unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
443        unsafe { GenericListViewArray::value_unchecked(self, index) }
444    }
445}
446
447/// SAFETY: Correctly implements the contract of Arrow Arrays
448unsafe impl<OffsetSize: OffsetSizeTrait> Array for GenericListViewArray<OffsetSize> {
449    fn as_any(&self) -> &dyn Any {
450        self
451    }
452
453    fn to_data(&self) -> ArrayData {
454        self.clone().into()
455    }
456
457    fn into_data(self) -> ArrayData {
458        self.into()
459    }
460
461    fn data_type(&self) -> &DataType {
462        &self.data_type
463    }
464
465    fn slice(&self, offset: usize, length: usize) -> ArrayRef {
466        Arc::new(self.slice(offset, length))
467    }
468
469    fn len(&self) -> usize {
470        self.sizes().len()
471    }
472
473    fn is_empty(&self) -> bool {
474        self.value_sizes.is_empty()
475    }
476
477    fn shrink_to_fit(&mut self) {
478        if let Some(nulls) = &mut self.nulls {
479            nulls.shrink_to_fit();
480        }
481        self.values.shrink_to_fit();
482        self.value_offsets.shrink_to_fit();
483        self.value_sizes.shrink_to_fit();
484    }
485
486    fn offset(&self) -> usize {
487        0
488    }
489
490    fn nulls(&self) -> Option<&NullBuffer> {
491        self.nulls.as_ref()
492    }
493
494    fn logical_null_count(&self) -> usize {
495        // More efficient that the default implementation
496        self.null_count()
497    }
498
499    fn get_buffer_memory_size(&self) -> usize {
500        let mut size = self.values.get_buffer_memory_size();
501        size += self.value_offsets.inner().capacity();
502        size += self.value_sizes.inner().capacity();
503        if let Some(n) = self.nulls.as_ref() {
504            size += n.buffer().capacity();
505        }
506        size
507    }
508
509    fn get_array_memory_size(&self) -> usize {
510        let mut size = std::mem::size_of::<Self>() + self.values.get_array_memory_size();
511        size += self.value_offsets.inner().capacity();
512        size += self.value_sizes.inner().capacity();
513        if let Some(n) = self.nulls.as_ref() {
514            size += n.buffer().capacity();
515        }
516        size
517    }
518
519    #[cfg(feature = "pool")]
520    fn claim(&self, pool: &dyn arrow_buffer::MemoryPool) {
521        self.value_offsets.claim(pool);
522        self.value_sizes.claim(pool);
523        self.values.claim(pool);
524        if let Some(nulls) = &self.nulls {
525            nulls.claim(pool);
526        }
527    }
528}
529
530impl<OffsetSize: OffsetSizeTrait> super::ListLikeArray for GenericListViewArray<OffsetSize> {
531    fn values(&self) -> &ArrayRef {
532        self.values()
533    }
534
535    fn element_range(&self, index: usize) -> std::ops::Range<usize> {
536        let offset = self.value_offsets()[index].as_usize();
537        let size = self.value_sizes()[index].as_usize();
538        offset..(offset + size)
539    }
540}
541
542impl<OffsetSize: OffsetSizeTrait> std::fmt::Debug for GenericListViewArray<OffsetSize> {
543    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
544        let prefix = OffsetSize::PREFIX;
545        write!(f, "{prefix}ListViewArray\n[\n")?;
546        print_long_array(self, f, |array, index, f| {
547            std::fmt::Debug::fmt(&array.value(index), f)
548        })?;
549        write!(f, "]")
550    }
551}
552
553impl<OffsetSize: OffsetSizeTrait> From<GenericListArray<OffsetSize>>
554    for GenericListViewArray<OffsetSize>
555{
556    fn from(value: GenericListArray<OffsetSize>) -> Self {
557        let (field, offsets, values, nulls) = value.into_parts();
558        let len = offsets.len() - 1;
559        let mut sizes = Vec::with_capacity(len);
560        let mut view_offsets = Vec::with_capacity(len);
561        for (i, offset) in offsets.iter().enumerate().take(len) {
562            view_offsets.push(*offset);
563            sizes.push(offsets[i + 1] - offsets[i]);
564        }
565
566        Self::new(
567            field,
568            ScalarBuffer::from(view_offsets),
569            ScalarBuffer::from(sizes),
570            values,
571            nulls,
572        )
573    }
574}
575
576impl<OffsetSize: OffsetSizeTrait> From<GenericListViewArray<OffsetSize>> for ArrayData {
577    fn from(array: GenericListViewArray<OffsetSize>) -> Self {
578        let len = array.len();
579        let builder = ArrayDataBuilder::new(array.data_type)
580            .len(len)
581            .nulls(array.nulls)
582            .buffers(vec![
583                array.value_offsets.into_inner(),
584                array.value_sizes.into_inner(),
585            ])
586            .child_data(vec![array.values.to_data()]);
587
588        unsafe { builder.build_unchecked() }
589    }
590}
591
592impl<OffsetSize: OffsetSizeTrait> From<ArrayData> for GenericListViewArray<OffsetSize> {
593    fn from(data: ArrayData) -> Self {
594        Self::try_new_from_array_data(data)
595            .expect("Expected infallible creation of GenericListViewArray from ArrayDataRef failed")
596    }
597}
598
599impl<OffsetSize: OffsetSizeTrait> From<FixedSizeListArray> for GenericListViewArray<OffsetSize> {
600    fn from(value: FixedSizeListArray) -> Self {
601        let (field, size) = match value.data_type() {
602            DataType::FixedSizeList(f, size) => (f, *size as usize),
603            _ => unreachable!(),
604        };
605        let mut acc = 0_usize;
606        let iter = std::iter::repeat_n(size, value.len());
607        let mut sizes = Vec::with_capacity(iter.size_hint().0);
608        let mut offsets = Vec::with_capacity(iter.size_hint().0);
609
610        for size in iter {
611            offsets.push(OffsetSize::usize_as(acc));
612            acc = acc.add(size);
613            sizes.push(OffsetSize::usize_as(size));
614        }
615        let sizes = ScalarBuffer::from(sizes);
616        let offsets = ScalarBuffer::from(offsets);
617        Self {
618            data_type: Self::DATA_TYPE_CONSTRUCTOR(field.clone()),
619            nulls: value.nulls().cloned(),
620            values: value.values().clone(),
621            value_offsets: offsets,
622            value_sizes: sizes,
623        }
624    }
625}
626
627impl<OffsetSize: OffsetSizeTrait> GenericListViewArray<OffsetSize> {
628    fn try_new_from_array_data(data: ArrayData) -> Result<Self, ArrowError> {
629        let (data_type, len, nulls, offset, buffers, child_data) = data.into_parts();
630
631        // ArrayData is valid, and verified type above
632        // buffer[0] is offsets, buffer[1] is sizes
633        let num_buffers = buffers.len();
634        let [offsets_buffer, sizes_buffer] : [Buffer; 2] = buffers.try_into().map_err(|_| {
635             ArrowError::InvalidArgumentError(format!(
636                "ListViewArray data should contain two buffers (value offsets & value sizes), had {num_buffers}",
637            ))
638        })?;
639
640        let num_child = child_data.len();
641        let [values]: [ArrayData; 1] = child_data.try_into().map_err(|_| {
642            ArrowError::InvalidArgumentError(format!(
643                "ListViewArray should contain a single child array (values array), had {num_child}",
644            ))
645        })?;
646
647        if let Some(child_data_type) = Self::get_type(&data_type) {
648            if values.data_type() != child_data_type {
649                return Err(ArrowError::InvalidArgumentError(format!(
650                    "{}ListViewArray's child datatype {:?} does not \
651                             correspond to the List's datatype {:?}",
652                    OffsetSize::PREFIX,
653                    values.data_type(),
654                    child_data_type
655                )));
656            }
657        } else {
658            return Err(ArrowError::InvalidArgumentError(format!(
659                "{}ListViewArray's datatype must be {}ListViewArray(). It is {:?}",
660                OffsetSize::PREFIX,
661                OffsetSize::PREFIX,
662                data_type
663            )));
664        }
665
666        let values = make_array(values);
667        let value_offsets = ScalarBuffer::new(offsets_buffer, offset, len);
668        let value_sizes = ScalarBuffer::new(sizes_buffer, offset, len);
669
670        Ok(Self {
671            data_type,
672            nulls,
673            values,
674            value_offsets,
675            value_sizes,
676        })
677    }
678}
679
680#[cfg(test)]
681mod tests {
682    use arrow_buffer::{BooleanBuffer, Buffer, NullBufferBuilder, ScalarBuffer, bit_util};
683    use arrow_schema::Field;
684
685    use crate::builder::{FixedSizeListBuilder, Int32Builder};
686    use crate::cast::AsArray;
687    use crate::types::Int32Type;
688    use crate::{Int32Array, Int64Array};
689
690    use super::*;
691
692    #[test]
693    fn test_empty_list_view_array() {
694        // Construct an empty value array
695        let vec: Vec<i32> = vec![];
696        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
697        let sizes = ScalarBuffer::from(vec![]);
698        let offsets = ScalarBuffer::from(vec![]);
699        let values = Int32Array::from(vec);
700        let list_array = LargeListViewArray::new(field, offsets, sizes, Arc::new(values), None);
701
702        assert_eq!(list_array.len(), 0)
703    }
704
705    #[test]
706    fn test_list_view_array() {
707        // Construct a value array
708        let value_data = ArrayData::builder(DataType::Int32)
709            .len(8)
710            .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7]))
711            .build()
712            .unwrap();
713
714        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
715        let sizes = ScalarBuffer::from(vec![3i32, 3, 2]);
716        let offsets = ScalarBuffer::from(vec![0i32, 3, 6]);
717        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7]);
718        let list_array = ListViewArray::new(field, offsets, sizes, Arc::new(values), None);
719
720        let values = list_array.values();
721        assert_eq!(value_data, values.to_data());
722        assert_eq!(DataType::Int32, list_array.value_type());
723        assert_eq!(3, list_array.len());
724        assert_eq!(0, list_array.null_count());
725        assert_eq!(6, list_array.value_offsets()[2]);
726        assert_eq!(2, list_array.value_sizes()[2]);
727        assert_eq!(2, list_array.value_size(2));
728        assert_eq!(0, list_array.value(0).as_primitive::<Int32Type>().value(0));
729        assert_eq!(
730            0,
731            unsafe { list_array.value_unchecked(0) }
732                .as_primitive::<Int32Type>()
733                .value(0)
734        );
735        for i in 0..3 {
736            assert!(list_array.is_valid(i));
737            assert!(!list_array.is_null(i));
738        }
739    }
740
741    #[test]
742    fn test_large_list_view_array() {
743        // Construct a value array
744        let value_data = ArrayData::builder(DataType::Int32)
745            .len(8)
746            .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7]))
747            .build()
748            .unwrap();
749
750        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
751        let sizes = ScalarBuffer::from(vec![3i64, 3, 2]);
752        let offsets = ScalarBuffer::from(vec![0i64, 3, 6]);
753        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7]);
754        let list_array = LargeListViewArray::new(field, offsets, sizes, Arc::new(values), None);
755
756        let values = list_array.values();
757        assert_eq!(value_data, values.to_data());
758        assert_eq!(DataType::Int32, list_array.value_type());
759        assert_eq!(3, list_array.len());
760        assert_eq!(0, list_array.null_count());
761        assert_eq!(6, list_array.value_offsets()[2]);
762        assert_eq!(2, list_array.value_sizes()[2]);
763        assert_eq!(2, list_array.value_size(2));
764        assert_eq!(0, list_array.value(0).as_primitive::<Int32Type>().value(0));
765        assert_eq!(
766            0,
767            unsafe { list_array.value_unchecked(0) }
768                .as_primitive::<Int32Type>()
769                .value(0)
770        );
771        for i in 0..3 {
772            assert!(list_array.is_valid(i));
773            assert!(!list_array.is_null(i));
774        }
775    }
776
777    #[test]
778    fn test_list_view_array_slice() {
779        // Construct a value array
780        let value_data = ArrayData::builder(DataType::Int32)
781            .len(10)
782            .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))
783            .build()
784            .unwrap();
785
786        // 01011001 00000001
787        let mut null_bits: [u8; 2] = [0; 2];
788        bit_util::set_bit(&mut null_bits, 0);
789        bit_util::set_bit(&mut null_bits, 3);
790        bit_util::set_bit(&mut null_bits, 4);
791        bit_util::set_bit(&mut null_bits, 6);
792        bit_util::set_bit(&mut null_bits, 8);
793        let buffer = BooleanBuffer::new(Buffer::from(null_bits), 0, 9);
794        let null_buffer = NullBuffer::new(buffer);
795
796        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
797        let sizes = ScalarBuffer::from(vec![2, 0, 0, 2, 2, 0, 3, 0, 1]);
798        let offsets = ScalarBuffer::from(vec![0, 2, 2, 2, 4, 6, 6, 9, 9]);
799        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
800        let list_array =
801            ListViewArray::new(field, offsets, sizes, Arc::new(values), Some(null_buffer));
802
803        let values = list_array.values();
804        assert_eq!(value_data, values.to_data());
805        assert_eq!(DataType::Int32, list_array.value_type());
806        assert_eq!(9, list_array.len());
807        assert_eq!(4, list_array.null_count());
808        assert_eq!(2, list_array.value_offsets()[3]);
809        assert_eq!(2, list_array.value_sizes()[3]);
810        assert_eq!(2, list_array.value_size(3));
811
812        let sliced_array = list_array.slice(1, 6);
813        assert_eq!(6, sliced_array.len());
814        assert_eq!(3, sliced_array.null_count());
815
816        for i in 0..sliced_array.len() {
817            if bit_util::get_bit(&null_bits, 1 + i) {
818                assert!(sliced_array.is_valid(i));
819            } else {
820                assert!(sliced_array.is_null(i));
821            }
822        }
823
824        // Check offset and length for each non-null value.
825        let sliced_list_array = sliced_array
826            .as_any()
827            .downcast_ref::<ListViewArray>()
828            .unwrap();
829        assert_eq!(2, sliced_list_array.value_offsets()[2]);
830        assert_eq!(2, sliced_list_array.value_sizes()[2]);
831        assert_eq!(2, sliced_list_array.value_size(2));
832
833        assert_eq!(4, sliced_list_array.value_offsets()[3]);
834        assert_eq!(2, sliced_list_array.value_sizes()[3]);
835        assert_eq!(2, sliced_list_array.value_size(3));
836
837        assert_eq!(6, sliced_list_array.value_offsets()[5]);
838        assert_eq!(3, sliced_list_array.value_sizes()[5]);
839        assert_eq!(3, sliced_list_array.value_size(5));
840    }
841
842    #[test]
843    fn test_large_list_view_array_slice() {
844        // Construct a value array
845        let value_data = ArrayData::builder(DataType::Int32)
846            .len(10)
847            .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))
848            .build()
849            .unwrap();
850
851        // 01011001 00000001
852        let mut null_bits: [u8; 2] = [0; 2];
853        bit_util::set_bit(&mut null_bits, 0);
854        bit_util::set_bit(&mut null_bits, 3);
855        bit_util::set_bit(&mut null_bits, 4);
856        bit_util::set_bit(&mut null_bits, 6);
857        bit_util::set_bit(&mut null_bits, 8);
858        let buffer = BooleanBuffer::new(Buffer::from(null_bits), 0, 9);
859        let null_buffer = NullBuffer::new(buffer);
860
861        // Construct a large list view array from the above two
862        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
863        let sizes = ScalarBuffer::from(vec![2i64, 0, 0, 2, 2, 0, 3, 0, 1]);
864        let offsets = ScalarBuffer::from(vec![0i64, 2, 2, 2, 4, 6, 6, 9, 9]);
865        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
866        let list_array =
867            LargeListViewArray::new(field, offsets, sizes, Arc::new(values), Some(null_buffer));
868
869        let values = list_array.values();
870        assert_eq!(value_data, values.to_data());
871        assert_eq!(DataType::Int32, list_array.value_type());
872        assert_eq!(9, list_array.len());
873        assert_eq!(4, list_array.null_count());
874        assert_eq!(2, list_array.value_offsets()[3]);
875        assert_eq!(2, list_array.value_sizes()[3]);
876        assert_eq!(2, list_array.value_size(3));
877
878        let sliced_array = list_array.slice(1, 6);
879        assert_eq!(6, sliced_array.len());
880        assert_eq!(3, sliced_array.null_count());
881
882        for i in 0..sliced_array.len() {
883            if bit_util::get_bit(&null_bits, 1 + i) {
884                assert!(sliced_array.is_valid(i));
885            } else {
886                assert!(sliced_array.is_null(i));
887            }
888        }
889
890        // Check offset and length for each non-null value.
891        let sliced_list_array = sliced_array
892            .as_any()
893            .downcast_ref::<LargeListViewArray>()
894            .unwrap();
895        assert_eq!(2, sliced_list_array.value_offsets()[2]);
896        assert_eq!(2, sliced_list_array.value_size(2));
897        assert_eq!(2, sliced_list_array.value_sizes()[2]);
898
899        assert_eq!(4, sliced_list_array.value_offsets()[3]);
900        assert_eq!(2, sliced_list_array.value_size(3));
901        assert_eq!(2, sliced_list_array.value_sizes()[3]);
902
903        assert_eq!(6, sliced_list_array.value_offsets()[5]);
904        assert_eq!(3, sliced_list_array.value_size(5));
905        assert_eq!(2, sliced_list_array.value_sizes()[3]);
906    }
907
908    #[test]
909    #[should_panic(expected = "index out of bounds: the len is 9 but the index is 10")]
910    fn test_list_view_array_index_out_of_bound() {
911        // 01011001 00000001
912        let mut null_bits: [u8; 2] = [0; 2];
913        bit_util::set_bit(&mut null_bits, 0);
914        bit_util::set_bit(&mut null_bits, 3);
915        bit_util::set_bit(&mut null_bits, 4);
916        bit_util::set_bit(&mut null_bits, 6);
917        bit_util::set_bit(&mut null_bits, 8);
918        let buffer = BooleanBuffer::new(Buffer::from(null_bits), 0, 9);
919        let null_buffer = NullBuffer::new(buffer);
920
921        // Construct a buffer for value offsets, for the nested array:
922        //  [[0, 1], null, null, [2, 3], [4, 5], null, [6, 7, 8], null, [9]]
923        // Construct a list array from the above two
924        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
925        let sizes = ScalarBuffer::from(vec![2i32, 0, 0, 2, 2, 0, 3, 0, 1]);
926        let offsets = ScalarBuffer::from(vec![0i32, 2, 2, 2, 4, 6, 6, 9, 9]);
927        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
928        let list_array =
929            ListViewArray::new(field, offsets, sizes, Arc::new(values), Some(null_buffer));
930
931        assert_eq!(9, list_array.len());
932        list_array.value(10);
933    }
934    #[test]
935    #[should_panic(
936        expected = "ListViewArray data should contain two buffers (value offsets & value sizes), had 0"
937    )]
938    #[cfg(not(feature = "force_validate"))]
939    fn test_list_view_array_invalid_buffer_len() {
940        let value_data = unsafe {
941            ArrayData::builder(DataType::Int32)
942                .len(8)
943                .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7]))
944                .build_unchecked()
945        };
946        let list_data_type =
947            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, false)));
948        let list_data = unsafe {
949            ArrayData::builder(list_data_type)
950                .len(3)
951                .add_child_data(value_data)
952                .build_unchecked()
953        };
954        drop(ListViewArray::from(list_data));
955    }
956
957    #[test]
958    #[should_panic(
959        expected = "ListViewArray data should contain two buffers (value offsets & value sizes), had 1"
960    )]
961    #[cfg(not(feature = "force_validate"))]
962    fn test_list_view_array_invalid_child_array_len() {
963        let value_offsets = Buffer::from_slice_ref([0, 2, 5, 7]);
964        let list_data_type =
965            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, false)));
966        let list_data = unsafe {
967            ArrayData::builder(list_data_type)
968                .len(3)
969                .add_buffer(value_offsets)
970                .build_unchecked()
971        };
972        drop(ListViewArray::from(list_data));
973    }
974
975    #[test]
976    fn test_list_view_array_offsets_need_not_start_at_zero() {
977        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
978        let sizes = ScalarBuffer::from(vec![0i32, 0, 3]);
979        let offsets = ScalarBuffer::from(vec![2i32, 2, 5]);
980        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7]);
981        let list_array = ListViewArray::new(field, offsets, sizes, Arc::new(values), None);
982
983        assert_eq!(list_array.value_size(0), 0);
984        assert_eq!(list_array.value_size(1), 0);
985        assert_eq!(list_array.value_size(2), 3);
986    }
987
988    #[test]
989    #[should_panic(expected = "Memory pointer is not aligned with the specified scalar type")]
990    #[cfg(not(feature = "force_validate"))]
991    fn test_list_view_array_alignment() {
992        let offset_buf = Buffer::from_slice_ref([0_u64]);
993        let offset_buf2 = offset_buf.slice(1);
994
995        let size_buf = Buffer::from_slice_ref([0_u64]);
996        let size_buf2 = size_buf.slice(1);
997
998        let values: [i32; 8] = [0; 8];
999        let value_data = unsafe {
1000            ArrayData::builder(DataType::Int32)
1001                .add_buffer(Buffer::from_slice_ref(values))
1002                .build_unchecked()
1003        };
1004
1005        let list_data_type =
1006            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, false)));
1007        let list_data = unsafe {
1008            ArrayData::builder(list_data_type)
1009                .add_buffer(offset_buf2)
1010                .add_buffer(size_buf2)
1011                .add_child_data(value_data)
1012                .build_unchecked()
1013        };
1014        drop(ListViewArray::from(list_data));
1015    }
1016
1017    #[test]
1018    fn test_empty_offsets() {
1019        let f = Arc::new(Field::new("element", DataType::Int32, true));
1020        let string = ListViewArray::from(
1021            ArrayData::builder(DataType::ListView(f.clone()))
1022                .buffers(vec![Buffer::from(&[]), Buffer::from(&[])])
1023                .add_child_data(ArrayData::new_empty(&DataType::Int32))
1024                .build()
1025                .unwrap(),
1026        );
1027        assert_eq!(string.value_offsets(), &[] as &[i32; 0]);
1028        assert_eq!(string.value_sizes(), &[] as &[i32; 0]);
1029
1030        let string = LargeListViewArray::from(
1031            ArrayData::builder(DataType::LargeListView(f))
1032                .buffers(vec![Buffer::from(&[]), Buffer::from(&[])])
1033                .add_child_data(ArrayData::new_empty(&DataType::Int32))
1034                .build()
1035                .unwrap(),
1036        );
1037        assert_eq!(string.len(), 0);
1038        assert_eq!(string.value_offsets(), &[] as &[i64; 0]);
1039        assert_eq!(string.value_sizes(), &[] as &[i64; 0]);
1040    }
1041
1042    #[test]
1043    fn test_try_new() {
1044        let offsets = ScalarBuffer::from(vec![0, 1, 4, 5]);
1045        let sizes = ScalarBuffer::from(vec![1, 3, 1, 0]);
1046        let values = Int32Array::new(vec![1, 2, 3, 4, 5].into(), None);
1047        let values = Arc::new(values) as ArrayRef;
1048
1049        let field = Arc::new(Field::new("element", DataType::Int32, false));
1050        ListViewArray::new(
1051            field.clone(),
1052            offsets.clone(),
1053            sizes.clone(),
1054            values.clone(),
1055            None,
1056        );
1057
1058        let nulls = NullBuffer::new_null(4);
1059        ListViewArray::new(
1060            field.clone(),
1061            offsets,
1062            sizes.clone(),
1063            values.clone(),
1064            Some(nulls),
1065        );
1066
1067        let nulls = NullBuffer::new_null(4);
1068        let offsets = ScalarBuffer::from(vec![0, 1, 2, 3, 4]);
1069        let sizes = ScalarBuffer::from(vec![1, 1, 1, 1, 0]);
1070        let err = LargeListViewArray::try_new(
1071            field,
1072            offsets.clone(),
1073            sizes.clone(),
1074            values.clone(),
1075            Some(nulls),
1076        )
1077        .unwrap_err();
1078
1079        assert_eq!(
1080            err.to_string(),
1081            "Invalid argument error: Incorrect length of null buffer for LargeListViewArray, expected 5 got 4"
1082        );
1083
1084        let field = Arc::new(Field::new("element", DataType::Int64, false));
1085        let err = LargeListViewArray::try_new(
1086            field.clone(),
1087            offsets.clone(),
1088            sizes.clone(),
1089            values.clone(),
1090            None,
1091        )
1092        .unwrap_err();
1093
1094        assert_eq!(
1095            err.to_string(),
1096            "Invalid argument error: LargeListViewArray expected data type Int64 got Int32 for \"element\""
1097        );
1098
1099        let nulls = NullBuffer::new_null(7);
1100        let values = Int64Array::new(vec![0; 7].into(), Some(nulls));
1101        let values = Arc::new(values);
1102
1103        let err = LargeListViewArray::try_new(
1104            field,
1105            offsets.clone(),
1106            sizes.clone(),
1107            values.clone(),
1108            None,
1109        )
1110        .unwrap_err();
1111
1112        assert_eq!(
1113            err.to_string(),
1114            "Invalid argument error: Non-nullable field of LargeListViewArray \"element\" cannot contain nulls"
1115        );
1116    }
1117
1118    #[test]
1119    fn test_from_fixed_size_list() {
1120        let mut builder = FixedSizeListBuilder::new(Int32Builder::new(), 3);
1121        builder.values().append_slice(&[1, 2, 3]);
1122        builder.append(true);
1123        builder.values().append_slice(&[0, 0, 0]);
1124        builder.append(false);
1125        builder.values().append_slice(&[4, 5, 6]);
1126        builder.append(true);
1127        let list: ListViewArray = builder.finish().into();
1128        let values: Vec<_> = list
1129            .iter()
1130            .map(|x| x.map(|x| x.as_primitive::<Int32Type>().values().to_vec()))
1131            .collect();
1132        assert_eq!(values, vec![Some(vec![1, 2, 3]), None, Some(vec![4, 5, 6])]);
1133        let offsets = list.value_offsets();
1134        assert_eq!(offsets, &[0, 3, 6]);
1135        let sizes = list.value_sizes();
1136        assert_eq!(sizes, &[3, 3, 3]);
1137    }
1138
1139    #[test]
1140    fn test_list_view_array_overlap_lists() {
1141        let value_data = unsafe {
1142            ArrayData::builder(DataType::Int32)
1143                .len(8)
1144                .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7]))
1145                .build_unchecked()
1146        };
1147        let list_data_type =
1148            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, false)));
1149        let list_data = unsafe {
1150            ArrayData::builder(list_data_type)
1151                .len(2)
1152                .add_buffer(Buffer::from_slice_ref([0, 3])) // offsets
1153                .add_buffer(Buffer::from_slice_ref([5, 5])) // sizes
1154                .add_child_data(value_data)
1155                .build_unchecked()
1156        };
1157        let array = ListViewArray::from(list_data);
1158
1159        assert_eq!(array.len(), 2);
1160        assert_eq!(array.value_size(0), 5);
1161        assert_eq!(array.value_size(1), 5);
1162
1163        let values: Vec<_> = array
1164            .iter()
1165            .map(|x| x.map(|x| x.as_primitive::<Int32Type>().values().to_vec()))
1166            .collect();
1167        assert_eq!(
1168            values,
1169            vec![Some(vec![0, 1, 2, 3, 4]), Some(vec![3, 4, 5, 6, 7])]
1170        );
1171    }
1172
1173    #[test]
1174    fn test_list_view_array_incomplete_offsets() {
1175        let value_data = unsafe {
1176            ArrayData::builder(DataType::Int32)
1177                .len(50)
1178                .add_buffer(Buffer::from_slice_ref((0..50).collect::<Vec<i32>>()))
1179                .build_unchecked()
1180        };
1181        let list_data_type =
1182            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, false)));
1183        let list_data = unsafe {
1184            ArrayData::builder(list_data_type)
1185                .len(3)
1186                .add_buffer(Buffer::from_slice_ref([0, 5, 10])) // offsets
1187                .add_buffer(Buffer::from_slice_ref([0, 5, 10])) // sizes
1188                .add_child_data(value_data)
1189                .build_unchecked()
1190        };
1191        let array = ListViewArray::from(list_data);
1192
1193        assert_eq!(array.len(), 3);
1194        assert_eq!(array.value_size(0), 0);
1195        assert_eq!(array.value_size(1), 5);
1196        assert_eq!(array.value_size(2), 10);
1197
1198        let values: Vec<_> = array
1199            .iter()
1200            .map(|x| x.map(|x| x.as_primitive::<Int32Type>().values().to_vec()))
1201            .collect();
1202        assert_eq!(
1203            values,
1204            vec![
1205                Some(vec![]),
1206                Some(vec![5, 6, 7, 8, 9]),
1207                Some(vec![10, 11, 12, 13, 14, 15, 16, 17, 18, 19])
1208            ]
1209        );
1210    }
1211
1212    #[test]
1213    fn test_list_view_array_empty_lists() {
1214        let value_data = unsafe {
1215            ArrayData::builder(DataType::Int32)
1216                .len(0)
1217                .add_buffer(Buffer::from_slice_ref::<i32, &[_; 0]>(&[]))
1218                .build_unchecked()
1219        };
1220        let list_data_type =
1221            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, false)));
1222        let list_data = unsafe {
1223            ArrayData::builder(list_data_type)
1224                .len(3)
1225                .add_buffer(Buffer::from_slice_ref([0, 0, 0])) // offsets
1226                .add_buffer(Buffer::from_slice_ref([0, 0, 0])) // sizes
1227                .add_child_data(value_data)
1228                .build_unchecked()
1229        };
1230        let array = ListViewArray::from(list_data);
1231
1232        assert_eq!(array.len(), 3);
1233        assert_eq!(array.value_size(0), 0);
1234        assert_eq!(array.value_size(1), 0);
1235        assert_eq!(array.value_size(2), 0);
1236
1237        let values: Vec<_> = array
1238            .iter()
1239            .map(|x| x.map(|x| x.as_primitive::<Int32Type>().values().to_vec()))
1240            .collect();
1241        assert_eq!(values, vec![Some(vec![]), Some(vec![]), Some(vec![])]);
1242    }
1243
1244    #[test]
1245    fn test_list_view_new_null_len() {
1246        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
1247        let array = ListViewArray::new_null(field, 5);
1248        assert_eq!(array.len(), 5);
1249    }
1250
1251    #[test]
1252    fn test_from_iter_primitive() {
1253        let data = vec![
1254            Some(vec![Some(0), Some(1), Some(2)]),
1255            None,
1256            Some(vec![Some(3), Some(4), Some(5)]),
1257            Some(vec![Some(6), Some(7)]),
1258        ];
1259        let list_array = ListViewArray::from_iter_primitive::<Int32Type, _, _>(data);
1260
1261        //  [[0, 1, 2], NULL, [3, 4, 5], [6, 7]]
1262        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7]);
1263        let offsets = ScalarBuffer::from(vec![0, 3, 3, 6]);
1264        let sizes = ScalarBuffer::from(vec![3, 0, 3, 2]);
1265        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
1266
1267        let mut nulls = NullBufferBuilder::new(4);
1268        nulls.append(true);
1269        nulls.append(false);
1270        nulls.append_n_non_nulls(2);
1271        let another = ListViewArray::new(field, offsets, sizes, Arc::new(values), nulls.finish());
1272
1273        assert_eq!(list_array, another)
1274    }
1275}
arrow_array/array/list_view_array.rs

arrow_array/array/
list_view_array.rs
