arrow_array/array/
list_view_array.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use arrow_buffer::{Buffer, NullBuffer, ScalarBuffer};
19use arrow_data::{ArrayData, ArrayDataBuilder};
20use arrow_schema::{ArrowError, DataType, FieldRef};
21use std::any::Any;
22use std::ops::Add;
23use std::sync::Arc;
24
25use crate::array::{make_array, print_long_array};
26use crate::builder::{GenericListViewBuilder, PrimitiveBuilder};
27use crate::iterator::GenericListViewArrayIter;
28use crate::{
29    Array, ArrayAccessor, ArrayRef, ArrowPrimitiveType, FixedSizeListArray, GenericListArray,
30    OffsetSizeTrait, new_empty_array,
31};
32
33/// A [`GenericListViewArray`] of variable size lists, storing offsets as `i32`.
34pub type ListViewArray = GenericListViewArray<i32>;
35
36/// A [`GenericListViewArray`] of variable size lists, storing offsets as `i64`.
37pub type LargeListViewArray = GenericListViewArray<i64>;
38
39/// An array of [variable length lists], specifically in the [list-view layout].
40///
41/// Differs from [`GenericListArray`] (which represents the [list layout]) in that
42/// the sizes of the child arrays are explicitly encoded in a separate buffer, instead
43/// of being derived from the difference between subsequent offsets in the offset buffer.
44///
45/// This allows the offsets (and subsequently child data) to be out of order. It also
46/// allows take / filter operations to be implemented without copying the underlying data.
47///
48/// # Representation
49///
50/// Given the same example array from [`GenericListArray`], it would be represented
51/// as such via a list-view layout array:
52///
53/// ```text
54///                                         ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
55///                                                                         ┌ ─ ─ ─ ─ ─ ─ ┐    │
56///  ┌─────────────┐  ┌───────┐             │     ┌───┐   ┌───┐   ┌───┐       ┌───┐ ┌───┐
57///  │   [A,B,C]   │  │ (0,3) │                   │ 1 │   │ 0 │   │ 3 │     │ │ 1 │ │ A │ │ 0  │
58///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
59///  │      []     │  │ (3,0) │                   │ 1 │   │ 3 │   │ 0 │     │ │ 1 │ │ B │ │ 1  │
60///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
61///  │    NULL     │  │ (?,?) │                   │ 0 │   │ ? │   │ ? │     │ │ 1 │ │ C │ │ 2  │
62///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
63///  │     [D]     │  │ (4,1) │                   │ 1 │   │ 4 │   │ 1 │     │ │ ? │ │ ? │ │ 3  │
64///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
65///  │  [NULL, F]  │  │ (5,2) │                   │ 1 │   │ 5 │   │ 2 │     │ │ 1 │ │ D │ │ 4  │
66///  └─────────────┘  └───────┘             │     └───┘   └───┘   └───┘       ├───┤ ├───┤
67///                                                                         │ │ 0 │ │ ? │ │ 5  │
68///     Logical       Logical               │  Validity  Offsets  Sizes       ├───┤ ├───┤
69///      Values       Offset                   (nulls)                      │ │ 1 │ │ F │ │ 6  │
70///                   & Size                │                                 └───┘ └───┘
71///                                                                         │    Values   │    │
72///                 (offsets[i],            │   ListViewArray                   (Array)
73///                  sizes[i])                                              └ ─ ─ ─ ─ ─ ─ ┘    │
74///                                         └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
75/// ```
76///
77/// Another way of representing the same array but taking advantage of the offsets being out of order:
78///
79/// ```text
80///                                         ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
81///                                                                         ┌ ─ ─ ─ ─ ─ ─ ┐    │
82///  ┌─────────────┐  ┌───────┐             │     ┌───┐   ┌───┐   ┌───┐       ┌───┐ ┌───┐
83///  │   [A,B,C]   │  │ (2,3) │                   │ 1 │   │ 2 │   │ 3 │     │ │ 0 │ │ ? │ │ 0  │
84///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
85///  │      []     │  │ (0,0) │                   │ 1 │   │ 0 │   │ 0 │     │ │ 1 │ │ F │ │ 1  │
86///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
87///  │    NULL     │  │ (?,?) │                   │ 0 │   │ ? │   │ ? │     │ │ 1 │ │ A │ │ 2  │
88///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
89///  │     [D]     │  │ (5,1) │                   │ 1 │   │ 5 │   │ 1 │     │ │ 1 │ │ B │ │ 3  │
90///  ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
91///  │  [NULL, F]  │  │ (0,2) │                   │ 1 │   │ 0 │   │ 2 │     │ │ 1 │ │ C │ │ 4  │
92///  └─────────────┘  └───────┘             │     └───┘   └───┘   └───┘       ├───┤ ├───┤
93///                                                                         │ │ 1 │ │ D │ │ 5  │
94///     Logical       Logical               │  Validity  Offsets  Sizes       └───┘ └───┘
95///      Values       Offset                   (nulls)                      │    Values   │    │
96///                   & Size                │                                   (Array)
97///                                                                         └ ─ ─ ─ ─ ─ ─ ┘    │
98///                 (offsets[i],            │   ListViewArray
99///                  sizes[i])                                                                 │
100///                                         └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
101/// ```
102///
103/// [`GenericListArray`]: crate::array::GenericListArray
104/// [variable length lists]: https://arrow.apache.org/docs/format/Columnar.html#variable-size-list-layout
105/// [list layout]: https://arrow.apache.org/docs/format/Columnar.html#list-layout
106/// [list-view layout]: https://arrow.apache.org/docs/format/Columnar.html#listview-layout
107#[derive(Clone)]
108pub struct GenericListViewArray<OffsetSize: OffsetSizeTrait> {
109    data_type: DataType,
110    nulls: Option<NullBuffer>,
111    values: ArrayRef,
112    // Unlike GenericListArray, we do not use OffsetBuffer here as offsets are not
113    // guaranteed to be monotonically increasing.
114    value_offsets: ScalarBuffer<OffsetSize>,
115    value_sizes: ScalarBuffer<OffsetSize>,
116}
117
118impl<OffsetSize: OffsetSizeTrait> GenericListViewArray<OffsetSize> {
119    /// The data type constructor of listview array.
120    /// The input is the schema of the child array and
121    /// the output is the [`DataType`], ListView or LargeListView.
122    pub const DATA_TYPE_CONSTRUCTOR: fn(FieldRef) -> DataType = if OffsetSize::IS_LARGE {
123        DataType::LargeListView
124    } else {
125        DataType::ListView
126    };
127
128    /// Create a new [`GenericListViewArray`] from the provided parts
129    ///
130    /// # Errors
131    ///
132    /// Errors if
133    ///
134    /// * `offsets.len() != sizes.len()`
135    /// * `offsets.len() != nulls.len()`
136    /// * `offsets[i] > values.len()`
137    /// * `!field.is_nullable() && values.is_nullable()`
138    /// * `field.data_type() != values.data_type()`
139    /// * `0 <= offsets[i] <= length of the child array`
140    /// * `0 <= offsets[i] + size[i] <= length of the child array`
141    pub fn try_new(
142        field: FieldRef,
143        offsets: ScalarBuffer<OffsetSize>,
144        sizes: ScalarBuffer<OffsetSize>,
145        values: ArrayRef,
146        nulls: Option<NullBuffer>,
147    ) -> Result<Self, ArrowError> {
148        let len = offsets.len();
149        if let Some(n) = nulls.as_ref()
150            && n.len() != len
151        {
152            return Err(ArrowError::InvalidArgumentError(format!(
153                "Incorrect length of null buffer for {}ListViewArray, expected {len} got {}",
154                OffsetSize::PREFIX,
155                n.len(),
156            )));
157        }
158        if len != sizes.len() {
159            return Err(ArrowError::InvalidArgumentError(format!(
160                "Length of offsets buffer and sizes buffer must be equal for {}ListViewArray, got {len} and {}",
161                OffsetSize::PREFIX,
162                sizes.len()
163            )));
164        }
165
166        for (offset, size) in offsets.iter().zip(sizes.iter()) {
167            let offset = offset.as_usize();
168            let size = size.as_usize();
169            if offset.checked_add(size).ok_or_else(|| {
170                ArrowError::InvalidArgumentError(format!(
171                    "Overflow in offset + size for {}ListViewArray",
172                    OffsetSize::PREFIX
173                ))
174            })? > values.len()
175            {
176                return Err(ArrowError::InvalidArgumentError(format!(
177                    "Offset + size for {}ListViewArray must be within the bounds of the child array, got offset: {offset}, size: {size}, child array length: {}",
178                    OffsetSize::PREFIX,
179                    values.len()
180                )));
181            }
182        }
183
184        if !field.is_nullable() && values.is_nullable() {
185            return Err(ArrowError::InvalidArgumentError(format!(
186                "Non-nullable field of {}ListViewArray {:?} cannot contain nulls",
187                OffsetSize::PREFIX,
188                field.name()
189            )));
190        }
191
192        if field.data_type() != values.data_type() {
193            return Err(ArrowError::InvalidArgumentError(format!(
194                "{}ListViewArray expected data type {} got {} for {:?}",
195                OffsetSize::PREFIX,
196                field.data_type(),
197                values.data_type(),
198                field.name()
199            )));
200        }
201
202        Ok(Self {
203            data_type: Self::DATA_TYPE_CONSTRUCTOR(field),
204            nulls,
205            values,
206            value_offsets: offsets,
207            value_sizes: sizes,
208        })
209    }
210
211    /// Create a new [`GenericListViewArray`] from the provided parts
212    ///
213    /// # Panics
214    ///
215    /// Panics if [`Self::try_new`] returns an error
216    pub fn new(
217        field: FieldRef,
218        offsets: ScalarBuffer<OffsetSize>,
219        sizes: ScalarBuffer<OffsetSize>,
220        values: ArrayRef,
221        nulls: Option<NullBuffer>,
222    ) -> Self {
223        Self::try_new(field, offsets, sizes, values, nulls).unwrap()
224    }
225
226    /// Create a new [`GenericListViewArray`] from the provided parts without validation
227    ///
228    /// See [`Self::try_new`] for the checked version of this function, and the
229    /// documentation of that function for the invariants that must be upheld.
230    ///
231    /// # Safety
232    ///
233    /// The parts must form a valid [`ListViewArray`] or [`LargeListViewArray`] according
234    /// to the Arrow spec.
235    pub unsafe fn new_unchecked(
236        field: FieldRef,
237        offsets: ScalarBuffer<OffsetSize>,
238        sizes: ScalarBuffer<OffsetSize>,
239        values: ArrayRef,
240        nulls: Option<NullBuffer>,
241    ) -> Self {
242        if cfg!(feature = "force_validate") {
243            return Self::new(field, offsets, sizes, values, nulls);
244        }
245
246        Self {
247            data_type: Self::DATA_TYPE_CONSTRUCTOR(field),
248            nulls,
249            values,
250            value_offsets: offsets,
251            value_sizes: sizes,
252        }
253    }
254
255    /// Create a new [`GenericListViewArray`] of length `len` where all values are null
256    pub fn new_null(field: FieldRef, len: usize) -> Self {
257        let values = new_empty_array(field.data_type());
258        Self {
259            data_type: Self::DATA_TYPE_CONSTRUCTOR(field),
260            nulls: Some(NullBuffer::new_null(len)),
261            value_offsets: ScalarBuffer::from(vec![OffsetSize::usize_as(0); len]),
262            value_sizes: ScalarBuffer::from(vec![OffsetSize::usize_as(0); len]),
263            values,
264        }
265    }
266
267    /// Deconstruct this array into its constituent parts
268    pub fn into_parts(
269        self,
270    ) -> (
271        FieldRef,
272        ScalarBuffer<OffsetSize>,
273        ScalarBuffer<OffsetSize>,
274        ArrayRef,
275        Option<NullBuffer>,
276    ) {
277        let (DataType::ListView(f) | DataType::LargeListView(f)) = self.data_type else {
278            unreachable!()
279        };
280        (
281            f,
282            self.value_offsets,
283            self.value_sizes,
284            self.values,
285            self.nulls,
286        )
287    }
288
289    /// The field that describes the values of this list.
290    pub fn value_field(&self) -> &FieldRef {
291        match &self.data_type {
292            DataType::ListView(f) | DataType::LargeListView(f) => f,
293            _ => unreachable!(),
294        }
295    }
296
297    /// Returns a reference to the offsets of this list
298    ///
299    /// Unlike [`Self::value_offsets`] this returns the [`ScalarBuffer`]
300    /// allowing for zero-copy cloning
301    #[inline]
302    pub fn offsets(&self) -> &ScalarBuffer<OffsetSize> {
303        &self.value_offsets
304    }
305
306    /// Returns a reference to the values of this list
307    #[inline]
308    pub fn values(&self) -> &ArrayRef {
309        &self.values
310    }
311
312    /// Returns a reference to the sizes of this list
313    ///
314    /// Unlike [`Self::value_sizes`] this returns the [`ScalarBuffer`]
315    /// allowing for zero-copy cloning
316    #[inline]
317    pub fn sizes(&self) -> &ScalarBuffer<OffsetSize> {
318        &self.value_sizes
319    }
320
321    /// Returns a clone of the value type of this list.
322    pub fn value_type(&self) -> DataType {
323        self.values.data_type().clone()
324    }
325
326    /// Returns ith value of this list view array.
327    ///
328    /// Note: This method does not check for nulls and the value is arbitrary
329    /// if [`is_null`](Self::is_null) returns true for the index.
330    ///
331    /// # Safety
332    /// Caller must ensure that the index is within the array bounds
333    pub unsafe fn value_unchecked(&self, i: usize) -> ArrayRef {
334        let offset = unsafe { self.value_offsets().get_unchecked(i).as_usize() };
335        let length = unsafe { self.value_sizes().get_unchecked(i).as_usize() };
336        self.values.slice(offset, length)
337    }
338
339    /// Returns ith value of this list view array.
340    ///
341    /// Note: This method does not check for nulls and the value is arbitrary
342    /// (but still well-defined) if [`is_null`](Self::is_null) returns true for the index.
343    ///
344    /// # Panics
345    /// Panics if `i >= self.len()`
346    pub fn value(&self, i: usize) -> ArrayRef {
347        let offset = self.value_offsets()[i].as_usize();
348        let length = self.value_sizes()[i].as_usize();
349        self.values.slice(offset, length)
350    }
351
352    /// Returns the offset values in the offsets buffer
353    #[inline]
354    pub fn value_offsets(&self) -> &[OffsetSize] {
355        &self.value_offsets
356    }
357
358    /// Returns the sizes values in the offsets buffer
359    #[inline]
360    pub fn value_sizes(&self) -> &[OffsetSize] {
361        &self.value_sizes
362    }
363
364    /// Returns the size for value at index `i`.
365    ///
366    /// # Panics
367    /// Panics if `i >= self.len()`
368    #[inline]
369    pub fn value_size(&self, i: usize) -> OffsetSize {
370        self.value_sizes[i]
371    }
372
373    /// Returns the offset for value at index `i`.
374    ///
375    /// # Panics
376    /// Panics if `i >= self.len()`
377    pub fn value_offset(&self, i: usize) -> OffsetSize {
378        self.value_offsets[i]
379    }
380
381    /// Constructs a new iterator
382    pub fn iter(&self) -> GenericListViewArrayIter<'_, OffsetSize> {
383        GenericListViewArrayIter::<'_, OffsetSize>::new(self)
384    }
385
386    #[inline]
387    fn get_type(data_type: &DataType) -> Option<&DataType> {
388        match (OffsetSize::IS_LARGE, data_type) {
389            (true, DataType::LargeListView(child)) | (false, DataType::ListView(child)) => {
390                Some(child.data_type())
391            }
392            _ => None,
393        }
394    }
395
396    /// Returns a zero-copy slice of this array with the indicated offset and length.
397    ///
398    /// # Panics
399    /// Panics if `offset + length > self.len()`
400    pub fn slice(&self, offset: usize, length: usize) -> Self {
401        Self {
402            data_type: self.data_type.clone(),
403            nulls: self.nulls.as_ref().map(|n| n.slice(offset, length)),
404            values: self.values.clone(),
405            value_offsets: self.value_offsets.slice(offset, length),
406            value_sizes: self.value_sizes.slice(offset, length),
407        }
408    }
409
410    /// Creates a [`GenericListViewArray`] from an iterator of primitive values
411    /// # Example
412    /// ```
413    /// # use arrow_array::ListViewArray;
414    /// # use arrow_array::types::Int32Type;
415    ///
416    /// let data = vec![
417    ///    Some(vec![Some(0), Some(1), Some(2)]),
418    ///    None,
419    ///    Some(vec![Some(3), None, Some(5)]),
420    ///    Some(vec![Some(6), Some(7)]),
421    /// ];
422    /// let list_array = ListViewArray::from_iter_primitive::<Int32Type, _, _>(data);
423    /// println!("{:?}", list_array);
424    /// ```
425    pub fn from_iter_primitive<T, P, I>(iter: I) -> Self
426    where
427        T: ArrowPrimitiveType,
428        P: IntoIterator<Item = Option<<T as ArrowPrimitiveType>::Native>>,
429        I: IntoIterator<Item = Option<P>>,
430    {
431        let iter = iter.into_iter();
432        let size_hint = iter.size_hint().0;
433        let mut builder =
434            GenericListViewBuilder::with_capacity(PrimitiveBuilder::<T>::new(), size_hint);
435
436        for i in iter {
437            match i {
438                Some(p) => {
439                    for t in p {
440                        builder.values().append_option(t);
441                    }
442                    builder.append(true);
443                }
444                None => builder.append(false),
445            }
446        }
447        builder.finish()
448    }
449}
450
451impl<'a, OffsetSize: OffsetSizeTrait> IntoIterator for &'a GenericListViewArray<OffsetSize> {
452    type Item = Option<ArrayRef>;
453    type IntoIter = GenericListViewArrayIter<'a, OffsetSize>;
454
455    fn into_iter(self) -> Self::IntoIter {
456        GenericListViewArrayIter::<'a, OffsetSize>::new(self)
457    }
458}
459
460impl<OffsetSize: OffsetSizeTrait> ArrayAccessor for &GenericListViewArray<OffsetSize> {
461    type Item = ArrayRef;
462
463    fn value(&self, index: usize) -> Self::Item {
464        GenericListViewArray::value(self, index)
465    }
466
467    unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
468        unsafe { GenericListViewArray::value_unchecked(self, index) }
469    }
470}
471
472/// SAFETY: Correctly implements the contract of Arrow Arrays
473unsafe impl<OffsetSize: OffsetSizeTrait> Array for GenericListViewArray<OffsetSize> {
474    fn as_any(&self) -> &dyn Any {
475        self
476    }
477
478    fn to_data(&self) -> ArrayData {
479        self.clone().into()
480    }
481
482    fn into_data(self) -> ArrayData {
483        self.into()
484    }
485
486    fn data_type(&self) -> &DataType {
487        &self.data_type
488    }
489
490    fn slice(&self, offset: usize, length: usize) -> ArrayRef {
491        Arc::new(self.slice(offset, length))
492    }
493
494    fn len(&self) -> usize {
495        self.sizes().len()
496    }
497
498    fn is_empty(&self) -> bool {
499        self.value_sizes.is_empty()
500    }
501
502    fn shrink_to_fit(&mut self) {
503        if let Some(nulls) = &mut self.nulls {
504            nulls.shrink_to_fit();
505        }
506        self.values.shrink_to_fit();
507        self.value_offsets.shrink_to_fit();
508        self.value_sizes.shrink_to_fit();
509    }
510
511    fn offset(&self) -> usize {
512        0
513    }
514
515    fn nulls(&self) -> Option<&NullBuffer> {
516        self.nulls.as_ref()
517    }
518
519    fn logical_null_count(&self) -> usize {
520        // More efficient that the default implementation
521        self.null_count()
522    }
523
524    fn get_buffer_memory_size(&self) -> usize {
525        let mut size = self.values.get_buffer_memory_size();
526        size += self.value_offsets.inner().capacity();
527        size += self.value_sizes.inner().capacity();
528        if let Some(n) = self.nulls.as_ref() {
529            size += n.buffer().capacity();
530        }
531        size
532    }
533
534    fn get_array_memory_size(&self) -> usize {
535        let mut size = std::mem::size_of::<Self>() + self.values.get_array_memory_size();
536        size += self.value_offsets.inner().capacity();
537        size += self.value_sizes.inner().capacity();
538        if let Some(n) = self.nulls.as_ref() {
539            size += n.buffer().capacity();
540        }
541        size
542    }
543
544    #[cfg(feature = "pool")]
545    fn claim(&self, pool: &dyn arrow_buffer::MemoryPool) {
546        self.value_offsets.claim(pool);
547        self.value_sizes.claim(pool);
548        self.values.claim(pool);
549        if let Some(nulls) = &self.nulls {
550            nulls.claim(pool);
551        }
552    }
553}
554
555impl<OffsetSize: OffsetSizeTrait> super::ListLikeArray for GenericListViewArray<OffsetSize> {
556    fn values(&self) -> &ArrayRef {
557        self.values()
558    }
559
560    fn element_range(&self, index: usize) -> std::ops::Range<usize> {
561        let offset = self.value_offsets()[index].as_usize();
562        let size = self.value_sizes()[index].as_usize();
563        offset..(offset + size)
564    }
565}
566
567impl<OffsetSize: OffsetSizeTrait> std::fmt::Debug for GenericListViewArray<OffsetSize> {
568    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
569        let prefix = OffsetSize::PREFIX;
570        write!(f, "{prefix}ListViewArray\n[\n")?;
571        print_long_array(self, f, |array, index, f| {
572            std::fmt::Debug::fmt(&array.value(index), f)
573        })?;
574        write!(f, "]")
575    }
576}
577
578impl<OffsetSize: OffsetSizeTrait> From<GenericListArray<OffsetSize>>
579    for GenericListViewArray<OffsetSize>
580{
581    fn from(value: GenericListArray<OffsetSize>) -> Self {
582        let (field, offsets, values, nulls) = value.into_parts();
583        let len = offsets.len() - 1;
584        let mut sizes = Vec::with_capacity(len);
585        let mut view_offsets = Vec::with_capacity(len);
586        for (i, offset) in offsets.iter().enumerate().take(len) {
587            view_offsets.push(*offset);
588            sizes.push(offsets[i + 1] - offsets[i]);
589        }
590
591        Self::new(
592            field,
593            ScalarBuffer::from(view_offsets),
594            ScalarBuffer::from(sizes),
595            values,
596            nulls,
597        )
598    }
599}
600
601impl<OffsetSize: OffsetSizeTrait> From<GenericListViewArray<OffsetSize>> for ArrayData {
602    fn from(array: GenericListViewArray<OffsetSize>) -> Self {
603        let len = array.len();
604        let builder = ArrayDataBuilder::new(array.data_type)
605            .len(len)
606            .nulls(array.nulls)
607            .buffers(vec![
608                array.value_offsets.into_inner(),
609                array.value_sizes.into_inner(),
610            ])
611            .child_data(vec![array.values.to_data()]);
612
613        unsafe { builder.build_unchecked() }
614    }
615}
616
617impl<OffsetSize: OffsetSizeTrait> From<ArrayData> for GenericListViewArray<OffsetSize> {
618    fn from(data: ArrayData) -> Self {
619        Self::try_new_from_array_data(data)
620            .expect("Expected infallible creation of GenericListViewArray from ArrayDataRef failed")
621    }
622}
623
624impl<OffsetSize: OffsetSizeTrait> From<FixedSizeListArray> for GenericListViewArray<OffsetSize> {
625    fn from(value: FixedSizeListArray) -> Self {
626        let (field, size) = match value.data_type() {
627            DataType::FixedSizeList(f, size) => (f, *size as usize),
628            _ => unreachable!(),
629        };
630        let mut acc = 0_usize;
631        let iter = std::iter::repeat_n(size, value.len());
632        let mut sizes = Vec::with_capacity(iter.size_hint().0);
633        let mut offsets = Vec::with_capacity(iter.size_hint().0);
634
635        for size in iter {
636            offsets.push(OffsetSize::usize_as(acc));
637            acc = acc.add(size);
638            sizes.push(OffsetSize::usize_as(size));
639        }
640        let sizes = ScalarBuffer::from(sizes);
641        let offsets = ScalarBuffer::from(offsets);
642        Self {
643            data_type: Self::DATA_TYPE_CONSTRUCTOR(field.clone()),
644            nulls: value.nulls().cloned(),
645            values: value.values().clone(),
646            value_offsets: offsets,
647            value_sizes: sizes,
648        }
649    }
650}
651
652impl<OffsetSize: OffsetSizeTrait> GenericListViewArray<OffsetSize> {
653    fn try_new_from_array_data(data: ArrayData) -> Result<Self, ArrowError> {
654        let (data_type, len, nulls, offset, buffers, child_data) = data.into_parts();
655
656        // ArrayData is valid, and verified type above
657        // buffer[0] is offsets, buffer[1] is sizes
658        let num_buffers = buffers.len();
659        let [offsets_buffer, sizes_buffer] : [Buffer; 2] = buffers.try_into().map_err(|_| {
660             ArrowError::InvalidArgumentError(format!(
661                "ListViewArray data should contain two buffers (value offsets & value sizes), had {num_buffers}",
662            ))
663        })?;
664
665        let num_child = child_data.len();
666        let [values]: [ArrayData; 1] = child_data.try_into().map_err(|_| {
667            ArrowError::InvalidArgumentError(format!(
668                "ListViewArray should contain a single child array (values array), had {num_child}",
669            ))
670        })?;
671
672        if let Some(child_data_type) = Self::get_type(&data_type) {
673            if values.data_type() != child_data_type {
674                return Err(ArrowError::InvalidArgumentError(format!(
675                    "{}ListViewArray's child datatype {:?} does not \
676                             correspond to the List's datatype {:?}",
677                    OffsetSize::PREFIX,
678                    values.data_type(),
679                    child_data_type
680                )));
681            }
682        } else {
683            return Err(ArrowError::InvalidArgumentError(format!(
684                "{}ListViewArray's datatype must be {}ListViewArray(). It is {:?}",
685                OffsetSize::PREFIX,
686                OffsetSize::PREFIX,
687                data_type
688            )));
689        }
690
691        let values = make_array(values);
692        let value_offsets = ScalarBuffer::new(offsets_buffer, offset, len);
693        let value_sizes = ScalarBuffer::new(sizes_buffer, offset, len);
694
695        Ok(Self {
696            data_type,
697            nulls,
698            values,
699            value_offsets,
700            value_sizes,
701        })
702    }
703}
704
705#[cfg(test)]
706mod tests {
707    use arrow_buffer::{BooleanBuffer, Buffer, NullBufferBuilder, ScalarBuffer, bit_util};
708    use arrow_schema::Field;
709
710    use crate::builder::{FixedSizeListBuilder, Int32Builder};
711    use crate::cast::AsArray;
712    use crate::types::Int32Type;
713    use crate::{Int32Array, Int64Array};
714
715    use super::*;
716
717    #[test]
718    fn test_empty_list_view_array() {
719        // Construct an empty value array
720        let vec: Vec<i32> = vec![];
721        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
722        let sizes = ScalarBuffer::from(vec![]);
723        let offsets = ScalarBuffer::from(vec![]);
724        let values = Int32Array::from(vec);
725        let list_array = LargeListViewArray::new(field, offsets, sizes, Arc::new(values), None);
726
727        assert_eq!(list_array.len(), 0)
728    }
729
730    #[test]
731    fn test_list_view_array() {
732        // Construct a value array
733        let value_data = ArrayData::builder(DataType::Int32)
734            .len(8)
735            .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7]))
736            .build()
737            .unwrap();
738
739        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
740        let sizes = ScalarBuffer::from(vec![3i32, 3, 2]);
741        let offsets = ScalarBuffer::from(vec![0i32, 3, 6]);
742        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7]);
743        let list_array = ListViewArray::new(field, offsets, sizes, Arc::new(values), None);
744
745        let values = list_array.values();
746        assert_eq!(value_data, values.to_data());
747        assert_eq!(DataType::Int32, list_array.value_type());
748        assert_eq!(3, list_array.len());
749        assert_eq!(0, list_array.null_count());
750        assert_eq!(6, list_array.value_offsets()[2]);
751        assert_eq!(2, list_array.value_sizes()[2]);
752        assert_eq!(2, list_array.value_size(2));
753        assert_eq!(0, list_array.value(0).as_primitive::<Int32Type>().value(0));
754        assert_eq!(
755            0,
756            unsafe { list_array.value_unchecked(0) }
757                .as_primitive::<Int32Type>()
758                .value(0)
759        );
760        for i in 0..3 {
761            assert!(list_array.is_valid(i));
762            assert!(!list_array.is_null(i));
763        }
764    }
765
766    #[test]
767    fn test_large_list_view_array() {
768        // Construct a value array
769        let value_data = ArrayData::builder(DataType::Int32)
770            .len(8)
771            .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7]))
772            .build()
773            .unwrap();
774
775        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
776        let sizes = ScalarBuffer::from(vec![3i64, 3, 2]);
777        let offsets = ScalarBuffer::from(vec![0i64, 3, 6]);
778        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7]);
779        let list_array = LargeListViewArray::new(field, offsets, sizes, Arc::new(values), None);
780
781        let values = list_array.values();
782        assert_eq!(value_data, values.to_data());
783        assert_eq!(DataType::Int32, list_array.value_type());
784        assert_eq!(3, list_array.len());
785        assert_eq!(0, list_array.null_count());
786        assert_eq!(6, list_array.value_offsets()[2]);
787        assert_eq!(2, list_array.value_sizes()[2]);
788        assert_eq!(2, list_array.value_size(2));
789        assert_eq!(0, list_array.value(0).as_primitive::<Int32Type>().value(0));
790        assert_eq!(
791            0,
792            unsafe { list_array.value_unchecked(0) }
793                .as_primitive::<Int32Type>()
794                .value(0)
795        );
796        for i in 0..3 {
797            assert!(list_array.is_valid(i));
798            assert!(!list_array.is_null(i));
799        }
800    }
801
802    #[test]
803    fn test_list_view_array_slice() {
804        // Construct a value array
805        let value_data = ArrayData::builder(DataType::Int32)
806            .len(10)
807            .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))
808            .build()
809            .unwrap();
810
811        // 01011001 00000001
812        let mut null_bits: [u8; 2] = [0; 2];
813        bit_util::set_bit(&mut null_bits, 0);
814        bit_util::set_bit(&mut null_bits, 3);
815        bit_util::set_bit(&mut null_bits, 4);
816        bit_util::set_bit(&mut null_bits, 6);
817        bit_util::set_bit(&mut null_bits, 8);
818        let buffer = BooleanBuffer::new(Buffer::from(null_bits), 0, 9);
819        let null_buffer = NullBuffer::new(buffer);
820
821        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
822        let sizes = ScalarBuffer::from(vec![2, 0, 0, 2, 2, 0, 3, 0, 1]);
823        let offsets = ScalarBuffer::from(vec![0, 2, 2, 2, 4, 6, 6, 9, 9]);
824        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
825        let list_array =
826            ListViewArray::new(field, offsets, sizes, Arc::new(values), Some(null_buffer));
827
828        let values = list_array.values();
829        assert_eq!(value_data, values.to_data());
830        assert_eq!(DataType::Int32, list_array.value_type());
831        assert_eq!(9, list_array.len());
832        assert_eq!(4, list_array.null_count());
833        assert_eq!(2, list_array.value_offsets()[3]);
834        assert_eq!(2, list_array.value_sizes()[3]);
835        assert_eq!(2, list_array.value_size(3));
836
837        let sliced_array = list_array.slice(1, 6);
838        assert_eq!(6, sliced_array.len());
839        assert_eq!(3, sliced_array.null_count());
840
841        for i in 0..sliced_array.len() {
842            if bit_util::get_bit(&null_bits, 1 + i) {
843                assert!(sliced_array.is_valid(i));
844            } else {
845                assert!(sliced_array.is_null(i));
846            }
847        }
848
849        // Check offset and length for each non-null value.
850        let sliced_list_array = sliced_array
851            .as_any()
852            .downcast_ref::<ListViewArray>()
853            .unwrap();
854        assert_eq!(2, sliced_list_array.value_offsets()[2]);
855        assert_eq!(2, sliced_list_array.value_sizes()[2]);
856        assert_eq!(2, sliced_list_array.value_size(2));
857
858        assert_eq!(4, sliced_list_array.value_offsets()[3]);
859        assert_eq!(2, sliced_list_array.value_sizes()[3]);
860        assert_eq!(2, sliced_list_array.value_size(3));
861
862        assert_eq!(6, sliced_list_array.value_offsets()[5]);
863        assert_eq!(3, sliced_list_array.value_sizes()[5]);
864        assert_eq!(3, sliced_list_array.value_size(5));
865    }
866
867    #[test]
868    fn test_large_list_view_array_slice() {
869        // Construct a value array
870        let value_data = ArrayData::builder(DataType::Int32)
871            .len(10)
872            .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]))
873            .build()
874            .unwrap();
875
876        // 01011001 00000001
877        let mut null_bits: [u8; 2] = [0; 2];
878        bit_util::set_bit(&mut null_bits, 0);
879        bit_util::set_bit(&mut null_bits, 3);
880        bit_util::set_bit(&mut null_bits, 4);
881        bit_util::set_bit(&mut null_bits, 6);
882        bit_util::set_bit(&mut null_bits, 8);
883        let buffer = BooleanBuffer::new(Buffer::from(null_bits), 0, 9);
884        let null_buffer = NullBuffer::new(buffer);
885
886        // Construct a large list view array from the above two
887        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
888        let sizes = ScalarBuffer::from(vec![2i64, 0, 0, 2, 2, 0, 3, 0, 1]);
889        let offsets = ScalarBuffer::from(vec![0i64, 2, 2, 2, 4, 6, 6, 9, 9]);
890        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
891        let list_array =
892            LargeListViewArray::new(field, offsets, sizes, Arc::new(values), Some(null_buffer));
893
894        let values = list_array.values();
895        assert_eq!(value_data, values.to_data());
896        assert_eq!(DataType::Int32, list_array.value_type());
897        assert_eq!(9, list_array.len());
898        assert_eq!(4, list_array.null_count());
899        assert_eq!(2, list_array.value_offsets()[3]);
900        assert_eq!(2, list_array.value_sizes()[3]);
901        assert_eq!(2, list_array.value_size(3));
902
903        let sliced_array = list_array.slice(1, 6);
904        assert_eq!(6, sliced_array.len());
905        assert_eq!(3, sliced_array.null_count());
906
907        for i in 0..sliced_array.len() {
908            if bit_util::get_bit(&null_bits, 1 + i) {
909                assert!(sliced_array.is_valid(i));
910            } else {
911                assert!(sliced_array.is_null(i));
912            }
913        }
914
915        // Check offset and length for each non-null value.
916        let sliced_list_array = sliced_array
917            .as_any()
918            .downcast_ref::<LargeListViewArray>()
919            .unwrap();
920        assert_eq!(2, sliced_list_array.value_offsets()[2]);
921        assert_eq!(2, sliced_list_array.value_size(2));
922        assert_eq!(2, sliced_list_array.value_sizes()[2]);
923
924        assert_eq!(4, sliced_list_array.value_offsets()[3]);
925        assert_eq!(2, sliced_list_array.value_size(3));
926        assert_eq!(2, sliced_list_array.value_sizes()[3]);
927
928        assert_eq!(6, sliced_list_array.value_offsets()[5]);
929        assert_eq!(3, sliced_list_array.value_size(5));
930        assert_eq!(2, sliced_list_array.value_sizes()[3]);
931    }
932
933    #[test]
934    #[should_panic(expected = "index out of bounds: the len is 9 but the index is 10")]
935    fn test_list_view_array_index_out_of_bound() {
936        // 01011001 00000001
937        let mut null_bits: [u8; 2] = [0; 2];
938        bit_util::set_bit(&mut null_bits, 0);
939        bit_util::set_bit(&mut null_bits, 3);
940        bit_util::set_bit(&mut null_bits, 4);
941        bit_util::set_bit(&mut null_bits, 6);
942        bit_util::set_bit(&mut null_bits, 8);
943        let buffer = BooleanBuffer::new(Buffer::from(null_bits), 0, 9);
944        let null_buffer = NullBuffer::new(buffer);
945
946        // Construct a buffer for value offsets, for the nested array:
947        //  [[0, 1], null, null, [2, 3], [4, 5], null, [6, 7, 8], null, [9]]
948        // Construct a list array from the above two
949        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
950        let sizes = ScalarBuffer::from(vec![2i32, 0, 0, 2, 2, 0, 3, 0, 1]);
951        let offsets = ScalarBuffer::from(vec![0i32, 2, 2, 2, 4, 6, 6, 9, 9]);
952        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
953        let list_array =
954            ListViewArray::new(field, offsets, sizes, Arc::new(values), Some(null_buffer));
955
956        assert_eq!(9, list_array.len());
957        list_array.value(10);
958    }
959    #[test]
960    #[should_panic(
961        expected = "ListViewArray data should contain two buffers (value offsets & value sizes), had 0"
962    )]
963    #[cfg(not(feature = "force_validate"))]
964    fn test_list_view_array_invalid_buffer_len() {
965        let value_data = unsafe {
966            ArrayData::builder(DataType::Int32)
967                .len(8)
968                .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7]))
969                .build_unchecked()
970        };
971        let list_data_type =
972            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, false)));
973        let list_data = unsafe {
974            ArrayData::builder(list_data_type)
975                .len(3)
976                .add_child_data(value_data)
977                .build_unchecked()
978        };
979        drop(ListViewArray::from(list_data));
980    }
981
982    #[test]
983    #[should_panic(
984        expected = "ListViewArray data should contain two buffers (value offsets & value sizes), had 1"
985    )]
986    #[cfg(not(feature = "force_validate"))]
987    fn test_list_view_array_invalid_child_array_len() {
988        let value_offsets = Buffer::from_slice_ref([0, 2, 5, 7]);
989        let list_data_type =
990            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, false)));
991        let list_data = unsafe {
992            ArrayData::builder(list_data_type)
993                .len(3)
994                .add_buffer(value_offsets)
995                .build_unchecked()
996        };
997        drop(ListViewArray::from(list_data));
998    }
999
1000    #[test]
1001    fn test_list_view_array_offsets_need_not_start_at_zero() {
1002        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
1003        let sizes = ScalarBuffer::from(vec![0i32, 0, 3]);
1004        let offsets = ScalarBuffer::from(vec![2i32, 2, 5]);
1005        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7]);
1006        let list_array = ListViewArray::new(field, offsets, sizes, Arc::new(values), None);
1007
1008        assert_eq!(list_array.value_size(0), 0);
1009        assert_eq!(list_array.value_size(1), 0);
1010        assert_eq!(list_array.value_size(2), 3);
1011    }
1012
1013    #[test]
1014    #[should_panic(expected = "Memory pointer is not aligned with the specified scalar type")]
1015    #[cfg(not(feature = "force_validate"))]
1016    fn test_list_view_array_alignment() {
1017        let offset_buf = Buffer::from_slice_ref([0_u64]);
1018        let offset_buf2 = offset_buf.slice(1);
1019
1020        let size_buf = Buffer::from_slice_ref([0_u64]);
1021        let size_buf2 = size_buf.slice(1);
1022
1023        let values: [i32; 8] = [0; 8];
1024        let value_data = unsafe {
1025            ArrayData::builder(DataType::Int32)
1026                .add_buffer(Buffer::from_slice_ref(values))
1027                .build_unchecked()
1028        };
1029
1030        let list_data_type =
1031            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, false)));
1032        let list_data = unsafe {
1033            ArrayData::builder(list_data_type)
1034                .add_buffer(offset_buf2)
1035                .add_buffer(size_buf2)
1036                .add_child_data(value_data)
1037                .build_unchecked()
1038        };
1039        drop(ListViewArray::from(list_data));
1040    }
1041
1042    #[test]
1043    fn test_empty_offsets() {
1044        let f = Arc::new(Field::new("element", DataType::Int32, true));
1045        let string = ListViewArray::from(
1046            ArrayData::builder(DataType::ListView(f.clone()))
1047                .buffers(vec![Buffer::from(&[]), Buffer::from(&[])])
1048                .add_child_data(ArrayData::new_empty(&DataType::Int32))
1049                .build()
1050                .unwrap(),
1051        );
1052        assert_eq!(string.value_offsets(), &[] as &[i32; 0]);
1053        assert_eq!(string.value_sizes(), &[] as &[i32; 0]);
1054
1055        let string = LargeListViewArray::from(
1056            ArrayData::builder(DataType::LargeListView(f))
1057                .buffers(vec![Buffer::from(&[]), Buffer::from(&[])])
1058                .add_child_data(ArrayData::new_empty(&DataType::Int32))
1059                .build()
1060                .unwrap(),
1061        );
1062        assert_eq!(string.len(), 0);
1063        assert_eq!(string.value_offsets(), &[] as &[i64; 0]);
1064        assert_eq!(string.value_sizes(), &[] as &[i64; 0]);
1065    }
1066
1067    #[test]
1068    fn test_try_new() {
1069        let offsets = ScalarBuffer::from(vec![0, 1, 4, 5]);
1070        let sizes = ScalarBuffer::from(vec![1, 3, 1, 0]);
1071        let values = Int32Array::new(vec![1, 2, 3, 4, 5].into(), None);
1072        let values = Arc::new(values) as ArrayRef;
1073
1074        let field = Arc::new(Field::new("element", DataType::Int32, false));
1075        ListViewArray::new(
1076            field.clone(),
1077            offsets.clone(),
1078            sizes.clone(),
1079            values.clone(),
1080            None,
1081        );
1082
1083        let nulls = NullBuffer::new_null(4);
1084        ListViewArray::new(
1085            field.clone(),
1086            offsets,
1087            sizes.clone(),
1088            values.clone(),
1089            Some(nulls),
1090        );
1091
1092        let nulls = NullBuffer::new_null(4);
1093        let offsets = ScalarBuffer::from(vec![0, 1, 2, 3, 4]);
1094        let sizes = ScalarBuffer::from(vec![1, 1, 1, 1, 0]);
1095        let err = LargeListViewArray::try_new(
1096            field,
1097            offsets.clone(),
1098            sizes.clone(),
1099            values.clone(),
1100            Some(nulls),
1101        )
1102        .unwrap_err();
1103
1104        assert_eq!(
1105            err.to_string(),
1106            "Invalid argument error: Incorrect length of null buffer for LargeListViewArray, expected 5 got 4"
1107        );
1108
1109        let field = Arc::new(Field::new("element", DataType::Int64, false));
1110        let err = LargeListViewArray::try_new(
1111            field.clone(),
1112            offsets.clone(),
1113            sizes.clone(),
1114            values.clone(),
1115            None,
1116        )
1117        .unwrap_err();
1118
1119        assert_eq!(
1120            err.to_string(),
1121            "Invalid argument error: LargeListViewArray expected data type Int64 got Int32 for \"element\""
1122        );
1123
1124        let nulls = NullBuffer::new_null(7);
1125        let values = Int64Array::new(vec![0; 7].into(), Some(nulls));
1126        let values = Arc::new(values);
1127
1128        let err = LargeListViewArray::try_new(
1129            field,
1130            offsets.clone(),
1131            sizes.clone(),
1132            values.clone(),
1133            None,
1134        )
1135        .unwrap_err();
1136
1137        assert_eq!(
1138            err.to_string(),
1139            "Invalid argument error: Non-nullable field of LargeListViewArray \"element\" cannot contain nulls"
1140        );
1141    }
1142
1143    #[test]
1144    fn test_from_fixed_size_list() {
1145        let mut builder = FixedSizeListBuilder::new(Int32Builder::new(), 3);
1146        builder.values().append_slice(&[1, 2, 3]);
1147        builder.append(true);
1148        builder.values().append_slice(&[0, 0, 0]);
1149        builder.append(false);
1150        builder.values().append_slice(&[4, 5, 6]);
1151        builder.append(true);
1152        let list: ListViewArray = builder.finish().into();
1153        let values: Vec<_> = list
1154            .iter()
1155            .map(|x| x.map(|x| x.as_primitive::<Int32Type>().values().to_vec()))
1156            .collect();
1157        assert_eq!(values, vec![Some(vec![1, 2, 3]), None, Some(vec![4, 5, 6])]);
1158        let offsets = list.value_offsets();
1159        assert_eq!(offsets, &[0, 3, 6]);
1160        let sizes = list.value_sizes();
1161        assert_eq!(sizes, &[3, 3, 3]);
1162    }
1163
1164    #[test]
1165    fn test_list_view_array_overlap_lists() {
1166        let value_data = unsafe {
1167            ArrayData::builder(DataType::Int32)
1168                .len(8)
1169                .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7]))
1170                .build_unchecked()
1171        };
1172        let list_data_type =
1173            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, false)));
1174        let list_data = unsafe {
1175            ArrayData::builder(list_data_type)
1176                .len(2)
1177                .add_buffer(Buffer::from_slice_ref([0, 3])) // offsets
1178                .add_buffer(Buffer::from_slice_ref([5, 5])) // sizes
1179                .add_child_data(value_data)
1180                .build_unchecked()
1181        };
1182        let array = ListViewArray::from(list_data);
1183
1184        assert_eq!(array.len(), 2);
1185        assert_eq!(array.value_size(0), 5);
1186        assert_eq!(array.value_size(1), 5);
1187
1188        let values: Vec<_> = array
1189            .iter()
1190            .map(|x| x.map(|x| x.as_primitive::<Int32Type>().values().to_vec()))
1191            .collect();
1192        assert_eq!(
1193            values,
1194            vec![Some(vec![0, 1, 2, 3, 4]), Some(vec![3, 4, 5, 6, 7])]
1195        );
1196    }
1197
1198    #[test]
1199    fn test_list_view_array_incomplete_offsets() {
1200        let value_data = unsafe {
1201            ArrayData::builder(DataType::Int32)
1202                .len(50)
1203                .add_buffer(Buffer::from_slice_ref((0..50).collect::<Vec<i32>>()))
1204                .build_unchecked()
1205        };
1206        let list_data_type =
1207            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, false)));
1208        let list_data = unsafe {
1209            ArrayData::builder(list_data_type)
1210                .len(3)
1211                .add_buffer(Buffer::from_slice_ref([0, 5, 10])) // offsets
1212                .add_buffer(Buffer::from_slice_ref([0, 5, 10])) // sizes
1213                .add_child_data(value_data)
1214                .build_unchecked()
1215        };
1216        let array = ListViewArray::from(list_data);
1217
1218        assert_eq!(array.len(), 3);
1219        assert_eq!(array.value_size(0), 0);
1220        assert_eq!(array.value_size(1), 5);
1221        assert_eq!(array.value_size(2), 10);
1222
1223        let values: Vec<_> = array
1224            .iter()
1225            .map(|x| x.map(|x| x.as_primitive::<Int32Type>().values().to_vec()))
1226            .collect();
1227        assert_eq!(
1228            values,
1229            vec![
1230                Some(vec![]),
1231                Some(vec![5, 6, 7, 8, 9]),
1232                Some(vec![10, 11, 12, 13, 14, 15, 16, 17, 18, 19])
1233            ]
1234        );
1235    }
1236
1237    #[test]
1238    fn test_list_view_array_empty_lists() {
1239        let value_data = unsafe {
1240            ArrayData::builder(DataType::Int32)
1241                .len(0)
1242                .add_buffer(Buffer::from_slice_ref::<i32, &[_; 0]>(&[]))
1243                .build_unchecked()
1244        };
1245        let list_data_type =
1246            DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, false)));
1247        let list_data = unsafe {
1248            ArrayData::builder(list_data_type)
1249                .len(3)
1250                .add_buffer(Buffer::from_slice_ref([0, 0, 0])) // offsets
1251                .add_buffer(Buffer::from_slice_ref([0, 0, 0])) // sizes
1252                .add_child_data(value_data)
1253                .build_unchecked()
1254        };
1255        let array = ListViewArray::from(list_data);
1256
1257        assert_eq!(array.len(), 3);
1258        assert_eq!(array.value_size(0), 0);
1259        assert_eq!(array.value_size(1), 0);
1260        assert_eq!(array.value_size(2), 0);
1261
1262        let values: Vec<_> = array
1263            .iter()
1264            .map(|x| x.map(|x| x.as_primitive::<Int32Type>().values().to_vec()))
1265            .collect();
1266        assert_eq!(values, vec![Some(vec![]), Some(vec![]), Some(vec![])]);
1267    }
1268
1269    #[test]
1270    fn test_list_view_new_null_len() {
1271        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
1272        let array = ListViewArray::new_null(field, 5);
1273        assert_eq!(array.len(), 5);
1274    }
1275
1276    #[test]
1277    fn test_from_iter_primitive() {
1278        let data = vec![
1279            Some(vec![Some(0), Some(1), Some(2)]),
1280            None,
1281            Some(vec![Some(3), Some(4), Some(5)]),
1282            Some(vec![Some(6), Some(7)]),
1283        ];
1284        let list_array = ListViewArray::from_iter_primitive::<Int32Type, _, _>(data);
1285
1286        //  [[0, 1, 2], NULL, [3, 4, 5], [6, 7]]
1287        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7]);
1288        let offsets = ScalarBuffer::from(vec![0, 3, 3, 6]);
1289        let sizes = ScalarBuffer::from(vec![3, 0, 3, 2]);
1290        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
1291
1292        let mut nulls = NullBufferBuilder::new(4);
1293        nulls.append(true);
1294        nulls.append(false);
1295        nulls.append_n_non_nulls(2);
1296        let another = ListViewArray::new(field, offsets, sizes, Arc::new(values), nulls.finish());
1297
1298        assert_eq!(list_array, another)
1299    }
1300}
arrow_array/array/list_view_array.rs

arrow_array/array/
list_view_array.rs
