Skip to main content

arrow_array/array/
byte_view_array.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use crate::array::print_long_array;
6use crate::builder::{ArrayBuilder, GenericByteViewBuilder};
7use crate::iterator::ArrayIter;
8use crate::types::bytes::ByteArrayNativeType;
9use crate::types::{BinaryViewType, ByteViewType, StringViewType};
10use crate::{Array, ArrayAccessor, ArrayRef, GenericByteArray, OffsetSizeTrait, Scalar};
11use arrow_buffer::{ArrowNativeType, Buffer, NullBuffer, ScalarBuffer};
12use arrow_data::{ArrayData, ArrayDataBuilder, ByteView, MAX_INLINE_VIEW_LEN};
13use arrow_schema::{ArrowError, DataType};
14use core::str;
15use num_traits::ToPrimitive;
16use std::any::Any;
17use std::cmp::Ordering;
18use std::fmt::Debug;
19use std::marker::PhantomData;
20use std::sync::Arc;
21
22use super::ByteArrayType;
23
24/// [Variable-size Binary View Layout]: An array of variable length bytes views.
25///
26/// This array type is used to store variable length byte data (e.g. Strings, Binary)
27/// and has efficient operations such as `take`, `filter`, and comparison.
28///
29/// [Variable-size Binary View Layout]: https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-view-layout
30///
31/// This is different from [`GenericByteArray`], which also stores variable
32/// length byte data, as it represents strings with an offset and length. `take`
33/// and `filter` like operations are implemented by manipulating the "views"
34/// (`u128`) without modifying the bytes. Each view also stores an inlined
35/// prefix which speed up comparisons.
36///
37/// # See Also
38///
39/// * [`StringViewArray`] for storing utf8 encoded string data
40/// * [`BinaryViewArray`] for storing bytes
41/// * [`ByteView`] to interpret `u128`s layout of the views.
42///
43/// [`ByteView`]: arrow_data::ByteView
44///
45/// # Layout: "views" and buffers
46///
47/// A `GenericByteViewArray` stores variable length byte strings. An array of
48/// `N` elements is stored as `N` fixed length "views" and a variable number
49/// of variable length "buffers".
50///
51/// Each view is a `u128` value whose layout is different depending on the
52/// length of the string stored at that location:
53///
54/// ```text
55///                         ┌──────┬────────────────────────┐
56///                         │length│      string value      │
57///    Strings (len <= 12)  │      │    (padded with 0)     │
58///                         └──────┴────────────────────────┘
59///                          0    31                      127
60///
61///                         ┌───────┬───────┬───────┬───────┐
62///                         │length │prefix │  buf  │offset │
63///    Strings (len > 12)   │       │       │ index │       │
64///                         └───────┴───────┴───────┴───────┘
65///                          0    31       63      95    127
66/// ```
67///
68/// * Strings with length <= 12 ([`MAX_INLINE_VIEW_LEN`]) are stored directly in
69///   the view. See [`Self::inline_value`] to access the inlined prefix from a
70///   short view.
71///
72/// * Strings with length > 12: The first four bytes are stored inline in the
73///   view and the entire string is stored in one of the buffers. See [`ByteView`]
74///   to access the fields of the these views.
75///
76/// As with other arrays, the optimized kernels in [`arrow_compute`] are likely
77/// the easiest and fastest way to work with this data. However, it is possible
78/// to access the views and buffers directly for more control.
79///
80/// For example
81///
82/// ```rust
83/// # use arrow_array::StringViewArray;
84/// # use arrow_array::Array;
85/// use arrow_data::ByteView;
86/// let array = StringViewArray::from(vec![
87///   "hello",
88///   "this string is longer than 12 bytes",
89///   "this string is also longer than 12 bytes"
90/// ]);
91///
92/// // ** Examine the first view (short string) **
93/// assert!(array.is_valid(0)); // Check for nulls
94/// let short_view: u128 = array.views()[0]; // "hello"
95/// // get length of the string
96/// let len = short_view as u32;
97/// assert_eq!(len, 5); // strings less than 12 bytes are stored in the view
98/// // SAFETY: `view` is a valid view
99/// let value = unsafe {
100///   StringViewArray::inline_value(&short_view, len as usize)
101/// };
102/// assert_eq!(value, b"hello");
103///
104/// // ** Examine the third view (long string) **
105/// assert!(array.is_valid(12)); // Check for nulls
106/// let long_view: u128 = array.views()[2]; // "this string is also longer than 12 bytes"
107/// let len = long_view as u32;
108/// assert_eq!(len, 40); // strings longer than 12 bytes are stored in the buffer
109/// let view = ByteView::from(long_view); // use ByteView to access the fields
110/// assert_eq!(view.length, 40);
111/// assert_eq!(view.buffer_index, 0);
112/// assert_eq!(view.offset, 35); // data starts after the first long string
113/// // Views for long strings store a 4 byte prefix
114/// let prefix = view.prefix.to_le_bytes();
115/// assert_eq!(&prefix, b"this");
116/// let value = array.value(2); // get the string value (see `value` implementation for how to access the bytes directly)
117/// assert_eq!(value, "this string is also longer than 12 bytes");
118/// ```
119///
120/// [`MAX_INLINE_VIEW_LEN`]: arrow_data::MAX_INLINE_VIEW_LEN
121/// [`arrow_compute`]: https://docs.rs/arrow/latest/arrow/compute/index.html
122///
123/// Unlike [`GenericByteArray`], there are no constraints on the offsets other
124/// than they must point into a valid buffer. However, they can be out of order,
125/// non continuous and overlapping.
126///
127/// For example, in the following diagram, the strings "FishWasInTownToday" and
128/// "CrumpleFacedFish" are both longer than 12 bytes and thus are stored in a
129/// separate buffer while the string "LavaMonster" is stored inlined in the
130/// view. In this case, the same bytes for "Fish" are used to store both strings.
131///
132/// [`ByteView`]: arrow_data::ByteView
133///
134/// ```text
135///                                                                            ┌───┐
136///                         ┌──────┬──────┬──────┬──────┐               offset │...│
137/// "FishWasInTownTodayYay" │  21  │ Fish │  0   │ 115  │─ ─              103  │Mr.│
138///                         └──────┴──────┴──────┴──────┘   │      ┌ ─ ─ ─ ─ ▶ │Cru│
139///                         ┌──────┬──────┬──────┬──────┐                      │mpl│
140/// "CrumpleFacedFish"      │  16  │ Crum │  0   │ 103  │─ ─│─ ─ ─ ┘           │eFa│
141///                         └──────┴──────┴──────┴──────┘                      │ced│
142///                         ┌──────┬────────────────────┐   └ ─ ─ ─ ─ ─ ─ ─ ─ ▶│Fis│
143/// "LavaMonster"           │  11  │   LavaMonster      │                      │hWa│
144///                         └──────┴────────────────────┘               offset │sIn│
145///                                                                       115  │Tow│
146///                                                                            │nTo│
147///                                                                            │day│
148///                                  u128 "views"                              │Yay│
149///                                                                   buffer 0 │...│
150///                                                                            └───┘
151/// ```
152pub struct GenericByteViewArray<T: ByteViewType + ?Sized> {
153    data_type: DataType,
154    views: ScalarBuffer<u128>,
155    buffers: Arc<[Buffer]>,
156    phantom: PhantomData<T>,
157    nulls: Option<NullBuffer>,
158}
159
160impl<T: ByteViewType + ?Sized> Clone for GenericByteViewArray<T> {
161    fn clone(&self) -> Self {
162        Self {
163            data_type: T::DATA_TYPE,
164            views: self.views.clone(),
165            buffers: self.buffers.clone(),
166            nulls: self.nulls.clone(),
167            phantom: Default::default(),
168        }
169    }
170}
171
172impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
173    /// Create a new [`GenericByteViewArray`] from the provided parts, panicking on failure
174    ///
175    /// # Panics
176    ///
177    /// Panics if [`GenericByteViewArray::try_new`] returns an error
178    pub fn new<U>(views: ScalarBuffer<u128>, buffers: U, nulls: Option<NullBuffer>) -> Self
179    where
180        U: Into<Arc<[Buffer]>>,
181    {
182        Self::try_new(views, buffers, nulls).unwrap()
183    }
184
185    /// Create a new [`GenericByteViewArray`] from the provided parts, returning an error on failure
186    ///
187    /// # Errors
188    ///
189    /// * `views.len() != nulls.len()`
190    /// * [ByteViewType::validate] fails
191    pub fn try_new<U>(
192        views: ScalarBuffer<u128>,
193        buffers: U,
194        nulls: Option<NullBuffer>,
195    ) -> Result<Self, ArrowError>
196    where
197        U: Into<Arc<[Buffer]>>,
198    {
199        let buffers: Arc<[Buffer]> = buffers.into();
200
201        T::validate(&views, &buffers)?;
202
203        if let Some(n) = nulls.as_ref()
204            && n.len() != views.len()
205        {
206            return Err(ArrowError::InvalidArgumentError(format!(
207                "Incorrect length of null buffer for {}ViewArray, expected {} got {}",
208                T::PREFIX,
209                views.len(),
210                n.len(),
211            )));
212        }
213
214        Ok(Self {
215            data_type: T::DATA_TYPE,
216            views,
217            buffers,
218            nulls,
219            phantom: Default::default(),
220        })
221    }
222
223    /// Create a new [`GenericByteViewArray`] from the provided parts, without validation
224    ///
225    /// # Safety
226    ///
227    /// Safe if [`Self::try_new`] would not error
228    pub unsafe fn new_unchecked(
229        views: ScalarBuffer<u128>,
230        buffers: Arc<[Buffer]>,
231        nulls: Option<NullBuffer>,
232    ) -> Self {
233        if cfg!(feature = "force_validate") {
234            return Self::new(views, buffers, nulls);
235        }
236
237        Self {
238            data_type: T::DATA_TYPE,
239            phantom: Default::default(),
240            views,
241            buffers,
242            nulls,
243        }
244    }
245
246    /// Create a new [`GenericByteViewArray`] of length `len` where all values are null
247    pub fn new_null(len: usize) -> Self {
248        Self {
249            data_type: T::DATA_TYPE,
250            views: vec![0; len].into(),
251            buffers: vec![].into(),
252            nulls: Some(NullBuffer::new_null(len)),
253            phantom: Default::default(),
254        }
255    }
256
257    /// Create a new [`Scalar`] from `value`
258    pub fn new_scalar(value: impl AsRef<T::Native>) -> Scalar<Self> {
259        Scalar::new(Self::from_iter_values(std::iter::once(value)))
260    }
261
262    /// Creates a [`GenericByteViewArray`] based on an iterator of values without nulls
263    pub fn from_iter_values<Ptr, I>(iter: I) -> Self
264    where
265        Ptr: AsRef<T::Native>,
266        I: IntoIterator<Item = Ptr>,
267    {
268        let iter = iter.into_iter();
269        let mut builder = GenericByteViewBuilder::<T>::with_capacity(iter.size_hint().0);
270        for v in iter {
271            builder.append_value(v);
272        }
273        builder.finish()
274    }
275
276    /// Deconstruct this array into its constituent parts
277    pub fn into_parts(self) -> (ScalarBuffer<u128>, Arc<[Buffer]>, Option<NullBuffer>) {
278        (self.views, self.buffers, self.nulls)
279    }
280
281    /// Returns the views buffer
282    #[inline]
283    pub fn views(&self) -> &ScalarBuffer<u128> {
284        &self.views
285    }
286
287    /// Returns the shared collection of buffers storing non-inline string or binary data.
288    ///
289    /// The returned `Arc` can be cloned to share the buffers with another array without
290    /// allocating a new collection or cloning the individual buffers. To consume this
291    /// array and take ownership of its buffers, use [`Self::into_parts`].
292    #[inline]
293    pub fn data_buffers(&self) -> &Arc<[Buffer]> {
294        &self.buffers
295    }
296
297    /// Returns the element at index `i`
298    ///
299    /// Note: This method does not check for nulls and the value is arbitrary
300    /// (but still well-defined) if [`is_null`](Self::is_null) returns true for the index.
301    ///
302    /// # Panics
303    /// Panics if index `i` is out of bounds.
304    pub fn value(&self, i: usize) -> &T::Native {
305        assert!(
306            i < self.len(),
307            "Trying to access an element at index {} from a {}ViewArray of length {}",
308            i,
309            T::PREFIX,
310            self.len()
311        );
312
313        unsafe { self.value_unchecked(i) }
314    }
315
316    /// Returns the element at index `i` without bounds checking
317    ///
318    /// Note: This method does not check for nulls and the value is arbitrary
319    /// if [`is_null`](Self::is_null) returns true for the index.
320    ///
321    /// # Safety
322    ///
323    /// Caller is responsible for ensuring that the index is within the bounds
324    /// of the array
325    pub unsafe fn value_unchecked(&self, idx: usize) -> &T::Native {
326        let v = unsafe { self.views.get_unchecked(idx) };
327        let len = *v as u32;
328        let b = if len <= MAX_INLINE_VIEW_LEN {
329            unsafe { Self::inline_value(v, len as usize) }
330        } else {
331            let view = ByteView::from(*v);
332            let data = unsafe { self.buffers.get_unchecked(view.buffer_index as usize) };
333            let offset = view.offset as usize;
334            unsafe { data.get_unchecked(offset..offset + len as usize) }
335        };
336        unsafe { T::Native::from_bytes_unchecked(b) }
337    }
338
339    /// Returns the first `len` bytes the inline value of the view.
340    ///
341    /// # Safety
342    /// - The `view` must be a valid element from `Self::views()` that adheres to the view layout.
343    /// - The `len` must be the length of the inlined value. It should never be larger than [`MAX_INLINE_VIEW_LEN`].
344    #[inline(always)]
345    pub unsafe fn inline_value(view: &u128, len: usize) -> &[u8] {
346        debug_assert!(len <= MAX_INLINE_VIEW_LEN as usize);
347        unsafe {
348            std::slice::from_raw_parts(
349                std::ptr::from_ref::<u128>(view)
350                    .cast::<u8>()
351                    .wrapping_add(4),
352                len,
353            )
354        }
355    }
356
357    /// Constructs a new iterator for iterating over the values of this array
358    pub fn iter(&self) -> ArrayIter<&Self> {
359        ArrayIter::new(self)
360    }
361
362    /// Returns an iterator over the bytes of this array, including null values
363    pub fn bytes_iter(&self) -> impl Iterator<Item = &[u8]> {
364        self.views.iter().map(move |v| {
365            let len = *v as u32;
366            if len <= MAX_INLINE_VIEW_LEN {
367                unsafe { Self::inline_value(v, len as usize) }
368            } else {
369                let view = ByteView::from(*v);
370                let data = &self.buffers[view.buffer_index as usize];
371                let offset = view.offset as usize;
372                unsafe { data.get_unchecked(offset..offset + len as usize) }
373            }
374        })
375    }
376
377    /// Returns an iterator over the first `prefix_len` bytes of each array
378    /// element, including null values.
379    ///
380    /// If `prefix_len` is larger than the element's length, the iterator will
381    /// return an empty slice (`&[]`).
382    pub fn prefix_bytes_iter(&self, prefix_len: usize) -> impl Iterator<Item = &[u8]> {
383        self.views().into_iter().map(move |v| {
384            let len = (*v as u32) as usize;
385
386            if len < prefix_len {
387                return &[] as &[u8];
388            }
389
390            if prefix_len <= 4 || len as u32 <= MAX_INLINE_VIEW_LEN {
391                unsafe { StringViewArray::inline_value(v, prefix_len) }
392            } else {
393                let view = ByteView::from(*v);
394                let data = unsafe {
395                    self.data_buffers()
396                        .get_unchecked(view.buffer_index as usize)
397                };
398                let offset = view.offset as usize;
399                unsafe { data.get_unchecked(offset..offset + prefix_len) }
400            }
401        })
402    }
403
404    /// Returns an iterator over the last `suffix_len` bytes of each array
405    /// element, including null values.
406    ///
407    /// Note that for [`StringViewArray`] the last bytes may start in the middle
408    /// of a UTF-8 codepoint, and thus may not be a valid `&str`.
409    ///
410    /// If `suffix_len` is larger than the element's length, the iterator will
411    /// return an empty slice (`&[]`).
412    pub fn suffix_bytes_iter(&self, suffix_len: usize) -> impl Iterator<Item = &[u8]> {
413        self.views().into_iter().map(move |v| {
414            let len = (*v as u32) as usize;
415
416            if len < suffix_len {
417                return &[] as &[u8];
418            }
419
420            if len as u32 <= MAX_INLINE_VIEW_LEN {
421                unsafe { &StringViewArray::inline_value(v, len)[len - suffix_len..] }
422            } else {
423                let view = ByteView::from(*v);
424                let data = unsafe {
425                    self.data_buffers()
426                        .get_unchecked(view.buffer_index as usize)
427                };
428                let offset = view.offset as usize;
429                unsafe { data.get_unchecked(offset + len - suffix_len..offset + len) }
430            }
431        })
432    }
433
434    /// Return an iterator over the length of each array element, including null values.
435    ///
436    /// Null values length would equal to the underlying bytes length and NOT 0
437    ///
438    /// Example of getting 0 for null values
439    /// ```rust
440    /// # use arrow_array::StringViewArray;
441    /// # use arrow_array::Array;
442    /// use arrow_data::ByteView;
443    ///
444    /// fn lengths_with_zero_for_nulls(view: &StringViewArray) -> impl Iterator<Item = u32> {
445    ///     view.lengths()
446    ///         .enumerate()
447    ///         .map(|(index, length)| if view.is_null(index) { 0 } else { length })
448    /// }
449    /// ```
450    pub fn lengths(&self) -> impl ExactSizeIterator<Item = u32> + Clone {
451        self.views().iter().map(|v| *v as u32)
452    }
453
454    /// Returns a zero-copy slice of this array with the indicated offset and length.
455    ///
456    /// # Panics
457    /// Panics if `offset + length > self.len()`
458    pub fn slice(&self, offset: usize, length: usize) -> Self {
459        Self {
460            data_type: T::DATA_TYPE,
461            views: self.views.slice(offset, length),
462            buffers: self.buffers.clone(),
463            nulls: self.nulls.as_ref().map(|n| n.slice(offset, length)),
464            phantom: Default::default(),
465        }
466    }
467
468    /// Returns a "compacted" version of this array
469    ///
470    /// The original array will *not* be modified
471    ///
472    /// # Garbage Collection
473    ///
474    /// Before GC:
475    /// ```text
476    ///                                        ┌──────┐
477    ///                                        │......│
478    ///                                        │......│
479    /// ┌────────────────────┐       ┌ ─ ─ ─ ▶ │Data1 │   Large buffer
480    /// │       View 1       │─ ─ ─ ─          │......│  with data that
481    /// ├────────────────────┤                 │......│ is not referred
482    /// │       View 2       │─ ─ ─ ─ ─ ─ ─ ─▶ │Data2 │ to by View 1 or
483    /// └────────────────────┘                 │......│      View 2
484    ///                                        │......│
485    ///    2 views, refer to                   │......│
486    ///   small portions of a                  └──────┘
487    ///      large buffer
488    /// ```
489    ///
490    /// After GC:
491    ///
492    /// ```text
493    /// ┌────────────────────┐                 ┌─────┐    After gc, only
494    /// │       View 1       │─ ─ ─ ─ ─ ─ ─ ─▶ │Data1│     data that is
495    /// ├────────────────────┤       ┌ ─ ─ ─ ▶ │Data2│    pointed to by
496    /// │       View 2       │─ ─ ─ ─          └─────┘     the views is
497    /// └────────────────────┘                                 left
498    ///
499    ///
500    ///         2 views
501    /// ```
502    /// This method will compact the data buffers by recreating the view array and only include the data
503    /// that is pointed to by the views.
504    ///
505    /// Note that it will copy the array regardless of whether the original array is compact.
506    /// Use with caution as this can be an expensive operation, only use it when you are sure that the view
507    /// array is significantly smaller than when it is originally created, e.g., after filtering or slicing.
508    ///
509    /// Note: this function does not attempt to canonicalize / deduplicate values. For this
510    /// feature see  [`GenericByteViewBuilder::with_deduplicate_strings`].
511    pub fn gc(&self) -> Self {
512        // A single Arrow data buffer is addressed by a `u32` offset, so no
513        // output buffer may exceed `i32::MAX` bytes. Above that the data is
514        // split across multiple buffers (the "slow path").
515        self.gc_with_max_buffer_size(i32::MAX as usize)
516    }
517
518    /// Like [`Self::gc`], but with a configurable maximum output buffer size.
519    ///
520    /// `gc` always uses `i32::MAX` (the largest a single Arrow buffer can be).
521    /// This method exists so the multi-buffer split path — which `gc` only
522    /// reaches once a column references more than 2 GiB of non-inline data —
523    /// can be exercised in tests with a small threshold and tiny inputs.
524    fn gc_with_max_buffer_size(&self, max_buffer_size: usize) -> Self {
525        // 1) Read basic properties once
526        let len = self.len(); // number of elements
527        let nulls = self.nulls().cloned(); // reuse & clone existing null bitmap
528
529        // 1.5) Fast path: if there are no buffers, just reuse original views and no data blocks
530        if self.data_buffers().is_empty() {
531            return unsafe {
532                GenericByteViewArray::new_unchecked(
533                    self.views().clone(),
534                    Arc::from([]), // empty data blocks
535                    nulls,
536                )
537            };
538        }
539
540        // 2) Calculate total size of all non-inline data and detect if any exists
541        let total_large = self.total_buffer_bytes_used();
542
543        // 2.5) Fast path: if there is no non-inline data, avoid buffer allocation & processing
544        if total_large == 0 {
545            // Views are inline-only or all null; just reuse original views and no data blocks
546            return unsafe {
547                GenericByteViewArray::new_unchecked(
548                    self.views().clone(),
549                    Arc::from([]), // empty data blocks
550                    nulls,
551                )
552            };
553        }
554
555        let (views_buf, data_blocks) = if total_large <= max_buffer_size {
556            // fast path, the entire data fits in a single buffer
557            // 3) Allocate exactly capacity for all non-inline data
558            let mut data_buf = Vec::with_capacity(total_large);
559
560            // 4) Iterate over views and process each inline/non-inline view
561            let views_buf: Vec<u128> = (0..len)
562                .map(|i| unsafe { self.copy_view_to_buffer(i, 0, &mut data_buf) })
563                .collect();
564            let data_block = Buffer::from_vec(data_buf);
565            let data_blocks = vec![data_block];
566            (views_buf, data_blocks)
567        } else {
568            // slow path: the non-inline data does not fit in a single buffer
569            // (a buffer offset is a `u32`, so no buffer may exceed `i32::MAX`),
570            // so it must be split across several buffers.
571
572            // A contiguous run of views destined for one output buffer.
573            struct GcCopyGroup {
574                total_buffer_bytes: usize,
575                total_len: usize,
576            }
577
578            // First pass: partition the views into contiguous groups, each of
579            // which fits within one output buffer. `total_len` counts *all*
580            // views in the group, not just the non-inline ones — this is
581            // essential for correctness, as gc must preserve the row count.
582            // Inline views reference no buffer and are copied unchanged, but
583            // they still belong to a group and must be accounted for. Recording
584            // each group's exact byte size lets the second pass size every
585            // buffer precisely, with no over-reservation or trailing waste.
586            let mut groups: Vec<GcCopyGroup> = Vec::new();
587            // Bytes accumulated for the current group. A view length is a `u32`
588            // and a single value may be larger than `max_buffer_size`, so the
589            // running sum is kept in `u64` to stay within the addressable
590            // buffer-offset space without wrapping.
591            let mut current_length: u64 = 0;
592            let mut current_elements = 0;
593
594            for view in self.views() {
595                let view_len = *view as u32;
596                if view_len > MAX_INLINE_VIEW_LEN {
597                    // Seal the current group before adding this view would push
598                    // its buffer past `max_buffer_size`.
599                    if current_length + view_len as u64 > max_buffer_size as u64 {
600                        groups.push(GcCopyGroup {
601                            total_buffer_bytes: current_length as usize,
602                            total_len: current_elements,
603                        });
604                        current_length = 0;
605                        current_elements = 0;
606                    }
607                    current_length += view_len as u64;
608                }
609                current_elements += 1;
610            }
611            if current_elements != 0 {
612                groups.push(GcCopyGroup {
613                    total_buffer_bytes: current_length as usize,
614                    total_len: current_elements,
615                });
616            }
617            debug_assert!(i32::try_from(groups.len()).is_ok());
618
619            // Second pass: copy each group into an exactly-sized buffer.
620            let mut views_buf = Vec::with_capacity(len);
621            let mut data_blocks = Vec::with_capacity(groups.len());
622            let mut current_view_idx = 0;
623
624            for (group_idx, group) in groups.iter().enumerate() {
625                let mut data_buf = Vec::with_capacity(group.total_buffer_bytes);
626
627                // Directly push views to avoid an intermediate Vec allocation.
628                let new_views =
629                    (current_view_idx..current_view_idx + group.total_len).map(|view_idx| {
630                        // SAFETY: `view_idx` came from iterating a valid range
631                        // within `len`, and every view refers to valid data.
632                        unsafe {
633                            self.copy_view_to_buffer(view_idx, group_idx as i32, &mut data_buf)
634                        }
635                    });
636                views_buf.extend(new_views);
637
638                data_blocks.push(Buffer::from_vec(data_buf));
639                current_view_idx += group.total_len;
640            }
641            (views_buf, data_blocks)
642        };
643
644        // 5) Wrap up views buffer
645        let views_scalar = ScalarBuffer::from(views_buf);
646
647        // SAFETY: views_scalar, data_blocks, and nulls are correctly aligned and sized
648        unsafe { GenericByteViewArray::new_unchecked(views_scalar, data_blocks.into(), nulls) }
649    }
650
651    /// Copy the i‑th view into `data_buf` if it refers to an out‑of‑line buffer.
652    ///
653    /// # Safety
654    ///
655    /// - `i < self.len()`.
656    /// - Every element in `self.views()` must currently refer to a valid slice
657    ///   inside one of `self.buffers`.
658    /// - `data_buf` must be ready to have additional bytes appended.
659    /// - After this call, the returned view will have its
660    ///   `buffer_index` reset to `buffer_idx` and its `offset` updated so that it points
661    ///   into the bytes just appended at the end of `data_buf`.
662    #[inline(always)]
663    unsafe fn copy_view_to_buffer(
664        &self,
665        i: usize,
666        buffer_idx: i32,
667        data_buf: &mut Vec<u8>,
668    ) -> u128 {
669        // SAFETY: `i < self.len()` ensures this is in‑bounds.
670        let raw_view = unsafe { *self.views().get_unchecked(i) };
671        let mut bv = ByteView::from(raw_view);
672
673        // Inline‑small views stay as‑is.
674        if bv.length <= MAX_INLINE_VIEW_LEN {
675            raw_view
676        } else {
677            // SAFETY: `bv.buffer_index` and `bv.offset..bv.offset+bv.length`
678            // must both lie within valid ranges for `self.buffers`.
679            let buffer = unsafe { self.buffers.get_unchecked(bv.buffer_index as usize) };
680            let start = bv.offset as usize;
681            let end = start + bv.length as usize;
682            let slice = unsafe { buffer.get_unchecked(start..end) };
683
684            // Copy out‑of‑line data into our single “0” buffer.
685            let new_offset = data_buf.len() as u32;
686            data_buf.extend_from_slice(slice);
687
688            bv.buffer_index = buffer_idx as u32;
689            bv.offset = new_offset;
690            bv.into()
691        }
692    }
693
694    /// Returns the total number of bytes of all non-null values in this array.
695    ///
696    /// Unlike [`Self::total_buffer_bytes_used`], this method includes inlined strings
697    /// (those with length ≤ [`MAX_INLINE_VIEW_LEN`]), making it suitable as a
698    /// capacity hint when pre-allocating output buffers.
699    ///
700    /// Null values are excluded from the sum.
701    ///
702    /// # Example
703    ///
704    /// ```rust
705    /// # use arrow_array::StringViewArray;
706    /// let array = StringViewArray::from_iter(vec![
707    ///     Some("hello"),   // 5 bytes, inlined
708    ///     None,            // excluded
709    ///     Some("large payload over 12 bytes"),  // 27 bytes, non-inlined
710    /// ]);
711    /// assert_eq!(array.total_bytes_len(), 5 + 27);
712    /// ```
713    pub fn total_bytes_len(&self) -> usize {
714        match self.nulls() {
715            None => self.views().iter().map(|v| (*v as u32) as usize).sum(),
716            Some(nulls) => self
717                .views()
718                .iter()
719                .zip(nulls.iter())
720                .map(|(v, is_valid)| if is_valid { (*v as u32) as usize } else { 0 })
721                .sum(),
722        }
723    }
724
725    /// Returns the total number of bytes used by all non inlined views in all
726    /// buffers.
727    ///
728    /// Note this does not account for views that point at the same underlying
729    /// data in buffers
730    ///
731    /// For example, if the array has three strings views:
732    /// * View with length = 9 (inlined)
733    /// * View with length = 32 (non inlined)
734    /// * View with length = 16 (non inlined)
735    ///
736    /// Then this method would report 48
737    pub fn total_buffer_bytes_used(&self) -> usize {
738        self.views()
739            .iter()
740            .map(|v| {
741                let len = *v as u32;
742                if len > MAX_INLINE_VIEW_LEN {
743                    len as usize
744                } else {
745                    0
746                }
747            })
748            .sum()
749    }
750
751    /// Compare two [`GenericByteViewArray`] at index `left_idx` and `right_idx`
752    ///
753    /// Comparing two ByteView types are non-trivial.
754    /// It takes a bit of patience to understand why we don't just compare two &[u8] directly.
755    ///
756    /// ByteView types give us the following two advantages, and we need to be careful not to lose them:
757    /// (1) For string/byte smaller than [`MAX_INLINE_VIEW_LEN`] bytes, the entire data is inlined in the view.
758    ///     Meaning that reading one array element requires only one memory access
759    ///     (two memory access required for StringArray, one for offset buffer, the other for value buffer).
760    ///
761    /// (2) For string/byte larger than [`MAX_INLINE_VIEW_LEN`] bytes, we can still be faster than (for certain operations) StringArray/ByteArray,
762    ///     thanks to the inlined 4 bytes.
763    ///     Consider equality check:
764    ///     If the first four bytes of the two strings are different, we can return false immediately (with just one memory access).
765    ///
766    /// If we directly compare two &[u8], we materialize the entire string (i.e., make multiple memory accesses), which might be unnecessary.
767    /// - Most of the time (eq, ord), we only need to look at the first 4 bytes to know the answer,
768    ///   e.g., if the inlined 4 bytes are different, we can directly return unequal without looking at the full string.
769    ///
770    /// # Order check flow
771    /// (1) if both string are smaller than [`MAX_INLINE_VIEW_LEN`] bytes, we can directly compare the data inlined to the view.
772    /// (2) if any of the string is larger than [`MAX_INLINE_VIEW_LEN`] bytes, we need to compare the full string.
773    ///     (2.1) if the inlined 4 bytes are different, we can return the result immediately.
774    ///     (2.2) o.w., we need to compare the full string.
775    ///
776    /// # Safety
777    /// The left/right_idx must within range of each array
778    pub unsafe fn compare_unchecked(
779        left: &GenericByteViewArray<T>,
780        left_idx: usize,
781        right: &GenericByteViewArray<T>,
782        right_idx: usize,
783    ) -> Ordering {
784        let l_view = unsafe { left.views().get_unchecked(left_idx) };
785        let l_byte_view = ByteView::from(*l_view);
786
787        let r_view = unsafe { right.views().get_unchecked(right_idx) };
788        let r_byte_view = ByteView::from(*r_view);
789
790        let l_len = l_byte_view.length;
791        let r_len = r_byte_view.length;
792
793        if l_len <= 12 && r_len <= 12 {
794            return Self::inline_key_fast(*l_view).cmp(&Self::inline_key_fast(*r_view));
795        }
796
797        // one of the string is larger than 12 bytes,
798        // we then try to compare the inlined data first
799
800        // Note: In theory, ByteView is only used for string which is larger than 12 bytes,
801        // but we can still use it to get the inlined prefix for shorter strings.
802        // The prefix is always the first 4 bytes of the view, for both short and long strings.
803        let l_inlined_be = l_byte_view.prefix.swap_bytes();
804        let r_inlined_be = r_byte_view.prefix.swap_bytes();
805        if l_inlined_be != r_inlined_be {
806            return l_inlined_be.cmp(&r_inlined_be);
807        }
808
809        // unfortunately, we need to compare the full data
810        let l_full_data: &[u8] = unsafe { left.value_unchecked(left_idx).as_ref() };
811        let r_full_data: &[u8] = unsafe { right.value_unchecked(right_idx).as_ref() };
812
813        l_full_data.cmp(r_full_data)
814    }
815
816    /// Builds a 128-bit composite key for an inline value:
817    ///
818    /// - High 96 bits: the inline data in big-endian byte order (for correct lexicographical sorting).
819    /// - Low  32 bits: the length in big-endian byte order, acting as a tiebreaker so shorter strings
820    ///   (or those with fewer meaningful bytes) always numerically sort before longer ones.
821    ///
822    /// This function extracts the length and the 12-byte inline string data from the raw
823    /// little-endian `u128` representation, converts them to big-endian ordering, and packs them
824    /// into a single `u128` value suitable for fast, branchless comparisons.
825    ///
826    /// # Why include length?
827    ///
828    /// A pure 96-bit content comparison can’t distinguish between two values whose inline bytes
829    /// compare equal—either because one is a true prefix of the other or because zero-padding
830    /// hides extra bytes. By tucking the 32-bit length into the lower bits, a single `u128` compare
831    /// handles both content and length in one go.
832    ///
833    /// Example: comparing "bar" (3 bytes) vs "bar\0" (4 bytes)
834    ///
835    /// | String     | Bytes 0–4 (length LE) | Bytes 4–16 (data + padding)    |
836    /// |------------|-----------------------|---------------------------------|
837    /// | `"bar"`   | `03 00 00 00`         | `62 61 72` + 9 × `00`           |
838    /// | `"bar\0"`| `04 00 00 00`         | `62 61 72 00` + 8 × `00`        |
839    ///
840    /// Both inline parts become `62 61 72 00…00`, so they tie on content. The length field
841    /// then differentiates:
842    ///
843    /// ```text
844    /// key("bar")   = 0x0000000000000000000062617200000003
845    /// key("bar\0") = 0x0000000000000000000062617200000004
846    /// ⇒ key("bar") < key("bar\0")
847    /// ```
848    /// - `raw` is treated as a 128-bit integer with its bits laid out as follows:
849    ///   - bits 0–31: length (little-endian)
850    ///   - bits 32–127: data (little-endian)
851    ///
852    /// # Inlining and Endianness
853    ///
854    /// This function uses platform-independent bitwise operations to construct a 128-bit key:
855    /// - `raw.swap_bytes() << 32` effectively clears the length bits and shifts the 12-byte inline data
856    ///   into the high 96 bits in Big-Endian order. This ensures the first byte of the string
857    ///   is the most significant byte of the resulting `u128`.
858    /// - `raw as u32` extracts the length as a numeric integer, which is then placed in the low 32 bits.
859    #[inline(always)]
860    pub fn inline_key_fast(raw: u128) -> u128 {
861        (raw.swap_bytes() << 32) | (raw as u32 as u128)
862    }
863}
864
865impl<T: ByteViewType + ?Sized> Debug for GenericByteViewArray<T> {
866    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
867        write!(f, "{}ViewArray\n[\n", T::PREFIX)?;
868        print_long_array(self, f, &mut |index, f| {
869            std::fmt::Debug::fmt(&self.value(index), f)
870        })?;
871        write!(f, "]")
872    }
873}
874
875/// SAFETY: Correctly implements the contract of Arrow Arrays
876unsafe impl<T: ByteViewType + ?Sized> Array for GenericByteViewArray<T> {
877    fn as_any(&self) -> &dyn Any {
878        self
879    }
880
881    fn to_data(&self) -> ArrayData {
882        self.clone().into()
883    }
884
885    fn into_data(self) -> ArrayData {
886        self.into()
887    }
888
889    fn data_type(&self) -> &DataType {
890        &self.data_type
891    }
892
893    fn slice(&self, offset: usize, length: usize) -> ArrayRef {
894        Arc::new(self.slice(offset, length))
895    }
896
897    fn len(&self) -> usize {
898        self.views.len()
899    }
900
901    fn is_empty(&self) -> bool {
902        self.views.is_empty()
903    }
904
905    fn shrink_to_fit(&mut self) {
906        self.views.shrink_to_fit();
907
908        // The goal of `shrink_to_fit` is to minimize the space used by any of
909        // its allocations. The use of `Arc::get_mut` over `Arc::make_mut` is
910        // because if the reference count is greater than 1, `Arc::make_mut`
911        // will first clone its contents. So, any large allocations will first
912        // be cloned before being shrunk, leaving the pre-cloned allocations
913        // intact, before adding the extra (used) space of the new clones.
914        if let Some(buffers) = Arc::get_mut(&mut self.buffers) {
915            buffers.iter_mut().for_each(|b| b.shrink_to_fit());
916        }
917
918        // With the assumption that this is a best-effort function, no attempt
919        // is made to shrink `self.buffers`, which it can't because it's type
920        // does not expose a `shrink_to_fit` method.
921
922        if let Some(nulls) = &mut self.nulls {
923            nulls.shrink_to_fit();
924        }
925    }
926
927    fn offset(&self) -> usize {
928        0
929    }
930
931    fn nulls(&self) -> Option<&NullBuffer> {
932        self.nulls.as_ref()
933    }
934
935    fn logical_null_count(&self) -> usize {
936        // More efficient that the default implementation
937        self.null_count()
938    }
939
940    fn get_buffer_memory_size(&self) -> usize {
941        let mut sum = self.buffers.iter().map(|b| b.capacity()).sum::<usize>();
942        sum += self.views.inner().capacity();
943        if let Some(x) = &self.nulls {
944            sum += x.buffer().capacity()
945        }
946        sum
947    }
948
949    fn get_array_memory_size(&self) -> usize {
950        std::mem::size_of::<Self>() + self.get_buffer_memory_size()
951    }
952
953    #[cfg(feature = "pool")]
954    fn claim(&self, pool: &dyn arrow_buffer::MemoryPool) {
955        self.views.claim(pool);
956        for buffer in self.buffers.iter() {
957            buffer.claim(pool);
958        }
959        if let Some(nulls) = &self.nulls {
960            nulls.claim(pool);
961        }
962    }
963}
964
965impl<'a, T: ByteViewType + ?Sized> ArrayAccessor for &'a GenericByteViewArray<T> {
966    type Item = &'a T::Native;
967
968    fn value(&self, index: usize) -> Self::Item {
969        GenericByteViewArray::value(self, index)
970    }
971
972    unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
973        unsafe { GenericByteViewArray::value_unchecked(self, index) }
974    }
975}
976
977impl<'a, T: ByteViewType + ?Sized> IntoIterator for &'a GenericByteViewArray<T> {
978    type Item = Option<&'a T::Native>;
979    type IntoIter = ArrayIter<Self>;
980
981    fn into_iter(self) -> Self::IntoIter {
982        ArrayIter::new(self)
983    }
984}
985
986impl<T: ByteViewType + ?Sized> From<ArrayData> for GenericByteViewArray<T> {
987    fn from(data: ArrayData) -> Self {
988        let (data_type, len, nulls, offset, buffers, _child_data) = data.into_parts();
989        assert_eq!(
990            data_type,
991            T::DATA_TYPE,
992            "Mismatched data type, expected {}, got {data_type}",
993            T::DATA_TYPE
994        );
995        let mut buffers = buffers.into_iter();
996        // first buffer is views, remaining are data buffers
997        let views = ScalarBuffer::new(buffers.next().unwrap(), offset, len);
998        Self {
999            data_type,
1000            views,
1001            buffers: Arc::from_iter(buffers),
1002            nulls,
1003            phantom: Default::default(),
1004        }
1005    }
1006}
1007
1008/// Efficiently convert a [`GenericByteArray`] to a [`GenericByteViewArray`]
1009///
1010/// For example this method can convert a [`StringArray`] to a
1011/// [`StringViewArray`].
1012///
1013/// If the offsets are all less than u32::MAX, the new [`GenericByteViewArray`]
1014/// is built without copying the underlying string data (views are created
1015/// directly into the existing buffer)
1016///
1017/// [`StringArray`]: crate::StringArray
1018impl<FROM, V> From<&GenericByteArray<FROM>> for GenericByteViewArray<V>
1019where
1020    FROM: ByteArrayType,
1021    FROM::Offset: OffsetSizeTrait + ToPrimitive,
1022    V: ByteViewType<Native = FROM::Native>,
1023{
1024    fn from(byte_array: &GenericByteArray<FROM>) -> Self {
1025        let offsets = byte_array.offsets();
1026
1027        let can_reuse_buffer = offsets.last().as_usize() < u32::MAX as usize;
1028
1029        if can_reuse_buffer {
1030            // build views directly pointing to the existing buffer
1031            let len = byte_array.len();
1032            let mut views_builder = GenericByteViewBuilder::<V>::with_capacity(len);
1033            let str_values_buf = byte_array.values().clone();
1034            let block = views_builder.append_block(str_values_buf);
1035            for (i, w) in offsets.windows(2).enumerate() {
1036                let offset = w[0].as_usize();
1037                let end = w[1].as_usize();
1038                let length = end - offset;
1039
1040                if byte_array.is_null(i) {
1041                    views_builder.append_null();
1042                } else {
1043                    // Safety: the input was a valid array so it valid UTF8 (if string). And
1044                    // all offsets were valid
1045                    unsafe {
1046                        views_builder.append_view_unchecked(block, offset as u32, length as u32)
1047                    }
1048                }
1049            }
1050            assert_eq!(views_builder.len(), len);
1051            views_builder.finish()
1052        } else {
1053            // Otherwise, create a new buffer for large strings
1054            // TODO: the original buffer could still be used
1055            // by making multiple slices of u32::MAX length
1056            GenericByteViewArray::<V>::from_iter(byte_array.iter())
1057        }
1058    }
1059}
1060
1061impl<T: ByteViewType + ?Sized> From<GenericByteViewArray<T>> for ArrayData {
1062    fn from(array: GenericByteViewArray<T>) -> Self {
1063        let len = array.len();
1064
1065        let mut buffers = array.buffers.to_vec();
1066        buffers.insert(0, array.views.into_inner());
1067
1068        let builder = ArrayDataBuilder::new(T::DATA_TYPE)
1069            .len(len)
1070            .buffers(buffers)
1071            .nulls(array.nulls);
1072
1073        unsafe { builder.build_unchecked() }
1074    }
1075}
1076
1077impl<'a, Ptr, T> FromIterator<&'a Option<Ptr>> for GenericByteViewArray<T>
1078where
1079    Ptr: AsRef<T::Native> + 'a,
1080    T: ByteViewType + ?Sized,
1081{
1082    fn from_iter<I: IntoIterator<Item = &'a Option<Ptr>>>(iter: I) -> Self {
1083        iter.into_iter()
1084            .map(|o| o.as_ref().map(|p| p.as_ref()))
1085            .collect()
1086    }
1087}
1088
1089impl<Ptr, T: ByteViewType + ?Sized> FromIterator<Option<Ptr>> for GenericByteViewArray<T>
1090where
1091    Ptr: AsRef<T::Native>,
1092{
1093    fn from_iter<I: IntoIterator<Item = Option<Ptr>>>(iter: I) -> Self {
1094        let iter = iter.into_iter();
1095        let mut builder = GenericByteViewBuilder::<T>::with_capacity(iter.size_hint().0);
1096        builder.extend(iter);
1097        builder.finish()
1098    }
1099}
1100
1101/// A [`GenericByteViewArray`] of `[u8]`
1102///
1103/// See [`GenericByteViewArray`] for format and layout details.
1104///
1105/// # Example
1106/// ```
1107/// use arrow_array::BinaryViewArray;
1108/// let array = BinaryViewArray::from_iter_values(vec![b"hello" as &[u8], b"world", b"lulu", b"large payload over 12 bytes"]);
1109/// assert_eq!(array.value(0), b"hello");
1110/// assert_eq!(array.value(3), b"large payload over 12 bytes");
1111/// ```
1112pub type BinaryViewArray = GenericByteViewArray<BinaryViewType>;
1113
1114impl BinaryViewArray {
1115    /// Convert the [`BinaryViewArray`] to [`StringViewArray`]
1116    /// If items not utf8 data, validate will fail and error returned.
1117    pub fn to_string_view(self) -> Result<StringViewArray, ArrowError> {
1118        StringViewType::validate(self.views(), self.data_buffers())?;
1119        unsafe { Ok(self.to_string_view_unchecked()) }
1120    }
1121
1122    /// Convert the [`BinaryViewArray`] to [`StringViewArray`]
1123    /// # Safety
1124    /// Caller is responsible for ensuring that items in array are utf8 data.
1125    pub unsafe fn to_string_view_unchecked(self) -> StringViewArray {
1126        unsafe { StringViewArray::new_unchecked(self.views, self.buffers, self.nulls) }
1127    }
1128}
1129
1130impl From<Vec<&[u8]>> for BinaryViewArray {
1131    fn from(v: Vec<&[u8]>) -> Self {
1132        Self::from_iter_values(v)
1133    }
1134}
1135
1136impl From<Vec<Option<&[u8]>>> for BinaryViewArray {
1137    fn from(v: Vec<Option<&[u8]>>) -> Self {
1138        v.into_iter().collect()
1139    }
1140}
1141
1142/// A [`GenericByteViewArray`] that stores utf8 data
1143///
1144/// See [`GenericByteViewArray`] for format and layout details.
1145///
1146/// # Example
1147/// ```
1148/// use arrow_array::StringViewArray;
1149/// let array = StringViewArray::from_iter_values(vec!["hello", "world", "lulu", "large payload over 12 bytes"]);
1150/// assert_eq!(array.value(0), "hello");
1151/// assert_eq!(array.value(3), "large payload over 12 bytes");
1152/// ```
1153pub type StringViewArray = GenericByteViewArray<StringViewType>;
1154
1155impl StringViewArray {
1156    /// Convert the [`StringViewArray`] to [`BinaryViewArray`]
1157    pub fn to_binary_view(self) -> BinaryViewArray {
1158        unsafe { BinaryViewArray::new_unchecked(self.views, self.buffers, self.nulls) }
1159    }
1160
1161    /// Returns true if all data within this array is ASCII
1162    pub fn is_ascii(&self) -> bool {
1163        // Alternative (but incorrect): directly check the underlying buffers
1164        // (1) Our string view might be sparse, i.e., a subset of the buffers,
1165        //      so even if the buffer is not ascii, we can still be ascii.
1166        // (2) It is quite difficult to know the range of each buffer (unlike StringArray)
1167        // This means that this operation is quite expensive, shall we cache the result?
1168        //  i.e. track `is_ascii` in the builder.
1169        self.iter().all(|v| match v {
1170            Some(v) => v.is_ascii(),
1171            None => true,
1172        })
1173    }
1174}
1175
1176impl From<Vec<&str>> for StringViewArray {
1177    fn from(v: Vec<&str>) -> Self {
1178        Self::from_iter_values(v)
1179    }
1180}
1181
1182impl From<Vec<Option<&str>>> for StringViewArray {
1183    fn from(v: Vec<Option<&str>>) -> Self {
1184        v.into_iter().collect()
1185    }
1186}
1187
1188impl From<Vec<String>> for StringViewArray {
1189    fn from(v: Vec<String>) -> Self {
1190        Self::from_iter_values(v)
1191    }
1192}
1193
1194impl From<Vec<Option<String>>> for StringViewArray {
1195    fn from(v: Vec<Option<String>>) -> Self {
1196        v.into_iter().collect()
1197    }
1198}
1199
1200#[cfg(test)]
1201mod tests {
1202    use crate::builder::{BinaryViewBuilder, StringViewBuilder};
1203    use crate::types::BinaryViewType;
1204    use crate::{
1205        Array, BinaryViewArray, GenericBinaryArray, GenericByteViewArray, StringViewArray,
1206    };
1207    use arrow_buffer::{Buffer, NullBuffer, ScalarBuffer};
1208    use arrow_data::{ArrayDataBuilder, ByteView, MAX_INLINE_VIEW_LEN};
1209    use arrow_schema::DataType;
1210    use rand::prelude::StdRng;
1211    use rand::{RngExt, SeedableRng};
1212    use std::str::from_utf8;
1213
1214    const BLOCK_SIZE: u32 = 8;
1215
1216    #[test]
1217    fn try_new_string() {
1218        let array = StringViewArray::from_iter_values(vec![
1219            "hello",
1220            "world",
1221            "lulu",
1222            "large payload over 12 bytes",
1223        ]);
1224        assert_eq!(array.value(0), "hello");
1225        assert_eq!(array.value(3), "large payload over 12 bytes");
1226    }
1227
1228    #[test]
1229    fn try_new_binary() {
1230        let array = BinaryViewArray::from_iter_values(vec![
1231            b"hello".as_slice(),
1232            b"world".as_slice(),
1233            b"lulu".as_slice(),
1234            b"large payload over 12 bytes".as_slice(),
1235        ]);
1236        assert_eq!(array.value(0), b"hello");
1237        assert_eq!(array.value(3), b"large payload over 12 bytes");
1238    }
1239
1240    #[test]
1241    fn try_new_empty_string() {
1242        // test empty array
1243        let array = {
1244            let mut builder = StringViewBuilder::new();
1245            builder.finish()
1246        };
1247        assert!(array.is_empty());
1248    }
1249
1250    #[test]
1251    fn try_new_empty_binary() {
1252        // test empty array
1253        let array = {
1254            let mut builder = BinaryViewBuilder::new();
1255            builder.finish()
1256        };
1257        assert!(array.is_empty());
1258    }
1259
1260    #[test]
1261    fn test_append_string() {
1262        // test builder append
1263        let array = {
1264            let mut builder = StringViewBuilder::new();
1265            builder.append_value("hello");
1266            builder.append_null();
1267            builder.append_option(Some("large payload over 12 bytes"));
1268            builder.finish()
1269        };
1270        assert_eq!(array.value(0), "hello");
1271        assert!(array.is_null(1));
1272        assert_eq!(array.value(2), "large payload over 12 bytes");
1273    }
1274
1275    #[test]
1276    fn test_append_binary() {
1277        // test builder append
1278        let array = {
1279            let mut builder = BinaryViewBuilder::new();
1280            builder.append_value(b"hello");
1281            builder.append_null();
1282            builder.append_option(Some(b"large payload over 12 bytes"));
1283            builder.finish()
1284        };
1285        assert_eq!(array.value(0), b"hello");
1286        assert!(array.is_null(1));
1287        assert_eq!(array.value(2), b"large payload over 12 bytes");
1288    }
1289
1290    #[test]
1291    fn test_in_progress_recreation() {
1292        let array = {
1293            // make a builder with small block size.
1294            let mut builder = StringViewBuilder::new().with_fixed_block_size(14);
1295            builder.append_value("large payload over 12 bytes");
1296            builder.append_option(Some("another large payload over 12 bytes that double than the first one, so that we can trigger the in_progress in builder re-created"));
1297            builder.finish()
1298        };
1299        assert_eq!(array.value(0), "large payload over 12 bytes");
1300        assert_eq!(
1301            array.value(1),
1302            "another large payload over 12 bytes that double than the first one, so that we can trigger the in_progress in builder re-created"
1303        );
1304        assert_eq!(2, array.buffers.len());
1305    }
1306
1307    #[test]
1308    #[should_panic(expected = "Invalid buffer index at 0: got index 3 but only has 1 buffers")]
1309    fn new_with_invalid_view_data() {
1310        let v = "large payload over 12 bytes";
1311        let view = ByteView::new(13, &v.as_bytes()[0..4])
1312            .with_buffer_index(3)
1313            .with_offset(1);
1314        let views = ScalarBuffer::from(vec![view.into()]);
1315        let buffers = vec![Buffer::from_slice_ref(v)];
1316        StringViewArray::new(views, buffers, None);
1317    }
1318
1319    #[test]
1320    #[should_panic(
1321        expected = "Encountered non-UTF-8 data at index 0: invalid utf-8 sequence of 1 bytes from index 0"
1322    )]
1323    fn new_with_invalid_utf8_data() {
1324        let v: Vec<u8> = vec![
1325            // invalid UTF8
1326            0xf0, 0x80, 0x80, 0x80, // more bytes to make it larger than 12
1327            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1328        ];
1329        let view = ByteView::new(v.len() as u32, &v[0..4]);
1330        let views = ScalarBuffer::from(vec![view.into()]);
1331        let buffers = vec![Buffer::from_slice_ref(v)];
1332        StringViewArray::new(views, buffers, None);
1333    }
1334
1335    #[test]
1336    #[should_panic(expected = "View at index 0 contained non-zero padding for string of length 1")]
1337    fn new_with_invalid_zero_padding() {
1338        let mut data = [0; 12];
1339        data[0] = b'H';
1340        data[11] = 1; // no zero padding
1341
1342        let mut view_buffer = [0; 16];
1343        view_buffer[0..4].copy_from_slice(&1u32.to_le_bytes());
1344        view_buffer[4..].copy_from_slice(&data);
1345
1346        let view = ByteView::from(u128::from_le_bytes(view_buffer));
1347        let views = ScalarBuffer::from(vec![view.into()]);
1348        let buffers = vec![];
1349        StringViewArray::new(views, buffers, None);
1350    }
1351
1352    #[test]
1353    #[should_panic(expected = "Mismatch between embedded prefix and data")]
1354    fn test_mismatch_between_embedded_prefix_and_data() {
1355        let input_str_1 = "Hello, Rustaceans!";
1356        let input_str_2 = "Hallo, Rustaceans!";
1357        let length = input_str_1.len() as u32;
1358        assert!(input_str_1.len() > 12);
1359
1360        let mut view_buffer = [0; 16];
1361        view_buffer[0..4].copy_from_slice(&length.to_le_bytes());
1362        view_buffer[4..8].copy_from_slice(&input_str_1.as_bytes()[0..4]);
1363        view_buffer[8..12].copy_from_slice(&0u32.to_le_bytes());
1364        view_buffer[12..].copy_from_slice(&0u32.to_le_bytes());
1365        let view = ByteView::from(u128::from_le_bytes(view_buffer));
1366        let views = ScalarBuffer::from(vec![view.into()]);
1367        let buffers = vec![Buffer::from_slice_ref(input_str_2.as_bytes())];
1368
1369        StringViewArray::new(views, buffers, None);
1370    }
1371
1372    #[test]
1373    fn test_gc() {
1374        let test_data = [
1375            Some("longer than 12 bytes"),
1376            Some("short"),
1377            Some("t"),
1378            Some("longer than 12 bytes"),
1379            None,
1380            Some("short"),
1381        ];
1382
1383        let array = {
1384            let mut builder = StringViewBuilder::new().with_fixed_block_size(8); // create multiple buffers
1385            test_data.into_iter().for_each(|v| builder.append_option(v));
1386            builder.finish()
1387        };
1388        assert!(array.buffers.len() > 1);
1389
1390        fn check_gc(to_test: &StringViewArray) {
1391            let gc = to_test.gc();
1392            assert_ne!(to_test.data_buffers().len(), gc.data_buffers().len());
1393
1394            to_test.iter().zip(gc.iter()).for_each(|(a, b)| {
1395                assert_eq!(a, b);
1396            });
1397            assert_eq!(to_test.len(), gc.len());
1398        }
1399
1400        check_gc(&array);
1401        check_gc(&array.slice(1, 3));
1402        check_gc(&array.slice(2, 1));
1403        check_gc(&array.slice(2, 2));
1404        check_gc(&array.slice(3, 1));
1405    }
1406
1407    /// 1) Empty array: no elements, expect gc to return empty with no data buffers
1408    #[test]
1409    fn test_gc_empty_array() {
1410        let array = StringViewBuilder::new()
1411            .with_fixed_block_size(BLOCK_SIZE)
1412            .finish();
1413        let gced = array.gc();
1414        // length and null count remain zero
1415        assert_eq!(gced.len(), 0);
1416        assert_eq!(gced.null_count(), 0);
1417        // no underlying data buffers should be allocated
1418        assert!(
1419            gced.data_buffers().is_empty(),
1420            "Expected no data buffers for empty array"
1421        );
1422    }
1423
1424    /// 2) All inline values (<= INLINE_LEN): capacity-only data buffer, same values
1425    #[test]
1426    fn test_gc_all_inline() {
1427        let mut builder = StringViewBuilder::new().with_fixed_block_size(BLOCK_SIZE);
1428        // append many short strings, each exactly INLINE_LEN long
1429        for _ in 0..100 {
1430            let s = "A".repeat(MAX_INLINE_VIEW_LEN as usize);
1431            builder.append_option(Some(&s));
1432        }
1433        let array = builder.finish();
1434        let gced = array.gc();
1435        // Since all views fit inline, data buffer is empty
1436        assert_eq!(
1437            gced.data_buffers().len(),
1438            0,
1439            "Should have no data buffers for inline values"
1440        );
1441        assert_eq!(gced.len(), 100);
1442        // verify element-wise equality
1443        array.iter().zip(gced.iter()).for_each(|(orig, got)| {
1444            assert_eq!(orig, got, "Inline value mismatch after gc");
1445        });
1446    }
1447
1448    /// 3) All large values (> INLINE_LEN): each must be copied into the new data buffer
1449    #[test]
1450    fn test_gc_all_large() {
1451        let mut builder = StringViewBuilder::new().with_fixed_block_size(BLOCK_SIZE);
1452        let large_str = "X".repeat(MAX_INLINE_VIEW_LEN as usize + 5);
1453        // append multiple large strings
1454        for _ in 0..50 {
1455            builder.append_option(Some(&large_str));
1456        }
1457        let array = builder.finish();
1458        let gced = array.gc();
1459        // New data buffers should be populated (one or more blocks)
1460        assert!(
1461            !gced.data_buffers().is_empty(),
1462            "Expected data buffers for large values"
1463        );
1464        assert_eq!(gced.len(), 50);
1465        // verify that every large string emerges unchanged
1466        array.iter().zip(gced.iter()).for_each(|(orig, got)| {
1467            assert_eq!(orig, got, "Large view mismatch after gc");
1468        });
1469    }
1470
1471    /// 4) All null elements: ensure null bitmap handling path is correct
1472    #[test]
1473    fn test_gc_all_nulls() {
1474        let mut builder = StringViewBuilder::new().with_fixed_block_size(BLOCK_SIZE);
1475        for _ in 0..20 {
1476            builder.append_null();
1477        }
1478        let array = builder.finish();
1479        let gced = array.gc();
1480        // length and null count match
1481        assert_eq!(gced.len(), 20);
1482        assert_eq!(gced.null_count(), 20);
1483        // data buffers remain empty for null-only array
1484        assert!(
1485            gced.data_buffers().is_empty(),
1486            "No data should be stored for nulls"
1487        );
1488    }
1489
1490    /// 5) Random mix of inline, large, and null values with slicing tests
1491    #[test]
1492    fn test_gc_random_mixed_and_slices() {
1493        let mut rng = StdRng::seed_from_u64(42);
1494        let mut builder = StringViewBuilder::new().with_fixed_block_size(BLOCK_SIZE);
1495        // Keep a Vec of original Option<String> for later comparison
1496        let mut original: Vec<Option<String>> = Vec::new();
1497
1498        for _ in 0..200 {
1499            if rng.random_bool(0.1) {
1500                // 10% nulls
1501                builder.append_null();
1502                original.push(None);
1503            } else {
1504                // random length between 0 and twice the inline limit
1505                let len = rng.random_range(0..(MAX_INLINE_VIEW_LEN * 2));
1506                let s = "A".repeat(len as usize);
1507                builder.append_option(Some(&s));
1508                original.push(Some(s));
1509            }
1510        }
1511
1512        let array = builder.finish();
1513        // Test multiple slice ranges to ensure offset logic is correct
1514        for (offset, slice_len) in &[(0, 50), (10, 100), (150, 30)] {
1515            let sliced = array.slice(*offset, *slice_len);
1516            let gced = sliced.gc();
1517            // Build expected slice of Option<&str>
1518            let expected: Vec<Option<&str>> = original[*offset..(*offset + *slice_len)]
1519                .iter()
1520                .map(|opt| opt.as_deref())
1521                .collect();
1522
1523            assert_eq!(gced.len(), *slice_len, "Slice length mismatch");
1524            // Compare element-wise
1525            gced.iter().zip(expected.iter()).for_each(|(got, expect)| {
1526                assert_eq!(got, *expect, "Value mismatch in mixed slice after gc");
1527            });
1528        }
1529    }
1530
1531    #[test]
1532    #[cfg_attr(miri, ignore)] // Takes too long
1533    fn test_gc_huge_array() {
1534        // Construct multiple 128 MiB BinaryView entries so total > 4 GiB
1535        let block_len: usize = 128 * 1024 * 1024; // 128 MiB per view
1536        let num_views: usize = 36;
1537
1538        // Create a single 128 MiB data block with a simple byte pattern
1539        let buffer = Buffer::from_vec(vec![0xAB; block_len]);
1540        let buffer2 = Buffer::from_vec(vec![0xFF; block_len]);
1541
1542        // Append this block and then add many views pointing to it
1543        let mut builder = BinaryViewBuilder::new();
1544        let block_id = builder.append_block(buffer);
1545        for _ in 0..num_views / 2 {
1546            builder
1547                .try_append_view(block_id, 0, block_len as u32)
1548                .expect("append view into 128MiB block");
1549        }
1550        let block_id2 = builder.append_block(buffer2);
1551        for _ in 0..num_views / 2 {
1552            builder
1553                .try_append_view(block_id2, 0, block_len as u32)
1554                .expect("append view into 128MiB block");
1555        }
1556
1557        let array = builder.finish();
1558        let total = array.total_buffer_bytes_used();
1559        assert!(
1560            total > u32::MAX as usize,
1561            "Expected total non-inline bytes to exceed 4 GiB, got {total}"
1562        );
1563
1564        // Run gc and verify correctness
1565        let gced = array.gc();
1566        assert_eq!(gced.len(), num_views, "Length mismatch after gc");
1567        assert_eq!(gced.null_count(), 0, "Null count mismatch after gc");
1568        assert_ne!(
1569            gced.data_buffers().len(),
1570            1,
1571            "gc with huge buffer should not consolidate data into a single buffer"
1572        );
1573
1574        // Element-wise equality check across the entire array
1575        array.iter().zip(gced.iter()).for_each(|(orig, got)| {
1576            assert_eq!(orig, got, "Value mismatch after gc on huge array");
1577        });
1578    }
1579
1580    #[test]
1581    fn test_gc_slow_path_preserves_inline_views() {
1582        // Regression test for a bug in the multi-buffer "slow path" of `gc()`,
1583        // which `gc` only reaches once a view column references more than
1584        // `i32::MAX` (~2.1 GiB) of non-inline data. That path grouped and copied
1585        // only the *non-inline* views and dropped every inline (short) view,
1586        // returning an array shorter than its input. GC is a pure size
1587        // optimisation and must never change the row count.
1588        //
1589        // Rather than allocate gigabytes to hit the real threshold, drive the
1590        // same code via `gc_with_max_buffer_size` with a tiny cap so a handful
1591        // of bytes forces the buffer split. Inline, large, and null views are
1592        // interspersed so the split lands between and around inline entries.
1593        let long = "this is definitely longer than twelve bytes"; // non-inline
1594        let test_data = [
1595            Some("short"),      // inline
1596            Some(long),         // large
1597            Some("s"),          // inline
1598            Some(long),         // large
1599            None,               // null
1600            Some(long),         // large
1601            Some("also short"), // inline
1602            Some(long),         // large
1603            Some("tail"),       // inline
1604        ];
1605        let array: StringViewArray = test_data.into_iter().collect();
1606
1607        // Cap each output buffer at ~1.5 large values so the copy is forced to
1608        // split across several buffers, exercising the slow path with no
1609        // multi-GiB allocation.
1610        let max_buffer_size = long.len() + long.len() / 2;
1611        let gced = array.gc_with_max_buffer_size(max_buffer_size);
1612
1613        // The core invariant: gc preserves the row count.
1614        assert_eq!(gced.len(), array.len(), "gc changed the row count");
1615        // The split must actually have happened, otherwise we'd be exercising
1616        // the fast path and could not catch the inline-dropping regression.
1617        assert!(
1618            gced.data_buffers().len() > 1,
1619            "expected output split across multiple buffers, got {}",
1620            gced.data_buffers().len()
1621        );
1622        // No output buffer may exceed the cap.
1623        for buf in gced.data_buffers().iter() {
1624            assert!(buf.len() <= max_buffer_size, "buffer exceeded max size");
1625        }
1626        // Every value (inline, large, and null) is unchanged and in order.
1627        array
1628            .iter()
1629            .zip(gced.iter())
1630            .enumerate()
1631            .for_each(|(i, (a, b))| {
1632                assert_eq!(a, b, "value mismatch at index {i} after gc");
1633            });
1634        // The result round-trips through full validation.
1635        gced.to_data().validate_full().unwrap();
1636    }
1637
1638    #[test]
1639    fn test_gc_slow_path_value_larger_than_buffer() {
1640        // A single non-inline value can be larger than one output buffer (a view
1641        // length is a `u32`, so a value may exceed `max_buffer_size`). Such a
1642        // value cannot be split, so it occupies a buffer of its own; gc must
1643        // still preserve every row and produce a valid array.
1644        //
1645        // The real threshold needs a multi-GiB value, so drive the same logic
1646        // with a `max_buffer_size` smaller than the value length.
1647        let big = "x".repeat(64); // non-inline, larger than the cap below
1648        let test_data = [
1649            Some("short"),      // inline
1650            Some(big.as_str()), // larger than max_buffer_size
1651            None,               // null
1652            Some("tail"),       // inline
1653            Some(big.as_str()), // larger than max_buffer_size
1654        ];
1655        let array: StringViewArray = test_data.into_iter().collect();
1656
1657        let max_buffer_size = 16; // smaller than a single `big` value
1658        let gced = array.gc_with_max_buffer_size(max_buffer_size);
1659
1660        // Row count is preserved.
1661        assert_eq!(gced.len(), array.len(), "gc changed the row count");
1662        // Each oversized value lands in its own buffer, so the copy splits.
1663        assert!(
1664            gced.data_buffers().len() > 1,
1665            "expected output split across multiple buffers, got {}",
1666            gced.data_buffers().len()
1667        );
1668        // Every value is unchanged and in order.
1669        array
1670            .iter()
1671            .zip(gced.iter())
1672            .enumerate()
1673            .for_each(|(i, (a, b))| {
1674                assert_eq!(a, b, "value mismatch at index {i} after gc");
1675            });
1676        // The result round-trips through full validation.
1677        gced.to_data().validate_full().unwrap();
1678    }
1679
1680    #[test]
1681    fn test_eq() {
1682        let test_data = [
1683            Some("longer than 12 bytes"),
1684            None,
1685            Some("short"),
1686            Some("again, this is longer than 12 bytes"),
1687        ];
1688
1689        let array1 = {
1690            let mut builder = StringViewBuilder::new().with_fixed_block_size(8);
1691            test_data.into_iter().for_each(|v| builder.append_option(v));
1692            builder.finish()
1693        };
1694        let array2 = {
1695            // create a new array with the same data but different layout
1696            let mut builder = StringViewBuilder::new().with_fixed_block_size(100);
1697            test_data.into_iter().for_each(|v| builder.append_option(v));
1698            builder.finish()
1699        };
1700        assert_eq!(array1, array1.clone());
1701        assert_eq!(array2, array2.clone());
1702        assert_eq!(array1, array2);
1703    }
1704
1705    /// Integration tests for `inline_key_fast` covering:
1706    ///
1707    /// 1. Monotonic ordering across increasing lengths and lexical variations.
1708    /// 2. Cross-check against `GenericBinaryArray` comparison to ensure semantic equivalence.
1709    ///
1710    /// This also includes a specific test for the “bar” vs. “bar\0” case, demonstrating why
1711    /// the length field is required even when all inline bytes fit in 12 bytes.
1712    ///
1713    /// The test includes strings that verify correct byte order (prevent reversal bugs),
1714    /// and length-based tie-breaking in the composite key.
1715    ///
1716    /// The test confirms that `inline_key_fast` produces keys which sort consistently
1717    /// with the expected lexicographical order of the raw byte arrays.
1718    #[test]
1719    fn test_inline_key_fast_various_lengths_and_lexical() {
1720        /// Helper to create a raw u128 value representing an inline ByteView:
1721        /// - `length`: number of meaningful bytes (must be ≤ 12)
1722        /// - `data`: the actual inline data bytes
1723        ///
1724        /// The first 4 bytes encode length in little-endian,
1725        /// the following 12 bytes contain the inline string data (unpadded).
1726        fn make_raw_inline(length: u32, data: &[u8]) -> u128 {
1727            assert!(length as usize <= 12, "Inline length must be ≤ 12");
1728            assert!(
1729                data.len() == length as usize,
1730                "Data length must match `length`"
1731            );
1732
1733            let mut raw_bytes = [0u8; 16];
1734            raw_bytes[0..4].copy_from_slice(&length.to_le_bytes()); // length stored little-endian
1735            raw_bytes[4..(4 + data.len())].copy_from_slice(data); // inline data
1736            u128::from_le_bytes(raw_bytes)
1737        }
1738
1739        // Test inputs: various lengths and lexical orders,
1740        // plus special cases for byte order and length tie-breaking
1741        let test_inputs: Vec<&[u8]> = vec![
1742            b"a",
1743            b"aa",
1744            b"aaa",
1745            b"aab",
1746            b"abcd",
1747            b"abcde",
1748            b"abcdef",
1749            b"abcdefg",
1750            b"abcdefgh",
1751            b"abcdefghi",
1752            b"abcdefghij",
1753            b"abcdefghijk",
1754            b"abcdefghijkl",
1755            // Tests for byte-order reversal bug:
1756            // Without the fix, "backend one" would compare as "eno dnekcab",
1757            // causing incorrect sort order relative to "backend two".
1758            b"backend one",
1759            b"backend two",
1760            // Tests length-tiebreaker logic:
1761            // "bar" (3 bytes) and "bar\0" (4 bytes) have identical inline data,
1762            // so only the length differentiates their ordering.
1763            b"bar",
1764            b"bar\0",
1765            // Additional lexical and length tie-breaking cases with same prefix, in correct lex order:
1766            b"than12Byt",
1767            b"than12Bytes",
1768            b"than12Bytes\0",
1769            b"than12Bytesx",
1770            b"than12Bytex",
1771            b"than12Bytez",
1772            // Additional lexical tests
1773            b"xyy",
1774            b"xyz",
1775            b"xza",
1776        ];
1777
1778        // Create a GenericBinaryArray for cross-comparison of lex order
1779        let array: GenericBinaryArray<i32> =
1780            GenericBinaryArray::from(test_inputs.iter().map(|s| Some(*s)).collect::<Vec<_>>());
1781
1782        for i in 0..array.len() - 1 {
1783            let v1 = array.value(i);
1784            let v2 = array.value(i + 1);
1785
1786            // Assert the array's natural lexical ordering is correct
1787            assert!(v1 < v2, "Array compare failed: {v1:?} !< {v2:?}");
1788
1789            // Assert the keys produced by inline_key_fast reflect the same ordering
1790            let key1 = GenericByteViewArray::<BinaryViewType>::inline_key_fast(make_raw_inline(
1791                v1.len() as u32,
1792                v1,
1793            ));
1794            let key2 = GenericByteViewArray::<BinaryViewType>::inline_key_fast(make_raw_inline(
1795                v2.len() as u32,
1796                v2,
1797            ));
1798
1799            assert!(
1800                key1 < key2,
1801                "Key compare failed: key({v1:?})=0x{key1:032x} !< key({v2:?})=0x{key2:032x}",
1802            );
1803        }
1804    }
1805
1806    #[test]
1807    fn empty_array_should_return_empty_lengths_iterator() {
1808        let empty = GenericByteViewArray::<BinaryViewType>::from(Vec::<&[u8]>::new());
1809
1810        let mut lengths_iter = empty.lengths();
1811        assert_eq!(lengths_iter.len(), 0);
1812        assert_eq!(lengths_iter.next(), None);
1813    }
1814
1815    #[test]
1816    fn array_lengths_should_return_correct_length_for_both_inlined_and_non_inlined() {
1817        let cases = GenericByteViewArray::<BinaryViewType>::from(vec![
1818            // Not inlined as longer than 12 bytes
1819            b"Supercalifragilisticexpialidocious" as &[u8],
1820            // Inlined as shorter than 12 bytes
1821            b"Hello",
1822            // Empty value
1823            b"",
1824            // Exactly 12 bytes
1825            b"abcdefghijkl",
1826        ]);
1827
1828        let mut lengths_iter = cases.lengths();
1829
1830        assert_eq!(lengths_iter.len(), cases.len());
1831
1832        let cases_iter = cases.iter();
1833
1834        for case in cases_iter {
1835            let case_value = case.unwrap();
1836            let length = lengths_iter.next().expect("Should have a length");
1837
1838            assert_eq!(case_value.len(), length as usize);
1839        }
1840
1841        assert_eq!(lengths_iter.next(), None, "Should not have more lengths");
1842    }
1843
1844    #[test]
1845    fn array_lengths_should_return_the_underlying_length_for_null_values() {
1846        let cases = GenericByteViewArray::<BinaryViewType>::from(vec![
1847            // Not inlined as longer than 12 bytes
1848            b"Supercalifragilisticexpialidocious" as &[u8],
1849            // Inlined as shorter than 12 bytes
1850            b"Hello",
1851            // Empty value
1852            b"",
1853            // Exactly 12 bytes
1854            b"abcdefghijkl",
1855        ]);
1856
1857        let (views, buffer, _) = cases.clone().into_parts();
1858
1859        // Keeping the values but just adding nulls on top
1860        let cases_with_all_nulls = GenericByteViewArray::<BinaryViewType>::new(
1861            views,
1862            buffer,
1863            Some(NullBuffer::new_null(cases.len())),
1864        );
1865
1866        let lengths_iter = cases.lengths();
1867        let mut all_nulls_lengths_iter = cases_with_all_nulls.lengths();
1868
1869        assert_eq!(lengths_iter.len(), all_nulls_lengths_iter.len());
1870
1871        for expected_length in lengths_iter {
1872            let actual_length = all_nulls_lengths_iter.next().expect("Should have a length");
1873
1874            assert_eq!(expected_length, actual_length);
1875        }
1876
1877        assert_eq!(
1878            all_nulls_lengths_iter.next(),
1879            None,
1880            "Should not have more lengths"
1881        );
1882    }
1883
1884    #[test]
1885    fn array_lengths_on_sliced_should_only_return_lengths_for_sliced_data() {
1886        let array = GenericByteViewArray::<BinaryViewType>::from(vec![
1887            b"aaaaaaaaaaaaaaaaaaaaaaaaaaa" as &[u8],
1888            b"Hello",
1889            b"something great",
1890            b"is",
1891            b"coming soon!",
1892            b"when you find what it is",
1893            b"let me know",
1894            b"cause",
1895            b"I",
1896            b"have no idea",
1897            b"what it",
1898            b"is",
1899        ]);
1900
1901        let sliced_array = array.slice(2, array.len() - 3);
1902
1903        let mut lengths_iter = sliced_array.lengths();
1904
1905        assert_eq!(lengths_iter.len(), sliced_array.len());
1906
1907        let values_iter = sliced_array.iter();
1908
1909        for value in values_iter {
1910            let value = value.unwrap();
1911            let length = lengths_iter.next().expect("Should have a length");
1912
1913            assert_eq!(value.len(), length as usize);
1914        }
1915
1916        assert_eq!(lengths_iter.next(), None, "Should not have more lengths");
1917    }
1918
1919    #[should_panic(expected = "Mismatched data type, expected Utf8View, got BinaryView")]
1920    #[test]
1921    fn invalid_casting_from_array_data() {
1922        // Should not be able to cast to StringViewArray due to invalid UTF-8
1923        let array_data = binary_view_array_with_invalid_utf8_data().into_data();
1924        let _ = StringViewArray::from(array_data);
1925    }
1926
1927    #[should_panic(expected = "invalid utf-8 sequence")]
1928    #[test]
1929    fn invalid_array_data() {
1930        let (views, buffers, nulls) = binary_view_array_with_invalid_utf8_data().into_parts();
1931
1932        // manually try and add invalid array data with Utf8View data type
1933        let mut builder = ArrayDataBuilder::new(DataType::Utf8View)
1934            .add_buffer(views.into_inner())
1935            .len(3);
1936        for buffer in buffers.iter() {
1937            builder = builder.add_buffer(buffer.clone())
1938        }
1939        builder = builder.nulls(nulls);
1940
1941        let data = builder.build().unwrap(); // should fail validation
1942        let _arr = StringViewArray::from(data);
1943    }
1944
1945    /// Returns a BinaryViewArray with one invalid UTF-8 value
1946    fn binary_view_array_with_invalid_utf8_data() -> BinaryViewArray {
1947        let array = GenericByteViewArray::<BinaryViewType>::from(vec![
1948            b"aaaaaaaaaaaaaaaaaaaaaaaaaaa" as &[u8],
1949            &[
1950                0xf0, 0x80, 0x80, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1951                0x00, 0x00,
1952            ],
1953            b"good",
1954        ]);
1955        assert!(from_utf8(array.value(0)).is_ok());
1956        assert!(from_utf8(array.value(1)).is_err()); // value 1 is invalid utf8
1957        assert!(from_utf8(array.value(2)).is_ok());
1958        array
1959    }
1960
1961    #[test]
1962    fn test_total_bytes_len() {
1963        // inlined: "hello"=5, "world"=5, "lulu"=4 → 14
1964        // non-inlined: "large payload over 12 bytes"=27
1965        // null: should not count
1966        let mut builder = StringViewBuilder::new();
1967        builder.append_value("hello");
1968        builder.append_value("world");
1969        builder.append_value("lulu");
1970        builder.append_null();
1971        builder.append_value("large payload over 12 bytes");
1972        let array = builder.finish();
1973        assert_eq!(array.total_bytes_len(), 5 + 5 + 4 + 27);
1974    }
1975
1976    #[test]
1977    fn test_total_bytes_len_empty() {
1978        let array = StringViewArray::from_iter::<Vec<Option<&str>>>(vec![]);
1979        assert_eq!(array.total_bytes_len(), 0);
1980    }
1981
1982    #[test]
1983    fn test_total_bytes_len_all_nulls() {
1984        let array = StringViewArray::new_null(5);
1985        assert_eq!(array.total_bytes_len(), 0);
1986    }
1987
1988    #[test]
1989    fn test_total_bytes_len_binary_view() {
1990        let array = BinaryViewArray::from_iter(vec![
1991            Some(b"hi".as_ref()),
1992            None,
1993            Some(b"large payload over 12 bytes".as_ref()),
1994        ]);
1995        assert_eq!(array.total_bytes_len(), 2 + 27);
1996    }
1997}