arrow_array/array/byte_view_array.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use crate::array::print_long_array;
6use crate::builder::{ArrayBuilder, GenericByteViewBuilder};
7use crate::iterator::ArrayIter;
8use crate::types::bytes::ByteArrayNativeType;
9use crate::types::{BinaryViewType, ByteViewType, StringViewType};
10use crate::{Array, ArrayAccessor, ArrayRef, GenericByteArray, OffsetSizeTrait, Scalar};
11use arrow_buffer::{ArrowNativeType, Buffer, NullBuffer, ScalarBuffer};
12use arrow_data::{ArrayData, ArrayDataBuilder, ByteView, MAX_INLINE_VIEW_LEN};
13use arrow_schema::{ArrowError, DataType};
14use core::str;
15use num_traits::ToPrimitive;
16use std::any::Any;
17use std::cmp::Ordering;
18use std::fmt::Debug;
19use std::marker::PhantomData;
20use std::sync::Arc;
21
22use super::ByteArrayType;
23
24/// [Variable-size Binary View Layout]: An array of variable length bytes views.
25///
26/// This array type is used to store variable length byte data (e.g. Strings, Binary)
27/// and has efficient operations such as `take`, `filter`, and comparison.
28///
29/// [Variable-size Binary View Layout]: https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-view-layout
30///
31/// This is different from [`GenericByteArray`], which also stores variable
32/// length byte data, as it represents strings with an offset and length. `take`
33/// and `filter` like operations are implemented by manipulating the "views"
34/// (`u128`) without modifying the bytes. Each view also stores an inlined
35/// prefix which speed up comparisons.
36///
37/// # See Also
38///
39/// * [`StringViewArray`] for storing utf8 encoded string data
40/// * [`BinaryViewArray`] for storing bytes
41/// * [`ByteView`] to interpret `u128`s layout of the views.
42///
43/// [`ByteView`]: arrow_data::ByteView
44///
45/// # Layout: "views" and buffers
46///
47/// A `GenericByteViewArray` stores variable length byte strings. An array of
48/// `N` elements is stored as `N` fixed length "views" and a variable number
49/// of variable length "buffers".
50///
51/// Each view is a `u128` value whose layout is different depending on the
52/// length of the string stored at that location:
53///
54/// ```text
55/// ┌──────┬────────────────────────┐
56/// │length│ string value │
57/// Strings (len <= 12) │ │ (padded with 0) │
58/// └──────┴────────────────────────┘
59/// 0 31 127
60///
61/// ┌───────┬───────┬───────┬───────┐
62/// │length │prefix │ buf │offset │
63/// Strings (len > 12) │ │ │ index │ │
64/// └───────┴───────┴───────┴───────┘
65/// 0 31 63 95 127
66/// ```
67///
68/// * Strings with length <= 12 ([`MAX_INLINE_VIEW_LEN`]) are stored directly in
69/// the view. See [`Self::inline_value`] to access the inlined prefix from a
70/// short view.
71///
72/// * Strings with length > 12: The first four bytes are stored inline in the
73/// view and the entire string is stored in one of the buffers. See [`ByteView`]
74/// to access the fields of the these views.
75///
76/// As with other arrays, the optimized kernels in [`arrow_compute`] are likely
77/// the easiest and fastest way to work with this data. However, it is possible
78/// to access the views and buffers directly for more control.
79///
80/// For example
81///
82/// ```rust
83/// # use arrow_array::StringViewArray;
84/// # use arrow_array::Array;
85/// use arrow_data::ByteView;
86/// let array = StringViewArray::from(vec![
87/// "hello",
88/// "this string is longer than 12 bytes",
89/// "this string is also longer than 12 bytes"
90/// ]);
91///
92/// // ** Examine the first view (short string) **
93/// assert!(array.is_valid(0)); // Check for nulls
94/// let short_view: u128 = array.views()[0]; // "hello"
95/// // get length of the string
96/// let len = short_view as u32;
97/// assert_eq!(len, 5); // strings less than 12 bytes are stored in the view
98/// // SAFETY: `view` is a valid view
99/// let value = unsafe {
100/// StringViewArray::inline_value(&short_view, len as usize)
101/// };
102/// assert_eq!(value, b"hello");
103///
104/// // ** Examine the third view (long string) **
105/// assert!(array.is_valid(12)); // Check for nulls
106/// let long_view: u128 = array.views()[2]; // "this string is also longer than 12 bytes"
107/// let len = long_view as u32;
108/// assert_eq!(len, 40); // strings longer than 12 bytes are stored in the buffer
109/// let view = ByteView::from(long_view); // use ByteView to access the fields
110/// assert_eq!(view.length, 40);
111/// assert_eq!(view.buffer_index, 0);
112/// assert_eq!(view.offset, 35); // data starts after the first long string
113/// // Views for long strings store a 4 byte prefix
114/// let prefix = view.prefix.to_le_bytes();
115/// assert_eq!(&prefix, b"this");
116/// let value = array.value(2); // get the string value (see `value` implementation for how to access the bytes directly)
117/// assert_eq!(value, "this string is also longer than 12 bytes");
118/// ```
119///
120/// [`MAX_INLINE_VIEW_LEN`]: arrow_data::MAX_INLINE_VIEW_LEN
121/// [`arrow_compute`]: https://docs.rs/arrow/latest/arrow/compute/index.html
122///
123/// Unlike [`GenericByteArray`], there are no constraints on the offsets other
124/// than they must point into a valid buffer. However, they can be out of order,
125/// non continuous and overlapping.
126///
127/// For example, in the following diagram, the strings "FishWasInTownToday" and
128/// "CrumpleFacedFish" are both longer than 12 bytes and thus are stored in a
129/// separate buffer while the string "LavaMonster" is stored inlined in the
130/// view. In this case, the same bytes for "Fish" are used to store both strings.
131///
132/// [`ByteView`]: arrow_data::ByteView
133///
134/// ```text
135/// ┌───┐
136/// ┌──────┬──────┬──────┬──────┐ offset │...│
137/// "FishWasInTownTodayYay" │ 21 │ Fish │ 0 │ 115 │─ ─ 103 │Mr.│
138/// └──────┴──────┴──────┴──────┘ │ ┌ ─ ─ ─ ─ ▶ │Cru│
139/// ┌──────┬──────┬──────┬──────┐ │mpl│
140/// "CrumpleFacedFish" │ 16 │ Crum │ 0 │ 103 │─ ─│─ ─ ─ ┘ │eFa│
141/// └──────┴──────┴──────┴──────┘ │ced│
142/// ┌──────┬────────────────────┐ └ ─ ─ ─ ─ ─ ─ ─ ─ ▶│Fis│
143/// "LavaMonster" │ 11 │ LavaMonster │ │hWa│
144/// └──────┴────────────────────┘ offset │sIn│
145/// 115 │Tow│
146/// │nTo│
147/// │day│
148/// u128 "views" │Yay│
149/// buffer 0 │...│
150/// └───┘
151/// ```
152pub struct GenericByteViewArray<T: ByteViewType + ?Sized> {
153 data_type: DataType,
154 views: ScalarBuffer<u128>,
155 buffers: Arc<[Buffer]>,
156 phantom: PhantomData<T>,
157 nulls: Option<NullBuffer>,
158}
159
160impl<T: ByteViewType + ?Sized> Clone for GenericByteViewArray<T> {
161 fn clone(&self) -> Self {
162 Self {
163 data_type: T::DATA_TYPE,
164 views: self.views.clone(),
165 buffers: self.buffers.clone(),
166 nulls: self.nulls.clone(),
167 phantom: Default::default(),
168 }
169 }
170}
171
172impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
173 /// Create a new [`GenericByteViewArray`] from the provided parts, panicking on failure
174 ///
175 /// # Panics
176 ///
177 /// Panics if [`GenericByteViewArray::try_new`] returns an error
178 pub fn new<U>(views: ScalarBuffer<u128>, buffers: U, nulls: Option<NullBuffer>) -> Self
179 where
180 U: Into<Arc<[Buffer]>>,
181 {
182 Self::try_new(views, buffers, nulls).unwrap()
183 }
184
185 /// Create a new [`GenericByteViewArray`] from the provided parts, returning an error on failure
186 ///
187 /// # Errors
188 ///
189 /// * `views.len() != nulls.len()`
190 /// * [ByteViewType::validate] fails
191 pub fn try_new<U>(
192 views: ScalarBuffer<u128>,
193 buffers: U,
194 nulls: Option<NullBuffer>,
195 ) -> Result<Self, ArrowError>
196 where
197 U: Into<Arc<[Buffer]>>,
198 {
199 let buffers: Arc<[Buffer]> = buffers.into();
200
201 T::validate(&views, &buffers)?;
202
203 if let Some(n) = nulls.as_ref()
204 && n.len() != views.len()
205 {
206 return Err(ArrowError::InvalidArgumentError(format!(
207 "Incorrect length of null buffer for {}ViewArray, expected {} got {}",
208 T::PREFIX,
209 views.len(),
210 n.len(),
211 )));
212 }
213
214 Ok(Self {
215 data_type: T::DATA_TYPE,
216 views,
217 buffers,
218 nulls,
219 phantom: Default::default(),
220 })
221 }
222
223 /// Create a new [`GenericByteViewArray`] from the provided parts, without validation
224 ///
225 /// # Safety
226 ///
227 /// Safe if [`Self::try_new`] would not error
228 pub unsafe fn new_unchecked(
229 views: ScalarBuffer<u128>,
230 buffers: Arc<[Buffer]>,
231 nulls: Option<NullBuffer>,
232 ) -> Self {
233 if cfg!(feature = "force_validate") {
234 return Self::new(views, buffers, nulls);
235 }
236
237 Self {
238 data_type: T::DATA_TYPE,
239 phantom: Default::default(),
240 views,
241 buffers,
242 nulls,
243 }
244 }
245
246 /// Create a new [`GenericByteViewArray`] of length `len` where all values are null
247 pub fn new_null(len: usize) -> Self {
248 Self {
249 data_type: T::DATA_TYPE,
250 views: vec![0; len].into(),
251 buffers: vec![].into(),
252 nulls: Some(NullBuffer::new_null(len)),
253 phantom: Default::default(),
254 }
255 }
256
257 /// Create a new [`Scalar`] from `value`
258 pub fn new_scalar(value: impl AsRef<T::Native>) -> Scalar<Self> {
259 Scalar::new(Self::from_iter_values(std::iter::once(value)))
260 }
261
262 /// Creates a [`GenericByteViewArray`] based on an iterator of values without nulls
263 pub fn from_iter_values<Ptr, I>(iter: I) -> Self
264 where
265 Ptr: AsRef<T::Native>,
266 I: IntoIterator<Item = Ptr>,
267 {
268 let iter = iter.into_iter();
269 let mut builder = GenericByteViewBuilder::<T>::with_capacity(iter.size_hint().0);
270 for v in iter {
271 builder.append_value(v);
272 }
273 builder.finish()
274 }
275
276 /// Deconstruct this array into its constituent parts
277 pub fn into_parts(self) -> (ScalarBuffer<u128>, Arc<[Buffer]>, Option<NullBuffer>) {
278 (self.views, self.buffers, self.nulls)
279 }
280
281 /// Returns the views buffer
282 #[inline]
283 pub fn views(&self) -> &ScalarBuffer<u128> {
284 &self.views
285 }
286
287 /// Returns the shared collection of buffers storing non-inline string or binary data.
288 ///
289 /// The returned `Arc` can be cloned to share the buffers with another array without
290 /// allocating a new collection or cloning the individual buffers. To consume this
291 /// array and take ownership of its buffers, use [`Self::into_parts`].
292 #[inline]
293 pub fn data_buffers(&self) -> &Arc<[Buffer]> {
294 &self.buffers
295 }
296
297 /// Returns the element at index `i`
298 ///
299 /// Note: This method does not check for nulls and the value is arbitrary
300 /// (but still well-defined) if [`is_null`](Self::is_null) returns true for the index.
301 ///
302 /// # Panics
303 /// Panics if index `i` is out of bounds.
304 pub fn value(&self, i: usize) -> &T::Native {
305 assert!(
306 i < self.len(),
307 "Trying to access an element at index {} from a {}ViewArray of length {}",
308 i,
309 T::PREFIX,
310 self.len()
311 );
312
313 unsafe { self.value_unchecked(i) }
314 }
315
316 /// Returns the element at index `i` without bounds checking
317 ///
318 /// Note: This method does not check for nulls and the value is arbitrary
319 /// if [`is_null`](Self::is_null) returns true for the index.
320 ///
321 /// # Safety
322 ///
323 /// Caller is responsible for ensuring that the index is within the bounds
324 /// of the array
325 pub unsafe fn value_unchecked(&self, idx: usize) -> &T::Native {
326 let v = unsafe { self.views.get_unchecked(idx) };
327 let len = *v as u32;
328 let b = if len <= MAX_INLINE_VIEW_LEN {
329 unsafe { Self::inline_value(v, len as usize) }
330 } else {
331 let view = ByteView::from(*v);
332 let data = unsafe { self.buffers.get_unchecked(view.buffer_index as usize) };
333 let offset = view.offset as usize;
334 unsafe { data.get_unchecked(offset..offset + len as usize) }
335 };
336 unsafe { T::Native::from_bytes_unchecked(b) }
337 }
338
339 /// Returns the first `len` bytes the inline value of the view.
340 ///
341 /// # Safety
342 /// - The `view` must be a valid element from `Self::views()` that adheres to the view layout.
343 /// - The `len` must be the length of the inlined value. It should never be larger than [`MAX_INLINE_VIEW_LEN`].
344 #[inline(always)]
345 pub unsafe fn inline_value(view: &u128, len: usize) -> &[u8] {
346 debug_assert!(len <= MAX_INLINE_VIEW_LEN as usize);
347 unsafe {
348 std::slice::from_raw_parts(
349 std::ptr::from_ref::<u128>(view)
350 .cast::<u8>()
351 .wrapping_add(4),
352 len,
353 )
354 }
355 }
356
357 /// Constructs a new iterator for iterating over the values of this array
358 pub fn iter(&self) -> ArrayIter<&Self> {
359 ArrayIter::new(self)
360 }
361
362 /// Returns an iterator over the bytes of this array, including null values
363 pub fn bytes_iter(&self) -> impl Iterator<Item = &[u8]> {
364 self.views.iter().map(move |v| {
365 let len = *v as u32;
366 if len <= MAX_INLINE_VIEW_LEN {
367 unsafe { Self::inline_value(v, len as usize) }
368 } else {
369 let view = ByteView::from(*v);
370 let data = &self.buffers[view.buffer_index as usize];
371 let offset = view.offset as usize;
372 unsafe { data.get_unchecked(offset..offset + len as usize) }
373 }
374 })
375 }
376
377 /// Returns an iterator over the first `prefix_len` bytes of each array
378 /// element, including null values.
379 ///
380 /// If `prefix_len` is larger than the element's length, the iterator will
381 /// return an empty slice (`&[]`).
382 pub fn prefix_bytes_iter(&self, prefix_len: usize) -> impl Iterator<Item = &[u8]> {
383 self.views().into_iter().map(move |v| {
384 let len = (*v as u32) as usize;
385
386 if len < prefix_len {
387 return &[] as &[u8];
388 }
389
390 if prefix_len <= 4 || len as u32 <= MAX_INLINE_VIEW_LEN {
391 unsafe { StringViewArray::inline_value(v, prefix_len) }
392 } else {
393 let view = ByteView::from(*v);
394 let data = unsafe {
395 self.data_buffers()
396 .get_unchecked(view.buffer_index as usize)
397 };
398 let offset = view.offset as usize;
399 unsafe { data.get_unchecked(offset..offset + prefix_len) }
400 }
401 })
402 }
403
404 /// Returns an iterator over the last `suffix_len` bytes of each array
405 /// element, including null values.
406 ///
407 /// Note that for [`StringViewArray`] the last bytes may start in the middle
408 /// of a UTF-8 codepoint, and thus may not be a valid `&str`.
409 ///
410 /// If `suffix_len` is larger than the element's length, the iterator will
411 /// return an empty slice (`&[]`).
412 pub fn suffix_bytes_iter(&self, suffix_len: usize) -> impl Iterator<Item = &[u8]> {
413 self.views().into_iter().map(move |v| {
414 let len = (*v as u32) as usize;
415
416 if len < suffix_len {
417 return &[] as &[u8];
418 }
419
420 if len as u32 <= MAX_INLINE_VIEW_LEN {
421 unsafe { &StringViewArray::inline_value(v, len)[len - suffix_len..] }
422 } else {
423 let view = ByteView::from(*v);
424 let data = unsafe {
425 self.data_buffers()
426 .get_unchecked(view.buffer_index as usize)
427 };
428 let offset = view.offset as usize;
429 unsafe { data.get_unchecked(offset + len - suffix_len..offset + len) }
430 }
431 })
432 }
433
434 /// Return an iterator over the length of each array element, including null values.
435 ///
436 /// Null values length would equal to the underlying bytes length and NOT 0
437 ///
438 /// Example of getting 0 for null values
439 /// ```rust
440 /// # use arrow_array::StringViewArray;
441 /// # use arrow_array::Array;
442 /// use arrow_data::ByteView;
443 ///
444 /// fn lengths_with_zero_for_nulls(view: &StringViewArray) -> impl Iterator<Item = u32> {
445 /// view.lengths()
446 /// .enumerate()
447 /// .map(|(index, length)| if view.is_null(index) { 0 } else { length })
448 /// }
449 /// ```
450 pub fn lengths(&self) -> impl ExactSizeIterator<Item = u32> + Clone {
451 self.views().iter().map(|v| *v as u32)
452 }
453
454 /// Returns a zero-copy slice of this array with the indicated offset and length.
455 ///
456 /// # Panics
457 /// Panics if `offset + length > self.len()`
458 pub fn slice(&self, offset: usize, length: usize) -> Self {
459 Self {
460 data_type: T::DATA_TYPE,
461 views: self.views.slice(offset, length),
462 buffers: self.buffers.clone(),
463 nulls: self.nulls.as_ref().map(|n| n.slice(offset, length)),
464 phantom: Default::default(),
465 }
466 }
467
468 /// Returns a "compacted" version of this array
469 ///
470 /// The original array will *not* be modified
471 ///
472 /// # Garbage Collection
473 ///
474 /// Before GC:
475 /// ```text
476 /// ┌──────┐
477 /// │......│
478 /// │......│
479 /// ┌────────────────────┐ ┌ ─ ─ ─ ▶ │Data1 │ Large buffer
480 /// │ View 1 │─ ─ ─ ─ │......│ with data that
481 /// ├────────────────────┤ │......│ is not referred
482 /// │ View 2 │─ ─ ─ ─ ─ ─ ─ ─▶ │Data2 │ to by View 1 or
483 /// └────────────────────┘ │......│ View 2
484 /// │......│
485 /// 2 views, refer to │......│
486 /// small portions of a └──────┘
487 /// large buffer
488 /// ```
489 ///
490 /// After GC:
491 ///
492 /// ```text
493 /// ┌────────────────────┐ ┌─────┐ After gc, only
494 /// │ View 1 │─ ─ ─ ─ ─ ─ ─ ─▶ │Data1│ data that is
495 /// ├────────────────────┤ ┌ ─ ─ ─ ▶ │Data2│ pointed to by
496 /// │ View 2 │─ ─ ─ ─ └─────┘ the views is
497 /// └────────────────────┘ left
498 ///
499 ///
500 /// 2 views
501 /// ```
502 /// This method will compact the data buffers by recreating the view array and only include the data
503 /// that is pointed to by the views.
504 ///
505 /// Note that it will copy the array regardless of whether the original array is compact.
506 /// Use with caution as this can be an expensive operation, only use it when you are sure that the view
507 /// array is significantly smaller than when it is originally created, e.g., after filtering or slicing.
508 ///
509 /// Note: this function does not attempt to canonicalize / deduplicate values. For this
510 /// feature see [`GenericByteViewBuilder::with_deduplicate_strings`].
511 pub fn gc(&self) -> Self {
512 // A single Arrow data buffer is addressed by a `u32` offset, so no
513 // output buffer may exceed `i32::MAX` bytes. Above that the data is
514 // split across multiple buffers (the "slow path").
515 self.gc_with_max_buffer_size(i32::MAX as usize)
516 }
517
518 /// Like [`Self::gc`], but with a configurable maximum output buffer size.
519 ///
520 /// `gc` always uses `i32::MAX` (the largest a single Arrow buffer can be).
521 /// This method exists so the multi-buffer split path — which `gc` only
522 /// reaches once a column references more than 2 GiB of non-inline data —
523 /// can be exercised in tests with a small threshold and tiny inputs.
524 fn gc_with_max_buffer_size(&self, max_buffer_size: usize) -> Self {
525 // 1) Read basic properties once
526 let len = self.len(); // number of elements
527 let nulls = self.nulls().cloned(); // reuse & clone existing null bitmap
528
529 // 1.5) Fast path: if there are no buffers, just reuse original views and no data blocks
530 if self.data_buffers().is_empty() {
531 return unsafe {
532 GenericByteViewArray::new_unchecked(
533 self.views().clone(),
534 Arc::from([]), // empty data blocks
535 nulls,
536 )
537 };
538 }
539
540 // 2) Calculate total size of all non-inline data and detect if any exists
541 let total_large = self.total_buffer_bytes_used();
542
543 // 2.5) Fast path: if there is no non-inline data, avoid buffer allocation & processing
544 if total_large == 0 {
545 // Views are inline-only or all null; just reuse original views and no data blocks
546 return unsafe {
547 GenericByteViewArray::new_unchecked(
548 self.views().clone(),
549 Arc::from([]), // empty data blocks
550 nulls,
551 )
552 };
553 }
554
555 let (views_buf, data_blocks) = if total_large <= max_buffer_size {
556 // fast path, the entire data fits in a single buffer
557 // 3) Allocate exactly capacity for all non-inline data
558 let mut data_buf = Vec::with_capacity(total_large);
559
560 // 4) Iterate over views and process each inline/non-inline view
561 let views_buf: Vec<u128> = (0..len)
562 .map(|i| unsafe { self.copy_view_to_buffer(i, 0, &mut data_buf) })
563 .collect();
564 let data_block = Buffer::from_vec(data_buf);
565 let data_blocks = vec![data_block];
566 (views_buf, data_blocks)
567 } else {
568 // slow path: the non-inline data does not fit in a single buffer
569 // (a buffer offset is a `u32`, so no buffer may exceed `i32::MAX`),
570 // so it must be split across several buffers.
571
572 // A contiguous run of views destined for one output buffer.
573 struct GcCopyGroup {
574 total_buffer_bytes: usize,
575 total_len: usize,
576 }
577
578 // First pass: partition the views into contiguous groups, each of
579 // which fits within one output buffer. `total_len` counts *all*
580 // views in the group, not just the non-inline ones — this is
581 // essential for correctness, as gc must preserve the row count.
582 // Inline views reference no buffer and are copied unchanged, but
583 // they still belong to a group and must be accounted for. Recording
584 // each group's exact byte size lets the second pass size every
585 // buffer precisely, with no over-reservation or trailing waste.
586 let mut groups: Vec<GcCopyGroup> = Vec::new();
587 // Bytes accumulated for the current group. A view length is a `u32`
588 // and a single value may be larger than `max_buffer_size`, so the
589 // running sum is kept in `u64` to stay within the addressable
590 // buffer-offset space without wrapping.
591 let mut current_length: u64 = 0;
592 let mut current_elements = 0;
593
594 for view in self.views() {
595 let view_len = *view as u32;
596 if view_len > MAX_INLINE_VIEW_LEN {
597 // Seal the current group before adding this view would push
598 // its buffer past `max_buffer_size`.
599 if current_length + view_len as u64 > max_buffer_size as u64 {
600 groups.push(GcCopyGroup {
601 total_buffer_bytes: current_length as usize,
602 total_len: current_elements,
603 });
604 current_length = 0;
605 current_elements = 0;
606 }
607 current_length += view_len as u64;
608 }
609 current_elements += 1;
610 }
611 if current_elements != 0 {
612 groups.push(GcCopyGroup {
613 total_buffer_bytes: current_length as usize,
614 total_len: current_elements,
615 });
616 }
617 debug_assert!(i32::try_from(groups.len()).is_ok());
618
619 // Second pass: copy each group into an exactly-sized buffer.
620 let mut views_buf = Vec::with_capacity(len);
621 let mut data_blocks = Vec::with_capacity(groups.len());
622 let mut current_view_idx = 0;
623
624 for (group_idx, group) in groups.iter().enumerate() {
625 let mut data_buf = Vec::with_capacity(group.total_buffer_bytes);
626
627 // Directly push views to avoid an intermediate Vec allocation.
628 let new_views =
629 (current_view_idx..current_view_idx + group.total_len).map(|view_idx| {
630 // SAFETY: `view_idx` came from iterating a valid range
631 // within `len`, and every view refers to valid data.
632 unsafe {
633 self.copy_view_to_buffer(view_idx, group_idx as i32, &mut data_buf)
634 }
635 });
636 views_buf.extend(new_views);
637
638 data_blocks.push(Buffer::from_vec(data_buf));
639 current_view_idx += group.total_len;
640 }
641 (views_buf, data_blocks)
642 };
643
644 // 5) Wrap up views buffer
645 let views_scalar = ScalarBuffer::from(views_buf);
646
647 // SAFETY: views_scalar, data_blocks, and nulls are correctly aligned and sized
648 unsafe { GenericByteViewArray::new_unchecked(views_scalar, data_blocks.into(), nulls) }
649 }
650
651 /// Copy the i‑th view into `data_buf` if it refers to an out‑of‑line buffer.
652 ///
653 /// # Safety
654 ///
655 /// - `i < self.len()`.
656 /// - Every element in `self.views()` must currently refer to a valid slice
657 /// inside one of `self.buffers`.
658 /// - `data_buf` must be ready to have additional bytes appended.
659 /// - After this call, the returned view will have its
660 /// `buffer_index` reset to `buffer_idx` and its `offset` updated so that it points
661 /// into the bytes just appended at the end of `data_buf`.
662 #[inline(always)]
663 unsafe fn copy_view_to_buffer(
664 &self,
665 i: usize,
666 buffer_idx: i32,
667 data_buf: &mut Vec<u8>,
668 ) -> u128 {
669 // SAFETY: `i < self.len()` ensures this is in‑bounds.
670 let raw_view = unsafe { *self.views().get_unchecked(i) };
671 let mut bv = ByteView::from(raw_view);
672
673 // Inline‑small views stay as‑is.
674 if bv.length <= MAX_INLINE_VIEW_LEN {
675 raw_view
676 } else {
677 // SAFETY: `bv.buffer_index` and `bv.offset..bv.offset+bv.length`
678 // must both lie within valid ranges for `self.buffers`.
679 let buffer = unsafe { self.buffers.get_unchecked(bv.buffer_index as usize) };
680 let start = bv.offset as usize;
681 let end = start + bv.length as usize;
682 let slice = unsafe { buffer.get_unchecked(start..end) };
683
684 // Copy out‑of‑line data into our single “0” buffer.
685 let new_offset = data_buf.len() as u32;
686 data_buf.extend_from_slice(slice);
687
688 bv.buffer_index = buffer_idx as u32;
689 bv.offset = new_offset;
690 bv.into()
691 }
692 }
693
694 /// Returns the total number of bytes of all non-null values in this array.
695 ///
696 /// Unlike [`Self::total_buffer_bytes_used`], this method includes inlined strings
697 /// (those with length ≤ [`MAX_INLINE_VIEW_LEN`]), making it suitable as a
698 /// capacity hint when pre-allocating output buffers.
699 ///
700 /// Null values are excluded from the sum.
701 ///
702 /// # Example
703 ///
704 /// ```rust
705 /// # use arrow_array::StringViewArray;
706 /// let array = StringViewArray::from_iter(vec![
707 /// Some("hello"), // 5 bytes, inlined
708 /// None, // excluded
709 /// Some("large payload over 12 bytes"), // 27 bytes, non-inlined
710 /// ]);
711 /// assert_eq!(array.total_bytes_len(), 5 + 27);
712 /// ```
713 pub fn total_bytes_len(&self) -> usize {
714 match self.nulls() {
715 None => self.views().iter().map(|v| (*v as u32) as usize).sum(),
716 Some(nulls) => self
717 .views()
718 .iter()
719 .zip(nulls.iter())
720 .map(|(v, is_valid)| if is_valid { (*v as u32) as usize } else { 0 })
721 .sum(),
722 }
723 }
724
725 /// Returns the total number of bytes used by all non inlined views in all
726 /// buffers.
727 ///
728 /// Note this does not account for views that point at the same underlying
729 /// data in buffers
730 ///
731 /// For example, if the array has three strings views:
732 /// * View with length = 9 (inlined)
733 /// * View with length = 32 (non inlined)
734 /// * View with length = 16 (non inlined)
735 ///
736 /// Then this method would report 48
737 pub fn total_buffer_bytes_used(&self) -> usize {
738 self.views()
739 .iter()
740 .map(|v| {
741 let len = *v as u32;
742 if len > MAX_INLINE_VIEW_LEN {
743 len as usize
744 } else {
745 0
746 }
747 })
748 .sum()
749 }
750
751 /// Compare two [`GenericByteViewArray`] at index `left_idx` and `right_idx`
752 ///
753 /// Comparing two ByteView types are non-trivial.
754 /// It takes a bit of patience to understand why we don't just compare two &[u8] directly.
755 ///
756 /// ByteView types give us the following two advantages, and we need to be careful not to lose them:
757 /// (1) For string/byte smaller than [`MAX_INLINE_VIEW_LEN`] bytes, the entire data is inlined in the view.
758 /// Meaning that reading one array element requires only one memory access
759 /// (two memory access required for StringArray, one for offset buffer, the other for value buffer).
760 ///
761 /// (2) For string/byte larger than [`MAX_INLINE_VIEW_LEN`] bytes, we can still be faster than (for certain operations) StringArray/ByteArray,
762 /// thanks to the inlined 4 bytes.
763 /// Consider equality check:
764 /// If the first four bytes of the two strings are different, we can return false immediately (with just one memory access).
765 ///
766 /// If we directly compare two &[u8], we materialize the entire string (i.e., make multiple memory accesses), which might be unnecessary.
767 /// - Most of the time (eq, ord), we only need to look at the first 4 bytes to know the answer,
768 /// e.g., if the inlined 4 bytes are different, we can directly return unequal without looking at the full string.
769 ///
770 /// # Order check flow
771 /// (1) if both string are smaller than [`MAX_INLINE_VIEW_LEN`] bytes, we can directly compare the data inlined to the view.
772 /// (2) if any of the string is larger than [`MAX_INLINE_VIEW_LEN`] bytes, we need to compare the full string.
773 /// (2.1) if the inlined 4 bytes are different, we can return the result immediately.
774 /// (2.2) o.w., we need to compare the full string.
775 ///
776 /// # Safety
777 /// The left/right_idx must within range of each array
778 pub unsafe fn compare_unchecked(
779 left: &GenericByteViewArray<T>,
780 left_idx: usize,
781 right: &GenericByteViewArray<T>,
782 right_idx: usize,
783 ) -> Ordering {
784 let l_view = unsafe { left.views().get_unchecked(left_idx) };
785 let l_byte_view = ByteView::from(*l_view);
786
787 let r_view = unsafe { right.views().get_unchecked(right_idx) };
788 let r_byte_view = ByteView::from(*r_view);
789
790 let l_len = l_byte_view.length;
791 let r_len = r_byte_view.length;
792
793 if l_len <= 12 && r_len <= 12 {
794 return Self::inline_key_fast(*l_view).cmp(&Self::inline_key_fast(*r_view));
795 }
796
797 // one of the string is larger than 12 bytes,
798 // we then try to compare the inlined data first
799
800 // Note: In theory, ByteView is only used for string which is larger than 12 bytes,
801 // but we can still use it to get the inlined prefix for shorter strings.
802 // The prefix is always the first 4 bytes of the view, for both short and long strings.
803 let l_inlined_be = l_byte_view.prefix.swap_bytes();
804 let r_inlined_be = r_byte_view.prefix.swap_bytes();
805 if l_inlined_be != r_inlined_be {
806 return l_inlined_be.cmp(&r_inlined_be);
807 }
808
809 // unfortunately, we need to compare the full data
810 let l_full_data: &[u8] = unsafe { left.value_unchecked(left_idx).as_ref() };
811 let r_full_data: &[u8] = unsafe { right.value_unchecked(right_idx).as_ref() };
812
813 l_full_data.cmp(r_full_data)
814 }
815
816 /// Builds a 128-bit composite key for an inline value:
817 ///
818 /// - High 96 bits: the inline data in big-endian byte order (for correct lexicographical sorting).
819 /// - Low 32 bits: the length in big-endian byte order, acting as a tiebreaker so shorter strings
820 /// (or those with fewer meaningful bytes) always numerically sort before longer ones.
821 ///
822 /// This function extracts the length and the 12-byte inline string data from the raw
823 /// little-endian `u128` representation, converts them to big-endian ordering, and packs them
824 /// into a single `u128` value suitable for fast, branchless comparisons.
825 ///
826 /// # Why include length?
827 ///
828 /// A pure 96-bit content comparison can’t distinguish between two values whose inline bytes
829 /// compare equal—either because one is a true prefix of the other or because zero-padding
830 /// hides extra bytes. By tucking the 32-bit length into the lower bits, a single `u128` compare
831 /// handles both content and length in one go.
832 ///
833 /// Example: comparing "bar" (3 bytes) vs "bar\0" (4 bytes)
834 ///
835 /// | String | Bytes 0–4 (length LE) | Bytes 4–16 (data + padding) |
836 /// |------------|-----------------------|---------------------------------|
837 /// | `"bar"` | `03 00 00 00` | `62 61 72` + 9 × `00` |
838 /// | `"bar\0"`| `04 00 00 00` | `62 61 72 00` + 8 × `00` |
839 ///
840 /// Both inline parts become `62 61 72 00…00`, so they tie on content. The length field
841 /// then differentiates:
842 ///
843 /// ```text
844 /// key("bar") = 0x0000000000000000000062617200000003
845 /// key("bar\0") = 0x0000000000000000000062617200000004
846 /// ⇒ key("bar") < key("bar\0")
847 /// ```
848 /// - `raw` is treated as a 128-bit integer with its bits laid out as follows:
849 /// - bits 0–31: length (little-endian)
850 /// - bits 32–127: data (little-endian)
851 ///
852 /// # Inlining and Endianness
853 ///
854 /// This function uses platform-independent bitwise operations to construct a 128-bit key:
855 /// - `raw.swap_bytes() << 32` effectively clears the length bits and shifts the 12-byte inline data
856 /// into the high 96 bits in Big-Endian order. This ensures the first byte of the string
857 /// is the most significant byte of the resulting `u128`.
858 /// - `raw as u32` extracts the length as a numeric integer, which is then placed in the low 32 bits.
859 #[inline(always)]
860 pub fn inline_key_fast(raw: u128) -> u128 {
861 (raw.swap_bytes() << 32) | (raw as u32 as u128)
862 }
863}
864
865impl<T: ByteViewType + ?Sized> Debug for GenericByteViewArray<T> {
866 fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
867 write!(f, "{}ViewArray\n[\n", T::PREFIX)?;
868 print_long_array(self, f, &mut |index, f| {
869 std::fmt::Debug::fmt(&self.value(index), f)
870 })?;
871 write!(f, "]")
872 }
873}
874
875/// SAFETY: Correctly implements the contract of Arrow Arrays
876unsafe impl<T: ByteViewType + ?Sized> Array for GenericByteViewArray<T> {
877 fn as_any(&self) -> &dyn Any {
878 self
879 }
880
881 fn to_data(&self) -> ArrayData {
882 self.clone().into()
883 }
884
885 fn into_data(self) -> ArrayData {
886 self.into()
887 }
888
889 fn data_type(&self) -> &DataType {
890 &self.data_type
891 }
892
893 fn slice(&self, offset: usize, length: usize) -> ArrayRef {
894 Arc::new(self.slice(offset, length))
895 }
896
897 fn len(&self) -> usize {
898 self.views.len()
899 }
900
901 fn is_empty(&self) -> bool {
902 self.views.is_empty()
903 }
904
905 fn shrink_to_fit(&mut self) {
906 self.views.shrink_to_fit();
907
908 // The goal of `shrink_to_fit` is to minimize the space used by any of
909 // its allocations. The use of `Arc::get_mut` over `Arc::make_mut` is
910 // because if the reference count is greater than 1, `Arc::make_mut`
911 // will first clone its contents. So, any large allocations will first
912 // be cloned before being shrunk, leaving the pre-cloned allocations
913 // intact, before adding the extra (used) space of the new clones.
914 if let Some(buffers) = Arc::get_mut(&mut self.buffers) {
915 buffers.iter_mut().for_each(|b| b.shrink_to_fit());
916 }
917
918 // With the assumption that this is a best-effort function, no attempt
919 // is made to shrink `self.buffers`, which it can't because it's type
920 // does not expose a `shrink_to_fit` method.
921
922 if let Some(nulls) = &mut self.nulls {
923 nulls.shrink_to_fit();
924 }
925 }
926
927 fn offset(&self) -> usize {
928 0
929 }
930
931 fn nulls(&self) -> Option<&NullBuffer> {
932 self.nulls.as_ref()
933 }
934
935 fn logical_null_count(&self) -> usize {
936 // More efficient that the default implementation
937 self.null_count()
938 }
939
940 fn get_buffer_memory_size(&self) -> usize {
941 let mut sum = self.buffers.iter().map(|b| b.capacity()).sum::<usize>();
942 sum += self.views.inner().capacity();
943 if let Some(x) = &self.nulls {
944 sum += x.buffer().capacity()
945 }
946 sum
947 }
948
949 fn get_array_memory_size(&self) -> usize {
950 std::mem::size_of::<Self>() + self.get_buffer_memory_size()
951 }
952
953 #[cfg(feature = "pool")]
954 fn claim(&self, pool: &dyn arrow_buffer::MemoryPool) {
955 self.views.claim(pool);
956 for buffer in self.buffers.iter() {
957 buffer.claim(pool);
958 }
959 if let Some(nulls) = &self.nulls {
960 nulls.claim(pool);
961 }
962 }
963}
964
965impl<'a, T: ByteViewType + ?Sized> ArrayAccessor for &'a GenericByteViewArray<T> {
966 type Item = &'a T::Native;
967
968 fn value(&self, index: usize) -> Self::Item {
969 GenericByteViewArray::value(self, index)
970 }
971
972 unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
973 unsafe { GenericByteViewArray::value_unchecked(self, index) }
974 }
975}
976
977impl<'a, T: ByteViewType + ?Sized> IntoIterator for &'a GenericByteViewArray<T> {
978 type Item = Option<&'a T::Native>;
979 type IntoIter = ArrayIter<Self>;
980
981 fn into_iter(self) -> Self::IntoIter {
982 ArrayIter::new(self)
983 }
984}
985
986impl<T: ByteViewType + ?Sized> From<ArrayData> for GenericByteViewArray<T> {
987 fn from(data: ArrayData) -> Self {
988 let (data_type, len, nulls, offset, buffers, _child_data) = data.into_parts();
989 assert_eq!(
990 data_type,
991 T::DATA_TYPE,
992 "Mismatched data type, expected {}, got {data_type}",
993 T::DATA_TYPE
994 );
995 let mut buffers = buffers.into_iter();
996 // first buffer is views, remaining are data buffers
997 let views = ScalarBuffer::new(buffers.next().unwrap(), offset, len);
998 Self {
999 data_type,
1000 views,
1001 buffers: Arc::from_iter(buffers),
1002 nulls,
1003 phantom: Default::default(),
1004 }
1005 }
1006}
1007
1008/// Efficiently convert a [`GenericByteArray`] to a [`GenericByteViewArray`]
1009///
1010/// For example this method can convert a [`StringArray`] to a
1011/// [`StringViewArray`].
1012///
1013/// If the offsets are all less than u32::MAX, the new [`GenericByteViewArray`]
1014/// is built without copying the underlying string data (views are created
1015/// directly into the existing buffer)
1016///
1017/// [`StringArray`]: crate::StringArray
1018impl<FROM, V> From<&GenericByteArray<FROM>> for GenericByteViewArray<V>
1019where
1020 FROM: ByteArrayType,
1021 FROM::Offset: OffsetSizeTrait + ToPrimitive,
1022 V: ByteViewType<Native = FROM::Native>,
1023{
1024 fn from(byte_array: &GenericByteArray<FROM>) -> Self {
1025 let offsets = byte_array.offsets();
1026
1027 let can_reuse_buffer = offsets.last().as_usize() < u32::MAX as usize;
1028
1029 if can_reuse_buffer {
1030 // build views directly pointing to the existing buffer
1031 let len = byte_array.len();
1032 let mut views_builder = GenericByteViewBuilder::<V>::with_capacity(len);
1033 let str_values_buf = byte_array.values().clone();
1034 let block = views_builder.append_block(str_values_buf);
1035 for (i, w) in offsets.windows(2).enumerate() {
1036 let offset = w[0].as_usize();
1037 let end = w[1].as_usize();
1038 let length = end - offset;
1039
1040 if byte_array.is_null(i) {
1041 views_builder.append_null();
1042 } else {
1043 // Safety: the input was a valid array so it valid UTF8 (if string). And
1044 // all offsets were valid
1045 unsafe {
1046 views_builder.append_view_unchecked(block, offset as u32, length as u32)
1047 }
1048 }
1049 }
1050 assert_eq!(views_builder.len(), len);
1051 views_builder.finish()
1052 } else {
1053 // Otherwise, create a new buffer for large strings
1054 // TODO: the original buffer could still be used
1055 // by making multiple slices of u32::MAX length
1056 GenericByteViewArray::<V>::from_iter(byte_array.iter())
1057 }
1058 }
1059}
1060
1061impl<T: ByteViewType + ?Sized> From<GenericByteViewArray<T>> for ArrayData {
1062 fn from(array: GenericByteViewArray<T>) -> Self {
1063 let len = array.len();
1064
1065 let mut buffers = array.buffers.to_vec();
1066 buffers.insert(0, array.views.into_inner());
1067
1068 let builder = ArrayDataBuilder::new(T::DATA_TYPE)
1069 .len(len)
1070 .buffers(buffers)
1071 .nulls(array.nulls);
1072
1073 unsafe { builder.build_unchecked() }
1074 }
1075}
1076
1077impl<'a, Ptr, T> FromIterator<&'a Option<Ptr>> for GenericByteViewArray<T>
1078where
1079 Ptr: AsRef<T::Native> + 'a,
1080 T: ByteViewType + ?Sized,
1081{
1082 fn from_iter<I: IntoIterator<Item = &'a Option<Ptr>>>(iter: I) -> Self {
1083 iter.into_iter()
1084 .map(|o| o.as_ref().map(|p| p.as_ref()))
1085 .collect()
1086 }
1087}
1088
1089impl<Ptr, T: ByteViewType + ?Sized> FromIterator<Option<Ptr>> for GenericByteViewArray<T>
1090where
1091 Ptr: AsRef<T::Native>,
1092{
1093 fn from_iter<I: IntoIterator<Item = Option<Ptr>>>(iter: I) -> Self {
1094 let iter = iter.into_iter();
1095 let mut builder = GenericByteViewBuilder::<T>::with_capacity(iter.size_hint().0);
1096 builder.extend(iter);
1097 builder.finish()
1098 }
1099}
1100
1101/// A [`GenericByteViewArray`] of `[u8]`
1102///
1103/// See [`GenericByteViewArray`] for format and layout details.
1104///
1105/// # Example
1106/// ```
1107/// use arrow_array::BinaryViewArray;
1108/// let array = BinaryViewArray::from_iter_values(vec![b"hello" as &[u8], b"world", b"lulu", b"large payload over 12 bytes"]);
1109/// assert_eq!(array.value(0), b"hello");
1110/// assert_eq!(array.value(3), b"large payload over 12 bytes");
1111/// ```
1112pub type BinaryViewArray = GenericByteViewArray<BinaryViewType>;
1113
1114impl BinaryViewArray {
1115 /// Convert the [`BinaryViewArray`] to [`StringViewArray`]
1116 /// If items not utf8 data, validate will fail and error returned.
1117 pub fn to_string_view(self) -> Result<StringViewArray, ArrowError> {
1118 StringViewType::validate(self.views(), self.data_buffers())?;
1119 unsafe { Ok(self.to_string_view_unchecked()) }
1120 }
1121
1122 /// Convert the [`BinaryViewArray`] to [`StringViewArray`]
1123 /// # Safety
1124 /// Caller is responsible for ensuring that items in array are utf8 data.
1125 pub unsafe fn to_string_view_unchecked(self) -> StringViewArray {
1126 unsafe { StringViewArray::new_unchecked(self.views, self.buffers, self.nulls) }
1127 }
1128}
1129
1130impl From<Vec<&[u8]>> for BinaryViewArray {
1131 fn from(v: Vec<&[u8]>) -> Self {
1132 Self::from_iter_values(v)
1133 }
1134}
1135
1136impl From<Vec<Option<&[u8]>>> for BinaryViewArray {
1137 fn from(v: Vec<Option<&[u8]>>) -> Self {
1138 v.into_iter().collect()
1139 }
1140}
1141
1142/// A [`GenericByteViewArray`] that stores utf8 data
1143///
1144/// See [`GenericByteViewArray`] for format and layout details.
1145///
1146/// # Example
1147/// ```
1148/// use arrow_array::StringViewArray;
1149/// let array = StringViewArray::from_iter_values(vec!["hello", "world", "lulu", "large payload over 12 bytes"]);
1150/// assert_eq!(array.value(0), "hello");
1151/// assert_eq!(array.value(3), "large payload over 12 bytes");
1152/// ```
1153pub type StringViewArray = GenericByteViewArray<StringViewType>;
1154
1155impl StringViewArray {
1156 /// Convert the [`StringViewArray`] to [`BinaryViewArray`]
1157 pub fn to_binary_view(self) -> BinaryViewArray {
1158 unsafe { BinaryViewArray::new_unchecked(self.views, self.buffers, self.nulls) }
1159 }
1160
1161 /// Returns true if all data within this array is ASCII
1162 pub fn is_ascii(&self) -> bool {
1163 // Alternative (but incorrect): directly check the underlying buffers
1164 // (1) Our string view might be sparse, i.e., a subset of the buffers,
1165 // so even if the buffer is not ascii, we can still be ascii.
1166 // (2) It is quite difficult to know the range of each buffer (unlike StringArray)
1167 // This means that this operation is quite expensive, shall we cache the result?
1168 // i.e. track `is_ascii` in the builder.
1169 self.iter().all(|v| match v {
1170 Some(v) => v.is_ascii(),
1171 None => true,
1172 })
1173 }
1174}
1175
1176impl From<Vec<&str>> for StringViewArray {
1177 fn from(v: Vec<&str>) -> Self {
1178 Self::from_iter_values(v)
1179 }
1180}
1181
1182impl From<Vec<Option<&str>>> for StringViewArray {
1183 fn from(v: Vec<Option<&str>>) -> Self {
1184 v.into_iter().collect()
1185 }
1186}
1187
1188impl From<Vec<String>> for StringViewArray {
1189 fn from(v: Vec<String>) -> Self {
1190 Self::from_iter_values(v)
1191 }
1192}
1193
1194impl From<Vec<Option<String>>> for StringViewArray {
1195 fn from(v: Vec<Option<String>>) -> Self {
1196 v.into_iter().collect()
1197 }
1198}
1199
1200#[cfg(test)]
1201mod tests {
1202 use crate::builder::{BinaryViewBuilder, StringViewBuilder};
1203 use crate::types::BinaryViewType;
1204 use crate::{
1205 Array, BinaryViewArray, GenericBinaryArray, GenericByteViewArray, StringViewArray,
1206 };
1207 use arrow_buffer::{Buffer, NullBuffer, ScalarBuffer};
1208 use arrow_data::{ArrayDataBuilder, ByteView, MAX_INLINE_VIEW_LEN};
1209 use arrow_schema::DataType;
1210 use rand::prelude::StdRng;
1211 use rand::{RngExt, SeedableRng};
1212 use std::str::from_utf8;
1213
1214 const BLOCK_SIZE: u32 = 8;
1215
1216 #[test]
1217 fn try_new_string() {
1218 let array = StringViewArray::from_iter_values(vec![
1219 "hello",
1220 "world",
1221 "lulu",
1222 "large payload over 12 bytes",
1223 ]);
1224 assert_eq!(array.value(0), "hello");
1225 assert_eq!(array.value(3), "large payload over 12 bytes");
1226 }
1227
1228 #[test]
1229 fn try_new_binary() {
1230 let array = BinaryViewArray::from_iter_values(vec![
1231 b"hello".as_slice(),
1232 b"world".as_slice(),
1233 b"lulu".as_slice(),
1234 b"large payload over 12 bytes".as_slice(),
1235 ]);
1236 assert_eq!(array.value(0), b"hello");
1237 assert_eq!(array.value(3), b"large payload over 12 bytes");
1238 }
1239
1240 #[test]
1241 fn try_new_empty_string() {
1242 // test empty array
1243 let array = {
1244 let mut builder = StringViewBuilder::new();
1245 builder.finish()
1246 };
1247 assert!(array.is_empty());
1248 }
1249
1250 #[test]
1251 fn try_new_empty_binary() {
1252 // test empty array
1253 let array = {
1254 let mut builder = BinaryViewBuilder::new();
1255 builder.finish()
1256 };
1257 assert!(array.is_empty());
1258 }
1259
1260 #[test]
1261 fn test_append_string() {
1262 // test builder append
1263 let array = {
1264 let mut builder = StringViewBuilder::new();
1265 builder.append_value("hello");
1266 builder.append_null();
1267 builder.append_option(Some("large payload over 12 bytes"));
1268 builder.finish()
1269 };
1270 assert_eq!(array.value(0), "hello");
1271 assert!(array.is_null(1));
1272 assert_eq!(array.value(2), "large payload over 12 bytes");
1273 }
1274
1275 #[test]
1276 fn test_append_binary() {
1277 // test builder append
1278 let array = {
1279 let mut builder = BinaryViewBuilder::new();
1280 builder.append_value(b"hello");
1281 builder.append_null();
1282 builder.append_option(Some(b"large payload over 12 bytes"));
1283 builder.finish()
1284 };
1285 assert_eq!(array.value(0), b"hello");
1286 assert!(array.is_null(1));
1287 assert_eq!(array.value(2), b"large payload over 12 bytes");
1288 }
1289
1290 #[test]
1291 fn test_in_progress_recreation() {
1292 let array = {
1293 // make a builder with small block size.
1294 let mut builder = StringViewBuilder::new().with_fixed_block_size(14);
1295 builder.append_value("large payload over 12 bytes");
1296 builder.append_option(Some("another large payload over 12 bytes that double than the first one, so that we can trigger the in_progress in builder re-created"));
1297 builder.finish()
1298 };
1299 assert_eq!(array.value(0), "large payload over 12 bytes");
1300 assert_eq!(
1301 array.value(1),
1302 "another large payload over 12 bytes that double than the first one, so that we can trigger the in_progress in builder re-created"
1303 );
1304 assert_eq!(2, array.buffers.len());
1305 }
1306
1307 #[test]
1308 #[should_panic(expected = "Invalid buffer index at 0: got index 3 but only has 1 buffers")]
1309 fn new_with_invalid_view_data() {
1310 let v = "large payload over 12 bytes";
1311 let view = ByteView::new(13, &v.as_bytes()[0..4])
1312 .with_buffer_index(3)
1313 .with_offset(1);
1314 let views = ScalarBuffer::from(vec![view.into()]);
1315 let buffers = vec![Buffer::from_slice_ref(v)];
1316 StringViewArray::new(views, buffers, None);
1317 }
1318
1319 #[test]
1320 #[should_panic(
1321 expected = "Encountered non-UTF-8 data at index 0: invalid utf-8 sequence of 1 bytes from index 0"
1322 )]
1323 fn new_with_invalid_utf8_data() {
1324 let v: Vec<u8> = vec![
1325 // invalid UTF8
1326 0xf0, 0x80, 0x80, 0x80, // more bytes to make it larger than 12
1327 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1328 ];
1329 let view = ByteView::new(v.len() as u32, &v[0..4]);
1330 let views = ScalarBuffer::from(vec![view.into()]);
1331 let buffers = vec![Buffer::from_slice_ref(v)];
1332 StringViewArray::new(views, buffers, None);
1333 }
1334
1335 #[test]
1336 #[should_panic(expected = "View at index 0 contained non-zero padding for string of length 1")]
1337 fn new_with_invalid_zero_padding() {
1338 let mut data = [0; 12];
1339 data[0] = b'H';
1340 data[11] = 1; // no zero padding
1341
1342 let mut view_buffer = [0; 16];
1343 view_buffer[0..4].copy_from_slice(&1u32.to_le_bytes());
1344 view_buffer[4..].copy_from_slice(&data);
1345
1346 let view = ByteView::from(u128::from_le_bytes(view_buffer));
1347 let views = ScalarBuffer::from(vec![view.into()]);
1348 let buffers = vec![];
1349 StringViewArray::new(views, buffers, None);
1350 }
1351
1352 #[test]
1353 #[should_panic(expected = "Mismatch between embedded prefix and data")]
1354 fn test_mismatch_between_embedded_prefix_and_data() {
1355 let input_str_1 = "Hello, Rustaceans!";
1356 let input_str_2 = "Hallo, Rustaceans!";
1357 let length = input_str_1.len() as u32;
1358 assert!(input_str_1.len() > 12);
1359
1360 let mut view_buffer = [0; 16];
1361 view_buffer[0..4].copy_from_slice(&length.to_le_bytes());
1362 view_buffer[4..8].copy_from_slice(&input_str_1.as_bytes()[0..4]);
1363 view_buffer[8..12].copy_from_slice(&0u32.to_le_bytes());
1364 view_buffer[12..].copy_from_slice(&0u32.to_le_bytes());
1365 let view = ByteView::from(u128::from_le_bytes(view_buffer));
1366 let views = ScalarBuffer::from(vec![view.into()]);
1367 let buffers = vec![Buffer::from_slice_ref(input_str_2.as_bytes())];
1368
1369 StringViewArray::new(views, buffers, None);
1370 }
1371
1372 #[test]
1373 fn test_gc() {
1374 let test_data = [
1375 Some("longer than 12 bytes"),
1376 Some("short"),
1377 Some("t"),
1378 Some("longer than 12 bytes"),
1379 None,
1380 Some("short"),
1381 ];
1382
1383 let array = {
1384 let mut builder = StringViewBuilder::new().with_fixed_block_size(8); // create multiple buffers
1385 test_data.into_iter().for_each(|v| builder.append_option(v));
1386 builder.finish()
1387 };
1388 assert!(array.buffers.len() > 1);
1389
1390 fn check_gc(to_test: &StringViewArray) {
1391 let gc = to_test.gc();
1392 assert_ne!(to_test.data_buffers().len(), gc.data_buffers().len());
1393
1394 to_test.iter().zip(gc.iter()).for_each(|(a, b)| {
1395 assert_eq!(a, b);
1396 });
1397 assert_eq!(to_test.len(), gc.len());
1398 }
1399
1400 check_gc(&array);
1401 check_gc(&array.slice(1, 3));
1402 check_gc(&array.slice(2, 1));
1403 check_gc(&array.slice(2, 2));
1404 check_gc(&array.slice(3, 1));
1405 }
1406
1407 /// 1) Empty array: no elements, expect gc to return empty with no data buffers
1408 #[test]
1409 fn test_gc_empty_array() {
1410 let array = StringViewBuilder::new()
1411 .with_fixed_block_size(BLOCK_SIZE)
1412 .finish();
1413 let gced = array.gc();
1414 // length and null count remain zero
1415 assert_eq!(gced.len(), 0);
1416 assert_eq!(gced.null_count(), 0);
1417 // no underlying data buffers should be allocated
1418 assert!(
1419 gced.data_buffers().is_empty(),
1420 "Expected no data buffers for empty array"
1421 );
1422 }
1423
1424 /// 2) All inline values (<= INLINE_LEN): capacity-only data buffer, same values
1425 #[test]
1426 fn test_gc_all_inline() {
1427 let mut builder = StringViewBuilder::new().with_fixed_block_size(BLOCK_SIZE);
1428 // append many short strings, each exactly INLINE_LEN long
1429 for _ in 0..100 {
1430 let s = "A".repeat(MAX_INLINE_VIEW_LEN as usize);
1431 builder.append_option(Some(&s));
1432 }
1433 let array = builder.finish();
1434 let gced = array.gc();
1435 // Since all views fit inline, data buffer is empty
1436 assert_eq!(
1437 gced.data_buffers().len(),
1438 0,
1439 "Should have no data buffers for inline values"
1440 );
1441 assert_eq!(gced.len(), 100);
1442 // verify element-wise equality
1443 array.iter().zip(gced.iter()).for_each(|(orig, got)| {
1444 assert_eq!(orig, got, "Inline value mismatch after gc");
1445 });
1446 }
1447
1448 /// 3) All large values (> INLINE_LEN): each must be copied into the new data buffer
1449 #[test]
1450 fn test_gc_all_large() {
1451 let mut builder = StringViewBuilder::new().with_fixed_block_size(BLOCK_SIZE);
1452 let large_str = "X".repeat(MAX_INLINE_VIEW_LEN as usize + 5);
1453 // append multiple large strings
1454 for _ in 0..50 {
1455 builder.append_option(Some(&large_str));
1456 }
1457 let array = builder.finish();
1458 let gced = array.gc();
1459 // New data buffers should be populated (one or more blocks)
1460 assert!(
1461 !gced.data_buffers().is_empty(),
1462 "Expected data buffers for large values"
1463 );
1464 assert_eq!(gced.len(), 50);
1465 // verify that every large string emerges unchanged
1466 array.iter().zip(gced.iter()).for_each(|(orig, got)| {
1467 assert_eq!(orig, got, "Large view mismatch after gc");
1468 });
1469 }
1470
1471 /// 4) All null elements: ensure null bitmap handling path is correct
1472 #[test]
1473 fn test_gc_all_nulls() {
1474 let mut builder = StringViewBuilder::new().with_fixed_block_size(BLOCK_SIZE);
1475 for _ in 0..20 {
1476 builder.append_null();
1477 }
1478 let array = builder.finish();
1479 let gced = array.gc();
1480 // length and null count match
1481 assert_eq!(gced.len(), 20);
1482 assert_eq!(gced.null_count(), 20);
1483 // data buffers remain empty for null-only array
1484 assert!(
1485 gced.data_buffers().is_empty(),
1486 "No data should be stored for nulls"
1487 );
1488 }
1489
1490 /// 5) Random mix of inline, large, and null values with slicing tests
1491 #[test]
1492 fn test_gc_random_mixed_and_slices() {
1493 let mut rng = StdRng::seed_from_u64(42);
1494 let mut builder = StringViewBuilder::new().with_fixed_block_size(BLOCK_SIZE);
1495 // Keep a Vec of original Option<String> for later comparison
1496 let mut original: Vec<Option<String>> = Vec::new();
1497
1498 for _ in 0..200 {
1499 if rng.random_bool(0.1) {
1500 // 10% nulls
1501 builder.append_null();
1502 original.push(None);
1503 } else {
1504 // random length between 0 and twice the inline limit
1505 let len = rng.random_range(0..(MAX_INLINE_VIEW_LEN * 2));
1506 let s = "A".repeat(len as usize);
1507 builder.append_option(Some(&s));
1508 original.push(Some(s));
1509 }
1510 }
1511
1512 let array = builder.finish();
1513 // Test multiple slice ranges to ensure offset logic is correct
1514 for (offset, slice_len) in &[(0, 50), (10, 100), (150, 30)] {
1515 let sliced = array.slice(*offset, *slice_len);
1516 let gced = sliced.gc();
1517 // Build expected slice of Option<&str>
1518 let expected: Vec<Option<&str>> = original[*offset..(*offset + *slice_len)]
1519 .iter()
1520 .map(|opt| opt.as_deref())
1521 .collect();
1522
1523 assert_eq!(gced.len(), *slice_len, "Slice length mismatch");
1524 // Compare element-wise
1525 gced.iter().zip(expected.iter()).for_each(|(got, expect)| {
1526 assert_eq!(got, *expect, "Value mismatch in mixed slice after gc");
1527 });
1528 }
1529 }
1530
1531 #[test]
1532 #[cfg_attr(miri, ignore)] // Takes too long
1533 fn test_gc_huge_array() {
1534 // Construct multiple 128 MiB BinaryView entries so total > 4 GiB
1535 let block_len: usize = 128 * 1024 * 1024; // 128 MiB per view
1536 let num_views: usize = 36;
1537
1538 // Create a single 128 MiB data block with a simple byte pattern
1539 let buffer = Buffer::from_vec(vec![0xAB; block_len]);
1540 let buffer2 = Buffer::from_vec(vec![0xFF; block_len]);
1541
1542 // Append this block and then add many views pointing to it
1543 let mut builder = BinaryViewBuilder::new();
1544 let block_id = builder.append_block(buffer);
1545 for _ in 0..num_views / 2 {
1546 builder
1547 .try_append_view(block_id, 0, block_len as u32)
1548 .expect("append view into 128MiB block");
1549 }
1550 let block_id2 = builder.append_block(buffer2);
1551 for _ in 0..num_views / 2 {
1552 builder
1553 .try_append_view(block_id2, 0, block_len as u32)
1554 .expect("append view into 128MiB block");
1555 }
1556
1557 let array = builder.finish();
1558 let total = array.total_buffer_bytes_used();
1559 assert!(
1560 total > u32::MAX as usize,
1561 "Expected total non-inline bytes to exceed 4 GiB, got {total}"
1562 );
1563
1564 // Run gc and verify correctness
1565 let gced = array.gc();
1566 assert_eq!(gced.len(), num_views, "Length mismatch after gc");
1567 assert_eq!(gced.null_count(), 0, "Null count mismatch after gc");
1568 assert_ne!(
1569 gced.data_buffers().len(),
1570 1,
1571 "gc with huge buffer should not consolidate data into a single buffer"
1572 );
1573
1574 // Element-wise equality check across the entire array
1575 array.iter().zip(gced.iter()).for_each(|(orig, got)| {
1576 assert_eq!(orig, got, "Value mismatch after gc on huge array");
1577 });
1578 }
1579
1580 #[test]
1581 fn test_gc_slow_path_preserves_inline_views() {
1582 // Regression test for a bug in the multi-buffer "slow path" of `gc()`,
1583 // which `gc` only reaches once a view column references more than
1584 // `i32::MAX` (~2.1 GiB) of non-inline data. That path grouped and copied
1585 // only the *non-inline* views and dropped every inline (short) view,
1586 // returning an array shorter than its input. GC is a pure size
1587 // optimisation and must never change the row count.
1588 //
1589 // Rather than allocate gigabytes to hit the real threshold, drive the
1590 // same code via `gc_with_max_buffer_size` with a tiny cap so a handful
1591 // of bytes forces the buffer split. Inline, large, and null views are
1592 // interspersed so the split lands between and around inline entries.
1593 let long = "this is definitely longer than twelve bytes"; // non-inline
1594 let test_data = [
1595 Some("short"), // inline
1596 Some(long), // large
1597 Some("s"), // inline
1598 Some(long), // large
1599 None, // null
1600 Some(long), // large
1601 Some("also short"), // inline
1602 Some(long), // large
1603 Some("tail"), // inline
1604 ];
1605 let array: StringViewArray = test_data.into_iter().collect();
1606
1607 // Cap each output buffer at ~1.5 large values so the copy is forced to
1608 // split across several buffers, exercising the slow path with no
1609 // multi-GiB allocation.
1610 let max_buffer_size = long.len() + long.len() / 2;
1611 let gced = array.gc_with_max_buffer_size(max_buffer_size);
1612
1613 // The core invariant: gc preserves the row count.
1614 assert_eq!(gced.len(), array.len(), "gc changed the row count");
1615 // The split must actually have happened, otherwise we'd be exercising
1616 // the fast path and could not catch the inline-dropping regression.
1617 assert!(
1618 gced.data_buffers().len() > 1,
1619 "expected output split across multiple buffers, got {}",
1620 gced.data_buffers().len()
1621 );
1622 // No output buffer may exceed the cap.
1623 for buf in gced.data_buffers().iter() {
1624 assert!(buf.len() <= max_buffer_size, "buffer exceeded max size");
1625 }
1626 // Every value (inline, large, and null) is unchanged and in order.
1627 array
1628 .iter()
1629 .zip(gced.iter())
1630 .enumerate()
1631 .for_each(|(i, (a, b))| {
1632 assert_eq!(a, b, "value mismatch at index {i} after gc");
1633 });
1634 // The result round-trips through full validation.
1635 gced.to_data().validate_full().unwrap();
1636 }
1637
1638 #[test]
1639 fn test_gc_slow_path_value_larger_than_buffer() {
1640 // A single non-inline value can be larger than one output buffer (a view
1641 // length is a `u32`, so a value may exceed `max_buffer_size`). Such a
1642 // value cannot be split, so it occupies a buffer of its own; gc must
1643 // still preserve every row and produce a valid array.
1644 //
1645 // The real threshold needs a multi-GiB value, so drive the same logic
1646 // with a `max_buffer_size` smaller than the value length.
1647 let big = "x".repeat(64); // non-inline, larger than the cap below
1648 let test_data = [
1649 Some("short"), // inline
1650 Some(big.as_str()), // larger than max_buffer_size
1651 None, // null
1652 Some("tail"), // inline
1653 Some(big.as_str()), // larger than max_buffer_size
1654 ];
1655 let array: StringViewArray = test_data.into_iter().collect();
1656
1657 let max_buffer_size = 16; // smaller than a single `big` value
1658 let gced = array.gc_with_max_buffer_size(max_buffer_size);
1659
1660 // Row count is preserved.
1661 assert_eq!(gced.len(), array.len(), "gc changed the row count");
1662 // Each oversized value lands in its own buffer, so the copy splits.
1663 assert!(
1664 gced.data_buffers().len() > 1,
1665 "expected output split across multiple buffers, got {}",
1666 gced.data_buffers().len()
1667 );
1668 // Every value is unchanged and in order.
1669 array
1670 .iter()
1671 .zip(gced.iter())
1672 .enumerate()
1673 .for_each(|(i, (a, b))| {
1674 assert_eq!(a, b, "value mismatch at index {i} after gc");
1675 });
1676 // The result round-trips through full validation.
1677 gced.to_data().validate_full().unwrap();
1678 }
1679
1680 #[test]
1681 fn test_eq() {
1682 let test_data = [
1683 Some("longer than 12 bytes"),
1684 None,
1685 Some("short"),
1686 Some("again, this is longer than 12 bytes"),
1687 ];
1688
1689 let array1 = {
1690 let mut builder = StringViewBuilder::new().with_fixed_block_size(8);
1691 test_data.into_iter().for_each(|v| builder.append_option(v));
1692 builder.finish()
1693 };
1694 let array2 = {
1695 // create a new array with the same data but different layout
1696 let mut builder = StringViewBuilder::new().with_fixed_block_size(100);
1697 test_data.into_iter().for_each(|v| builder.append_option(v));
1698 builder.finish()
1699 };
1700 assert_eq!(array1, array1.clone());
1701 assert_eq!(array2, array2.clone());
1702 assert_eq!(array1, array2);
1703 }
1704
1705 /// Integration tests for `inline_key_fast` covering:
1706 ///
1707 /// 1. Monotonic ordering across increasing lengths and lexical variations.
1708 /// 2. Cross-check against `GenericBinaryArray` comparison to ensure semantic equivalence.
1709 ///
1710 /// This also includes a specific test for the “bar” vs. “bar\0” case, demonstrating why
1711 /// the length field is required even when all inline bytes fit in 12 bytes.
1712 ///
1713 /// The test includes strings that verify correct byte order (prevent reversal bugs),
1714 /// and length-based tie-breaking in the composite key.
1715 ///
1716 /// The test confirms that `inline_key_fast` produces keys which sort consistently
1717 /// with the expected lexicographical order of the raw byte arrays.
1718 #[test]
1719 fn test_inline_key_fast_various_lengths_and_lexical() {
1720 /// Helper to create a raw u128 value representing an inline ByteView:
1721 /// - `length`: number of meaningful bytes (must be ≤ 12)
1722 /// - `data`: the actual inline data bytes
1723 ///
1724 /// The first 4 bytes encode length in little-endian,
1725 /// the following 12 bytes contain the inline string data (unpadded).
1726 fn make_raw_inline(length: u32, data: &[u8]) -> u128 {
1727 assert!(length as usize <= 12, "Inline length must be ≤ 12");
1728 assert!(
1729 data.len() == length as usize,
1730 "Data length must match `length`"
1731 );
1732
1733 let mut raw_bytes = [0u8; 16];
1734 raw_bytes[0..4].copy_from_slice(&length.to_le_bytes()); // length stored little-endian
1735 raw_bytes[4..(4 + data.len())].copy_from_slice(data); // inline data
1736 u128::from_le_bytes(raw_bytes)
1737 }
1738
1739 // Test inputs: various lengths and lexical orders,
1740 // plus special cases for byte order and length tie-breaking
1741 let test_inputs: Vec<&[u8]> = vec![
1742 b"a",
1743 b"aa",
1744 b"aaa",
1745 b"aab",
1746 b"abcd",
1747 b"abcde",
1748 b"abcdef",
1749 b"abcdefg",
1750 b"abcdefgh",
1751 b"abcdefghi",
1752 b"abcdefghij",
1753 b"abcdefghijk",
1754 b"abcdefghijkl",
1755 // Tests for byte-order reversal bug:
1756 // Without the fix, "backend one" would compare as "eno dnekcab",
1757 // causing incorrect sort order relative to "backend two".
1758 b"backend one",
1759 b"backend two",
1760 // Tests length-tiebreaker logic:
1761 // "bar" (3 bytes) and "bar\0" (4 bytes) have identical inline data,
1762 // so only the length differentiates their ordering.
1763 b"bar",
1764 b"bar\0",
1765 // Additional lexical and length tie-breaking cases with same prefix, in correct lex order:
1766 b"than12Byt",
1767 b"than12Bytes",
1768 b"than12Bytes\0",
1769 b"than12Bytesx",
1770 b"than12Bytex",
1771 b"than12Bytez",
1772 // Additional lexical tests
1773 b"xyy",
1774 b"xyz",
1775 b"xza",
1776 ];
1777
1778 // Create a GenericBinaryArray for cross-comparison of lex order
1779 let array: GenericBinaryArray<i32> =
1780 GenericBinaryArray::from(test_inputs.iter().map(|s| Some(*s)).collect::<Vec<_>>());
1781
1782 for i in 0..array.len() - 1 {
1783 let v1 = array.value(i);
1784 let v2 = array.value(i + 1);
1785
1786 // Assert the array's natural lexical ordering is correct
1787 assert!(v1 < v2, "Array compare failed: {v1:?} !< {v2:?}");
1788
1789 // Assert the keys produced by inline_key_fast reflect the same ordering
1790 let key1 = GenericByteViewArray::<BinaryViewType>::inline_key_fast(make_raw_inline(
1791 v1.len() as u32,
1792 v1,
1793 ));
1794 let key2 = GenericByteViewArray::<BinaryViewType>::inline_key_fast(make_raw_inline(
1795 v2.len() as u32,
1796 v2,
1797 ));
1798
1799 assert!(
1800 key1 < key2,
1801 "Key compare failed: key({v1:?})=0x{key1:032x} !< key({v2:?})=0x{key2:032x}",
1802 );
1803 }
1804 }
1805
1806 #[test]
1807 fn empty_array_should_return_empty_lengths_iterator() {
1808 let empty = GenericByteViewArray::<BinaryViewType>::from(Vec::<&[u8]>::new());
1809
1810 let mut lengths_iter = empty.lengths();
1811 assert_eq!(lengths_iter.len(), 0);
1812 assert_eq!(lengths_iter.next(), None);
1813 }
1814
1815 #[test]
1816 fn array_lengths_should_return_correct_length_for_both_inlined_and_non_inlined() {
1817 let cases = GenericByteViewArray::<BinaryViewType>::from(vec![
1818 // Not inlined as longer than 12 bytes
1819 b"Supercalifragilisticexpialidocious" as &[u8],
1820 // Inlined as shorter than 12 bytes
1821 b"Hello",
1822 // Empty value
1823 b"",
1824 // Exactly 12 bytes
1825 b"abcdefghijkl",
1826 ]);
1827
1828 let mut lengths_iter = cases.lengths();
1829
1830 assert_eq!(lengths_iter.len(), cases.len());
1831
1832 let cases_iter = cases.iter();
1833
1834 for case in cases_iter {
1835 let case_value = case.unwrap();
1836 let length = lengths_iter.next().expect("Should have a length");
1837
1838 assert_eq!(case_value.len(), length as usize);
1839 }
1840
1841 assert_eq!(lengths_iter.next(), None, "Should not have more lengths");
1842 }
1843
1844 #[test]
1845 fn array_lengths_should_return_the_underlying_length_for_null_values() {
1846 let cases = GenericByteViewArray::<BinaryViewType>::from(vec![
1847 // Not inlined as longer than 12 bytes
1848 b"Supercalifragilisticexpialidocious" as &[u8],
1849 // Inlined as shorter than 12 bytes
1850 b"Hello",
1851 // Empty value
1852 b"",
1853 // Exactly 12 bytes
1854 b"abcdefghijkl",
1855 ]);
1856
1857 let (views, buffer, _) = cases.clone().into_parts();
1858
1859 // Keeping the values but just adding nulls on top
1860 let cases_with_all_nulls = GenericByteViewArray::<BinaryViewType>::new(
1861 views,
1862 buffer,
1863 Some(NullBuffer::new_null(cases.len())),
1864 );
1865
1866 let lengths_iter = cases.lengths();
1867 let mut all_nulls_lengths_iter = cases_with_all_nulls.lengths();
1868
1869 assert_eq!(lengths_iter.len(), all_nulls_lengths_iter.len());
1870
1871 for expected_length in lengths_iter {
1872 let actual_length = all_nulls_lengths_iter.next().expect("Should have a length");
1873
1874 assert_eq!(expected_length, actual_length);
1875 }
1876
1877 assert_eq!(
1878 all_nulls_lengths_iter.next(),
1879 None,
1880 "Should not have more lengths"
1881 );
1882 }
1883
1884 #[test]
1885 fn array_lengths_on_sliced_should_only_return_lengths_for_sliced_data() {
1886 let array = GenericByteViewArray::<BinaryViewType>::from(vec![
1887 b"aaaaaaaaaaaaaaaaaaaaaaaaaaa" as &[u8],
1888 b"Hello",
1889 b"something great",
1890 b"is",
1891 b"coming soon!",
1892 b"when you find what it is",
1893 b"let me know",
1894 b"cause",
1895 b"I",
1896 b"have no idea",
1897 b"what it",
1898 b"is",
1899 ]);
1900
1901 let sliced_array = array.slice(2, array.len() - 3);
1902
1903 let mut lengths_iter = sliced_array.lengths();
1904
1905 assert_eq!(lengths_iter.len(), sliced_array.len());
1906
1907 let values_iter = sliced_array.iter();
1908
1909 for value in values_iter {
1910 let value = value.unwrap();
1911 let length = lengths_iter.next().expect("Should have a length");
1912
1913 assert_eq!(value.len(), length as usize);
1914 }
1915
1916 assert_eq!(lengths_iter.next(), None, "Should not have more lengths");
1917 }
1918
1919 #[should_panic(expected = "Mismatched data type, expected Utf8View, got BinaryView")]
1920 #[test]
1921 fn invalid_casting_from_array_data() {
1922 // Should not be able to cast to StringViewArray due to invalid UTF-8
1923 let array_data = binary_view_array_with_invalid_utf8_data().into_data();
1924 let _ = StringViewArray::from(array_data);
1925 }
1926
1927 #[should_panic(expected = "invalid utf-8 sequence")]
1928 #[test]
1929 fn invalid_array_data() {
1930 let (views, buffers, nulls) = binary_view_array_with_invalid_utf8_data().into_parts();
1931
1932 // manually try and add invalid array data with Utf8View data type
1933 let mut builder = ArrayDataBuilder::new(DataType::Utf8View)
1934 .add_buffer(views.into_inner())
1935 .len(3);
1936 for buffer in buffers.iter() {
1937 builder = builder.add_buffer(buffer.clone())
1938 }
1939 builder = builder.nulls(nulls);
1940
1941 let data = builder.build().unwrap(); // should fail validation
1942 let _arr = StringViewArray::from(data);
1943 }
1944
1945 /// Returns a BinaryViewArray with one invalid UTF-8 value
1946 fn binary_view_array_with_invalid_utf8_data() -> BinaryViewArray {
1947 let array = GenericByteViewArray::<BinaryViewType>::from(vec![
1948 b"aaaaaaaaaaaaaaaaaaaaaaaaaaa" as &[u8],
1949 &[
1950 0xf0, 0x80, 0x80, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1951 0x00, 0x00,
1952 ],
1953 b"good",
1954 ]);
1955 assert!(from_utf8(array.value(0)).is_ok());
1956 assert!(from_utf8(array.value(1)).is_err()); // value 1 is invalid utf8
1957 assert!(from_utf8(array.value(2)).is_ok());
1958 array
1959 }
1960
1961 #[test]
1962 fn test_total_bytes_len() {
1963 // inlined: "hello"=5, "world"=5, "lulu"=4 → 14
1964 // non-inlined: "large payload over 12 bytes"=27
1965 // null: should not count
1966 let mut builder = StringViewBuilder::new();
1967 builder.append_value("hello");
1968 builder.append_value("world");
1969 builder.append_value("lulu");
1970 builder.append_null();
1971 builder.append_value("large payload over 12 bytes");
1972 let array = builder.finish();
1973 assert_eq!(array.total_bytes_len(), 5 + 5 + 4 + 27);
1974 }
1975
1976 #[test]
1977 fn test_total_bytes_len_empty() {
1978 let array = StringViewArray::from_iter::<Vec<Option<&str>>>(vec![]);
1979 assert_eq!(array.total_bytes_len(), 0);
1980 }
1981
1982 #[test]
1983 fn test_total_bytes_len_all_nulls() {
1984 let array = StringViewArray::new_null(5);
1985 assert_eq!(array.total_bytes_len(), 0);
1986 }
1987
1988 #[test]
1989 fn test_total_bytes_len_binary_view() {
1990 let array = BinaryViewArray::from_iter(vec![
1991 Some(b"hi".as_ref()),
1992 None,
1993 Some(b"large payload over 12 bytes".as_ref()),
1994 ]);
1995 assert_eq!(array.total_bytes_len(), 2 + 27);
1996 }
1997}