arrow_array/array/
byte_array.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use crate::array::{get_offsets_from_buffer, print_long_array};
19use crate::builder::GenericByteBuilder;
20use crate::iterator::ArrayIter;
21use crate::types::ByteArrayType;
22use crate::types::bytes::ByteArrayNativeType;
23use crate::{Array, ArrayAccessor, ArrayRef, OffsetSizeTrait, Scalar};
24use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer};
25use arrow_buffer::{NullBuffer, OffsetBuffer};
26use arrow_data::{ArrayData, ArrayDataBuilder};
27use arrow_schema::{ArrowError, DataType};
28use std::any::Any;
29use std::sync::Arc;
30
31/// An array of [variable length byte arrays](https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout)
32///
33/// See [`StringArray`] and [`LargeStringArray`] for storing utf8 encoded string data
34///
35/// See [`BinaryArray`] and [`LargeBinaryArray`] for storing arbitrary bytes
36///
37/// # Example: From a Vec
38///
39/// ```
40/// # use arrow_array::{Array, GenericByteArray, types::Utf8Type};
41/// let arr: GenericByteArray<Utf8Type> = vec!["hello", "world", ""].into();
42/// assert_eq!(arr.value_data(), b"helloworld");
43/// assert_eq!(arr.value_offsets(), &[0, 5, 10, 10]);
44/// let values: Vec<_> = arr.iter().collect();
45/// assert_eq!(values, &[Some("hello"), Some("world"), Some("")]);
46/// ```
47///
48/// # Example: From an optional Vec
49///
50/// ```
51/// # use arrow_array::{Array, GenericByteArray, types::Utf8Type};
52/// let arr: GenericByteArray<Utf8Type> = vec![Some("hello"), Some("world"), Some(""), None].into();
53/// assert_eq!(arr.value_data(), b"helloworld");
54/// assert_eq!(arr.value_offsets(), &[0, 5, 10, 10, 10]);
55/// let values: Vec<_> = arr.iter().collect();
56/// assert_eq!(values, &[Some("hello"), Some("world"), Some(""), None]);
57/// ```
58///
59/// # Example: From an iterator of option
60///
61/// ```
62/// # use arrow_array::{Array, GenericByteArray, types::Utf8Type};
63/// let arr: GenericByteArray<Utf8Type> = (0..5).map(|x| (x % 2 == 0).then(|| x.to_string())).collect();
64/// let values: Vec<_> = arr.iter().collect();
65/// assert_eq!(values, &[Some("0"), None, Some("2"), None, Some("4")]);
66/// ```
67///
68/// # Example: Using Builder
69///
70/// ```
71/// # use arrow_array::Array;
72/// # use arrow_array::builder::GenericByteBuilder;
73/// # use arrow_array::types::Utf8Type;
74/// let mut builder = GenericByteBuilder::<Utf8Type>::new();
75/// builder.append_value("hello");
76/// builder.append_null();
77/// builder.append_value("world");
78/// let array = builder.finish();
79/// let values: Vec<_> = array.iter().collect();
80/// assert_eq!(values, &[Some("hello"), None, Some("world")]);
81/// ```
82///
83/// [`StringArray`]: crate::StringArray
84/// [`LargeStringArray`]: crate::LargeStringArray
85/// [`BinaryArray`]: crate::BinaryArray
86/// [`LargeBinaryArray`]: crate::LargeBinaryArray
87pub struct GenericByteArray<T: ByteArrayType> {
88    data_type: DataType,
89    value_offsets: OffsetBuffer<T::Offset>,
90    value_data: Buffer,
91    nulls: Option<NullBuffer>,
92}
93
94impl<T: ByteArrayType> Clone for GenericByteArray<T> {
95    fn clone(&self) -> Self {
96        Self {
97            data_type: T::DATA_TYPE,
98            value_offsets: self.value_offsets.clone(),
99            value_data: self.value_data.clone(),
100            nulls: self.nulls.clone(),
101        }
102    }
103}
104
105impl<T: ByteArrayType> GenericByteArray<T> {
106    /// Data type of the array.
107    pub const DATA_TYPE: DataType = T::DATA_TYPE;
108
109    /// Create a new [`GenericByteArray`] from the provided parts, panicking on failure
110    ///
111    /// # Panics
112    ///
113    /// Panics if [`GenericByteArray::try_new`] returns an error
114    pub fn new(
115        offsets: OffsetBuffer<T::Offset>,
116        values: Buffer,
117        nulls: Option<NullBuffer>,
118    ) -> Self {
119        Self::try_new(offsets, values, nulls).unwrap()
120    }
121
122    /// Create a new [`GenericByteArray`] from the provided parts, returning an error on failure
123    ///
124    /// # Errors
125    ///
126    /// * `offsets.len() - 1 != nulls.len()`
127    /// * Any consecutive pair of `offsets` does not denote a valid slice of `values`
128    pub fn try_new(
129        offsets: OffsetBuffer<T::Offset>,
130        values: Buffer,
131        nulls: Option<NullBuffer>,
132    ) -> Result<Self, ArrowError> {
133        let len = offsets.len() - 1;
134
135        // Verify that each pair of offsets is a valid slices of values
136        T::validate(&offsets, &values)?;
137
138        if let Some(n) = nulls.as_ref()
139            && n.len() != len
140        {
141            return Err(ArrowError::InvalidArgumentError(format!(
142                "Incorrect length of null buffer for {}{}Array, expected {len} got {}",
143                T::Offset::PREFIX,
144                T::PREFIX,
145                n.len(),
146            )));
147        }
148
149        Ok(Self {
150            data_type: T::DATA_TYPE,
151            value_offsets: offsets,
152            value_data: values,
153            nulls,
154        })
155    }
156
157    /// Create a new [`GenericByteArray`] from the provided parts, without validation
158    ///
159    /// # Safety
160    ///
161    /// Safe if [`Self::try_new`] would not error
162    pub unsafe fn new_unchecked(
163        offsets: OffsetBuffer<T::Offset>,
164        values: Buffer,
165        nulls: Option<NullBuffer>,
166    ) -> Self {
167        if cfg!(feature = "force_validate") {
168            return Self::new(offsets, values, nulls);
169        }
170        Self {
171            data_type: T::DATA_TYPE,
172            value_offsets: offsets,
173            value_data: values,
174            nulls,
175        }
176    }
177
178    /// Create a new [`GenericByteArray`] of length `len` where all values are null
179    pub fn new_null(len: usize) -> Self {
180        Self {
181            data_type: T::DATA_TYPE,
182            value_offsets: OffsetBuffer::new_zeroed(len),
183            value_data: MutableBuffer::new(0).into(),
184            nulls: Some(NullBuffer::new_null(len)),
185        }
186    }
187
188    /// Create a new [`Scalar`] from `v`
189    pub fn new_scalar(value: impl AsRef<T::Native>) -> Scalar<Self> {
190        Scalar::new(Self::from_iter_values(std::iter::once(value)))
191    }
192
193    /// Create a new [`GenericByteArray`] where `value` is repeated `repeat_count` times.
194    ///
195    /// # Panics
196    /// This will panic if value's length multiplied by `repeat_count` overflows usize.
197    ///
198    pub fn new_repeated(value: impl AsRef<T::Native>, repeat_count: usize) -> Self {
199        let s: &[u8] = value.as_ref().as_ref();
200        let value_offsets = OffsetBuffer::from_repeated_length(s.len(), repeat_count);
201        let bytes: Buffer = {
202            let mut mutable_buffer = MutableBuffer::with_capacity(0);
203            mutable_buffer.repeat_slice_n_times(s, repeat_count);
204
205            mutable_buffer.into()
206        };
207
208        Self {
209            data_type: T::DATA_TYPE,
210            value_data: bytes,
211            value_offsets,
212            nulls: None,
213        }
214    }
215
216    /// Creates a [`GenericByteArray`] based on an iterator of values without nulls
217    ///
218    /// # Panics
219    /// Panics if the total length of the values exceeds `T::Offset::MAX`
220    pub fn from_iter_values<Ptr, I>(iter: I) -> Self
221    where
222        Ptr: AsRef<T::Native>,
223        I: IntoIterator<Item = Ptr>,
224    {
225        let iter = iter.into_iter();
226        // The size hint is only used to pre-allocate: an iterator is free to yield
227        // a different number of items than it reports.
228        let (lower, upper) = iter.size_hint();
229        let capacity = upper.unwrap_or(lower);
230
231        let mut offsets = MutableBuffer::new(
232            capacity
233                .saturating_add(1)
234                .saturating_mul(std::mem::size_of::<T::Offset>()),
235        );
236        offsets.push(T::Offset::usize_as(0));
237
238        let mut values = MutableBuffer::new(0);
239        for s in iter {
240            let s: &[u8] = s.as_ref().as_ref();
241            values.extend_from_slice(s);
242            offsets.push(T::Offset::usize_as(values.len()));
243        }
244
245        T::Offset::from_usize(values.len()).expect("offset overflow");
246        let offsets = Buffer::from(offsets);
247
248        // Safety: valid by construction
249        let value_offsets = unsafe { OffsetBuffer::new_unchecked(offsets.into()) };
250
251        Self {
252            data_type: T::DATA_TYPE,
253            value_data: values.into(),
254            value_offsets,
255            nulls: None,
256        }
257    }
258
259    /// Deconstruct this array into its constituent parts
260    pub fn into_parts(self) -> (OffsetBuffer<T::Offset>, Buffer, Option<NullBuffer>) {
261        (self.value_offsets, self.value_data, self.nulls)
262    }
263
264    /// Returns the length for value at index `i`.
265    /// # Panics
266    /// Panics if index `i` is out of bounds.
267    #[inline]
268    pub fn value_length(&self, i: usize) -> T::Offset {
269        let offsets = self.value_offsets();
270        offsets[i + 1] - offsets[i]
271    }
272
273    /// Returns a reference to the offsets of this array
274    ///
275    /// Unlike [`Self::value_offsets`] this returns the [`OffsetBuffer`]
276    /// allowing for zero-copy cloning
277    #[inline]
278    pub fn offsets(&self) -> &OffsetBuffer<T::Offset> {
279        &self.value_offsets
280    }
281
282    /// Returns the values of this array
283    ///
284    /// Unlike [`Self::value_data`] this returns the [`Buffer`]
285    /// allowing for zero-copy cloning. Like [`Self::value_data`], it can
286    /// include bytes that are not part of any value.
287    #[inline]
288    pub fn values(&self) -> &Buffer {
289        &self.value_data
290    }
291
292    /// Returns the raw value data
293    ///
294    /// This can include bytes before the first offset or after the last one,
295    /// for example when the array is a slice of a larger array. Those bytes are
296    /// not part of any value, and in a string array they need not be valid UTF-8.
297    pub fn value_data(&self) -> &[u8] {
298        self.value_data.as_slice()
299    }
300
301    /// Returns true if all data within this array is ASCII
302    pub fn is_ascii(&self) -> bool {
303        let offsets = &self.value_offsets;
304        self.value_data()[offsets.first().as_usize()..offsets.last().as_usize()].is_ascii()
305    }
306
307    /// Returns the offset values in the offsets buffer
308    #[inline]
309    pub fn value_offsets(&self) -> &[T::Offset] {
310        &self.value_offsets
311    }
312
313    /// Returns the element at index `i`
314    ///
315    /// Note: This method does not check for nulls and the value is arbitrary
316    /// if [`is_null`](Self::is_null) returns true for the index.
317    ///
318    /// # Safety
319    /// Caller is responsible for ensuring that the index is within the bounds of the array
320    pub unsafe fn value_unchecked(&self, i: usize) -> &T::Native {
321        let end = *unsafe { self.value_offsets().get_unchecked(i + 1) };
322        let start = *unsafe { self.value_offsets().get_unchecked(i) };
323
324        // Soundness
325        // pointer alignment & location is ensured by RawPtrBox
326        // buffer bounds/offset is ensured by the value_offset invariants
327
328        // Safety of `to_isize().unwrap()`
329        // `start` and `end` are &OffsetSize, which is a generic type that implements the
330        // OffsetSizeTrait. Currently, only i32 and i64 implement OffsetSizeTrait,
331        // both of which should cleanly cast to isize on an architecture that supports
332        // 32/64-bit offsets
333        let b = unsafe {
334            std::slice::from_raw_parts(
335                self.value_data
336                    .as_ptr()
337                    .offset(start.to_isize().unwrap_unchecked()),
338                (end - start).to_usize().unwrap_unchecked(),
339            )
340        };
341
342        // SAFETY:
343        // ArrayData is valid
344        unsafe { T::Native::from_bytes_unchecked(b) }
345    }
346
347    /// Returns the element at index `i`
348    ///
349    /// Note: This method does not check for nulls and the value is arbitrary
350    /// (but still well-defined) if [`is_null`](Self::is_null) returns true for the index.
351    ///
352    /// # Panics
353    /// Panics if index `i` is out of bounds.
354    pub fn value(&self, i: usize) -> &T::Native {
355        assert!(
356            i < self.len(),
357            "Trying to access an element at index {} from a {}{}Array of length {}",
358            i,
359            T::Offset::PREFIX,
360            T::PREFIX,
361            self.len()
362        );
363        // SAFETY:
364        // Verified length above
365        unsafe { self.value_unchecked(i) }
366    }
367
368    /// constructs a new iterator
369    pub fn iter(&self) -> ArrayIter<&Self> {
370        ArrayIter::new(self)
371    }
372
373    /// Returns a zero-copy slice of this array with the indicated offset and length.
374    ///
375    /// # Panics
376    /// Panics if `offset + length > self.len()`
377    pub fn slice(&self, offset: usize, length: usize) -> Self {
378        Self {
379            data_type: T::DATA_TYPE,
380            value_offsets: self.value_offsets.slice(offset, length),
381            value_data: self.value_data.clone(),
382            nulls: self.nulls.as_ref().map(|n| n.slice(offset, length)),
383        }
384    }
385
386    /// Converts this array into a [`GenericByteBuilder`] that reuses its buffers,
387    /// if possible.
388    ///
389    /// Returns `Err(self)` if the offsets or values cannot be reused, for
390    /// example because they are shared with another array, or because this is a
391    /// sliced array with a non-zero first offset.
392    ///
393    /// The null buffer is copied if it cannot be reused.
394    pub fn into_builder(self) -> Result<GenericByteBuilder<T>, Self> {
395        let (offsets, values, nulls) = self.into_parts();
396
397        // Puts the parts back together into the original array
398        let rebuild = |offsets: Buffer, values: Buffer, nulls: Option<NullBuffer>| {
399            // SAFETY: the parts come unchanged from `self`
400            unsafe {
401                Self::new_unchecked(OffsetBuffer::new_unchecked(offsets.into()), values, nulls)
402            }
403        };
404
405        // A buffer can be reused only if this array is its only owner and it
406        // starts at the beginning of its allocation. Otherwise, the original
407        // array is returned.
408        let offsets = match offsets.into_inner().into_inner().into_mutable() {
409            Ok(offsets) => offsets,
410            Err(offsets) => return Err(rebuild(offsets, values, nulls)),
411        };
412        let values = match values.into_mutable() {
413            Ok(values) => values,
414            Err(values) => return Err(rebuild(offsets.into(), values, nulls)),
415        };
416
417        // The builder needs a null buffer whose first bit is for the first row.
418        // That is the case when the bit offset is 0, and then the buffer is reused
419        // if this array is its only owner. Otherwise, the bits are copied, which is
420        // cheap as there is one bit per row.
421        let nulls = nulls.map(|nulls| {
422            let bits = nulls.into_inner();
423            if bits.offset() == 0 {
424                let len = bits.len();
425                bits.into_inner()
426                    .into_mutable()
427                    .unwrap_or_else(|buffer| buffer[..len.div_ceil(8)].to_vec().into())
428            } else {
429                // `sliced` returns the bits starting at the first row
430                bits.sliced().to_vec().into()
431            }
432        });
433
434        // SAFETY: the buffers come from a valid array
435        Ok(unsafe { GenericByteBuilder::new_from_buffer(offsets, values, nulls) })
436    }
437}
438
439impl<T: ByteArrayType> std::fmt::Debug for GenericByteArray<T> {
440    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
441        write!(f, "{}{}Array\n[\n", T::Offset::PREFIX, T::PREFIX)?;
442        print_long_array(self, f, &mut |index, f| {
443            std::fmt::Debug::fmt(&self.value(index), f)
444        })?;
445        write!(f, "]")
446    }
447}
448
449/// SAFETY: Correctly implements the contract of Arrow Arrays
450unsafe impl<T: ByteArrayType> Array for GenericByteArray<T> {
451    fn as_any(&self) -> &dyn Any {
452        self
453    }
454
455    fn to_data(&self) -> ArrayData {
456        self.clone().into()
457    }
458
459    fn into_data(self) -> ArrayData {
460        self.into()
461    }
462
463    fn data_type(&self) -> &DataType {
464        &self.data_type
465    }
466
467    fn slice(&self, offset: usize, length: usize) -> ArrayRef {
468        Arc::new(self.slice(offset, length))
469    }
470
471    fn len(&self) -> usize {
472        self.value_offsets.len() - 1
473    }
474
475    fn is_empty(&self) -> bool {
476        self.value_offsets.len() <= 1
477    }
478
479    fn shrink_to_fit(&mut self) {
480        self.value_offsets.shrink_to_fit();
481        self.value_data.shrink_to_fit();
482        if let Some(nulls) = &mut self.nulls {
483            nulls.shrink_to_fit();
484        }
485    }
486
487    fn offset(&self) -> usize {
488        0
489    }
490
491    fn nulls(&self) -> Option<&NullBuffer> {
492        self.nulls.as_ref()
493    }
494
495    fn logical_null_count(&self) -> usize {
496        // More efficient that the default implementation
497        self.null_count()
498    }
499
500    fn get_buffer_memory_size(&self) -> usize {
501        let mut sum = self.value_offsets.inner().inner().capacity();
502        sum += self.value_data.capacity();
503        if let Some(x) = &self.nulls {
504            sum += x.buffer().capacity()
505        }
506        sum
507    }
508
509    fn get_array_memory_size(&self) -> usize {
510        std::mem::size_of::<Self>() + self.get_buffer_memory_size()
511    }
512
513    #[cfg(feature = "pool")]
514    fn claim(&self, pool: &dyn arrow_buffer::MemoryPool) {
515        self.value_offsets.claim(pool);
516        self.value_data.claim(pool);
517        if let Some(nulls) = &self.nulls {
518            nulls.claim(pool);
519        }
520    }
521}
522
523impl<'a, T: ByteArrayType> ArrayAccessor for &'a GenericByteArray<T> {
524    type Item = &'a T::Native;
525
526    fn value(&self, index: usize) -> Self::Item {
527        GenericByteArray::value(self, index)
528    }
529
530    unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
531        unsafe { GenericByteArray::value_unchecked(self, index) }
532    }
533}
534
535impl<T: ByteArrayType> From<ArrayData> for GenericByteArray<T> {
536    fn from(data: ArrayData) -> Self {
537        let (data_type, len, nulls, offset, mut buffers, _child_data) = data.into_parts();
538        assert_eq!(
539            data_type,
540            Self::DATA_TYPE,
541            "{}{}Array expects DataType::{}",
542            T::Offset::PREFIX,
543            T::PREFIX,
544            Self::DATA_TYPE
545        );
546        assert_eq!(
547            buffers.len(),
548            2,
549            "{}{}Array data should contain 2 buffers only (offsets and values)",
550            T::Offset::PREFIX,
551            T::PREFIX,
552        );
553        // buffers are offset then value, so pop in reverse
554        let value_data = buffers.pop().expect("checked above");
555        let offset_buffer = buffers.pop().expect("checked above");
556
557        // SAFETY:
558        // ArrayData is valid, and verified type above
559        let value_offsets = unsafe { get_offsets_from_buffer(offset_buffer, offset, len) };
560        Self {
561            data_type,
562            value_offsets,
563            value_data,
564            nulls,
565        }
566    }
567}
568
569impl<T: ByteArrayType> From<GenericByteArray<T>> for ArrayData {
570    fn from(array: GenericByteArray<T>) -> Self {
571        let len = array.len();
572
573        let offsets = array.value_offsets.into_inner().into_inner();
574        let builder = ArrayDataBuilder::new(array.data_type)
575            .len(len)
576            .buffers(vec![offsets, array.value_data])
577            .nulls(array.nulls);
578
579        unsafe { builder.build_unchecked() }
580    }
581}
582
583impl<'a, T: ByteArrayType> IntoIterator for &'a GenericByteArray<T> {
584    type Item = Option<&'a T::Native>;
585    type IntoIter = ArrayIter<Self>;
586
587    fn into_iter(self) -> Self::IntoIter {
588        ArrayIter::new(self)
589    }
590}
591
592impl<'a, Ptr, T: ByteArrayType> FromIterator<&'a Option<Ptr>> for GenericByteArray<T>
593where
594    Ptr: AsRef<T::Native> + 'a,
595{
596    fn from_iter<I: IntoIterator<Item = &'a Option<Ptr>>>(iter: I) -> Self {
597        iter.into_iter()
598            .map(|o| o.as_ref().map(|p| p.as_ref()))
599            .collect()
600    }
601}
602
603impl<Ptr, T: ByteArrayType> FromIterator<Option<Ptr>> for GenericByteArray<T>
604where
605    Ptr: AsRef<T::Native>,
606{
607    fn from_iter<I: IntoIterator<Item = Option<Ptr>>>(iter: I) -> Self {
608        let iter = iter.into_iter();
609        let mut builder = GenericByteBuilder::with_capacity(iter.size_hint().0, 1024);
610        builder.extend(iter);
611        builder.finish()
612    }
613}
614
615#[cfg(test)]
616mod tests {
617    use crate::{Array, BinaryArray, StringArray};
618    use arrow_buffer::{Buffer, NullBuffer, OffsetBuffer};
619
620    /// `from_iter_values` must work with iterators that report no upper size bound,
621    /// and must not trust the size hint it does get.
622    #[test]
623    fn from_iter_values_untrusted_size_hint() {
624        // No upper bound at all:
625        let no_upper_bound = (0..20).filter(|i| i % 2 == 0).map(|i| format!("v{i}"));
626        assert_eq!(no_upper_bound.size_hint(), (0, Some(20)));
627        let array = StringArray::from_iter_values(no_upper_bound);
628        assert_eq!(array.len(), 10);
629        assert_eq!(array.value(0), "v0");
630        assert_eq!(array.value(9), "v18");
631
632        // Upper bound larger than the number of yielded values:
633        let too_large_upper_bound = (0..).map(|i| format!("v{i}")).take_while(|v| v != "v3");
634        assert_eq!(too_large_upper_bound.size_hint(), (0, None));
635        let array = StringArray::from_iter_values(too_large_upper_bound);
636        assert_eq!(array.len(), 3);
637        assert_eq!(array.value(2), "v2");
638    }
639
640    #[test]
641    fn try_new() {
642        let data = Buffer::from_slice_ref("helloworld");
643        let offsets = OffsetBuffer::new(vec![0, 5, 10].into());
644        StringArray::new(offsets.clone(), data.clone(), None);
645
646        let nulls = NullBuffer::new_null(3);
647        let err =
648            StringArray::try_new(offsets.clone(), data.clone(), Some(nulls.clone())).unwrap_err();
649        assert_eq!(
650            err.to_string(),
651            "Invalid argument error: Incorrect length of null buffer for StringArray, expected 2 got 3"
652        );
653
654        let err = BinaryArray::try_new(offsets.clone(), data.clone(), Some(nulls)).unwrap_err();
655        assert_eq!(
656            err.to_string(),
657            "Invalid argument error: Incorrect length of null buffer for BinaryArray, expected 2 got 3"
658        );
659
660        let non_utf8_data = Buffer::from_slice_ref(b"he\xFFloworld");
661        let err = StringArray::try_new(offsets.clone(), non_utf8_data.clone(), None).unwrap_err();
662        assert_eq!(
663            err.to_string(),
664            "Invalid argument error: Encountered non UTF-8 data: invalid utf-8 sequence of 1 bytes from index 2"
665        );
666
667        BinaryArray::new(offsets, non_utf8_data, None);
668
669        let offsets = OffsetBuffer::new(vec![0, 5, 11].into());
670        let err = StringArray::try_new(offsets.clone(), data.clone(), None).unwrap_err();
671        assert_eq!(
672            err.to_string(),
673            "Invalid argument error: Offset of 11 exceeds length of values 10"
674        );
675
676        let err = BinaryArray::try_new(offsets.clone(), data, None).unwrap_err();
677        assert_eq!(
678            err.to_string(),
679            "Invalid argument error: Maximum offset of 11 is larger than values of length 10"
680        );
681
682        let non_ascii_data = Buffer::from_slice_ref("heìloworld");
683        StringArray::new(offsets.clone(), non_ascii_data.clone(), None);
684        BinaryArray::new(offsets, non_ascii_data.clone(), None);
685
686        let offsets = OffsetBuffer::new(vec![0, 3, 10].into());
687        let err = StringArray::try_new(offsets.clone(), non_ascii_data.clone(), None).unwrap_err();
688        assert_eq!(
689            err.to_string(),
690            "Invalid argument error: Split UTF-8 codepoint at offset 3"
691        );
692
693        BinaryArray::new(offsets, non_ascii_data, None);
694    }
695
696    #[test]
697    fn create_repeated() {
698        let arr = BinaryArray::new_repeated(b"hello", 3);
699        assert_eq!(arr.len(), 3);
700        assert_eq!(arr.value(0), b"hello");
701        assert_eq!(arr.value(1), b"hello");
702        assert_eq!(arr.value(2), b"hello");
703
704        let arr = StringArray::new_repeated("world", 2);
705        assert_eq!(arr.len(), 2);
706        assert_eq!(arr.value(0), "world");
707        assert_eq!(arr.value(1), "world");
708    }
709
710    #[test]
711    #[should_panic(expected = "total length overflow: does not fit in usize")]
712    fn create_repeated_usize_overflow_1() {
713        let _arr = BinaryArray::new_repeated(b"hello", (usize::MAX / "hello".len()) + 1);
714    }
715
716    #[test]
717    #[should_panic(expected = "total length overflow: does not fit in usize")]
718    fn create_repeated_usize_overflow_2() {
719        let _arr = BinaryArray::new_repeated(b"hello", usize::MAX);
720    }
721
722    #[test]
723    #[should_panic(expected = "offset overflow")]
724    fn create_repeated_i32_offset_overflow_1() {
725        let _arr = BinaryArray::new_repeated(b"hello", usize::MAX / "hello".len());
726    }
727
728    #[test]
729    #[should_panic(expected = "offset overflow")]
730    fn create_repeated_i32_offset_overflow_2() {
731        let _arr = BinaryArray::new_repeated(b"hello", ((i32::MAX as usize) / "hello".len()) + 1);
732    }
733}
arrow_array/array/byte_array.rs

arrow_array/array/
byte_array.rs
