arrow_array/array/
binary_array.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use crate::types::{ByteArrayType, GenericBinaryType};
6use crate::{Array, GenericByteArray, GenericListArray, GenericStringArray, OffsetSizeTrait};
7use arrow_data::ArrayData;
8use arrow_schema::DataType;
9
10/// A [`GenericByteArray`] for storing `[u8]`
11pub type GenericBinaryArray<OffsetSize> = GenericByteArray<GenericBinaryType<OffsetSize>>;
12
13impl<OffsetSize: OffsetSizeTrait> GenericBinaryArray<OffsetSize> {
14    /// Creates a [GenericBinaryArray] from a vector of byte slices
15    ///
16    /// See also [`Self::from_iter_values`]
17    pub fn from_vec(v: Vec<&[u8]>) -> Self {
18        Self::from_iter_values(v)
19    }
20
21    /// Creates a [GenericBinaryArray] from a vector of Optional (null) byte slices
22    pub fn from_opt_vec(v: Vec<Option<&[u8]>>) -> Self {
23        v.into_iter().collect()
24    }
25
26    fn from_list(v: GenericListArray<OffsetSize>) -> Self {
27        let v = v.into_data();
28        assert_eq!(
29            v.child_data().len(),
30            1,
31            "BinaryArray can only be created from list array of u8 values \
32             (i.e. List<PrimitiveArray<u8>>)."
33        );
34        let child_data = &v.child_data()[0];
35
36        assert_eq!(
37            child_data.child_data().len(),
38            0,
39            "BinaryArray can only be created from list array of u8 values \
40             (i.e. List<PrimitiveArray<u8>>)."
41        );
42        assert_eq!(
43            child_data.data_type(),
44            &DataType::UInt8,
45            "BinaryArray can only be created from List<u8> arrays, mismatched data types."
46        );
47        assert_eq!(
48            child_data.null_count(),
49            0,
50            "The child array cannot contain null values."
51        );
52
53        let builder = ArrayData::builder(Self::DATA_TYPE)
54            .len(v.len())
55            .offset(v.offset())
56            .add_buffer(v.buffers()[0].clone())
57            .add_buffer(child_data.buffers()[0].slice(child_data.offset()))
58            .nulls(v.nulls().cloned());
59
60        let data = unsafe { builder.build_unchecked() };
61        Self::from(data)
62    }
63
64    /// Returns an iterator that returns the values of `array.value(i)` for an iterator with each element `i`
65    pub fn take_iter<'a>(
66        &'a self,
67        indexes: impl Iterator<Item = Option<usize>> + 'a,
68    ) -> impl Iterator<Item = Option<&'a [u8]>> {
69        indexes.map(|opt_index| opt_index.map(|index| self.value(index)))
70    }
71
72    /// Returns an iterator that returns the values of `array.value(i)` for an iterator with each element `i`
73    /// # Safety
74    ///
75    /// caller must ensure that the indexes in the iterator are less than the `array.len()`
76    pub unsafe fn take_iter_unchecked<'a>(
77        &'a self,
78        indexes: impl Iterator<Item = Option<usize>> + 'a,
79    ) -> impl Iterator<Item = Option<&'a [u8]>> {
80        unsafe { indexes.map(|opt_index| opt_index.map(|index| self.value_unchecked(index))) }
81    }
82}
83
84impl<OffsetSize: OffsetSizeTrait> From<Vec<Option<&[u8]>>> for GenericBinaryArray<OffsetSize> {
85    fn from(v: Vec<Option<&[u8]>>) -> Self {
86        Self::from_opt_vec(v)
87    }
88}
89
90impl<OffsetSize: OffsetSizeTrait> From<Vec<&[u8]>> for GenericBinaryArray<OffsetSize> {
91    fn from(v: Vec<&[u8]>) -> Self {
92        Self::from_iter_values(v)
93    }
94}
95
96impl<T: OffsetSizeTrait> From<GenericListArray<T>> for GenericBinaryArray<T> {
97    fn from(v: GenericListArray<T>) -> Self {
98        Self::from_list(v)
99    }
100}
101
102impl<OffsetSize: OffsetSizeTrait> From<GenericStringArray<OffsetSize>>
103    for GenericBinaryArray<OffsetSize>
104{
105    fn from(value: GenericStringArray<OffsetSize>) -> Self {
106        let builder = value
107            .into_data()
108            .into_builder()
109            .data_type(GenericBinaryType::<OffsetSize>::DATA_TYPE);
110
111        // Safety:
112        // A StringArray is a valid BinaryArray
113        Self::from(unsafe { builder.build_unchecked() })
114    }
115}
116
117/// A [`GenericBinaryArray`] of `[u8]` using `i32` offsets
118///
119/// The byte length of each element is represented by an i32.
120///
121/// # Examples
122///
123/// Create a BinaryArray from a vector of byte slices.
124///
125/// ```
126/// use arrow_array::{Array, BinaryArray};
127/// let values: Vec<&[u8]> =
128///     vec![b"one", b"two", b"", b"three"];
129/// let array = BinaryArray::from_vec(values);
130/// assert_eq!(4, array.len());
131/// assert_eq!(b"one", array.value(0));
132/// assert_eq!(b"two", array.value(1));
133/// assert_eq!(b"", array.value(2));
134/// assert_eq!(b"three", array.value(3));
135/// ```
136///
137/// Create a BinaryArray from a vector of Optional (null) byte slices.
138///
139/// ```
140/// use arrow_array::{Array, BinaryArray};
141/// let values: Vec<Option<&[u8]>> =
142///     vec![Some(b"one"), Some(b"two"), None, Some(b""), Some(b"three")];
143/// let array = BinaryArray::from_opt_vec(values);
144/// assert_eq!(5, array.len());
145/// assert_eq!(b"one", array.value(0));
146/// assert_eq!(b"two", array.value(1));
147/// assert_eq!(b"", array.value(3));
148/// assert_eq!(b"three", array.value(4));
149/// assert!(!array.is_null(0));
150/// assert!(!array.is_null(1));
151/// assert!(array.is_null(2));
152/// assert!(!array.is_null(3));
153/// assert!(!array.is_null(4));
154/// ```
155///
156/// See [`GenericByteArray`] for more information and examples
157pub type BinaryArray = GenericBinaryArray<i32>;
158
159/// A [`GenericBinaryArray`] of `[u8]` using `i64` offsets
160///
161/// # Examples
162///
163/// Create a LargeBinaryArray from a vector of byte slices.
164///
165/// ```
166/// use arrow_array::{Array, LargeBinaryArray};
167/// let values: Vec<&[u8]> =
168///     vec![b"one", b"two", b"", b"three"];
169/// let array = LargeBinaryArray::from_vec(values);
170/// assert_eq!(4, array.len());
171/// assert_eq!(b"one", array.value(0));
172/// assert_eq!(b"two", array.value(1));
173/// assert_eq!(b"", array.value(2));
174/// assert_eq!(b"three", array.value(3));
175/// ```
176///
177/// Create a LargeBinaryArray from a vector of Optional (null) byte slices.
178///
179/// ```
180/// use arrow_array::{Array, LargeBinaryArray};
181/// let values: Vec<Option<&[u8]>> =
182///     vec![Some(b"one"), Some(b"two"), None, Some(b""), Some(b"three")];
183/// let array = LargeBinaryArray::from_opt_vec(values);
184/// assert_eq!(5, array.len());
185/// assert_eq!(b"one", array.value(0));
186/// assert_eq!(b"two", array.value(1));
187/// assert_eq!(b"", array.value(3));
188/// assert_eq!(b"three", array.value(4));
189/// assert!(!array.is_null(0));
190/// assert!(!array.is_null(1));
191/// assert!(array.is_null(2));
192/// assert!(!array.is_null(3));
193/// assert!(!array.is_null(4));
194/// ```
195///
196/// See [`GenericByteArray`] for more information and examples
197pub type LargeBinaryArray = GenericBinaryArray<i64>;
198
199#[cfg(test)]
200mod tests {
201    use super::*;
202    use crate::{ListArray, StringArray};
203    use arrow_buffer::Buffer;
204    use arrow_schema::Field;
205    use std::sync::Arc;
206
207    #[test]
208    fn test_binary_array() {
209        let values = b"helloparquet";
210        let offsets: [i32; 4] = [0, 5, 5, 12];
211
212        // Array data: ["hello", "", "parquet"]
213        let array_data = ArrayData::builder(DataType::Binary)
214            .len(3)
215            .add_buffer(Buffer::from_slice_ref(offsets))
216            .add_buffer(Buffer::from_slice_ref(values))
217            .build()
218            .unwrap();
219        let binary_array = BinaryArray::from(array_data);
220        assert_eq!(3, binary_array.len());
221        assert_eq!(0, binary_array.null_count());
222        assert_eq!([b'h', b'e', b'l', b'l', b'o'], binary_array.value(0));
223        assert_eq!([b'h', b'e', b'l', b'l', b'o'], unsafe {
224            binary_array.value_unchecked(0)
225        });
226        assert_eq!([] as [u8; 0], binary_array.value(1));
227        assert_eq!([] as [u8; 0], unsafe { binary_array.value_unchecked(1) });
228        assert_eq!(
229            [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
230            binary_array.value(2)
231        );
232        assert_eq!([b'p', b'a', b'r', b'q', b'u', b'e', b't'], unsafe {
233            binary_array.value_unchecked(2)
234        });
235        assert_eq!(5, binary_array.value_offsets()[2]);
236        assert_eq!(7, binary_array.value_length(2));
237        for i in 0..3 {
238            assert!(binary_array.is_valid(i));
239            assert!(!binary_array.is_null(i));
240        }
241    }
242
243    #[test]
244    fn test_binary_array_with_offsets() {
245        let values = b"helloparquet";
246        let offsets: [i32; 4] = [0, 5, 5, 12];
247
248        // Test binary array with offset
249        let array_data = ArrayData::builder(DataType::Binary)
250            .len(2)
251            .offset(1)
252            .add_buffer(Buffer::from_slice_ref(offsets))
253            .add_buffer(Buffer::from_slice_ref(values))
254            .build()
255            .unwrap();
256        let binary_array = BinaryArray::from(array_data);
257        assert_eq!(
258            [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
259            binary_array.value(1)
260        );
261        assert_eq!(5, binary_array.value_offsets()[0]);
262        assert_eq!(0, binary_array.value_length(0));
263        assert_eq!(5, binary_array.value_offsets()[1]);
264        assert_eq!(7, binary_array.value_length(1));
265    }
266
267    #[test]
268    fn test_large_binary_array() {
269        let values = b"helloparquet";
270        let offsets: [i64; 4] = [0, 5, 5, 12];
271
272        // Array data: ["hello", "", "parquet"]
273        let array_data = ArrayData::builder(DataType::LargeBinary)
274            .len(3)
275            .add_buffer(Buffer::from_slice_ref(offsets))
276            .add_buffer(Buffer::from_slice_ref(values))
277            .build()
278            .unwrap();
279        let binary_array = LargeBinaryArray::from(array_data);
280        assert_eq!(3, binary_array.len());
281        assert_eq!(0, binary_array.null_count());
282        assert_eq!([b'h', b'e', b'l', b'l', b'o'], binary_array.value(0));
283        assert_eq!([b'h', b'e', b'l', b'l', b'o'], unsafe {
284            binary_array.value_unchecked(0)
285        });
286        assert_eq!([] as [u8; 0], binary_array.value(1));
287        assert_eq!([] as [u8; 0], unsafe { binary_array.value_unchecked(1) });
288        assert_eq!(
289            [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
290            binary_array.value(2)
291        );
292        assert_eq!([b'p', b'a', b'r', b'q', b'u', b'e', b't'], unsafe {
293            binary_array.value_unchecked(2)
294        });
295        assert_eq!(5, binary_array.value_offsets()[2]);
296        assert_eq!(7, binary_array.value_length(2));
297        for i in 0..3 {
298            assert!(binary_array.is_valid(i));
299            assert!(!binary_array.is_null(i));
300        }
301    }
302
303    #[test]
304    fn test_large_binary_array_with_offsets() {
305        let values = b"helloparquet";
306        let offsets: [i64; 4] = [0, 5, 5, 12];
307
308        // Test binary array with offset
309        let array_data = ArrayData::builder(DataType::LargeBinary)
310            .len(2)
311            .offset(1)
312            .add_buffer(Buffer::from_slice_ref(offsets))
313            .add_buffer(Buffer::from_slice_ref(values))
314            .build()
315            .unwrap();
316        let binary_array = LargeBinaryArray::from(array_data);
317        assert_eq!(
318            [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
319            binary_array.value(1)
320        );
321        assert_eq!([b'p', b'a', b'r', b'q', b'u', b'e', b't'], unsafe {
322            binary_array.value_unchecked(1)
323        });
324        assert_eq!(5, binary_array.value_offsets()[0]);
325        assert_eq!(0, binary_array.value_length(0));
326        assert_eq!(5, binary_array.value_offsets()[1]);
327        assert_eq!(7, binary_array.value_length(1));
328    }
329
330    fn _test_generic_binary_array_from_list_array<O: OffsetSizeTrait>() {
331        let values = b"helloparquet";
332        let child_data = ArrayData::builder(DataType::UInt8)
333            .len(12)
334            .add_buffer(Buffer::from(values))
335            .build()
336            .unwrap();
337        let offsets = [0, 5, 5, 12].map(|n| O::from_usize(n).unwrap());
338
339        // Array data: ["hello", "", "parquet"]
340        let array_data1 = ArrayData::builder(GenericBinaryArray::<O>::DATA_TYPE)
341            .len(3)
342            .add_buffer(Buffer::from_slice_ref(offsets))
343            .add_buffer(Buffer::from_slice_ref(values))
344            .build()
345            .unwrap();
346        let binary_array1 = GenericBinaryArray::<O>::from(array_data1);
347
348        let data_type = GenericListArray::<O>::DATA_TYPE_CONSTRUCTOR(Arc::new(
349            Field::new_list_field(DataType::UInt8, false),
350        ));
351
352        let array_data2 = ArrayData::builder(data_type)
353            .len(3)
354            .add_buffer(Buffer::from_slice_ref(offsets))
355            .add_child_data(child_data)
356            .build()
357            .unwrap();
358        let list_array = GenericListArray::<O>::from(array_data2);
359        let binary_array2 = GenericBinaryArray::<O>::from(list_array);
360
361        assert_eq!(binary_array1.len(), binary_array2.len());
362        assert_eq!(binary_array1.null_count(), binary_array2.null_count());
363        assert_eq!(binary_array1.value_offsets(), binary_array2.value_offsets());
364        for i in 0..binary_array1.len() {
365            assert_eq!(binary_array1.value(i), binary_array2.value(i));
366            assert_eq!(binary_array1.value(i), unsafe {
367                binary_array2.value_unchecked(i)
368            });
369            assert_eq!(binary_array1.value_length(i), binary_array2.value_length(i));
370        }
371    }
372
373    #[test]
374    fn test_binary_array_from_list_array() {
375        _test_generic_binary_array_from_list_array::<i32>();
376    }
377
378    #[test]
379    fn test_large_binary_array_from_list_array() {
380        _test_generic_binary_array_from_list_array::<i64>();
381    }
382
383    fn _test_generic_binary_array_from_list_array_with_offset<O: OffsetSizeTrait>() {
384        let values = b"HelloArrowAndParquet";
385        // b"ArrowAndParquet"
386        let child_data = ArrayData::builder(DataType::UInt8)
387            .len(15)
388            .offset(5)
389            .add_buffer(Buffer::from(values))
390            .build()
391            .unwrap();
392
393        let offsets = [0, 5, 8, 15].map(|n| O::from_usize(n).unwrap());
394        let null_buffer = Buffer::from_slice_ref([0b101]);
395        let data_type = GenericListArray::<O>::DATA_TYPE_CONSTRUCTOR(Arc::new(
396            Field::new_list_field(DataType::UInt8, false),
397        ));
398
399        // [None, Some(b"Parquet")]
400        let array_data = ArrayData::builder(data_type)
401            .len(2)
402            .offset(1)
403            .add_buffer(Buffer::from_slice_ref(offsets))
404            .null_bit_buffer(Some(null_buffer))
405            .add_child_data(child_data)
406            .build()
407            .unwrap();
408        let list_array = GenericListArray::<O>::from(array_data);
409        let binary_array = GenericBinaryArray::<O>::from(list_array);
410
411        assert_eq!(2, binary_array.len());
412        assert_eq!(1, binary_array.null_count());
413        assert!(binary_array.is_null(0));
414        assert!(binary_array.is_valid(1));
415        assert_eq!(b"Parquet", binary_array.value(1));
416    }
417
418    #[test]
419    fn test_binary_array_from_list_array_with_offset() {
420        _test_generic_binary_array_from_list_array_with_offset::<i32>();
421    }
422
423    #[test]
424    fn test_large_binary_array_from_list_array_with_offset() {
425        _test_generic_binary_array_from_list_array_with_offset::<i64>();
426    }
427
428    fn _test_generic_binary_array_from_list_array_with_child_nulls_failed<O: OffsetSizeTrait>() {
429        let values = b"HelloArrow";
430        let child_data = ArrayData::builder(DataType::UInt8)
431            .len(10)
432            .add_buffer(Buffer::from(values))
433            .null_bit_buffer(Some(Buffer::from_slice_ref([0b1010101010])))
434            .build()
435            .unwrap();
436
437        let offsets = [0, 5, 10].map(|n| O::from_usize(n).unwrap());
438        let data_type = GenericListArray::<O>::DATA_TYPE_CONSTRUCTOR(Arc::new(
439            Field::new_list_field(DataType::UInt8, true),
440        ));
441
442        // [None, Some(b"Parquet")]
443        let array_data = ArrayData::builder(data_type)
444            .len(2)
445            .add_buffer(Buffer::from_slice_ref(offsets))
446            .add_child_data(child_data)
447            .build()
448            .unwrap();
449        let list_array = GenericListArray::<O>::from(array_data);
450        drop(GenericBinaryArray::<O>::from(list_array));
451    }
452
453    #[test]
454    #[should_panic(expected = "The child array cannot contain null values.")]
455    fn test_binary_array_from_list_array_with_child_nulls_failed() {
456        _test_generic_binary_array_from_list_array_with_child_nulls_failed::<i32>();
457    }
458
459    #[test]
460    #[should_panic(expected = "The child array cannot contain null values.")]
461    fn test_large_binary_array_from_list_array_with_child_nulls_failed() {
462        _test_generic_binary_array_from_list_array_with_child_nulls_failed::<i64>();
463    }
464
465    fn test_generic_binary_array_from_opt_vec<T: OffsetSizeTrait>() {
466        let values: Vec<Option<&[u8]>> =
467            vec![Some(b"one"), Some(b"two"), None, Some(b""), Some(b"three")];
468        let array = GenericBinaryArray::<T>::from_opt_vec(values);
469        assert_eq!(array.len(), 5);
470        assert_eq!(array.value(0), b"one");
471        assert_eq!(array.value(1), b"two");
472        assert_eq!(array.value(3), b"");
473        assert_eq!(array.value(4), b"three");
474        assert!(!array.is_null(0));
475        assert!(!array.is_null(1));
476        assert!(array.is_null(2));
477        assert!(!array.is_null(3));
478        assert!(!array.is_null(4));
479    }
480
481    #[test]
482    fn test_large_binary_array_from_opt_vec() {
483        test_generic_binary_array_from_opt_vec::<i64>()
484    }
485
486    #[test]
487    fn test_binary_array_from_opt_vec() {
488        test_generic_binary_array_from_opt_vec::<i32>()
489    }
490
491    #[test]
492    fn test_binary_array_from_unbound_iter() {
493        // iterator that doesn't declare (upper) size bound
494        let value_iter = (0..)
495            .scan(0usize, |pos, i| {
496                if *pos < 10 {
497                    *pos += 1;
498                    Some(Some(format!("value {i}")))
499                } else {
500                    // actually returns up to 10 values
501                    None
502                }
503            })
504            // limited using take()
505            .take(100);
506
507        let (_, upper_size_bound) = value_iter.size_hint();
508        // the upper bound, defined by take above, is 100
509        assert_eq!(upper_size_bound, Some(100));
510        let binary_array: BinaryArray = value_iter.collect();
511        // but the actual number of items in the array should be 10
512        assert_eq!(binary_array.len(), 10);
513    }
514
515    #[test]
516    #[should_panic(
517        expected = "BinaryArray can only be created from List<u8> arrays, mismatched data types."
518    )]
519    fn test_binary_array_from_incorrect_list_array() {
520        let values: [u32; 12] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
521        let values_data = ArrayData::builder(DataType::UInt32)
522            .len(12)
523            .add_buffer(Buffer::from_slice_ref(values))
524            .build()
525            .unwrap();
526        let offsets: [i32; 4] = [0, 5, 5, 12];
527
528        let data_type = DataType::List(Arc::new(Field::new_list_field(DataType::UInt32, false)));
529        let array_data = ArrayData::builder(data_type)
530            .len(3)
531            .add_buffer(Buffer::from_slice_ref(offsets))
532            .add_child_data(values_data)
533            .build()
534            .unwrap();
535        let list_array = ListArray::from(array_data);
536        drop(BinaryArray::from(list_array));
537    }
538
539    #[test]
540    #[should_panic(
541        expected = "Trying to access an element at index 4 from a BinaryArray of length 3"
542    )]
543    fn test_binary_array_get_value_index_out_of_bound() {
544        let values: [u8; 12] = [104, 101, 108, 108, 111, 112, 97, 114, 113, 117, 101, 116];
545        let offsets: [i32; 4] = [0, 5, 5, 12];
546        let array_data = ArrayData::builder(DataType::Binary)
547            .len(3)
548            .add_buffer(Buffer::from_slice_ref(offsets))
549            .add_buffer(Buffer::from_slice_ref(values))
550            .build()
551            .unwrap();
552        let binary_array = BinaryArray::from(array_data);
553        binary_array.value(4);
554    }
555
556    #[test]
557    #[should_panic(expected = "LargeBinaryArray expects DataType::LargeBinary")]
558    fn test_binary_array_validation() {
559        let array = BinaryArray::from_iter_values([&[1, 2]]);
560        let _ = LargeBinaryArray::from(array.into_data());
561    }
562
563    #[test]
564    fn test_binary_array_all_null() {
565        let data = vec![None];
566        let array = BinaryArray::from(data);
567        array
568            .into_data()
569            .validate_full()
570            .expect("All null array has valid array data");
571    }
572
573    #[test]
574    fn test_large_binary_array_all_null() {
575        let data = vec![None];
576        let array = LargeBinaryArray::from(data);
577        array
578            .into_data()
579            .validate_full()
580            .expect("All null array has valid array data");
581    }
582
583    #[test]
584    fn test_empty_offsets() {
585        let string = BinaryArray::from(
586            ArrayData::builder(DataType::Binary)
587                .buffers(vec![Buffer::from(&[]), Buffer::from(&[])])
588                .build()
589                .unwrap(),
590        );
591        assert_eq!(string.value_offsets(), &[0]);
592        let string = LargeBinaryArray::from(
593            ArrayData::builder(DataType::LargeBinary)
594                .buffers(vec![Buffer::from(&[]), Buffer::from(&[])])
595                .build()
596                .unwrap(),
597        );
598        assert_eq!(string.len(), 0);
599        assert_eq!(string.value_offsets(), &[0]);
600    }
601
602    #[test]
603    fn test_to_from_string() {
604        let s = StringArray::from_iter_values(["a", "b", "c", "d"]);
605        let b = BinaryArray::from(s.clone());
606        let sa = StringArray::from(b); // Performs UTF-8 validation again
607
608        assert_eq!(s, sa);
609    }
610}
arrow_array/array/binary_array.rs

arrow_array/array/
binary_array.rs
