arrow_array/array/
fixed_size_binary_array.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use crate::array::print_long_array;
19use crate::iterator::FixedSizeBinaryIter;
20use crate::{Array, ArrayAccessor, ArrayRef, FixedSizeListArray, Scalar};
21use arrow_buffer::buffer::NullBuffer;
22use arrow_buffer::{ArrowNativeType, BooleanBuffer, Buffer, MutableBuffer, bit_util};
23use arrow_data::{ArrayData, ArrayDataBuilder};
24use arrow_schema::{ArrowError, DataType};
25use std::any::Any;
26use std::sync::Arc;
27
28/// An array of [fixed size binary arrays](https://arrow.apache.org/docs/format/Columnar.html#fixed-size-primitive-layout)
29///
30/// # Examples
31///
32/// Create an array from an iterable argument of byte slices.
33///
34/// ```
35///    use arrow_array::{Array, FixedSizeBinaryArray};
36///    let input_arg = vec![ vec![1, 2], vec![3, 4], vec![5, 6] ];
37///    let arr = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
38///
39///    assert_eq!(3, arr.len());
40///
41/// ```
42/// Create an array from an iterable argument of sparse byte slices.
43/// Sparsity means that the input argument can contain `None` items.
44/// ```
45///    use arrow_array::{Array, FixedSizeBinaryArray};
46///    let input_arg = vec![ None, Some(vec![7, 8]), Some(vec![9, 10]), None, Some(vec![13, 14]) ];
47///    let arr = FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 2).unwrap();
48///    assert_eq!(5, arr.len())
49///
50/// ```
51///
52#[derive(Clone)]
53pub struct FixedSizeBinaryArray {
54    data_type: DataType, // Must be DataType::FixedSizeBinary(value_length)
55    value_data: Buffer,
56    nulls: Option<NullBuffer>,
57    len: usize,
58    value_length: i32,
59}
60
61impl FixedSizeBinaryArray {
62    /// Create a new [`FixedSizeBinaryArray`] with `size` element size, panicking on failure
63    ///
64    /// # Panics
65    ///
66    /// Panics if [`Self::try_new`] returns an error
67    pub fn new(size: i32, values: Buffer, nulls: Option<NullBuffer>) -> Self {
68        Self::try_new(size, values, nulls).unwrap()
69    }
70
71    /// Create a new [`Scalar`] from `value`
72    pub fn new_scalar(value: impl AsRef<[u8]>) -> Scalar<Self> {
73        let v = value.as_ref();
74        Scalar::new(Self::new(v.len() as _, Buffer::from(v), None))
75    }
76
77    /// Create a new [`FixedSizeBinaryArray`] from the provided parts, returning an error on failure
78    ///
79    /// # Errors
80    ///
81    /// * `size < 0`
82    /// * `values.len() / size != nulls.len()`
83    pub fn try_new(
84        size: i32,
85        values: Buffer,
86        nulls: Option<NullBuffer>,
87    ) -> Result<Self, ArrowError> {
88        let data_type = DataType::FixedSizeBinary(size);
89        let s = size.to_usize().ok_or_else(|| {
90            ArrowError::InvalidArgumentError(format!("Size cannot be negative, got {size}"))
91        })?;
92
93        let len = values.len() / s;
94        if let Some(n) = nulls.as_ref() {
95            if n.len() != len {
96                return Err(ArrowError::InvalidArgumentError(format!(
97                    "Incorrect length of null buffer for FixedSizeBinaryArray, expected {} got {}",
98                    len,
99                    n.len(),
100                )));
101            }
102        }
103
104        Ok(Self {
105            data_type,
106            value_data: values,
107            value_length: size,
108            nulls,
109            len,
110        })
111    }
112
113    /// Create a new [`FixedSizeBinaryArray`] of length `len` where all values are null
114    ///
115    /// # Panics
116    ///
117    /// Panics if
118    ///
119    /// * `size < 0`
120    /// * `size * len` would overflow `usize`
121    pub fn new_null(size: i32, len: usize) -> Self {
122        const BITS_IN_A_BYTE: usize = 8;
123        let capacity_in_bytes = size.to_usize().unwrap().checked_mul(len).unwrap();
124        Self {
125            data_type: DataType::FixedSizeBinary(size),
126            value_data: MutableBuffer::new_null(capacity_in_bytes * BITS_IN_A_BYTE).into(),
127            nulls: Some(NullBuffer::new_null(len)),
128            value_length: size,
129            len,
130        }
131    }
132
133    /// Deconstruct this array into its constituent parts
134    pub fn into_parts(self) -> (i32, Buffer, Option<NullBuffer>) {
135        (self.value_length, self.value_data, self.nulls)
136    }
137
138    /// Returns the element at index `i` as a byte slice.
139    ///
140    /// Note: This method does not check for nulls and the value is arbitrary
141    /// (but still well-defined) if [`is_null`](Self::is_null) returns true for the index.
142    ///
143    /// # Panics
144    /// Panics if index `i` is out of bounds.
145    pub fn value(&self, i: usize) -> &[u8] {
146        assert!(
147            i < self.len(),
148            "Trying to access an element at index {} from a FixedSizeBinaryArray of length {}",
149            i,
150            self.len()
151        );
152        let offset = i + self.offset();
153        unsafe {
154            let pos = self.value_offset_at(offset);
155            std::slice::from_raw_parts(
156                self.value_data.as_ptr().offset(pos as isize),
157                (self.value_offset_at(offset + 1) - pos) as usize,
158            )
159        }
160    }
161
162    /// Returns the element at index `i` as a byte slice.
163    ///
164    /// Note: This method does not check for nulls and the value is arbitrary
165    /// if [`is_null`](Self::is_null) returns true for the index.
166    ///
167    /// # Safety
168    ///
169    /// Caller is responsible for ensuring that the index is within the bounds
170    /// of the array
171    pub unsafe fn value_unchecked(&self, i: usize) -> &[u8] {
172        let offset = i + self.offset();
173        let pos = self.value_offset_at(offset);
174        unsafe {
175            std::slice::from_raw_parts(
176                self.value_data.as_ptr().offset(pos as isize),
177                (self.value_offset_at(offset + 1) - pos) as usize,
178            )
179        }
180    }
181
182    /// Returns the offset for the element at index `i`.
183    ///
184    /// Note this doesn't do any bound checking, for performance reason.
185    #[inline]
186    pub fn value_offset(&self, i: usize) -> i32 {
187        self.value_offset_at(self.offset() + i)
188    }
189
190    /// Returns the length for an element.
191    ///
192    /// All elements have the same length as the array is a fixed size.
193    #[inline]
194    pub fn value_length(&self) -> i32 {
195        self.value_length
196    }
197
198    /// Returns the values of this array.
199    ///
200    /// Unlike [`Self::value_data`] this returns the [`Buffer`]
201    /// allowing for zero-copy cloning.
202    #[inline]
203    pub fn values(&self) -> &Buffer {
204        &self.value_data
205    }
206
207    /// Returns the raw value data.
208    pub fn value_data(&self) -> &[u8] {
209        self.value_data.as_slice()
210    }
211
212    /// Returns a zero-copy slice of this array with the indicated offset and length.
213    pub fn slice(&self, offset: usize, len: usize) -> Self {
214        assert!(
215            offset.saturating_add(len) <= self.len,
216            "the length + offset of the sliced FixedSizeBinaryArray cannot exceed the existing length"
217        );
218
219        let size = self.value_length as usize;
220
221        Self {
222            data_type: self.data_type.clone(),
223            nulls: self.nulls.as_ref().map(|n| n.slice(offset, len)),
224            value_length: self.value_length,
225            value_data: self.value_data.slice_with_length(offset * size, len * size),
226            len,
227        }
228    }
229
230    /// Create an array from an iterable argument of sparse byte slices.
231    /// Sparsity means that items returned by the iterator are optional, i.e input argument can
232    /// contain `None` items.
233    ///
234    /// # Examples
235    ///
236    /// ```
237    /// use arrow_array::FixedSizeBinaryArray;
238    /// let input_arg = vec![
239    ///     None,
240    ///     Some(vec![7, 8]),
241    ///     Some(vec![9, 10]),
242    ///     None,
243    ///     Some(vec![13, 14]),
244    ///     None,
245    /// ];
246    /// let array = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.into_iter()).unwrap();
247    /// ```
248    ///
249    /// # Errors
250    ///
251    /// Returns error if argument has length zero, or sizes of nested slices don't match.
252    #[deprecated(
253        since = "28.0.0",
254        note = "This function will fail if the iterator produces only None values; prefer `try_from_sparse_iter_with_size`"
255    )]
256    pub fn try_from_sparse_iter<T, U>(mut iter: T) -> Result<Self, ArrowError>
257    where
258        T: Iterator<Item = Option<U>>,
259        U: AsRef<[u8]>,
260    {
261        let mut len = 0;
262        let mut size = None;
263        let mut byte = 0;
264
265        let iter_size_hint = iter.size_hint().0;
266        let mut null_buf = MutableBuffer::new(bit_util::ceil(iter_size_hint, 8));
267        let mut buffer = MutableBuffer::new(0);
268
269        let mut prepend = 0;
270        iter.try_for_each(|item| -> Result<(), ArrowError> {
271            // extend null bitmask by one byte per each 8 items
272            if byte == 0 {
273                null_buf.push(0u8);
274                byte = 8;
275            }
276            byte -= 1;
277
278            if let Some(slice) = item {
279                let slice = slice.as_ref();
280                if let Some(size) = size {
281                    if size != slice.len() {
282                        return Err(ArrowError::InvalidArgumentError(format!(
283                            "Nested array size mismatch: one is {}, and the other is {}",
284                            size,
285                            slice.len()
286                        )));
287                    }
288                } else {
289                    let len = slice.len();
290                    size = Some(len);
291                    // Now that we know how large each element is we can reserve
292                    // sufficient capacity in the underlying mutable buffer for
293                    // the data.
294                    buffer.reserve(iter_size_hint * len);
295                    buffer.extend_zeros(slice.len() * prepend);
296                }
297                bit_util::set_bit(null_buf.as_slice_mut(), len);
298                buffer.extend_from_slice(slice);
299            } else if let Some(size) = size {
300                buffer.extend_zeros(size);
301            } else {
302                prepend += 1;
303            }
304
305            len += 1;
306
307            Ok(())
308        })?;
309
310        if len == 0 {
311            return Err(ArrowError::InvalidArgumentError(
312                "Input iterable argument has no data".to_owned(),
313            ));
314        }
315
316        let null_buf = BooleanBuffer::new(null_buf.into(), 0, len);
317        let nulls = Some(NullBuffer::new(null_buf)).filter(|n| n.null_count() > 0);
318
319        let size = size.unwrap_or(0) as i32;
320        Ok(Self {
321            data_type: DataType::FixedSizeBinary(size),
322            value_data: buffer.into(),
323            nulls,
324            value_length: size,
325            len,
326        })
327    }
328
329    /// Create an array from an iterable argument of sparse byte slices.
330    /// Sparsity means that items returned by the iterator are optional, i.e input argument can
331    /// contain `None` items. In cases where the iterator returns only `None` values, this
332    /// also takes a size parameter to ensure that the a valid FixedSizeBinaryArray is still
333    /// created.
334    ///
335    /// # Examples
336    ///
337    /// ```
338    /// use arrow_array::FixedSizeBinaryArray;
339    /// let input_arg = vec![
340    ///     None,
341    ///     Some(vec![7, 8]),
342    ///     Some(vec![9, 10]),
343    ///     None,
344    ///     Some(vec![13, 14]),
345    ///     None,
346    /// ];
347    /// let array = FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 2).unwrap();
348    /// ```
349    ///
350    /// # Errors
351    ///
352    /// Returns error if argument has length zero, or sizes of nested slices don't match.
353    pub fn try_from_sparse_iter_with_size<T, U>(mut iter: T, size: i32) -> Result<Self, ArrowError>
354    where
355        T: Iterator<Item = Option<U>>,
356        U: AsRef<[u8]>,
357    {
358        let mut len = 0;
359        let mut byte = 0;
360
361        let iter_size_hint = iter.size_hint().0;
362        let mut null_buf = MutableBuffer::new(bit_util::ceil(iter_size_hint, 8));
363        let mut buffer = MutableBuffer::new(iter_size_hint * (size as usize));
364
365        iter.try_for_each(|item| -> Result<(), ArrowError> {
366            // extend null bitmask by one byte per each 8 items
367            if byte == 0 {
368                null_buf.push(0u8);
369                byte = 8;
370            }
371            byte -= 1;
372
373            if let Some(slice) = item {
374                let slice = slice.as_ref();
375                if size as usize != slice.len() {
376                    return Err(ArrowError::InvalidArgumentError(format!(
377                        "Nested array size mismatch: one is {}, and the other is {}",
378                        size,
379                        slice.len()
380                    )));
381                }
382
383                bit_util::set_bit(null_buf.as_slice_mut(), len);
384                buffer.extend_from_slice(slice);
385            } else {
386                buffer.extend_zeros(size as usize);
387            }
388
389            len += 1;
390
391            Ok(())
392        })?;
393
394        let null_buf = BooleanBuffer::new(null_buf.into(), 0, len);
395        let nulls = Some(NullBuffer::new(null_buf)).filter(|n| n.null_count() > 0);
396
397        Ok(Self {
398            data_type: DataType::FixedSizeBinary(size),
399            value_data: buffer.into(),
400            nulls,
401            len,
402            value_length: size,
403        })
404    }
405
406    /// Create an array from an iterable argument of byte slices.
407    ///
408    /// # Examples
409    ///
410    /// ```
411    /// use arrow_array::FixedSizeBinaryArray;
412    /// let input_arg = vec![
413    ///     vec![1, 2],
414    ///     vec![3, 4],
415    ///     vec![5, 6],
416    /// ];
417    /// let array = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
418    /// ```
419    ///
420    /// # Errors
421    ///
422    /// Returns error if argument has length zero, or sizes of nested slices don't match.
423    pub fn try_from_iter<T, U>(mut iter: T) -> Result<Self, ArrowError>
424    where
425        T: Iterator<Item = U>,
426        U: AsRef<[u8]>,
427    {
428        let mut len = 0;
429        let mut size = None;
430        let iter_size_hint = iter.size_hint().0;
431        let mut buffer = MutableBuffer::new(0);
432
433        iter.try_for_each(|item| -> Result<(), ArrowError> {
434            let slice = item.as_ref();
435            if let Some(size) = size {
436                if size != slice.len() {
437                    return Err(ArrowError::InvalidArgumentError(format!(
438                        "Nested array size mismatch: one is {}, and the other is {}",
439                        size,
440                        slice.len()
441                    )));
442                }
443            } else {
444                let len = slice.len();
445                size = Some(len);
446                buffer.reserve(iter_size_hint * len);
447            }
448
449            buffer.extend_from_slice(slice);
450
451            len += 1;
452
453            Ok(())
454        })?;
455
456        if len == 0 {
457            return Err(ArrowError::InvalidArgumentError(
458                "Input iterable argument has no data".to_owned(),
459            ));
460        }
461
462        let size = size.unwrap_or(0).try_into().unwrap();
463        Ok(Self {
464            data_type: DataType::FixedSizeBinary(size),
465            value_data: buffer.into(),
466            nulls: None,
467            value_length: size,
468            len,
469        })
470    }
471
472    #[inline]
473    fn value_offset_at(&self, i: usize) -> i32 {
474        self.value_length * i as i32
475    }
476
477    /// constructs a new iterator
478    pub fn iter(&self) -> FixedSizeBinaryIter<'_> {
479        FixedSizeBinaryIter::new(self)
480    }
481}
482
483impl From<ArrayData> for FixedSizeBinaryArray {
484    fn from(data: ArrayData) -> Self {
485        assert_eq!(
486            data.buffers().len(),
487            1,
488            "FixedSizeBinaryArray data should contain 1 buffer only (values)"
489        );
490        let value_length = match data.data_type() {
491            DataType::FixedSizeBinary(len) => *len,
492            _ => panic!("Expected data type to be FixedSizeBinary"),
493        };
494
495        let size = value_length as usize;
496        let value_data =
497            data.buffers()[0].slice_with_length(data.offset() * size, data.len() * size);
498
499        Self {
500            data_type: data.data_type().clone(),
501            nulls: data.nulls().cloned(),
502            len: data.len(),
503            value_data,
504            value_length,
505        }
506    }
507}
508
509impl From<FixedSizeBinaryArray> for ArrayData {
510    fn from(array: FixedSizeBinaryArray) -> Self {
511        let builder = ArrayDataBuilder::new(array.data_type)
512            .len(array.len)
513            .buffers(vec![array.value_data])
514            .nulls(array.nulls);
515
516        unsafe { builder.build_unchecked() }
517    }
518}
519
520/// Creates a `FixedSizeBinaryArray` from `FixedSizeList<u8>` array
521impl From<FixedSizeListArray> for FixedSizeBinaryArray {
522    fn from(v: FixedSizeListArray) -> Self {
523        let value_len = v.value_length();
524        let v = v.into_data();
525        assert_eq!(
526            v.child_data().len(),
527            1,
528            "FixedSizeBinaryArray can only be created from list array of u8 values \
529             (i.e. FixedSizeList<PrimitiveArray<u8>>)."
530        );
531        let child_data = &v.child_data()[0];
532
533        assert_eq!(
534            child_data.child_data().len(),
535            0,
536            "FixedSizeBinaryArray can only be created from list array of u8 values \
537             (i.e. FixedSizeList<PrimitiveArray<u8>>)."
538        );
539        assert_eq!(
540            child_data.data_type(),
541            &DataType::UInt8,
542            "FixedSizeBinaryArray can only be created from FixedSizeList<u8> arrays, mismatched data types."
543        );
544        assert_eq!(
545            child_data.null_count(),
546            0,
547            "The child array cannot contain null values."
548        );
549
550        let builder = ArrayData::builder(DataType::FixedSizeBinary(value_len))
551            .len(v.len())
552            .offset(v.offset())
553            .add_buffer(child_data.buffers()[0].slice(child_data.offset()))
554            .nulls(v.nulls().cloned());
555
556        let data = unsafe { builder.build_unchecked() };
557        Self::from(data)
558    }
559}
560
561impl From<Vec<Option<&[u8]>>> for FixedSizeBinaryArray {
562    fn from(v: Vec<Option<&[u8]>>) -> Self {
563        #[allow(deprecated)]
564        Self::try_from_sparse_iter(v.into_iter()).unwrap()
565    }
566}
567
568impl From<Vec<&[u8]>> for FixedSizeBinaryArray {
569    fn from(v: Vec<&[u8]>) -> Self {
570        Self::try_from_iter(v.into_iter()).unwrap()
571    }
572}
573
574impl<const N: usize> From<Vec<&[u8; N]>> for FixedSizeBinaryArray {
575    fn from(v: Vec<&[u8; N]>) -> Self {
576        Self::try_from_iter(v.into_iter()).unwrap()
577    }
578}
579
580impl std::fmt::Debug for FixedSizeBinaryArray {
581    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
582        write!(f, "FixedSizeBinaryArray<{}>\n[\n", self.value_length())?;
583        print_long_array(self, f, |array, index, f| {
584            std::fmt::Debug::fmt(&array.value(index), f)
585        })?;
586        write!(f, "]")
587    }
588}
589
590impl Array for FixedSizeBinaryArray {
591    fn as_any(&self) -> &dyn Any {
592        self
593    }
594
595    fn to_data(&self) -> ArrayData {
596        self.clone().into()
597    }
598
599    fn into_data(self) -> ArrayData {
600        self.into()
601    }
602
603    fn data_type(&self) -> &DataType {
604        &self.data_type
605    }
606
607    fn slice(&self, offset: usize, length: usize) -> ArrayRef {
608        Arc::new(self.slice(offset, length))
609    }
610
611    fn len(&self) -> usize {
612        self.len
613    }
614
615    fn is_empty(&self) -> bool {
616        self.len == 0
617    }
618
619    fn shrink_to_fit(&mut self) {
620        self.value_data.shrink_to_fit();
621        if let Some(nulls) = &mut self.nulls {
622            nulls.shrink_to_fit();
623        }
624    }
625
626    fn offset(&self) -> usize {
627        0
628    }
629
630    fn nulls(&self) -> Option<&NullBuffer> {
631        self.nulls.as_ref()
632    }
633
634    fn logical_null_count(&self) -> usize {
635        // More efficient that the default implementation
636        self.null_count()
637    }
638
639    fn get_buffer_memory_size(&self) -> usize {
640        let mut sum = self.value_data.capacity();
641        if let Some(n) = &self.nulls {
642            sum += n.buffer().capacity();
643        }
644        sum
645    }
646
647    fn get_array_memory_size(&self) -> usize {
648        std::mem::size_of::<Self>() + self.get_buffer_memory_size()
649    }
650}
651
652impl<'a> ArrayAccessor for &'a FixedSizeBinaryArray {
653    type Item = &'a [u8];
654
655    fn value(&self, index: usize) -> Self::Item {
656        FixedSizeBinaryArray::value(self, index)
657    }
658
659    unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
660        unsafe { FixedSizeBinaryArray::value_unchecked(self, index) }
661    }
662}
663
664impl<'a> IntoIterator for &'a FixedSizeBinaryArray {
665    type Item = Option<&'a [u8]>;
666    type IntoIter = FixedSizeBinaryIter<'a>;
667
668    fn into_iter(self) -> Self::IntoIter {
669        FixedSizeBinaryIter::<'a>::new(self)
670    }
671}
672
673#[cfg(test)]
674mod tests {
675    use crate::RecordBatch;
676    use arrow_schema::{Field, Schema};
677
678    use super::*;
679
680    #[test]
681    fn test_fixed_size_binary_array() {
682        let values: [u8; 15] = *b"hellotherearrow";
683
684        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
685            .len(3)
686            .add_buffer(Buffer::from(&values))
687            .build()
688            .unwrap();
689        let fixed_size_binary_array = FixedSizeBinaryArray::from(array_data);
690        assert_eq!(3, fixed_size_binary_array.len());
691        assert_eq!(0, fixed_size_binary_array.null_count());
692        assert_eq!(
693            [b'h', b'e', b'l', b'l', b'o'],
694            fixed_size_binary_array.value(0)
695        );
696        assert_eq!(
697            [b't', b'h', b'e', b'r', b'e'],
698            fixed_size_binary_array.value(1)
699        );
700        assert_eq!(
701            [b'a', b'r', b'r', b'o', b'w'],
702            fixed_size_binary_array.value(2)
703        );
704        assert_eq!(5, fixed_size_binary_array.value_length());
705        assert_eq!(10, fixed_size_binary_array.value_offset(2));
706        for i in 0..3 {
707            assert!(fixed_size_binary_array.is_valid(i));
708            assert!(!fixed_size_binary_array.is_null(i));
709        }
710
711        // Test binary array with offset
712        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
713            .len(2)
714            .offset(1)
715            .add_buffer(Buffer::from(&values))
716            .build()
717            .unwrap();
718        let fixed_size_binary_array = FixedSizeBinaryArray::from(array_data);
719        assert_eq!(
720            [b't', b'h', b'e', b'r', b'e'],
721            fixed_size_binary_array.value(0)
722        );
723        assert_eq!(
724            [b'a', b'r', b'r', b'o', b'w'],
725            fixed_size_binary_array.value(1)
726        );
727        assert_eq!(2, fixed_size_binary_array.len());
728        assert_eq!(0, fixed_size_binary_array.value_offset(0));
729        assert_eq!(5, fixed_size_binary_array.value_length());
730        assert_eq!(5, fixed_size_binary_array.value_offset(1));
731    }
732
733    #[test]
734    fn test_fixed_size_binary_array_from_fixed_size_list_array() {
735        let values = [0_u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13];
736        let values_data = ArrayData::builder(DataType::UInt8)
737            .len(12)
738            .offset(2)
739            .add_buffer(Buffer::from_slice_ref(values))
740            .build()
741            .unwrap();
742        // [null, [10, 11, 12, 13]]
743        let array_data = unsafe {
744            ArrayData::builder(DataType::FixedSizeList(
745                Arc::new(Field::new_list_field(DataType::UInt8, false)),
746                4,
747            ))
748            .len(2)
749            .offset(1)
750            .add_child_data(values_data)
751            .null_bit_buffer(Some(Buffer::from_slice_ref([0b101])))
752            .build_unchecked()
753        };
754        let list_array = FixedSizeListArray::from(array_data);
755        let binary_array = FixedSizeBinaryArray::from(list_array);
756
757        assert_eq!(2, binary_array.len());
758        assert_eq!(1, binary_array.null_count());
759        assert!(binary_array.is_null(0));
760        assert!(binary_array.is_valid(1));
761        assert_eq!(&[10, 11, 12, 13], binary_array.value(1));
762    }
763
764    #[test]
765    #[should_panic(
766        expected = "FixedSizeBinaryArray can only be created from FixedSizeList<u8> arrays"
767    )]
768    // Different error messages, so skip for now
769    // https://github.com/apache/arrow-rs/issues/1545
770    #[cfg(not(feature = "force_validate"))]
771    fn test_fixed_size_binary_array_from_incorrect_fixed_size_list_array() {
772        let values: [u32; 12] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
773        let values_data = ArrayData::builder(DataType::UInt32)
774            .len(12)
775            .add_buffer(Buffer::from_slice_ref(values))
776            .build()
777            .unwrap();
778
779        let array_data = unsafe {
780            ArrayData::builder(DataType::FixedSizeList(
781                Arc::new(Field::new_list_field(DataType::Binary, false)),
782                4,
783            ))
784            .len(3)
785            .add_child_data(values_data)
786            .build_unchecked()
787        };
788        let list_array = FixedSizeListArray::from(array_data);
789        drop(FixedSizeBinaryArray::from(list_array));
790    }
791
792    #[test]
793    #[should_panic(expected = "The child array cannot contain null values.")]
794    fn test_fixed_size_binary_array_from_fixed_size_list_array_with_child_nulls_failed() {
795        let values = [0_u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
796        let values_data = ArrayData::builder(DataType::UInt8)
797            .len(12)
798            .add_buffer(Buffer::from_slice_ref(values))
799            .null_bit_buffer(Some(Buffer::from_slice_ref([0b101010101010])))
800            .build()
801            .unwrap();
802
803        let array_data = unsafe {
804            ArrayData::builder(DataType::FixedSizeList(
805                Arc::new(Field::new_list_field(DataType::UInt8, false)),
806                4,
807            ))
808            .len(3)
809            .add_child_data(values_data)
810            .build_unchecked()
811        };
812        let list_array = FixedSizeListArray::from(array_data);
813        drop(FixedSizeBinaryArray::from(list_array));
814    }
815
816    #[test]
817    fn test_fixed_size_binary_array_fmt_debug() {
818        let values: [u8; 15] = *b"hellotherearrow";
819
820        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
821            .len(3)
822            .add_buffer(Buffer::from(&values))
823            .build()
824            .unwrap();
825        let arr = FixedSizeBinaryArray::from(array_data);
826        assert_eq!(
827            "FixedSizeBinaryArray<5>\n[\n  [104, 101, 108, 108, 111],\n  [116, 104, 101, 114, 101],\n  [97, 114, 114, 111, 119],\n]",
828            format!("{arr:?}")
829        );
830    }
831
832    #[test]
833    fn test_fixed_size_binary_array_from_iter() {
834        let input_arg = vec![vec![1, 2], vec![3, 4], vec![5, 6]];
835        let arr = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
836
837        assert_eq!(2, arr.value_length());
838        assert_eq!(3, arr.len())
839    }
840
841    #[test]
842    fn test_all_none_fixed_size_binary_array_from_sparse_iter() {
843        let none_option: Option<[u8; 32]> = None;
844        let input_arg = vec![none_option, none_option, none_option];
845        #[allow(deprecated)]
846        let arr = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.into_iter()).unwrap();
847        assert_eq!(0, arr.value_length());
848        assert_eq!(3, arr.len())
849    }
850
851    #[test]
852    fn test_fixed_size_binary_array_from_sparse_iter() {
853        let input_arg = vec![
854            None,
855            Some(vec![7, 8]),
856            Some(vec![9, 10]),
857            None,
858            Some(vec![13, 14]),
859        ];
860        #[allow(deprecated)]
861        let arr = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.iter().cloned()).unwrap();
862        assert_eq!(2, arr.value_length());
863        assert_eq!(5, arr.len());
864
865        let arr =
866            FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 2).unwrap();
867        assert_eq!(2, arr.value_length());
868        assert_eq!(5, arr.len());
869    }
870
871    #[test]
872    fn test_fixed_size_binary_array_from_sparse_iter_with_size_all_none() {
873        let input_arg = vec![None, None, None, None, None] as Vec<Option<Vec<u8>>>;
874
875        let arr = FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 16)
876            .unwrap();
877        assert_eq!(16, arr.value_length());
878        assert_eq!(5, arr.len())
879    }
880
881    #[test]
882    fn test_fixed_size_binary_array_from_vec() {
883        let values = vec!["one".as_bytes(), b"two", b"six", b"ten"];
884        let array = FixedSizeBinaryArray::from(values);
885        assert_eq!(array.len(), 4);
886        assert_eq!(array.null_count(), 0);
887        assert_eq!(array.logical_null_count(), 0);
888        assert_eq!(array.value(0), b"one");
889        assert_eq!(array.value(1), b"two");
890        assert_eq!(array.value(2), b"six");
891        assert_eq!(array.value(3), b"ten");
892        assert!(!array.is_null(0));
893        assert!(!array.is_null(1));
894        assert!(!array.is_null(2));
895        assert!(!array.is_null(3));
896    }
897
898    #[test]
899    #[should_panic(expected = "Nested array size mismatch: one is 3, and the other is 5")]
900    fn test_fixed_size_binary_array_from_vec_incorrect_length() {
901        let values = vec!["one".as_bytes(), b"two", b"three", b"four"];
902        let _ = FixedSizeBinaryArray::from(values);
903    }
904
905    #[test]
906    fn test_fixed_size_binary_array_from_opt_vec() {
907        let values = vec![
908            Some("one".as_bytes()),
909            Some(b"two"),
910            None,
911            Some(b"six"),
912            Some(b"ten"),
913        ];
914        let array = FixedSizeBinaryArray::from(values);
915        assert_eq!(array.len(), 5);
916        assert_eq!(array.value(0), b"one");
917        assert_eq!(array.value(1), b"two");
918        assert_eq!(array.value(3), b"six");
919        assert_eq!(array.value(4), b"ten");
920        assert!(!array.is_null(0));
921        assert!(!array.is_null(1));
922        assert!(array.is_null(2));
923        assert!(!array.is_null(3));
924        assert!(!array.is_null(4));
925    }
926
927    #[test]
928    #[should_panic(expected = "Nested array size mismatch: one is 3, and the other is 5")]
929    fn test_fixed_size_binary_array_from_opt_vec_incorrect_length() {
930        let values = vec![
931            Some("one".as_bytes()),
932            Some(b"two"),
933            None,
934            Some(b"three"),
935            Some(b"four"),
936        ];
937        let _ = FixedSizeBinaryArray::from(values);
938    }
939
940    #[test]
941    fn fixed_size_binary_array_all_null() {
942        let data = vec![None] as Vec<Option<String>>;
943        let array =
944            FixedSizeBinaryArray::try_from_sparse_iter_with_size(data.into_iter(), 0).unwrap();
945        array
946            .into_data()
947            .validate_full()
948            .expect("All null array has valid array data");
949    }
950
951    #[test]
952    // Test for https://github.com/apache/arrow-rs/issues/1390
953    fn fixed_size_binary_array_all_null_in_batch_with_schema() {
954        let schema = Schema::new(vec![Field::new("a", DataType::FixedSizeBinary(2), true)]);
955
956        let none_option: Option<[u8; 2]> = None;
957        let item = FixedSizeBinaryArray::try_from_sparse_iter_with_size(
958            vec![none_option, none_option, none_option].into_iter(),
959            2,
960        )
961        .unwrap();
962
963        // Should not panic
964        RecordBatch::try_new(Arc::new(schema), vec![Arc::new(item)]).unwrap();
965    }
966
967    #[test]
968    #[should_panic(
969        expected = "Trying to access an element at index 4 from a FixedSizeBinaryArray of length 3"
970    )]
971    fn test_fixed_size_binary_array_get_value_index_out_of_bound() {
972        let values = vec![Some("one".as_bytes()), Some(b"two"), None];
973        let array = FixedSizeBinaryArray::from(values);
974
975        array.value(4);
976    }
977
978    #[test]
979    fn test_constructors() {
980        let buffer = Buffer::from_vec(vec![0_u8; 10]);
981        let a = FixedSizeBinaryArray::new(2, buffer.clone(), None);
982        assert_eq!(a.len(), 5);
983
984        let nulls = NullBuffer::new_null(5);
985        FixedSizeBinaryArray::new(2, buffer.clone(), Some(nulls));
986
987        let null_array = FixedSizeBinaryArray::new_null(4, 3);
988        assert_eq!(null_array.len(), 3);
989        assert_eq!(null_array.values().len(), 12);
990
991        let a = FixedSizeBinaryArray::new(3, buffer.clone(), None);
992        assert_eq!(a.len(), 3);
993
994        let nulls = NullBuffer::new_null(3);
995        FixedSizeBinaryArray::new(3, buffer.clone(), Some(nulls));
996
997        let err = FixedSizeBinaryArray::try_new(-1, buffer.clone(), None).unwrap_err();
998
999        assert_eq!(
1000            err.to_string(),
1001            "Invalid argument error: Size cannot be negative, got -1"
1002        );
1003
1004        let nulls = NullBuffer::new_null(3);
1005        let err = FixedSizeBinaryArray::try_new(2, buffer, Some(nulls)).unwrap_err();
1006        assert_eq!(
1007            err.to_string(),
1008            "Invalid argument error: Incorrect length of null buffer for FixedSizeBinaryArray, expected 5 got 3"
1009        );
1010    }
1011}
arrow_array/array/fixed_size_binary_array.rs

arrow_array/array/
fixed_size_binary_array.rs
