arrow_array/array/
fixed_size_binary_array.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use crate::array::print_long_array;
19use crate::iterator::FixedSizeBinaryIter;
20use crate::{Array, ArrayAccessor, ArrayRef, FixedSizeListArray, Scalar};
21use arrow_buffer::buffer::NullBuffer;
22use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer, bit_util};
23use arrow_data::{ArrayData, ArrayDataBuilder};
24use arrow_schema::{ArrowError, DataType};
25use std::any::Any;
26use std::sync::Arc;
27
28/// An array of [fixed size binary arrays](https://arrow.apache.org/docs/format/Columnar.html#fixed-size-primitive-layout)
29///
30/// # Examples
31///
32/// Create an array from an iterable argument of byte slices.
33///
34/// ```
35///    use arrow_array::{Array, FixedSizeBinaryArray};
36///    let input_arg = vec![ vec![1, 2], vec![3, 4], vec![5, 6] ];
37///    let arr = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
38///
39///    assert_eq!(3, arr.len());
40///
41/// ```
42/// Create an array from an iterable argument of sparse byte slices.
43/// Sparsity means that the input argument can contain `None` items.
44/// ```
45///    use arrow_array::{Array, FixedSizeBinaryArray};
46///    let input_arg = vec![ None, Some(vec![7, 8]), Some(vec![9, 10]), None, Some(vec![13, 14]) ];
47///    let arr = FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 2).unwrap();
48///    assert_eq!(5, arr.len())
49///
50/// ```
51///
52#[derive(Clone)]
53pub struct FixedSizeBinaryArray {
54    data_type: DataType, // Must be DataType::FixedSizeBinary(value_length)
55    value_data: Buffer,
56    nulls: Option<NullBuffer>,
57    len: usize,
58    value_length: i32,
59}
60
61impl FixedSizeBinaryArray {
62    /// Create a new [`FixedSizeBinaryArray`] with `size` element size, panicking on failure
63    ///
64    /// # Panics
65    ///
66    /// Panics if [`Self::try_new`] returns an error
67    pub fn new(size: i32, values: Buffer, nulls: Option<NullBuffer>) -> Self {
68        Self::try_new(size, values, nulls).unwrap()
69    }
70
71    /// Create a new [`Scalar`] from `value`
72    pub fn new_scalar(value: impl AsRef<[u8]>) -> Scalar<Self> {
73        let v = value.as_ref();
74        Scalar::new(Self::new(v.len() as _, Buffer::from(v), None))
75    }
76
77    /// Create a new [`FixedSizeBinaryArray`] from the provided parts, returning an error on failure
78    ///
79    /// Creating an arrow with `size == 0` will try to get the length from the null buffer. If
80    /// no null buffer is provided, the resulting array will have length zero.
81    ///
82    /// # Errors
83    ///
84    /// * `size < 0`
85    /// * `values.len() / size != nulls.len()`
86    /// * `size == 0 && values.len() != 0`
87    pub fn try_new(
88        size: i32,
89        values: Buffer,
90        nulls: Option<NullBuffer>,
91    ) -> Result<Self, ArrowError> {
92        let data_type = DataType::FixedSizeBinary(size);
93        let s = size.to_usize().ok_or_else(|| {
94            ArrowError::InvalidArgumentError(format!("Size cannot be negative, got {size}"))
95        })?;
96
97        let len = match values.len().checked_div(s) {
98            Some(len) => {
99                if let Some(n) = nulls.as_ref() {
100                    if n.len() != len {
101                        return Err(ArrowError::InvalidArgumentError(format!(
102                            "Incorrect length of null buffer for FixedSizeBinaryArray, expected {} got {}",
103                            len,
104                            n.len(),
105                        )));
106                    }
107                }
108
109                len
110            }
111            None => {
112                if !values.is_empty() {
113                    return Err(ArrowError::InvalidArgumentError(
114                        "Buffer cannot have non-zero length if the item size is zero".to_owned(),
115                    ));
116                }
117
118                // If the item size is zero, try to determine the length from the null buffer
119                nulls.as_ref().map(|n| n.len()).unwrap_or(0)
120            }
121        };
122
123        Ok(Self {
124            data_type,
125            value_data: values,
126            value_length: size,
127            nulls,
128            len,
129        })
130    }
131
132    /// Create a new [`FixedSizeBinaryArray`] of length `len` where all values are null
133    ///
134    /// # Panics
135    ///
136    /// Panics if
137    ///
138    /// * `size < 0`
139    /// * `size * len` would overflow `usize`
140    pub fn new_null(size: i32, len: usize) -> Self {
141        const BITS_IN_A_BYTE: usize = 8;
142        let capacity_in_bytes = size.to_usize().unwrap().checked_mul(len).unwrap();
143        Self {
144            data_type: DataType::FixedSizeBinary(size),
145            value_data: MutableBuffer::new_null(capacity_in_bytes * BITS_IN_A_BYTE).into(),
146            nulls: Some(NullBuffer::new_null(len)),
147            value_length: size,
148            len,
149        }
150    }
151
152    /// Deconstruct this array into its constituent parts
153    pub fn into_parts(self) -> (i32, Buffer, Option<NullBuffer>) {
154        (self.value_length, self.value_data, self.nulls)
155    }
156
157    /// Returns the element at index `i` as a byte slice.
158    ///
159    /// Note: This method does not check for nulls and the value is arbitrary
160    /// (but still well-defined) if [`is_null`](Self::is_null) returns true for the index.
161    ///
162    /// # Panics
163    /// Panics if index `i` is out of bounds.
164    pub fn value(&self, i: usize) -> &[u8] {
165        assert!(
166            i < self.len(),
167            "Trying to access an element at index {} from a FixedSizeBinaryArray of length {}",
168            i,
169            self.len()
170        );
171        let offset = i + self.offset();
172        unsafe {
173            let pos = self.value_offset_at(offset);
174            std::slice::from_raw_parts(
175                self.value_data.as_ptr().offset(pos as isize),
176                (self.value_offset_at(offset + 1) - pos) as usize,
177            )
178        }
179    }
180
181    /// Returns the element at index `i` as a byte slice.
182    ///
183    /// Note: This method does not check for nulls and the value is arbitrary
184    /// if [`is_null`](Self::is_null) returns true for the index.
185    ///
186    /// # Safety
187    ///
188    /// Caller is responsible for ensuring that the index is within the bounds
189    /// of the array
190    pub unsafe fn value_unchecked(&self, i: usize) -> &[u8] {
191        let offset = i + self.offset();
192        let pos = self.value_offset_at(offset);
193        unsafe {
194            std::slice::from_raw_parts(
195                self.value_data.as_ptr().offset(pos as isize),
196                (self.value_offset_at(offset + 1) - pos) as usize,
197            )
198        }
199    }
200
201    /// Returns the offset for the element at index `i`.
202    ///
203    /// Note this doesn't do any bound checking, for performance reason.
204    #[inline]
205    pub fn value_offset(&self, i: usize) -> i32 {
206        self.value_offset_at(self.offset() + i)
207    }
208
209    /// Returns the length for an element.
210    ///
211    /// All elements have the same length as the array is a fixed size.
212    #[inline]
213    pub fn value_length(&self) -> i32 {
214        self.value_length
215    }
216
217    /// Returns the values of this array.
218    ///
219    /// Unlike [`Self::value_data`] this returns the [`Buffer`]
220    /// allowing for zero-copy cloning.
221    #[inline]
222    pub fn values(&self) -> &Buffer {
223        &self.value_data
224    }
225
226    /// Returns the raw value data.
227    pub fn value_data(&self) -> &[u8] {
228        self.value_data.as_slice()
229    }
230
231    /// Returns a zero-copy slice of this array with the indicated offset and length.
232    pub fn slice(&self, offset: usize, len: usize) -> Self {
233        assert!(
234            offset.saturating_add(len) <= self.len,
235            "the length + offset of the sliced FixedSizeBinaryArray cannot exceed the existing length"
236        );
237
238        let size = self.value_length as usize;
239
240        Self {
241            data_type: self.data_type.clone(),
242            nulls: self.nulls.as_ref().map(|n| n.slice(offset, len)),
243            value_length: self.value_length,
244            value_data: self.value_data.slice_with_length(offset * size, len * size),
245            len,
246        }
247    }
248
249    /// Create an array from an iterable argument of sparse byte slices.
250    /// Sparsity means that items returned by the iterator are optional, i.e input argument can
251    /// contain `None` items.
252    ///
253    /// # Examples
254    ///
255    /// ```
256    /// use arrow_array::FixedSizeBinaryArray;
257    /// let input_arg = vec![
258    ///     None,
259    ///     Some(vec![7, 8]),
260    ///     Some(vec![9, 10]),
261    ///     None,
262    ///     Some(vec![13, 14]),
263    ///     None,
264    /// ];
265    /// let array = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.into_iter()).unwrap();
266    /// ```
267    ///
268    /// # Errors
269    ///
270    /// Returns error if argument has length zero, or sizes of nested slices don't match.
271    #[deprecated(
272        since = "28.0.0",
273        note = "This function will fail if the iterator produces only None values; prefer `try_from_sparse_iter_with_size`"
274    )]
275    pub fn try_from_sparse_iter<T, U>(mut iter: T) -> Result<Self, ArrowError>
276    where
277        T: Iterator<Item = Option<U>>,
278        U: AsRef<[u8]>,
279    {
280        let mut len = 0;
281        let mut size = None;
282        let mut byte = 0;
283
284        let iter_size_hint = iter.size_hint().0;
285        let mut null_buf = MutableBuffer::new(bit_util::ceil(iter_size_hint, 8));
286        let mut buffer = MutableBuffer::new(0);
287
288        let mut prepend = 0;
289        iter.try_for_each(|item| -> Result<(), ArrowError> {
290            // extend null bitmask by one byte per each 8 items
291            if byte == 0 {
292                null_buf.push(0u8);
293                byte = 8;
294            }
295            byte -= 1;
296
297            if let Some(slice) = item {
298                let slice = slice.as_ref();
299                if let Some(size) = size {
300                    if size != slice.len() {
301                        return Err(ArrowError::InvalidArgumentError(format!(
302                            "Nested array size mismatch: one is {}, and the other is {}",
303                            size,
304                            slice.len()
305                        )));
306                    }
307                } else {
308                    let len = slice.len();
309                    size = Some(len);
310                    // Now that we know how large each element is we can reserve
311                    // sufficient capacity in the underlying mutable buffer for
312                    // the data.
313                    buffer.reserve(iter_size_hint * len);
314                    buffer.extend_zeros(slice.len() * prepend);
315                }
316                bit_util::set_bit(null_buf.as_slice_mut(), len);
317                buffer.extend_from_slice(slice);
318            } else if let Some(size) = size {
319                buffer.extend_zeros(size);
320            } else {
321                prepend += 1;
322            }
323
324            len += 1;
325
326            Ok(())
327        })?;
328
329        if len == 0 {
330            return Err(ArrowError::InvalidArgumentError(
331                "Input iterable argument has no data".to_owned(),
332            ));
333        }
334
335        let nulls = NullBuffer::from_unsliced_buffer(null_buf, len);
336
337        let size = size.unwrap_or(0) as i32;
338        Ok(Self {
339            data_type: DataType::FixedSizeBinary(size),
340            value_data: buffer.into(),
341            nulls,
342            value_length: size,
343            len,
344        })
345    }
346
347    /// Create an array from an iterable argument of sparse byte slices.
348    /// Sparsity means that items returned by the iterator are optional, i.e input argument can
349    /// contain `None` items. In cases where the iterator returns only `None` values, this
350    /// also takes a size parameter to ensure that the a valid FixedSizeBinaryArray is still
351    /// created.
352    ///
353    /// # Examples
354    ///
355    /// ```
356    /// use arrow_array::FixedSizeBinaryArray;
357    /// let input_arg = vec![
358    ///     None,
359    ///     Some(vec![7, 8]),
360    ///     Some(vec![9, 10]),
361    ///     None,
362    ///     Some(vec![13, 14]),
363    ///     None,
364    /// ];
365    /// let array = FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 2).unwrap();
366    /// ```
367    ///
368    /// # Errors
369    ///
370    /// Returns error if argument has length zero, or sizes of nested slices don't match.
371    pub fn try_from_sparse_iter_with_size<T, U>(mut iter: T, size: i32) -> Result<Self, ArrowError>
372    where
373        T: Iterator<Item = Option<U>>,
374        U: AsRef<[u8]>,
375    {
376        let mut len = 0;
377        let mut byte = 0;
378
379        let iter_size_hint = iter.size_hint().0;
380        let mut null_buf = MutableBuffer::new(bit_util::ceil(iter_size_hint, 8));
381        let mut buffer = MutableBuffer::new(iter_size_hint * (size as usize));
382
383        iter.try_for_each(|item| -> Result<(), ArrowError> {
384            // extend null bitmask by one byte per each 8 items
385            if byte == 0 {
386                null_buf.push(0u8);
387                byte = 8;
388            }
389            byte -= 1;
390
391            if let Some(slice) = item {
392                let slice = slice.as_ref();
393                if size as usize != slice.len() {
394                    return Err(ArrowError::InvalidArgumentError(format!(
395                        "Nested array size mismatch: one is {}, and the other is {}",
396                        size,
397                        slice.len()
398                    )));
399                }
400
401                bit_util::set_bit(null_buf.as_slice_mut(), len);
402                buffer.extend_from_slice(slice);
403            } else {
404                buffer.extend_zeros(size as usize);
405            }
406
407            len += 1;
408
409            Ok(())
410        })?;
411
412        let nulls = NullBuffer::from_unsliced_buffer(null_buf, len);
413
414        Ok(Self {
415            data_type: DataType::FixedSizeBinary(size),
416            value_data: buffer.into(),
417            nulls,
418            len,
419            value_length: size,
420        })
421    }
422
423    /// Create an array from an iterable argument of byte slices.
424    ///
425    /// # Examples
426    ///
427    /// ```
428    /// use arrow_array::FixedSizeBinaryArray;
429    /// let input_arg = vec![
430    ///     vec![1, 2],
431    ///     vec![3, 4],
432    ///     vec![5, 6],
433    /// ];
434    /// let array = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
435    /// ```
436    ///
437    /// # Errors
438    ///
439    /// Returns error if argument has length zero, or sizes of nested slices don't match.
440    pub fn try_from_iter<T, U>(mut iter: T) -> Result<Self, ArrowError>
441    where
442        T: Iterator<Item = U>,
443        U: AsRef<[u8]>,
444    {
445        let mut len = 0;
446        let mut size = None;
447        let iter_size_hint = iter.size_hint().0;
448        let mut buffer = MutableBuffer::new(0);
449
450        iter.try_for_each(|item| -> Result<(), ArrowError> {
451            let slice = item.as_ref();
452            if let Some(size) = size {
453                if size != slice.len() {
454                    return Err(ArrowError::InvalidArgumentError(format!(
455                        "Nested array size mismatch: one is {}, and the other is {}",
456                        size,
457                        slice.len()
458                    )));
459                }
460            } else {
461                let len = slice.len();
462                size = Some(len);
463                buffer.reserve(iter_size_hint * len);
464            }
465
466            buffer.extend_from_slice(slice);
467
468            len += 1;
469
470            Ok(())
471        })?;
472
473        if len == 0 {
474            return Err(ArrowError::InvalidArgumentError(
475                "Input iterable argument has no data".to_owned(),
476            ));
477        }
478
479        let size = size.unwrap_or(0).try_into().unwrap();
480        Ok(Self {
481            data_type: DataType::FixedSizeBinary(size),
482            value_data: buffer.into(),
483            nulls: None,
484            value_length: size,
485            len,
486        })
487    }
488
489    #[inline]
490    fn value_offset_at(&self, i: usize) -> i32 {
491        self.value_length * i as i32
492    }
493
494    /// constructs a new iterator
495    pub fn iter(&self) -> FixedSizeBinaryIter<'_> {
496        FixedSizeBinaryIter::new(self)
497    }
498}
499
500impl From<ArrayData> for FixedSizeBinaryArray {
501    fn from(data: ArrayData) -> Self {
502        let (data_type, len, nulls, offset, buffers, _child_data) = data.into_parts();
503
504        assert_eq!(
505            buffers.len(),
506            1,
507            "FixedSizeBinaryArray data should contain 1 buffer only (values)"
508        );
509        let value_length = match data_type {
510            DataType::FixedSizeBinary(len) => len,
511            _ => panic!("Expected data type to be FixedSizeBinary"),
512        };
513
514        let size = value_length as usize;
515        let value_data = buffers[0].slice_with_length(offset * size, len * size);
516
517        Self {
518            data_type,
519            nulls,
520            len,
521            value_data,
522            value_length,
523        }
524    }
525}
526
527impl From<FixedSizeBinaryArray> for ArrayData {
528    fn from(array: FixedSizeBinaryArray) -> Self {
529        let builder = ArrayDataBuilder::new(array.data_type)
530            .len(array.len)
531            .buffers(vec![array.value_data])
532            .nulls(array.nulls);
533
534        unsafe { builder.build_unchecked() }
535    }
536}
537
538/// Creates a `FixedSizeBinaryArray` from `FixedSizeList<u8>` array
539impl From<FixedSizeListArray> for FixedSizeBinaryArray {
540    fn from(v: FixedSizeListArray) -> Self {
541        let value_len = v.value_length();
542        let v = v.into_data();
543        assert_eq!(
544            v.child_data().len(),
545            1,
546            "FixedSizeBinaryArray can only be created from list array of u8 values \
547             (i.e. FixedSizeList<PrimitiveArray<u8>>)."
548        );
549        let child_data = &v.child_data()[0];
550
551        assert_eq!(
552            child_data.child_data().len(),
553            0,
554            "FixedSizeBinaryArray can only be created from list array of u8 values \
555             (i.e. FixedSizeList<PrimitiveArray<u8>>)."
556        );
557        assert_eq!(
558            child_data.data_type(),
559            &DataType::UInt8,
560            "FixedSizeBinaryArray can only be created from FixedSizeList<u8> arrays, mismatched data types."
561        );
562        assert_eq!(
563            child_data.null_count(),
564            0,
565            "The child array cannot contain null values."
566        );
567
568        let builder = ArrayData::builder(DataType::FixedSizeBinary(value_len))
569            .len(v.len())
570            .offset(v.offset())
571            .add_buffer(child_data.buffers()[0].slice(child_data.offset()))
572            .nulls(v.nulls().cloned());
573
574        let data = unsafe { builder.build_unchecked() };
575        Self::from(data)
576    }
577}
578
579impl From<Vec<Option<&[u8]>>> for FixedSizeBinaryArray {
580    fn from(v: Vec<Option<&[u8]>>) -> Self {
581        #[allow(deprecated)]
582        Self::try_from_sparse_iter(v.into_iter()).unwrap()
583    }
584}
585
586impl From<Vec<&[u8]>> for FixedSizeBinaryArray {
587    fn from(v: Vec<&[u8]>) -> Self {
588        Self::try_from_iter(v.into_iter()).unwrap()
589    }
590}
591
592impl<const N: usize> From<Vec<&[u8; N]>> for FixedSizeBinaryArray {
593    fn from(v: Vec<&[u8; N]>) -> Self {
594        Self::try_from_iter(v.into_iter()).unwrap()
595    }
596}
597
598impl std::fmt::Debug for FixedSizeBinaryArray {
599    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
600        write!(f, "FixedSizeBinaryArray<{}>\n[\n", self.value_length())?;
601        print_long_array(self, f, |array, index, f| {
602            std::fmt::Debug::fmt(&array.value(index), f)
603        })?;
604        write!(f, "]")
605    }
606}
607
608/// SAFETY: Correctly implements the contract of Arrow Arrays
609unsafe impl Array for FixedSizeBinaryArray {
610    fn as_any(&self) -> &dyn Any {
611        self
612    }
613
614    fn to_data(&self) -> ArrayData {
615        self.clone().into()
616    }
617
618    fn into_data(self) -> ArrayData {
619        self.into()
620    }
621
622    fn data_type(&self) -> &DataType {
623        &self.data_type
624    }
625
626    fn slice(&self, offset: usize, length: usize) -> ArrayRef {
627        Arc::new(self.slice(offset, length))
628    }
629
630    fn len(&self) -> usize {
631        self.len
632    }
633
634    fn is_empty(&self) -> bool {
635        self.len == 0
636    }
637
638    fn shrink_to_fit(&mut self) {
639        self.value_data.shrink_to_fit();
640        if let Some(nulls) = &mut self.nulls {
641            nulls.shrink_to_fit();
642        }
643    }
644
645    fn offset(&self) -> usize {
646        0
647    }
648
649    fn nulls(&self) -> Option<&NullBuffer> {
650        self.nulls.as_ref()
651    }
652
653    fn logical_null_count(&self) -> usize {
654        // More efficient that the default implementation
655        self.null_count()
656    }
657
658    fn get_buffer_memory_size(&self) -> usize {
659        let mut sum = self.value_data.capacity();
660        if let Some(n) = &self.nulls {
661            sum += n.buffer().capacity();
662        }
663        sum
664    }
665
666    fn get_array_memory_size(&self) -> usize {
667        std::mem::size_of::<Self>() + self.get_buffer_memory_size()
668    }
669
670    #[cfg(feature = "pool")]
671    fn claim(&self, pool: &dyn arrow_buffer::MemoryPool) {
672        self.value_data.claim(pool);
673        if let Some(nulls) = &self.nulls {
674            nulls.claim(pool);
675        }
676    }
677}
678
679impl<'a> ArrayAccessor for &'a FixedSizeBinaryArray {
680    type Item = &'a [u8];
681
682    fn value(&self, index: usize) -> Self::Item {
683        FixedSizeBinaryArray::value(self, index)
684    }
685
686    unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
687        unsafe { FixedSizeBinaryArray::value_unchecked(self, index) }
688    }
689}
690
691impl<'a> IntoIterator for &'a FixedSizeBinaryArray {
692    type Item = Option<&'a [u8]>;
693    type IntoIter = FixedSizeBinaryIter<'a>;
694
695    fn into_iter(self) -> Self::IntoIter {
696        FixedSizeBinaryIter::<'a>::new(self)
697    }
698}
699
700#[cfg(test)]
701mod tests {
702    use super::*;
703    use crate::RecordBatch;
704    use arrow_schema::{Field, Schema};
705
706    #[test]
707    fn test_fixed_size_binary_array() {
708        let values: [u8; 15] = *b"hellotherearrow";
709
710        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
711            .len(3)
712            .add_buffer(Buffer::from(&values))
713            .build()
714            .unwrap();
715        let fixed_size_binary_array = FixedSizeBinaryArray::from(array_data);
716        assert_eq!(3, fixed_size_binary_array.len());
717        assert_eq!(0, fixed_size_binary_array.null_count());
718        assert_eq!(
719            [b'h', b'e', b'l', b'l', b'o'],
720            fixed_size_binary_array.value(0)
721        );
722        assert_eq!(
723            [b't', b'h', b'e', b'r', b'e'],
724            fixed_size_binary_array.value(1)
725        );
726        assert_eq!(
727            [b'a', b'r', b'r', b'o', b'w'],
728            fixed_size_binary_array.value(2)
729        );
730        assert_eq!(5, fixed_size_binary_array.value_length());
731        assert_eq!(10, fixed_size_binary_array.value_offset(2));
732        for i in 0..3 {
733            assert!(fixed_size_binary_array.is_valid(i));
734            assert!(!fixed_size_binary_array.is_null(i));
735        }
736
737        // Test binary array with offset
738        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
739            .len(2)
740            .offset(1)
741            .add_buffer(Buffer::from(&values))
742            .build()
743            .unwrap();
744        let fixed_size_binary_array = FixedSizeBinaryArray::from(array_data);
745        assert_eq!(
746            [b't', b'h', b'e', b'r', b'e'],
747            fixed_size_binary_array.value(0)
748        );
749        assert_eq!(
750            [b'a', b'r', b'r', b'o', b'w'],
751            fixed_size_binary_array.value(1)
752        );
753        assert_eq!(2, fixed_size_binary_array.len());
754        assert_eq!(0, fixed_size_binary_array.value_offset(0));
755        assert_eq!(5, fixed_size_binary_array.value_length());
756        assert_eq!(5, fixed_size_binary_array.value_offset(1));
757    }
758
759    #[test]
760    fn test_fixed_size_binary_array_from_fixed_size_list_array() {
761        let values = [0_u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13];
762        let values_data = ArrayData::builder(DataType::UInt8)
763            .len(12)
764            .offset(2)
765            .add_buffer(Buffer::from_slice_ref(values))
766            .build()
767            .unwrap();
768        // [null, [10, 11, 12, 13]]
769        let array_data = unsafe {
770            ArrayData::builder(DataType::FixedSizeList(
771                Arc::new(Field::new_list_field(DataType::UInt8, false)),
772                4,
773            ))
774            .len(2)
775            .offset(1)
776            .add_child_data(values_data)
777            .null_bit_buffer(Some(Buffer::from_slice_ref([0b101])))
778            .build_unchecked()
779        };
780        let list_array = FixedSizeListArray::from(array_data);
781        let binary_array = FixedSizeBinaryArray::from(list_array);
782
783        assert_eq!(2, binary_array.len());
784        assert_eq!(1, binary_array.null_count());
785        assert!(binary_array.is_null(0));
786        assert!(binary_array.is_valid(1));
787        assert_eq!(&[10, 11, 12, 13], binary_array.value(1));
788    }
789
790    #[test]
791    #[should_panic(
792        expected = "FixedSizeBinaryArray can only be created from FixedSizeList<u8> arrays"
793    )]
794    // Different error messages, so skip for now
795    // https://github.com/apache/arrow-rs/issues/1545
796    #[cfg(not(feature = "force_validate"))]
797    fn test_fixed_size_binary_array_from_incorrect_fixed_size_list_array() {
798        let values: [u32; 12] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
799        let values_data = ArrayData::builder(DataType::UInt32)
800            .len(12)
801            .add_buffer(Buffer::from_slice_ref(values))
802            .build()
803            .unwrap();
804
805        let array_data = unsafe {
806            ArrayData::builder(DataType::FixedSizeList(
807                Arc::new(Field::new_list_field(DataType::Binary, false)),
808                4,
809            ))
810            .len(3)
811            .add_child_data(values_data)
812            .build_unchecked()
813        };
814        let list_array = FixedSizeListArray::from(array_data);
815        drop(FixedSizeBinaryArray::from(list_array));
816    }
817
818    #[test]
819    #[should_panic(expected = "The child array cannot contain null values.")]
820    fn test_fixed_size_binary_array_from_fixed_size_list_array_with_child_nulls_failed() {
821        let values = [0_u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
822        let values_data = ArrayData::builder(DataType::UInt8)
823            .len(12)
824            .add_buffer(Buffer::from_slice_ref(values))
825            .null_bit_buffer(Some(Buffer::from_slice_ref([0b101010101010])))
826            .build()
827            .unwrap();
828
829        let array_data = unsafe {
830            ArrayData::builder(DataType::FixedSizeList(
831                Arc::new(Field::new_list_field(DataType::UInt8, false)),
832                4,
833            ))
834            .len(3)
835            .add_child_data(values_data)
836            .build_unchecked()
837        };
838        let list_array = FixedSizeListArray::from(array_data);
839        drop(FixedSizeBinaryArray::from(list_array));
840    }
841
842    #[test]
843    fn test_fixed_size_binary_array_fmt_debug() {
844        let values: [u8; 15] = *b"hellotherearrow";
845
846        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
847            .len(3)
848            .add_buffer(Buffer::from(&values))
849            .build()
850            .unwrap();
851        let arr = FixedSizeBinaryArray::from(array_data);
852        assert_eq!(
853            "FixedSizeBinaryArray<5>\n[\n  [104, 101, 108, 108, 111],\n  [116, 104, 101, 114, 101],\n  [97, 114, 114, 111, 119],\n]",
854            format!("{arr:?}")
855        );
856    }
857
858    #[test]
859    fn test_fixed_size_binary_array_from_iter() {
860        let input_arg = vec![vec![1, 2], vec![3, 4], vec![5, 6]];
861        let arr = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
862
863        assert_eq!(2, arr.value_length());
864        assert_eq!(3, arr.len())
865    }
866
867    #[test]
868    fn test_all_none_fixed_size_binary_array_from_sparse_iter() {
869        let none_option: Option<[u8; 32]> = None;
870        let input_arg = vec![none_option, none_option, none_option];
871        #[allow(deprecated)]
872        let arr = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.into_iter()).unwrap();
873        assert_eq!(0, arr.value_length());
874        assert_eq!(3, arr.len())
875    }
876
877    #[test]
878    fn test_fixed_size_binary_array_from_sparse_iter() {
879        let input_arg = vec![
880            None,
881            Some(vec![7, 8]),
882            Some(vec![9, 10]),
883            None,
884            Some(vec![13, 14]),
885        ];
886        #[allow(deprecated)]
887        let arr = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.iter().cloned()).unwrap();
888        assert_eq!(2, arr.value_length());
889        assert_eq!(5, arr.len());
890
891        let arr =
892            FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 2).unwrap();
893        assert_eq!(2, arr.value_length());
894        assert_eq!(5, arr.len());
895    }
896
897    #[test]
898    fn test_fixed_size_binary_array_from_sparse_iter_with_size_all_none() {
899        let input_arg = vec![None, None, None, None, None] as Vec<Option<Vec<u8>>>;
900
901        let arr = FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 16)
902            .unwrap();
903        assert_eq!(16, arr.value_length());
904        assert_eq!(5, arr.len())
905    }
906
907    #[test]
908    fn test_fixed_size_binary_array_from_vec() {
909        let values = vec!["one".as_bytes(), b"two", b"six", b"ten"];
910        let array = FixedSizeBinaryArray::from(values);
911        assert_eq!(array.len(), 4);
912        assert_eq!(array.null_count(), 0);
913        assert_eq!(array.logical_null_count(), 0);
914        assert_eq!(array.value(0), b"one");
915        assert_eq!(array.value(1), b"two");
916        assert_eq!(array.value(2), b"six");
917        assert_eq!(array.value(3), b"ten");
918        assert!(!array.is_null(0));
919        assert!(!array.is_null(1));
920        assert!(!array.is_null(2));
921        assert!(!array.is_null(3));
922    }
923
924    #[test]
925    #[should_panic(expected = "Nested array size mismatch: one is 3, and the other is 5")]
926    fn test_fixed_size_binary_array_from_vec_incorrect_length() {
927        let values = vec!["one".as_bytes(), b"two", b"three", b"four"];
928        let _ = FixedSizeBinaryArray::from(values);
929    }
930
931    #[test]
932    fn test_fixed_size_binary_array_from_opt_vec() {
933        let values = vec![
934            Some("one".as_bytes()),
935            Some(b"two"),
936            None,
937            Some(b"six"),
938            Some(b"ten"),
939        ];
940        let array = FixedSizeBinaryArray::from(values);
941        assert_eq!(array.len(), 5);
942        assert_eq!(array.value(0), b"one");
943        assert_eq!(array.value(1), b"two");
944        assert_eq!(array.value(3), b"six");
945        assert_eq!(array.value(4), b"ten");
946        assert!(!array.is_null(0));
947        assert!(!array.is_null(1));
948        assert!(array.is_null(2));
949        assert!(!array.is_null(3));
950        assert!(!array.is_null(4));
951    }
952
953    #[test]
954    #[should_panic(expected = "Nested array size mismatch: one is 3, and the other is 5")]
955    fn test_fixed_size_binary_array_from_opt_vec_incorrect_length() {
956        let values = vec![
957            Some("one".as_bytes()),
958            Some(b"two"),
959            None,
960            Some(b"three"),
961            Some(b"four"),
962        ];
963        let _ = FixedSizeBinaryArray::from(values);
964    }
965
966    #[test]
967    fn fixed_size_binary_array_all_null() {
968        let data = vec![None] as Vec<Option<String>>;
969        let array =
970            FixedSizeBinaryArray::try_from_sparse_iter_with_size(data.into_iter(), 0).unwrap();
971        array
972            .into_data()
973            .validate_full()
974            .expect("All null array has valid array data");
975    }
976
977    #[test]
978    // Test for https://github.com/apache/arrow-rs/issues/1390
979    fn fixed_size_binary_array_all_null_in_batch_with_schema() {
980        let schema = Schema::new(vec![Field::new("a", DataType::FixedSizeBinary(2), true)]);
981
982        let none_option: Option<[u8; 2]> = None;
983        let item = FixedSizeBinaryArray::try_from_sparse_iter_with_size(
984            vec![none_option, none_option, none_option].into_iter(),
985            2,
986        )
987        .unwrap();
988
989        // Should not panic
990        RecordBatch::try_new(Arc::new(schema), vec![Arc::new(item)]).unwrap();
991    }
992
993    #[test]
994    #[should_panic(
995        expected = "Trying to access an element at index 4 from a FixedSizeBinaryArray of length 3"
996    )]
997    fn test_fixed_size_binary_array_get_value_index_out_of_bound() {
998        let values = vec![Some("one".as_bytes()), Some(b"two"), None];
999        let array = FixedSizeBinaryArray::from(values);
1000
1001        array.value(4);
1002    }
1003
1004    #[test]
1005    fn test_constructors() {
1006        let buffer = Buffer::from_vec(vec![0_u8; 10]);
1007        let a = FixedSizeBinaryArray::new(2, buffer.clone(), None);
1008        assert_eq!(a.len(), 5);
1009
1010        let nulls = NullBuffer::new_null(5);
1011        FixedSizeBinaryArray::new(2, buffer.clone(), Some(nulls));
1012
1013        let null_array = FixedSizeBinaryArray::new_null(4, 3);
1014        assert_eq!(null_array.len(), 3);
1015        assert_eq!(null_array.values().len(), 12);
1016
1017        let a = FixedSizeBinaryArray::new(3, buffer.clone(), None);
1018        assert_eq!(a.len(), 3);
1019
1020        let nulls = NullBuffer::new_null(3);
1021        FixedSizeBinaryArray::new(3, buffer.clone(), Some(nulls));
1022
1023        let err = FixedSizeBinaryArray::try_new(-1, buffer.clone(), None).unwrap_err();
1024
1025        assert_eq!(
1026            err.to_string(),
1027            "Invalid argument error: Size cannot be negative, got -1"
1028        );
1029
1030        let nulls = NullBuffer::new_null(3);
1031        let err = FixedSizeBinaryArray::try_new(2, buffer.clone(), Some(nulls)).unwrap_err();
1032        assert_eq!(
1033            err.to_string(),
1034            "Invalid argument error: Incorrect length of null buffer for FixedSizeBinaryArray, expected 5 got 3"
1035        );
1036
1037        let zero_sized = FixedSizeBinaryArray::new(0, Buffer::default(), None);
1038        assert_eq!(zero_sized.len(), 0);
1039        assert_eq!(zero_sized.null_count(), 0);
1040        assert_eq!(zero_sized.values().len(), 0);
1041
1042        let nulls = NullBuffer::new_null(3);
1043        let zero_sized_with_nulls = FixedSizeBinaryArray::new(0, Buffer::default(), Some(nulls));
1044        assert_eq!(zero_sized_with_nulls.len(), 3);
1045        assert_eq!(zero_sized_with_nulls.null_count(), 3);
1046        assert_eq!(zero_sized_with_nulls.values().len(), 0);
1047
1048        let zero_sized_with_non_empty_buffer_err =
1049            FixedSizeBinaryArray::try_new(0, buffer, None).unwrap_err();
1050        assert_eq!(
1051            zero_sized_with_non_empty_buffer_err.to_string(),
1052            "Invalid argument error: Buffer cannot have non-zero length if the item size is zero"
1053        );
1054    }
1055}
arrow_array/array/fixed_size_binary_array.rs

arrow_array/array/
fixed_size_binary_array.rs
