arrow_array/array/
fixed_size_binary_array.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use crate::array::print_long_array;
19use crate::iterator::FixedSizeBinaryIter;
20use crate::{Array, ArrayAccessor, ArrayRef, FixedSizeListArray, Scalar};
21use arrow_buffer::buffer::NullBuffer;
22use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer, bit_util};
23use arrow_data::{ArrayData, ArrayDataBuilder};
24use arrow_schema::{ArrowError, DataType};
25use std::any::Any;
26use std::sync::Arc;
27
28/// An array of [fixed size binary arrays](https://arrow.apache.org/docs/format/Columnar.html#fixed-size-primitive-layout)
29///
30/// # Examples
31///
32/// Create an array from an iterable argument of byte slices.
33///
34/// ```
35///    use arrow_array::{Array, FixedSizeBinaryArray};
36///    let input_arg = vec![ vec![1, 2], vec![3, 4], vec![5, 6] ];
37///    let arr = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
38///
39///    assert_eq!(3, arr.len());
40///
41/// ```
42/// Create an array from an iterable argument of sparse byte slices.
43/// Sparsity means that the input argument can contain `None` items.
44/// ```
45///    use arrow_array::{Array, FixedSizeBinaryArray};
46///    let input_arg = vec![ None, Some(vec![7, 8]), Some(vec![9, 10]), None, Some(vec![13, 14]) ];
47///    let arr = FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 2).unwrap();
48///    assert_eq!(5, arr.len())
49///
50/// ```
51///
52#[derive(Clone)]
53pub struct FixedSizeBinaryArray {
54    data_type: DataType, // Must be DataType::FixedSizeBinary(value_length)
55    value_data: Buffer,
56    nulls: Option<NullBuffer>,
57    len: usize,
58    value_length: i32,
59}
60
61impl FixedSizeBinaryArray {
62    /// Create a new [`FixedSizeBinaryArray`] with `size` element size, panicking on failure
63    ///
64    /// # Panics
65    ///
66    /// Panics if [`Self::try_new`] returns an error
67    pub fn new(size: i32, values: Buffer, nulls: Option<NullBuffer>) -> Self {
68        Self::try_new(size, values, nulls).unwrap()
69    }
70
71    /// Create a new [`Scalar`] from `value`
72    pub fn new_scalar(value: impl AsRef<[u8]>) -> Scalar<Self> {
73        let v = value.as_ref();
74        Scalar::new(Self::new(v.len() as _, Buffer::from(v), None))
75    }
76
77    /// Create a new [`FixedSizeBinaryArray`] from the provided parts, returning an error on failure
78    ///
79    /// Creating an arrow with `size == 0` will try to get the length from the null buffer. If
80    /// no null buffer is provided, the resulting array will have length zero.
81    ///
82    /// # Errors
83    ///
84    /// * `size < 0`
85    /// * `values.len() / size != nulls.len()`
86    /// * `size == 0 && values.len() != 0`
87    pub fn try_new(
88        size: i32,
89        values: Buffer,
90        nulls: Option<NullBuffer>,
91    ) -> Result<Self, ArrowError> {
92        let data_type = DataType::FixedSizeBinary(size);
93        let s = size.to_usize().ok_or_else(|| {
94            ArrowError::InvalidArgumentError(format!("Size cannot be negative, got {size}"))
95        })?;
96
97        let len = if s == 0 {
98            if !values.is_empty() {
99                return Err(ArrowError::InvalidArgumentError(
100                    "Buffer cannot have non-zero length if the item size is zero".to_owned(),
101                ));
102            }
103
104            // If the item size is zero, try to determine the length from the null buffer
105            nulls.as_ref().map(|n| n.len()).unwrap_or(0)
106        } else {
107            values.len() / s
108        };
109        if let Some(n) = nulls.as_ref() {
110            if n.len() != len {
111                return Err(ArrowError::InvalidArgumentError(format!(
112                    "Incorrect length of null buffer for FixedSizeBinaryArray, expected {} got {}",
113                    len,
114                    n.len(),
115                )));
116            }
117        }
118
119        Ok(Self {
120            data_type,
121            value_data: values,
122            value_length: size,
123            nulls,
124            len,
125        })
126    }
127
128    /// Create a new [`FixedSizeBinaryArray`] of length `len` where all values are null
129    ///
130    /// # Panics
131    ///
132    /// Panics if
133    ///
134    /// * `size < 0`
135    /// * `size * len` would overflow `usize`
136    pub fn new_null(size: i32, len: usize) -> Self {
137        const BITS_IN_A_BYTE: usize = 8;
138        let capacity_in_bytes = size.to_usize().unwrap().checked_mul(len).unwrap();
139        Self {
140            data_type: DataType::FixedSizeBinary(size),
141            value_data: MutableBuffer::new_null(capacity_in_bytes * BITS_IN_A_BYTE).into(),
142            nulls: Some(NullBuffer::new_null(len)),
143            value_length: size,
144            len,
145        }
146    }
147
148    /// Deconstruct this array into its constituent parts
149    pub fn into_parts(self) -> (i32, Buffer, Option<NullBuffer>) {
150        (self.value_length, self.value_data, self.nulls)
151    }
152
153    /// Returns the element at index `i` as a byte slice.
154    ///
155    /// Note: This method does not check for nulls and the value is arbitrary
156    /// (but still well-defined) if [`is_null`](Self::is_null) returns true for the index.
157    ///
158    /// # Panics
159    /// Panics if index `i` is out of bounds.
160    pub fn value(&self, i: usize) -> &[u8] {
161        assert!(
162            i < self.len(),
163            "Trying to access an element at index {} from a FixedSizeBinaryArray of length {}",
164            i,
165            self.len()
166        );
167        let offset = i + self.offset();
168        unsafe {
169            let pos = self.value_offset_at(offset);
170            std::slice::from_raw_parts(
171                self.value_data.as_ptr().offset(pos as isize),
172                (self.value_offset_at(offset + 1) - pos) as usize,
173            )
174        }
175    }
176
177    /// Returns the element at index `i` as a byte slice.
178    ///
179    /// Note: This method does not check for nulls and the value is arbitrary
180    /// if [`is_null`](Self::is_null) returns true for the index.
181    ///
182    /// # Safety
183    ///
184    /// Caller is responsible for ensuring that the index is within the bounds
185    /// of the array
186    pub unsafe fn value_unchecked(&self, i: usize) -> &[u8] {
187        let offset = i + self.offset();
188        let pos = self.value_offset_at(offset);
189        unsafe {
190            std::slice::from_raw_parts(
191                self.value_data.as_ptr().offset(pos as isize),
192                (self.value_offset_at(offset + 1) - pos) as usize,
193            )
194        }
195    }
196
197    /// Returns the offset for the element at index `i`.
198    ///
199    /// Note this doesn't do any bound checking, for performance reason.
200    #[inline]
201    pub fn value_offset(&self, i: usize) -> i32 {
202        self.value_offset_at(self.offset() + i)
203    }
204
205    /// Returns the length for an element.
206    ///
207    /// All elements have the same length as the array is a fixed size.
208    #[inline]
209    pub fn value_length(&self) -> i32 {
210        self.value_length
211    }
212
213    /// Returns the values of this array.
214    ///
215    /// Unlike [`Self::value_data`] this returns the [`Buffer`]
216    /// allowing for zero-copy cloning.
217    #[inline]
218    pub fn values(&self) -> &Buffer {
219        &self.value_data
220    }
221
222    /// Returns the raw value data.
223    pub fn value_data(&self) -> &[u8] {
224        self.value_data.as_slice()
225    }
226
227    /// Returns a zero-copy slice of this array with the indicated offset and length.
228    pub fn slice(&self, offset: usize, len: usize) -> Self {
229        assert!(
230            offset.saturating_add(len) <= self.len,
231            "the length + offset of the sliced FixedSizeBinaryArray cannot exceed the existing length"
232        );
233
234        let size = self.value_length as usize;
235
236        Self {
237            data_type: self.data_type.clone(),
238            nulls: self.nulls.as_ref().map(|n| n.slice(offset, len)),
239            value_length: self.value_length,
240            value_data: self.value_data.slice_with_length(offset * size, len * size),
241            len,
242        }
243    }
244
245    /// Create an array from an iterable argument of sparse byte slices.
246    /// Sparsity means that items returned by the iterator are optional, i.e input argument can
247    /// contain `None` items.
248    ///
249    /// # Examples
250    ///
251    /// ```
252    /// use arrow_array::FixedSizeBinaryArray;
253    /// let input_arg = vec![
254    ///     None,
255    ///     Some(vec![7, 8]),
256    ///     Some(vec![9, 10]),
257    ///     None,
258    ///     Some(vec![13, 14]),
259    ///     None,
260    /// ];
261    /// let array = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.into_iter()).unwrap();
262    /// ```
263    ///
264    /// # Errors
265    ///
266    /// Returns error if argument has length zero, or sizes of nested slices don't match.
267    #[deprecated(
268        since = "28.0.0",
269        note = "This function will fail if the iterator produces only None values; prefer `try_from_sparse_iter_with_size`"
270    )]
271    pub fn try_from_sparse_iter<T, U>(mut iter: T) -> Result<Self, ArrowError>
272    where
273        T: Iterator<Item = Option<U>>,
274        U: AsRef<[u8]>,
275    {
276        let mut len = 0;
277        let mut size = None;
278        let mut byte = 0;
279
280        let iter_size_hint = iter.size_hint().0;
281        let mut null_buf = MutableBuffer::new(bit_util::ceil(iter_size_hint, 8));
282        let mut buffer = MutableBuffer::new(0);
283
284        let mut prepend = 0;
285        iter.try_for_each(|item| -> Result<(), ArrowError> {
286            // extend null bitmask by one byte per each 8 items
287            if byte == 0 {
288                null_buf.push(0u8);
289                byte = 8;
290            }
291            byte -= 1;
292
293            if let Some(slice) = item {
294                let slice = slice.as_ref();
295                if let Some(size) = size {
296                    if size != slice.len() {
297                        return Err(ArrowError::InvalidArgumentError(format!(
298                            "Nested array size mismatch: one is {}, and the other is {}",
299                            size,
300                            slice.len()
301                        )));
302                    }
303                } else {
304                    let len = slice.len();
305                    size = Some(len);
306                    // Now that we know how large each element is we can reserve
307                    // sufficient capacity in the underlying mutable buffer for
308                    // the data.
309                    buffer.reserve(iter_size_hint * len);
310                    buffer.extend_zeros(slice.len() * prepend);
311                }
312                bit_util::set_bit(null_buf.as_slice_mut(), len);
313                buffer.extend_from_slice(slice);
314            } else if let Some(size) = size {
315                buffer.extend_zeros(size);
316            } else {
317                prepend += 1;
318            }
319
320            len += 1;
321
322            Ok(())
323        })?;
324
325        if len == 0 {
326            return Err(ArrowError::InvalidArgumentError(
327                "Input iterable argument has no data".to_owned(),
328            ));
329        }
330
331        let nulls = NullBuffer::from_unsliced_buffer(null_buf, len);
332
333        let size = size.unwrap_or(0) as i32;
334        Ok(Self {
335            data_type: DataType::FixedSizeBinary(size),
336            value_data: buffer.into(),
337            nulls,
338            value_length: size,
339            len,
340        })
341    }
342
343    /// Create an array from an iterable argument of sparse byte slices.
344    /// Sparsity means that items returned by the iterator are optional, i.e input argument can
345    /// contain `None` items. In cases where the iterator returns only `None` values, this
346    /// also takes a size parameter to ensure that the a valid FixedSizeBinaryArray is still
347    /// created.
348    ///
349    /// # Examples
350    ///
351    /// ```
352    /// use arrow_array::FixedSizeBinaryArray;
353    /// let input_arg = vec![
354    ///     None,
355    ///     Some(vec![7, 8]),
356    ///     Some(vec![9, 10]),
357    ///     None,
358    ///     Some(vec![13, 14]),
359    ///     None,
360    /// ];
361    /// let array = FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 2).unwrap();
362    /// ```
363    ///
364    /// # Errors
365    ///
366    /// Returns error if argument has length zero, or sizes of nested slices don't match.
367    pub fn try_from_sparse_iter_with_size<T, U>(mut iter: T, size: i32) -> Result<Self, ArrowError>
368    where
369        T: Iterator<Item = Option<U>>,
370        U: AsRef<[u8]>,
371    {
372        let mut len = 0;
373        let mut byte = 0;
374
375        let iter_size_hint = iter.size_hint().0;
376        let mut null_buf = MutableBuffer::new(bit_util::ceil(iter_size_hint, 8));
377        let mut buffer = MutableBuffer::new(iter_size_hint * (size as usize));
378
379        iter.try_for_each(|item| -> Result<(), ArrowError> {
380            // extend null bitmask by one byte per each 8 items
381            if byte == 0 {
382                null_buf.push(0u8);
383                byte = 8;
384            }
385            byte -= 1;
386
387            if let Some(slice) = item {
388                let slice = slice.as_ref();
389                if size as usize != slice.len() {
390                    return Err(ArrowError::InvalidArgumentError(format!(
391                        "Nested array size mismatch: one is {}, and the other is {}",
392                        size,
393                        slice.len()
394                    )));
395                }
396
397                bit_util::set_bit(null_buf.as_slice_mut(), len);
398                buffer.extend_from_slice(slice);
399            } else {
400                buffer.extend_zeros(size as usize);
401            }
402
403            len += 1;
404
405            Ok(())
406        })?;
407
408        let nulls = NullBuffer::from_unsliced_buffer(null_buf, len);
409
410        Ok(Self {
411            data_type: DataType::FixedSizeBinary(size),
412            value_data: buffer.into(),
413            nulls,
414            len,
415            value_length: size,
416        })
417    }
418
419    /// Create an array from an iterable argument of byte slices.
420    ///
421    /// # Examples
422    ///
423    /// ```
424    /// use arrow_array::FixedSizeBinaryArray;
425    /// let input_arg = vec![
426    ///     vec![1, 2],
427    ///     vec![3, 4],
428    ///     vec![5, 6],
429    /// ];
430    /// let array = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
431    /// ```
432    ///
433    /// # Errors
434    ///
435    /// Returns error if argument has length zero, or sizes of nested slices don't match.
436    pub fn try_from_iter<T, U>(mut iter: T) -> Result<Self, ArrowError>
437    where
438        T: Iterator<Item = U>,
439        U: AsRef<[u8]>,
440    {
441        let mut len = 0;
442        let mut size = None;
443        let iter_size_hint = iter.size_hint().0;
444        let mut buffer = MutableBuffer::new(0);
445
446        iter.try_for_each(|item| -> Result<(), ArrowError> {
447            let slice = item.as_ref();
448            if let Some(size) = size {
449                if size != slice.len() {
450                    return Err(ArrowError::InvalidArgumentError(format!(
451                        "Nested array size mismatch: one is {}, and the other is {}",
452                        size,
453                        slice.len()
454                    )));
455                }
456            } else {
457                let len = slice.len();
458                size = Some(len);
459                buffer.reserve(iter_size_hint * len);
460            }
461
462            buffer.extend_from_slice(slice);
463
464            len += 1;
465
466            Ok(())
467        })?;
468
469        if len == 0 {
470            return Err(ArrowError::InvalidArgumentError(
471                "Input iterable argument has no data".to_owned(),
472            ));
473        }
474
475        let size = size.unwrap_or(0).try_into().unwrap();
476        Ok(Self {
477            data_type: DataType::FixedSizeBinary(size),
478            value_data: buffer.into(),
479            nulls: None,
480            value_length: size,
481            len,
482        })
483    }
484
485    #[inline]
486    fn value_offset_at(&self, i: usize) -> i32 {
487        self.value_length * i as i32
488    }
489
490    /// constructs a new iterator
491    pub fn iter(&self) -> FixedSizeBinaryIter<'_> {
492        FixedSizeBinaryIter::new(self)
493    }
494}
495
496impl From<ArrayData> for FixedSizeBinaryArray {
497    fn from(data: ArrayData) -> Self {
498        let (data_type, len, nulls, offset, buffers, _child_data) = data.into_parts();
499
500        assert_eq!(
501            buffers.len(),
502            1,
503            "FixedSizeBinaryArray data should contain 1 buffer only (values)"
504        );
505        let value_length = match data_type {
506            DataType::FixedSizeBinary(len) => len,
507            _ => panic!("Expected data type to be FixedSizeBinary"),
508        };
509
510        let size = value_length as usize;
511        let value_data = buffers[0].slice_with_length(offset * size, len * size);
512
513        Self {
514            data_type,
515            nulls,
516            len,
517            value_data,
518            value_length,
519        }
520    }
521}
522
523impl From<FixedSizeBinaryArray> for ArrayData {
524    fn from(array: FixedSizeBinaryArray) -> Self {
525        let builder = ArrayDataBuilder::new(array.data_type)
526            .len(array.len)
527            .buffers(vec![array.value_data])
528            .nulls(array.nulls);
529
530        unsafe { builder.build_unchecked() }
531    }
532}
533
534/// Creates a `FixedSizeBinaryArray` from `FixedSizeList<u8>` array
535impl From<FixedSizeListArray> for FixedSizeBinaryArray {
536    fn from(v: FixedSizeListArray) -> Self {
537        let value_len = v.value_length();
538        let v = v.into_data();
539        assert_eq!(
540            v.child_data().len(),
541            1,
542            "FixedSizeBinaryArray can only be created from list array of u8 values \
543             (i.e. FixedSizeList<PrimitiveArray<u8>>)."
544        );
545        let child_data = &v.child_data()[0];
546
547        assert_eq!(
548            child_data.child_data().len(),
549            0,
550            "FixedSizeBinaryArray can only be created from list array of u8 values \
551             (i.e. FixedSizeList<PrimitiveArray<u8>>)."
552        );
553        assert_eq!(
554            child_data.data_type(),
555            &DataType::UInt8,
556            "FixedSizeBinaryArray can only be created from FixedSizeList<u8> arrays, mismatched data types."
557        );
558        assert_eq!(
559            child_data.null_count(),
560            0,
561            "The child array cannot contain null values."
562        );
563
564        let builder = ArrayData::builder(DataType::FixedSizeBinary(value_len))
565            .len(v.len())
566            .offset(v.offset())
567            .add_buffer(child_data.buffers()[0].slice(child_data.offset()))
568            .nulls(v.nulls().cloned());
569
570        let data = unsafe { builder.build_unchecked() };
571        Self::from(data)
572    }
573}
574
575impl From<Vec<Option<&[u8]>>> for FixedSizeBinaryArray {
576    fn from(v: Vec<Option<&[u8]>>) -> Self {
577        #[allow(deprecated)]
578        Self::try_from_sparse_iter(v.into_iter()).unwrap()
579    }
580}
581
582impl From<Vec<&[u8]>> for FixedSizeBinaryArray {
583    fn from(v: Vec<&[u8]>) -> Self {
584        Self::try_from_iter(v.into_iter()).unwrap()
585    }
586}
587
588impl<const N: usize> From<Vec<&[u8; N]>> for FixedSizeBinaryArray {
589    fn from(v: Vec<&[u8; N]>) -> Self {
590        Self::try_from_iter(v.into_iter()).unwrap()
591    }
592}
593
594impl std::fmt::Debug for FixedSizeBinaryArray {
595    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
596        write!(f, "FixedSizeBinaryArray<{}>\n[\n", self.value_length())?;
597        print_long_array(self, f, |array, index, f| {
598            std::fmt::Debug::fmt(&array.value(index), f)
599        })?;
600        write!(f, "]")
601    }
602}
603
604/// SAFETY: Correctly implements the contract of Arrow Arrays
605unsafe impl Array for FixedSizeBinaryArray {
606    fn as_any(&self) -> &dyn Any {
607        self
608    }
609
610    fn to_data(&self) -> ArrayData {
611        self.clone().into()
612    }
613
614    fn into_data(self) -> ArrayData {
615        self.into()
616    }
617
618    fn data_type(&self) -> &DataType {
619        &self.data_type
620    }
621
622    fn slice(&self, offset: usize, length: usize) -> ArrayRef {
623        Arc::new(self.slice(offset, length))
624    }
625
626    fn len(&self) -> usize {
627        self.len
628    }
629
630    fn is_empty(&self) -> bool {
631        self.len == 0
632    }
633
634    fn shrink_to_fit(&mut self) {
635        self.value_data.shrink_to_fit();
636        if let Some(nulls) = &mut self.nulls {
637            nulls.shrink_to_fit();
638        }
639    }
640
641    fn offset(&self) -> usize {
642        0
643    }
644
645    fn nulls(&self) -> Option<&NullBuffer> {
646        self.nulls.as_ref()
647    }
648
649    fn logical_null_count(&self) -> usize {
650        // More efficient that the default implementation
651        self.null_count()
652    }
653
654    fn get_buffer_memory_size(&self) -> usize {
655        let mut sum = self.value_data.capacity();
656        if let Some(n) = &self.nulls {
657            sum += n.buffer().capacity();
658        }
659        sum
660    }
661
662    fn get_array_memory_size(&self) -> usize {
663        std::mem::size_of::<Self>() + self.get_buffer_memory_size()
664    }
665
666    #[cfg(feature = "pool")]
667    fn claim(&self, pool: &dyn arrow_buffer::MemoryPool) {
668        self.value_data.claim(pool);
669        if let Some(nulls) = &self.nulls {
670            nulls.claim(pool);
671        }
672    }
673}
674
675impl<'a> ArrayAccessor for &'a FixedSizeBinaryArray {
676    type Item = &'a [u8];
677
678    fn value(&self, index: usize) -> Self::Item {
679        FixedSizeBinaryArray::value(self, index)
680    }
681
682    unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
683        unsafe { FixedSizeBinaryArray::value_unchecked(self, index) }
684    }
685}
686
687impl<'a> IntoIterator for &'a FixedSizeBinaryArray {
688    type Item = Option<&'a [u8]>;
689    type IntoIter = FixedSizeBinaryIter<'a>;
690
691    fn into_iter(self) -> Self::IntoIter {
692        FixedSizeBinaryIter::<'a>::new(self)
693    }
694}
695
696#[cfg(test)]
697mod tests {
698    use super::*;
699    use crate::RecordBatch;
700    use arrow_schema::{Field, Schema};
701
702    #[test]
703    fn test_fixed_size_binary_array() {
704        let values: [u8; 15] = *b"hellotherearrow";
705
706        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
707            .len(3)
708            .add_buffer(Buffer::from(&values))
709            .build()
710            .unwrap();
711        let fixed_size_binary_array = FixedSizeBinaryArray::from(array_data);
712        assert_eq!(3, fixed_size_binary_array.len());
713        assert_eq!(0, fixed_size_binary_array.null_count());
714        assert_eq!(
715            [b'h', b'e', b'l', b'l', b'o'],
716            fixed_size_binary_array.value(0)
717        );
718        assert_eq!(
719            [b't', b'h', b'e', b'r', b'e'],
720            fixed_size_binary_array.value(1)
721        );
722        assert_eq!(
723            [b'a', b'r', b'r', b'o', b'w'],
724            fixed_size_binary_array.value(2)
725        );
726        assert_eq!(5, fixed_size_binary_array.value_length());
727        assert_eq!(10, fixed_size_binary_array.value_offset(2));
728        for i in 0..3 {
729            assert!(fixed_size_binary_array.is_valid(i));
730            assert!(!fixed_size_binary_array.is_null(i));
731        }
732
733        // Test binary array with offset
734        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
735            .len(2)
736            .offset(1)
737            .add_buffer(Buffer::from(&values))
738            .build()
739            .unwrap();
740        let fixed_size_binary_array = FixedSizeBinaryArray::from(array_data);
741        assert_eq!(
742            [b't', b'h', b'e', b'r', b'e'],
743            fixed_size_binary_array.value(0)
744        );
745        assert_eq!(
746            [b'a', b'r', b'r', b'o', b'w'],
747            fixed_size_binary_array.value(1)
748        );
749        assert_eq!(2, fixed_size_binary_array.len());
750        assert_eq!(0, fixed_size_binary_array.value_offset(0));
751        assert_eq!(5, fixed_size_binary_array.value_length());
752        assert_eq!(5, fixed_size_binary_array.value_offset(1));
753    }
754
755    #[test]
756    fn test_fixed_size_binary_array_from_fixed_size_list_array() {
757        let values = [0_u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13];
758        let values_data = ArrayData::builder(DataType::UInt8)
759            .len(12)
760            .offset(2)
761            .add_buffer(Buffer::from_slice_ref(values))
762            .build()
763            .unwrap();
764        // [null, [10, 11, 12, 13]]
765        let array_data = unsafe {
766            ArrayData::builder(DataType::FixedSizeList(
767                Arc::new(Field::new_list_field(DataType::UInt8, false)),
768                4,
769            ))
770            .len(2)
771            .offset(1)
772            .add_child_data(values_data)
773            .null_bit_buffer(Some(Buffer::from_slice_ref([0b101])))
774            .build_unchecked()
775        };
776        let list_array = FixedSizeListArray::from(array_data);
777        let binary_array = FixedSizeBinaryArray::from(list_array);
778
779        assert_eq!(2, binary_array.len());
780        assert_eq!(1, binary_array.null_count());
781        assert!(binary_array.is_null(0));
782        assert!(binary_array.is_valid(1));
783        assert_eq!(&[10, 11, 12, 13], binary_array.value(1));
784    }
785
786    #[test]
787    #[should_panic(
788        expected = "FixedSizeBinaryArray can only be created from FixedSizeList<u8> arrays"
789    )]
790    // Different error messages, so skip for now
791    // https://github.com/apache/arrow-rs/issues/1545
792    #[cfg(not(feature = "force_validate"))]
793    fn test_fixed_size_binary_array_from_incorrect_fixed_size_list_array() {
794        let values: [u32; 12] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
795        let values_data = ArrayData::builder(DataType::UInt32)
796            .len(12)
797            .add_buffer(Buffer::from_slice_ref(values))
798            .build()
799            .unwrap();
800
801        let array_data = unsafe {
802            ArrayData::builder(DataType::FixedSizeList(
803                Arc::new(Field::new_list_field(DataType::Binary, false)),
804                4,
805            ))
806            .len(3)
807            .add_child_data(values_data)
808            .build_unchecked()
809        };
810        let list_array = FixedSizeListArray::from(array_data);
811        drop(FixedSizeBinaryArray::from(list_array));
812    }
813
814    #[test]
815    #[should_panic(expected = "The child array cannot contain null values.")]
816    fn test_fixed_size_binary_array_from_fixed_size_list_array_with_child_nulls_failed() {
817        let values = [0_u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
818        let values_data = ArrayData::builder(DataType::UInt8)
819            .len(12)
820            .add_buffer(Buffer::from_slice_ref(values))
821            .null_bit_buffer(Some(Buffer::from_slice_ref([0b101010101010])))
822            .build()
823            .unwrap();
824
825        let array_data = unsafe {
826            ArrayData::builder(DataType::FixedSizeList(
827                Arc::new(Field::new_list_field(DataType::UInt8, false)),
828                4,
829            ))
830            .len(3)
831            .add_child_data(values_data)
832            .build_unchecked()
833        };
834        let list_array = FixedSizeListArray::from(array_data);
835        drop(FixedSizeBinaryArray::from(list_array));
836    }
837
838    #[test]
839    fn test_fixed_size_binary_array_fmt_debug() {
840        let values: [u8; 15] = *b"hellotherearrow";
841
842        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
843            .len(3)
844            .add_buffer(Buffer::from(&values))
845            .build()
846            .unwrap();
847        let arr = FixedSizeBinaryArray::from(array_data);
848        assert_eq!(
849            "FixedSizeBinaryArray<5>\n[\n  [104, 101, 108, 108, 111],\n  [116, 104, 101, 114, 101],\n  [97, 114, 114, 111, 119],\n]",
850            format!("{arr:?}")
851        );
852    }
853
854    #[test]
855    fn test_fixed_size_binary_array_from_iter() {
856        let input_arg = vec![vec![1, 2], vec![3, 4], vec![5, 6]];
857        let arr = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
858
859        assert_eq!(2, arr.value_length());
860        assert_eq!(3, arr.len())
861    }
862
863    #[test]
864    fn test_all_none_fixed_size_binary_array_from_sparse_iter() {
865        let none_option: Option<[u8; 32]> = None;
866        let input_arg = vec![none_option, none_option, none_option];
867        #[allow(deprecated)]
868        let arr = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.into_iter()).unwrap();
869        assert_eq!(0, arr.value_length());
870        assert_eq!(3, arr.len())
871    }
872
873    #[test]
874    fn test_fixed_size_binary_array_from_sparse_iter() {
875        let input_arg = vec![
876            None,
877            Some(vec![7, 8]),
878            Some(vec![9, 10]),
879            None,
880            Some(vec![13, 14]),
881        ];
882        #[allow(deprecated)]
883        let arr = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.iter().cloned()).unwrap();
884        assert_eq!(2, arr.value_length());
885        assert_eq!(5, arr.len());
886
887        let arr =
888            FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 2).unwrap();
889        assert_eq!(2, arr.value_length());
890        assert_eq!(5, arr.len());
891    }
892
893    #[test]
894    fn test_fixed_size_binary_array_from_sparse_iter_with_size_all_none() {
895        let input_arg = vec![None, None, None, None, None] as Vec<Option<Vec<u8>>>;
896
897        let arr = FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 16)
898            .unwrap();
899        assert_eq!(16, arr.value_length());
900        assert_eq!(5, arr.len())
901    }
902
903    #[test]
904    fn test_fixed_size_binary_array_from_vec() {
905        let values = vec!["one".as_bytes(), b"two", b"six", b"ten"];
906        let array = FixedSizeBinaryArray::from(values);
907        assert_eq!(array.len(), 4);
908        assert_eq!(array.null_count(), 0);
909        assert_eq!(array.logical_null_count(), 0);
910        assert_eq!(array.value(0), b"one");
911        assert_eq!(array.value(1), b"two");
912        assert_eq!(array.value(2), b"six");
913        assert_eq!(array.value(3), b"ten");
914        assert!(!array.is_null(0));
915        assert!(!array.is_null(1));
916        assert!(!array.is_null(2));
917        assert!(!array.is_null(3));
918    }
919
920    #[test]
921    #[should_panic(expected = "Nested array size mismatch: one is 3, and the other is 5")]
922    fn test_fixed_size_binary_array_from_vec_incorrect_length() {
923        let values = vec!["one".as_bytes(), b"two", b"three", b"four"];
924        let _ = FixedSizeBinaryArray::from(values);
925    }
926
927    #[test]
928    fn test_fixed_size_binary_array_from_opt_vec() {
929        let values = vec![
930            Some("one".as_bytes()),
931            Some(b"two"),
932            None,
933            Some(b"six"),
934            Some(b"ten"),
935        ];
936        let array = FixedSizeBinaryArray::from(values);
937        assert_eq!(array.len(), 5);
938        assert_eq!(array.value(0), b"one");
939        assert_eq!(array.value(1), b"two");
940        assert_eq!(array.value(3), b"six");
941        assert_eq!(array.value(4), b"ten");
942        assert!(!array.is_null(0));
943        assert!(!array.is_null(1));
944        assert!(array.is_null(2));
945        assert!(!array.is_null(3));
946        assert!(!array.is_null(4));
947    }
948
949    #[test]
950    #[should_panic(expected = "Nested array size mismatch: one is 3, and the other is 5")]
951    fn test_fixed_size_binary_array_from_opt_vec_incorrect_length() {
952        let values = vec![
953            Some("one".as_bytes()),
954            Some(b"two"),
955            None,
956            Some(b"three"),
957            Some(b"four"),
958        ];
959        let _ = FixedSizeBinaryArray::from(values);
960    }
961
962    #[test]
963    fn fixed_size_binary_array_all_null() {
964        let data = vec![None] as Vec<Option<String>>;
965        let array =
966            FixedSizeBinaryArray::try_from_sparse_iter_with_size(data.into_iter(), 0).unwrap();
967        array
968            .into_data()
969            .validate_full()
970            .expect("All null array has valid array data");
971    }
972
973    #[test]
974    // Test for https://github.com/apache/arrow-rs/issues/1390
975    fn fixed_size_binary_array_all_null_in_batch_with_schema() {
976        let schema = Schema::new(vec![Field::new("a", DataType::FixedSizeBinary(2), true)]);
977
978        let none_option: Option<[u8; 2]> = None;
979        let item = FixedSizeBinaryArray::try_from_sparse_iter_with_size(
980            vec![none_option, none_option, none_option].into_iter(),
981            2,
982        )
983        .unwrap();
984
985        // Should not panic
986        RecordBatch::try_new(Arc::new(schema), vec![Arc::new(item)]).unwrap();
987    }
988
989    #[test]
990    #[should_panic(
991        expected = "Trying to access an element at index 4 from a FixedSizeBinaryArray of length 3"
992    )]
993    fn test_fixed_size_binary_array_get_value_index_out_of_bound() {
994        let values = vec![Some("one".as_bytes()), Some(b"two"), None];
995        let array = FixedSizeBinaryArray::from(values);
996
997        array.value(4);
998    }
999
1000    #[test]
1001    fn test_constructors() {
1002        let buffer = Buffer::from_vec(vec![0_u8; 10]);
1003        let a = FixedSizeBinaryArray::new(2, buffer.clone(), None);
1004        assert_eq!(a.len(), 5);
1005
1006        let nulls = NullBuffer::new_null(5);
1007        FixedSizeBinaryArray::new(2, buffer.clone(), Some(nulls));
1008
1009        let null_array = FixedSizeBinaryArray::new_null(4, 3);
1010        assert_eq!(null_array.len(), 3);
1011        assert_eq!(null_array.values().len(), 12);
1012
1013        let a = FixedSizeBinaryArray::new(3, buffer.clone(), None);
1014        assert_eq!(a.len(), 3);
1015
1016        let nulls = NullBuffer::new_null(3);
1017        FixedSizeBinaryArray::new(3, buffer.clone(), Some(nulls));
1018
1019        let err = FixedSizeBinaryArray::try_new(-1, buffer.clone(), None).unwrap_err();
1020
1021        assert_eq!(
1022            err.to_string(),
1023            "Invalid argument error: Size cannot be negative, got -1"
1024        );
1025
1026        let nulls = NullBuffer::new_null(3);
1027        let err = FixedSizeBinaryArray::try_new(2, buffer.clone(), Some(nulls)).unwrap_err();
1028        assert_eq!(
1029            err.to_string(),
1030            "Invalid argument error: Incorrect length of null buffer for FixedSizeBinaryArray, expected 5 got 3"
1031        );
1032
1033        let zero_sized = FixedSizeBinaryArray::new(0, Buffer::default(), None);
1034        assert_eq!(zero_sized.len(), 0);
1035
1036        let nulls = NullBuffer::new_null(3);
1037        let zero_sized_with_nulls = FixedSizeBinaryArray::new(0, Buffer::default(), Some(nulls));
1038        assert_eq!(zero_sized_with_nulls.len(), 3);
1039
1040        let zero_sized_with_non_empty_buffer_err =
1041            FixedSizeBinaryArray::try_new(0, buffer, None).unwrap_err();
1042        assert_eq!(
1043            zero_sized_with_non_empty_buffer_err.to_string(),
1044            "Invalid argument error: Buffer cannot have non-zero length if the item size is zero"
1045        );
1046    }
1047}
arrow_array/array/fixed_size_binary_array.rs

arrow_array/array/
fixed_size_binary_array.rs
