arrow_array/builder/
generic_bytes_builder.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use crate::builder::ArrayBuilder;
19use crate::types::{ByteArrayType, GenericBinaryType, GenericStringType};
20use crate::{Array, ArrayRef, GenericByteArray, OffsetSizeTrait};
21use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer, NullBufferBuilder, ScalarBuffer};
22use arrow_data::ArrayDataBuilder;
23use arrow_schema::ArrowError;
24use std::any::Any;
25use std::sync::Arc;
26
27/// Builder for [`GenericByteArray`]
28///
29/// For building strings, see docs on [`GenericStringBuilder`].
30/// For building binary, see docs on [`GenericBinaryBuilder`].
31pub struct GenericByteBuilder<T: ByteArrayType> {
32    value_builder: Vec<u8>,
33    offsets_builder: Vec<T::Offset>,
34    null_buffer_builder: NullBufferBuilder,
35}
36
37impl<T: ByteArrayType> GenericByteBuilder<T> {
38    /// Creates a new [`GenericByteBuilder`].
39    pub fn new() -> Self {
40        Self::with_capacity(1024, 1024)
41    }
42
43    /// Creates a new [`GenericByteBuilder`].
44    ///
45    /// - `item_capacity` is the number of items to pre-allocate.
46    ///   The size of the preallocated buffer of offsets is the number of items plus one.
47    /// - `data_capacity` is the total number of bytes of data to pre-allocate
48    ///   (for all items, not per item).
49    pub fn with_capacity(item_capacity: usize, data_capacity: usize) -> Self {
50        let mut offsets_builder = Vec::with_capacity(item_capacity + 1);
51        offsets_builder.push(T::Offset::from_usize(0).unwrap());
52        Self {
53            value_builder: Vec::with_capacity(data_capacity),
54            offsets_builder,
55            null_buffer_builder: NullBufferBuilder::new(item_capacity),
56        }
57    }
58
59    /// Creates a new  [`GenericByteBuilder`] from buffers.
60    ///
61    /// # Safety
62    ///
63    /// This doesn't verify buffer contents as it assumes the buffers are from
64    /// existing and valid [`GenericByteArray`].
65    pub unsafe fn new_from_buffer(
66        offsets_buffer: MutableBuffer,
67        value_buffer: MutableBuffer,
68        null_buffer: Option<MutableBuffer>,
69    ) -> Self {
70        let offsets_builder: Vec<T::Offset> =
71            ScalarBuffer::<T::Offset>::from(offsets_buffer).into();
72        let value_builder: Vec<u8> = ScalarBuffer::<u8>::from(value_buffer).into();
73
74        let null_buffer_builder = null_buffer
75            .map(|buffer| NullBufferBuilder::new_from_buffer(buffer, offsets_builder.len() - 1))
76            .unwrap_or_else(|| NullBufferBuilder::new_with_len(offsets_builder.len() - 1));
77
78        Self {
79            offsets_builder,
80            value_builder,
81            null_buffer_builder,
82        }
83    }
84
85    #[inline]
86    fn next_offset(&self) -> T::Offset {
87        T::Offset::from_usize(self.value_builder.len()).expect("byte array offset overflow")
88    }
89
90    /// Appends a value into the builder.
91    ///
92    /// See the [GenericStringBuilder] documentation for examples of
93    /// incrementally building string values with multiple `write!` calls.
94    ///
95    /// # Panics
96    ///
97    /// Panics if the resulting length of [`Self::values_slice`] would exceed
98    /// `T::Offset::MAX` bytes.
99    ///
100    /// For example, this can happen with [`StringArray`] or [`BinaryArray`]
101    /// where the total length of all values exceeds 2GB
102    ///
103    /// [`StringArray`]: crate::StringArray
104    /// [`BinaryArray`]: crate::BinaryArray
105    #[inline]
106    pub fn append_value(&mut self, value: impl AsRef<T::Native>) {
107        self.value_builder
108            .extend_from_slice(value.as_ref().as_ref());
109        self.null_buffer_builder.append(true);
110        self.offsets_builder.push(self.next_offset());
111    }
112
113    /// Appends a value of type `T` into the builder `n` times.
114    ///
115    /// See [`Self::append_value`] for more panic information.
116    #[inline]
117    pub fn append_value_n(&mut self, value: impl AsRef<T::Native>, n: usize) {
118        let bytes: &[u8] = value.as_ref().as_ref();
119        self.value_builder.reserve(bytes.len() * n);
120        self.offsets_builder.reserve(n);
121        for _ in 0..n {
122            self.value_builder.extend_from_slice(bytes);
123            self.offsets_builder.push(self.next_offset());
124        }
125        self.null_buffer_builder.append_n_non_nulls(n);
126    }
127
128    /// Append an `Option` value into the builder.
129    ///
130    /// - A `None` value will append a null value.
131    /// - A `Some` value will append the value.
132    ///
133    /// See [`Self::append_value`] for more panic information.
134    #[inline]
135    pub fn append_option(&mut self, value: Option<impl AsRef<T::Native>>) {
136        match value {
137            None => self.append_null(),
138            Some(v) => self.append_value(v),
139        };
140    }
141
142    /// Append a null value into the builder.
143    #[inline]
144    pub fn append_null(&mut self) {
145        self.null_buffer_builder.append(false);
146        self.offsets_builder.push(self.next_offset());
147    }
148
149    /// Appends `n` `null`s into the builder.
150    #[inline]
151    pub fn append_nulls(&mut self, n: usize) {
152        self.null_buffer_builder.append_n_nulls(n);
153        let next_offset = self.next_offset();
154        self.offsets_builder
155            .extend(std::iter::repeat_n(next_offset, n));
156    }
157
158    /// Appends array values and null to this builder as is
159    /// (this means that underlying null values are copied as is).
160    #[inline]
161    pub fn append_array(&mut self, array: &GenericByteArray<T>) -> Result<(), ArrowError> {
162        use num_traits::CheckedAdd;
163        if array.len() == 0 {
164            return Ok(());
165        }
166
167        let offsets = array.offsets();
168
169        // If the offsets are contiguous, we can append them directly avoiding the need to align
170        // for example, when the first appended array is not sliced (starts at offset 0)
171        if self.next_offset() == offsets[0] {
172            self.offsets_builder.extend_from_slice(&offsets[1..]);
173        } else {
174            // Shifting all the offsets
175            let shift: T::Offset = self.next_offset() - offsets[0];
176
177            if shift.checked_add(&offsets[offsets.len() - 1]).is_none() {
178                return Err(ArrowError::OffsetOverflowError(
179                    shift.as_usize() + offsets[offsets.len() - 1].as_usize(),
180                ));
181            }
182
183            self.offsets_builder
184                .extend(offsets[1..].iter().map(|&offset| offset + shift));
185        }
186
187        // Append underlying values, starting from the first offset and ending at the last offset
188        self.value_builder.extend_from_slice(
189            &array.values().as_slice()[offsets[0].as_usize()..offsets[array.len()].as_usize()],
190        );
191
192        if let Some(null_buffer) = array.nulls() {
193            self.null_buffer_builder.append_buffer(null_buffer);
194        } else {
195            self.null_buffer_builder.append_n_non_nulls(array.len());
196        }
197        Ok(())
198    }
199
200    /// Builds the [`GenericByteArray`] and reset this builder.
201    pub fn finish(&mut self) -> GenericByteArray<T> {
202        let array_type = T::DATA_TYPE;
203        let array_builder = ArrayDataBuilder::new(array_type)
204            .len(self.len())
205            .add_buffer(std::mem::take(&mut self.offsets_builder).into())
206            .add_buffer(std::mem::take(&mut self.value_builder).into())
207            .nulls(self.null_buffer_builder.finish());
208
209        self.offsets_builder.push(self.next_offset());
210        let array_data = unsafe { array_builder.build_unchecked() };
211        GenericByteArray::from(array_data)
212    }
213
214    /// Builds the [`GenericByteArray`] without resetting the builder.
215    pub fn finish_cloned(&self) -> GenericByteArray<T> {
216        let array_type = T::DATA_TYPE;
217        let offset_buffer = Buffer::from_slice_ref(self.offsets_builder.as_slice());
218        let value_buffer = Buffer::from_slice_ref(self.value_builder.as_slice());
219        let array_builder = ArrayDataBuilder::new(array_type)
220            .len(self.len())
221            .add_buffer(offset_buffer)
222            .add_buffer(value_buffer)
223            .nulls(self.null_buffer_builder.finish_cloned());
224
225        let array_data = unsafe { array_builder.build_unchecked() };
226        GenericByteArray::from(array_data)
227    }
228
229    /// Returns the current values buffer as a slice
230    pub fn values_slice(&self) -> &[u8] {
231        self.value_builder.as_slice()
232    }
233
234    /// Returns the current values buffer capacity, in bytes.
235    pub fn values_capacity(&self) -> usize {
236        self.value_builder.capacity()
237    }
238
239    /// Returns the current offsets buffer as a slice
240    pub fn offsets_slice(&self) -> &[T::Offset] {
241        self.offsets_builder.as_slice()
242    }
243
244    /// Returns the current offsets buffer capacity, in offsets.
245    pub fn offsets_capacity(&self) -> usize {
246        self.offsets_builder.capacity()
247    }
248
249    /// Returns the current null buffer as a slice
250    pub fn validity_slice(&self) -> Option<&[u8]> {
251        self.null_buffer_builder.as_slice()
252    }
253
254    /// Returns the current null buffer allocated capacity, in bytes.
255    pub fn validity_capacity(&self) -> usize {
256        self.null_buffer_builder.allocated_size()
257    }
258
259    /// Returns the current null buffer as a mutable slice
260    pub fn validity_slice_mut(&mut self) -> Option<&mut [u8]> {
261        self.null_buffer_builder.as_slice_mut()
262    }
263}
264
265impl<T: ByteArrayType> std::fmt::Debug for GenericByteBuilder<T> {
266    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
267        write!(f, "{}{}Builder", T::Offset::PREFIX, T::PREFIX)?;
268        f.debug_struct("")
269            .field("value_builder", &self.value_builder)
270            .field("offsets_builder", &self.offsets_builder)
271            .field("null_buffer_builder", &self.null_buffer_builder)
272            .finish()
273    }
274}
275
276impl<T: ByteArrayType> Default for GenericByteBuilder<T> {
277    fn default() -> Self {
278        Self::new()
279    }
280}
281
282impl<T: ByteArrayType> ArrayBuilder for GenericByteBuilder<T> {
283    /// Returns the number of binary slots in the builder
284    fn len(&self) -> usize {
285        self.null_buffer_builder.len()
286    }
287
288    /// Builds the array and reset this builder.
289    fn finish(&mut self) -> ArrayRef {
290        Arc::new(self.finish())
291    }
292
293    /// Builds the array without resetting the builder.
294    fn finish_cloned(&self) -> ArrayRef {
295        Arc::new(self.finish_cloned())
296    }
297
298    /// Returns the builder as a non-mutable `Any` reference.
299    fn as_any(&self) -> &dyn Any {
300        self
301    }
302
303    /// Returns the builder as a mutable `Any` reference.
304    fn as_any_mut(&mut self) -> &mut dyn Any {
305        self
306    }
307
308    /// Returns the boxed builder as a box of `Any`.
309    fn into_box_any(self: Box<Self>) -> Box<dyn Any> {
310        self
311    }
312}
313
314impl<T: ByteArrayType, V: AsRef<T::Native>> Extend<Option<V>> for GenericByteBuilder<T> {
315    #[inline]
316    fn extend<I: IntoIterator<Item = Option<V>>>(&mut self, iter: I) {
317        for v in iter {
318            self.append_option(v)
319        }
320    }
321}
322
323/// Array builder for [`GenericStringArray`][crate::GenericStringArray]
324///
325/// Values can be appended using [`GenericByteBuilder::append_value`], and nulls with
326/// [`GenericByteBuilder::append_null`].
327///
328/// This builder also implements [`std::fmt::Write`] with any written data
329/// included in the next appended value. This allows using [`std::fmt::Display`]
330/// with standard Rust idioms like `write!` and `writeln!` to write data
331/// directly to the builder without intermediate allocations.
332///
333/// # Example writing strings with `append_value`
334/// ```
335/// # use arrow_array::builder::GenericStringBuilder;
336/// let mut builder = GenericStringBuilder::<i32>::new();
337///
338/// // Write one string value
339/// builder.append_value("foobarbaz");
340///
341/// // Write a second string
342/// builder.append_value("v2");
343///
344/// let array = builder.finish();
345/// assert_eq!(array.value(0), "foobarbaz");
346/// assert_eq!(array.value(1), "v2");
347/// ```
348///
349/// # Example incrementally writing strings with `std::fmt::Write`
350///
351/// ```
352/// # use std::fmt::Write;
353/// # use arrow_array::builder::GenericStringBuilder;
354/// let mut builder = GenericStringBuilder::<i32>::new();
355///
356/// // Write data in multiple `write!` calls
357/// write!(builder, "foo").unwrap();
358/// write!(builder, "bar").unwrap();
359/// // The next call to append_value finishes the current string
360/// // including all previously written strings.
361/// builder.append_value("baz");
362///
363/// // Write second value with a single write call
364/// write!(builder, "v2").unwrap();
365/// // finish the value by calling append_value with an empty string
366/// builder.append_value("");
367///
368/// let array = builder.finish();
369/// assert_eq!(array.value(0), "foobarbaz");
370/// assert_eq!(array.value(1), "v2");
371/// ```
372pub type GenericStringBuilder<O> = GenericByteBuilder<GenericStringType<O>>;
373
374impl<O: OffsetSizeTrait> std::fmt::Write for GenericStringBuilder<O> {
375    fn write_str(&mut self, s: &str) -> std::fmt::Result {
376        self.value_builder.extend_from_slice(s.as_bytes());
377        Ok(())
378    }
379}
380
381/// A byte size value representing the number of bytes to allocate per string in [`GenericStringBuilder`]
382///
383/// To create a [`GenericStringBuilder`] using `.with_capacity` we are required to provide: \
384/// - `item_capacity` - the row count \
385/// - `data_capacity` - total string byte count \
386///
387/// We will use the `AVERAGE_STRING_LENGTH` * row_count for `data_capacity`. \
388///
389/// These capacities are preallocation hints used to improve performance,
390/// but consequences of passing a hint too large or too small should be negligible.
391const AVERAGE_STRING_LENGTH: usize = 16;
392/// Trait for string-like array builders
393///
394/// This trait provides unified interface for builders that append string-like data
395/// such as [`GenericStringBuilder<O>`] and [`crate::builder::StringViewBuilder`]
396pub trait StringLikeArrayBuilder: ArrayBuilder {
397    /// Returns a human-readable type name for the builder.
398    fn type_name() -> &'static str;
399
400    /// Creates a new builder with the given row capacity.
401    fn with_capacity(capacity: usize) -> Self;
402
403    /// Appends a non-null string value to the builder.
404    fn append_value(&mut self, value: &str);
405
406    /// Appends a null value to the builder.
407    fn append_null(&mut self);
408}
409
410impl<O: OffsetSizeTrait> StringLikeArrayBuilder for GenericStringBuilder<O> {
411    fn type_name() -> &'static str {
412        std::any::type_name::<Self>()
413    }
414    fn with_capacity(capacity: usize) -> Self {
415        Self::with_capacity(capacity, capacity * AVERAGE_STRING_LENGTH)
416    }
417    fn append_value(&mut self, value: &str) {
418        Self::append_value(self, value);
419    }
420    fn append_null(&mut self) {
421        Self::append_null(self);
422    }
423}
424
425/// A byte size value representing the number of bytes to allocate per binary in [`GenericBinaryBuilder`]
426///
427/// To create a [`GenericBinaryBuilder`] using `.with_capacity` we are required to provide: \
428/// - `item_capacity` - the row count \
429/// - `data_capacity` - total binary byte count \
430///
431/// We will use the `AVERAGE_BINARY_LENGTH` * row_count for `data_capacity`. \
432///
433/// These capacities are preallocation hints used to improve performance,
434/// but consequences of passing a hint too large or too small should be negligible.
435const AVERAGE_BINARY_LENGTH: usize = 128;
436/// Trait for binary-like array builders
437///
438/// This trait provides unified interface for builders that append binary-like data
439/// such as [`GenericBinaryBuilder<O>`] and [`crate::builder::BinaryViewBuilder`]
440pub trait BinaryLikeArrayBuilder: ArrayBuilder {
441    /// Returns a human-readable type name for the builder.
442    fn type_name() -> &'static str;
443
444    /// Creates a new builder with the given row capacity.
445    fn with_capacity(capacity: usize) -> Self;
446
447    /// Appends a non-null string value to the builder.
448    fn append_value(&mut self, value: &[u8]);
449
450    /// Appends a null value to the builder.
451    fn append_null(&mut self);
452}
453
454impl<O: OffsetSizeTrait> BinaryLikeArrayBuilder for GenericBinaryBuilder<O> {
455    fn type_name() -> &'static str {
456        std::any::type_name::<Self>()
457    }
458    fn with_capacity(capacity: usize) -> Self {
459        Self::with_capacity(capacity, capacity * AVERAGE_BINARY_LENGTH)
460    }
461    fn append_value(&mut self, value: &[u8]) {
462        Self::append_value(self, value);
463    }
464    fn append_null(&mut self) {
465        Self::append_null(self);
466    }
467}
468
469///  Array builder for [`GenericBinaryArray`][crate::GenericBinaryArray]
470///
471/// Values can be appended using [`GenericByteBuilder::append_value`], and nulls with
472/// [`GenericByteBuilder::append_null`].
473///
474/// # Example
475/// ```
476/// # use arrow_array::builder::GenericBinaryBuilder;
477/// let mut builder = GenericBinaryBuilder::<i32>::new();
478///
479/// // Write data
480/// builder.append_value("foo");
481///
482/// // Write second value
483/// builder.append_value(&[0,1,2]);
484///
485/// let array = builder.finish();
486/// // binary values
487/// assert_eq!(array.value(0), b"foo");
488/// assert_eq!(array.value(1), b"\x00\x01\x02");
489/// ```
490///
491/// # Example incrementally writing bytes with `write_bytes`
492///
493/// ```
494/// # use std::io::Write;
495/// # use arrow_array::builder::GenericBinaryBuilder;
496/// let mut builder = GenericBinaryBuilder::<i32>::new();
497///
498/// // Write data in multiple `write_bytes` calls
499/// write!(builder, "foo").unwrap();
500/// write!(builder, "bar").unwrap();
501/// // The next call to append_value finishes the current string
502/// // including all previously written strings.
503/// builder.append_value("baz");
504///
505/// // Write second value with a single write call
506/// write!(builder, "v2").unwrap();
507/// // finish the value by calling append_value with an empty string
508/// builder.append_value("");
509///
510/// let array = builder.finish();
511/// assert_eq!(array.value(0), "foobarbaz".as_bytes());
512/// assert_eq!(array.value(1), "v2".as_bytes());
513/// ```
514pub type GenericBinaryBuilder<O> = GenericByteBuilder<GenericBinaryType<O>>;
515
516impl<O: OffsetSizeTrait> std::io::Write for GenericBinaryBuilder<O> {
517    fn write(&mut self, bs: &[u8]) -> std::io::Result<usize> {
518        self.value_builder.extend_from_slice(bs);
519        Ok(bs.len())
520    }
521
522    fn flush(&mut self) -> std::io::Result<()> {
523        Ok(())
524    }
525}
526
527#[cfg(test)]
528mod tests {
529    use super::*;
530    use crate::GenericStringArray;
531    use crate::array::Array;
532    use arrow_buffer::NullBuffer;
533    use std::fmt::Write as _;
534    use std::io::Write as _;
535
536    fn _test_generic_binary_builder<O: OffsetSizeTrait>() {
537        let mut builder = GenericBinaryBuilder::<O>::new();
538
539        builder.append_value(b"hello");
540        builder.append_value(b"");
541        builder.append_null();
542        builder.append_value(b"rust");
543
544        let array = builder.finish();
545
546        assert_eq!(4, array.len());
547        assert_eq!(1, array.null_count());
548        assert_eq!(b"hello", array.value(0));
549        assert_eq!([] as [u8; 0], array.value(1));
550        assert!(array.is_null(2));
551        assert_eq!(b"rust", array.value(3));
552        assert_eq!(O::from_usize(5).unwrap(), array.value_offsets()[2]);
553        assert_eq!(O::from_usize(4).unwrap(), array.value_length(3));
554    }
555
556    #[test]
557    fn test_binary_builder() {
558        _test_generic_binary_builder::<i32>()
559    }
560
561    #[test]
562    fn test_large_binary_builder() {
563        _test_generic_binary_builder::<i64>()
564    }
565
566    fn _test_generic_binary_builder_all_nulls<O: OffsetSizeTrait>() {
567        let mut builder = GenericBinaryBuilder::<O>::new();
568        builder.append_null();
569        builder.append_null();
570        builder.append_null();
571        builder.append_nulls(2);
572        assert_eq!(5, builder.len());
573        assert!(!builder.is_empty());
574
575        let array = builder.finish();
576        assert_eq!(5, array.null_count());
577        assert_eq!(5, array.len());
578        assert!(array.is_null(0));
579        assert!(array.is_null(1));
580        assert!(array.is_null(2));
581        assert!(array.is_null(3));
582        assert!(array.is_null(4));
583    }
584
585    #[test]
586    fn test_binary_builder_all_nulls() {
587        _test_generic_binary_builder_all_nulls::<i32>()
588    }
589
590    #[test]
591    fn test_large_binary_builder_all_nulls() {
592        _test_generic_binary_builder_all_nulls::<i64>()
593    }
594
595    fn _test_generic_binary_builder_reset<O: OffsetSizeTrait>() {
596        let mut builder = GenericBinaryBuilder::<O>::new();
597
598        builder.append_value(b"hello");
599        builder.append_value(b"");
600        builder.append_null();
601        builder.append_value(b"rust");
602        builder.finish();
603
604        assert!(builder.is_empty());
605
606        builder.append_value(b"parquet");
607        builder.append_null();
608        builder.append_value(b"arrow");
609        builder.append_value(b"");
610        builder.append_nulls(2);
611        builder.append_value(b"hi");
612        let array = builder.finish();
613
614        assert_eq!(7, array.len());
615        assert_eq!(3, array.null_count());
616        assert_eq!(b"parquet", array.value(0));
617        assert!(array.is_null(1));
618        assert!(array.is_null(4));
619        assert!(array.is_null(5));
620        assert_eq!(b"arrow", array.value(2));
621        assert_eq!(b"", array.value(1));
622        assert_eq!(b"hi", array.value(6));
623
624        assert_eq!(O::zero(), array.value_offsets()[0]);
625        assert_eq!(O::from_usize(7).unwrap(), array.value_offsets()[2]);
626        assert_eq!(O::from_usize(14).unwrap(), array.value_offsets()[7]);
627        assert_eq!(O::from_usize(5).unwrap(), array.value_length(2));
628    }
629
630    #[test]
631    fn test_binary_builder_reset() {
632        _test_generic_binary_builder_reset::<i32>()
633    }
634
635    #[test]
636    fn test_large_binary_builder_reset() {
637        _test_generic_binary_builder_reset::<i64>()
638    }
639
640    fn _test_generic_string_array_builder<O: OffsetSizeTrait>() {
641        let mut builder = GenericStringBuilder::<O>::new();
642        let owned = "arrow".to_owned();
643
644        builder.append_value("hello");
645        builder.append_value("");
646        builder.append_value(&owned);
647        builder.append_null();
648        builder.append_option(Some("rust"));
649        builder.append_option(None::<&str>);
650        builder.append_option(None::<String>);
651        builder.append_nulls(2);
652        builder.append_value("parquet");
653        assert_eq!(10, builder.len());
654
655        assert_eq!(
656            GenericStringArray::<O>::from(vec![
657                Some("hello"),
658                Some(""),
659                Some("arrow"),
660                None,
661                Some("rust"),
662                None,
663                None,
664                None,
665                None,
666                Some("parquet")
667            ]),
668            builder.finish()
669        );
670    }
671
672    #[test]
673    fn test_string_array_builder() {
674        _test_generic_string_array_builder::<i32>()
675    }
676
677    #[test]
678    fn test_large_string_array_builder() {
679        _test_generic_string_array_builder::<i64>()
680    }
681
682    fn _test_generic_string_array_builder_finish<O: OffsetSizeTrait>() {
683        let mut builder = GenericStringBuilder::<O>::with_capacity(3, 11);
684
685        builder.append_value("hello");
686        builder.append_value("rust");
687        builder.append_null();
688
689        builder.finish();
690        assert!(builder.is_empty());
691        assert_eq!(&[O::zero()], builder.offsets_slice());
692
693        builder.append_value("arrow");
694        builder.append_value("parquet");
695        let arr = builder.finish();
696        // array should not have null buffer because there is not `null` value.
697        assert!(arr.nulls().is_none());
698        assert_eq!(GenericStringArray::<O>::from(vec!["arrow", "parquet"]), arr,)
699    }
700
701    #[test]
702    fn test_string_array_builder_finish() {
703        _test_generic_string_array_builder_finish::<i32>()
704    }
705
706    #[test]
707    fn test_large_string_array_builder_finish() {
708        _test_generic_string_array_builder_finish::<i64>()
709    }
710
711    fn _test_generic_string_array_builder_finish_cloned<O: OffsetSizeTrait>() {
712        let mut builder = GenericStringBuilder::<O>::with_capacity(3, 11);
713
714        builder.append_value("hello");
715        builder.append_value("rust");
716        builder.append_null();
717
718        let mut arr = builder.finish_cloned();
719        assert!(!builder.is_empty());
720        assert_eq!(3, arr.len());
721
722        builder.append_value("arrow");
723        builder.append_value("parquet");
724        arr = builder.finish();
725
726        assert!(arr.nulls().is_some());
727        assert_eq!(&[O::zero()], builder.offsets_slice());
728        assert_eq!(5, arr.len());
729    }
730
731    #[test]
732    fn test_string_array_builder_finish_cloned() {
733        _test_generic_string_array_builder_finish_cloned::<i32>()
734    }
735
736    #[test]
737    fn test_large_string_array_builder_finish_cloned() {
738        _test_generic_string_array_builder_finish_cloned::<i64>()
739    }
740
741    #[test]
742    fn test_extend() {
743        let mut builder = GenericStringBuilder::<i32>::new();
744        builder.extend(["a", "b", "c", "", "a", "b", "c"].into_iter().map(Some));
745        builder.extend(["d", "cupcakes", "hello"].into_iter().map(Some));
746        let array = builder.finish();
747        assert_eq!(array.value_offsets(), &[0, 1, 2, 3, 3, 4, 5, 6, 7, 15, 20]);
748        assert_eq!(array.value_data(), b"abcabcdcupcakeshello");
749    }
750
751    #[test]
752    fn test_write_str() {
753        let mut builder = GenericStringBuilder::<i32>::new();
754        write!(builder, "foo").unwrap();
755        builder.append_value("");
756        writeln!(builder, "bar").unwrap();
757        builder.append_value("");
758        write!(builder, "fiz").unwrap();
759        write!(builder, "buz").unwrap();
760        builder.append_value("");
761        let a = builder.finish();
762        let r: Vec<_> = a.iter().flatten().collect();
763        assert_eq!(r, &["foo", "bar\n", "fizbuz"])
764    }
765
766    #[test]
767    fn test_write_bytes() {
768        let mut builder = GenericBinaryBuilder::<i32>::new();
769        write!(builder, "foo").unwrap();
770        builder.append_value("");
771        writeln!(builder, "bar").unwrap();
772        builder.append_value("");
773        write!(builder, "fiz").unwrap();
774        write!(builder, "buz").unwrap();
775        builder.append_value("");
776        let a = builder.finish();
777        let r: Vec<_> = a.iter().flatten().collect();
778        assert_eq!(
779            r,
780            &["foo".as_bytes(), "bar\n".as_bytes(), "fizbuz".as_bytes()]
781        )
782    }
783
784    #[test]
785    fn test_append_array_without_nulls() {
786        let input = vec![
787            "hello", "world", "how", "are", "you", "doing", "today", "I", "am", "doing", "well",
788            "thank", "you", "for", "asking",
789        ];
790        let arr1 = GenericStringArray::<i32>::from(input[..3].to_vec());
791        let arr2 = GenericStringArray::<i32>::from(input[3..7].to_vec());
792        let arr3 = GenericStringArray::<i32>::from(input[7..].to_vec());
793
794        let mut builder = GenericStringBuilder::<i32>::new();
795        builder.append_array(&arr1).unwrap();
796        builder.append_array(&arr2).unwrap();
797        builder.append_array(&arr3).unwrap();
798
799        let actual = builder.finish();
800        let expected = GenericStringArray::<i32>::from(input);
801
802        assert_eq!(actual, expected);
803    }
804
805    #[test]
806    fn test_append_array_with_nulls() {
807        let input = vec![
808            Some("hello"),
809            None,
810            Some("how"),
811            None,
812            None,
813            None,
814            None,
815            Some("I"),
816            Some("am"),
817            Some("doing"),
818            Some("well"),
819        ];
820        let arr1 = GenericStringArray::<i32>::from(input[..3].to_vec());
821        let arr2 = GenericStringArray::<i32>::from(input[3..7].to_vec());
822        let arr3 = GenericStringArray::<i32>::from(input[7..].to_vec());
823
824        let mut builder = GenericStringBuilder::<i32>::new();
825        builder.append_array(&arr1).unwrap();
826        builder.append_array(&arr2).unwrap();
827        builder.append_array(&arr3).unwrap();
828
829        let actual = builder.finish();
830        let expected = GenericStringArray::<i32>::from(input);
831
832        assert_eq!(actual, expected);
833    }
834
835    #[test]
836    fn test_append_empty_array() {
837        let arr = GenericStringArray::<i32>::from(Vec::<&str>::new());
838        let mut builder = GenericStringBuilder::<i32>::new();
839        builder.append_array(&arr).unwrap();
840        let result = builder.finish();
841        assert_eq!(result.len(), 0);
842    }
843
844    #[test]
845    fn test_append_array_with_offset_not_starting_at_0() {
846        let input = vec![
847            Some("hello"),
848            None,
849            Some("how"),
850            None,
851            None,
852            None,
853            None,
854            Some("I"),
855            Some("am"),
856            Some("doing"),
857            Some("well"),
858        ];
859        let full_array = GenericStringArray::<i32>::from(input);
860        let sliced = full_array.slice(1, 4);
861
862        assert_ne!(sliced.offsets()[0].as_usize(), 0);
863        assert_ne!(sliced.offsets().last(), full_array.offsets().last());
864
865        let mut builder = GenericStringBuilder::<i32>::new();
866        builder.append_array(&sliced).unwrap();
867        let actual = builder.finish();
868
869        let expected = GenericStringArray::<i32>::from(vec![None, Some("how"), None, None]);
870
871        assert_eq!(actual, expected);
872    }
873
874    #[test]
875    fn test_append_underlying_null_values_added_as_is() {
876        let input_1_array_with_nulls = {
877            let input = vec![
878                "hello", "world", "how", "are", "you", "doing", "today", "I", "am",
879            ];
880            let (offsets, buffer, _) = GenericStringArray::<i32>::from(input).into_parts();
881
882            GenericStringArray::<i32>::new(
883                offsets,
884                buffer,
885                Some(NullBuffer::from(&[
886                    true, false, true, false, false, true, true, true, false,
887                ])),
888            )
889        };
890        let input_2_array_with_nulls = {
891            let input = vec!["doing", "well", "thank", "you", "for", "asking"];
892            let (offsets, buffer, _) = GenericStringArray::<i32>::from(input).into_parts();
893
894            GenericStringArray::<i32>::new(
895                offsets,
896                buffer,
897                Some(NullBuffer::from(&[false, false, true, false, true, true])),
898            )
899        };
900
901        let mut builder = GenericStringBuilder::<i32>::new();
902        builder.append_array(&input_1_array_with_nulls).unwrap();
903        builder.append_array(&input_2_array_with_nulls).unwrap();
904
905        let actual = builder.finish();
906        let expected = GenericStringArray::<i32>::from(vec![
907            Some("hello"),
908            None, // world
909            Some("how"),
910            None, // are
911            None, // you
912            Some("doing"),
913            Some("today"),
914            Some("I"),
915            None, // am
916            None, // doing
917            None, // well
918            Some("thank"),
919            None, // "you",
920            Some("for"),
921            Some("asking"),
922        ]);
923
924        assert_eq!(actual, expected);
925
926        let expected_underlying_buffer = Buffer::from(
927            [
928                "hello", "world", "how", "are", "you", "doing", "today", "I", "am", "doing",
929                "well", "thank", "you", "for", "asking",
930            ]
931            .join("")
932            .as_bytes(),
933        );
934        assert_eq!(actual.values(), &expected_underlying_buffer);
935    }
936
937    #[test]
938    fn append_array_with_continues_indices() {
939        let input = vec![
940            "hello", "world", "how", "are", "you", "doing", "today", "I", "am", "doing", "well",
941            "thank", "you", "for", "asking",
942        ];
943        let full_array = GenericStringArray::<i32>::from(input);
944        let slice1 = full_array.slice(0, 3);
945        let slice2 = full_array.slice(3, 4);
946        let slice3 = full_array.slice(7, full_array.len() - 7);
947
948        let mut builder = GenericStringBuilder::<i32>::new();
949        builder.append_array(&slice1).unwrap();
950        builder.append_array(&slice2).unwrap();
951        builder.append_array(&slice3).unwrap();
952
953        let actual = builder.finish();
954
955        assert_eq!(actual, full_array);
956    }
957
958    #[test]
959    fn test_append_array_offset_overflow_precise() {
960        let mut builder = GenericStringBuilder::<i32>::new();
961
962        let initial_string = "x".repeat(i32::MAX as usize - 100);
963        builder.append_value(&initial_string);
964
965        let overflow_string = "y".repeat(200);
966        let overflow_array = GenericStringArray::<i32>::from(vec![overflow_string.as_str()]);
967
968        let result = builder.append_array(&overflow_array);
969
970        assert!(matches!(result, Err(ArrowError::OffsetOverflowError(_))));
971    }
972
973    #[test]
974    fn test_append_value_n() {
975        let mut builder = GenericStringBuilder::<i32>::new();
976        builder.append_value("hello");
977        builder.append_value_n("world", 3);
978        builder.append_null();
979        let array = builder.finish();
980
981        assert_eq!(5, array.len());
982        assert_eq!(1, array.null_count());
983        assert_eq!("hello", array.value(0));
984        assert_eq!("world", array.value(1));
985        assert_eq!("world", array.value(2));
986        assert_eq!("world", array.value(3));
987        assert!(array.is_null(4));
988    }
989}
arrow_array/builder/generic_bytes_builder.rs

arrow_array/builder/
generic_bytes_builder.rs
