Skip to main content

arrow_array/builder/
generic_bytes_builder.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use crate::builder::ArrayBuilder;
6use crate::types::{ByteArrayType, GenericBinaryType, GenericStringType};
7use crate::{Array, ArrayRef, GenericByteArray, OffsetSizeTrait};
8use arrow_buffer::{
9    ArrowNativeType, Buffer, MutableBuffer, NullBufferBuilder, OffsetBuffer, ScalarBuffer,
10};
11use arrow_schema::ArrowError;
12use std::any::Any;
13use std::sync::Arc;
14
15/// Builder for [`GenericByteArray`]
16///
17/// For building strings, see docs on [`GenericStringBuilder`].
18/// For building binary, see docs on [`GenericBinaryBuilder`].
19pub struct GenericByteBuilder<T: ByteArrayType> {
20    value_builder: Vec<u8>,
21    offsets_builder: Vec<T::Offset>,
22    null_buffer_builder: NullBufferBuilder,
23}
24
25impl<T: ByteArrayType> GenericByteBuilder<T> {
26    /// Creates a new [`GenericByteBuilder`].
27    pub fn new() -> Self {
28        Self::with_capacity(1024, 1024)
29    }
30
31    /// Creates a new [`GenericByteBuilder`].
32    ///
33    /// - `item_capacity` is the number of items to pre-allocate.
34    ///   The size of the preallocated buffer of offsets is the number of items plus one.
35    /// - `data_capacity` is the total number of bytes of data to pre-allocate
36    ///   (for all items, not per item).
37    pub fn with_capacity(item_capacity: usize, data_capacity: usize) -> Self {
38        let mut offsets_builder = Vec::with_capacity(item_capacity + 1);
39        offsets_builder.push(T::Offset::from_usize(0).unwrap());
40        Self {
41            value_builder: Vec::with_capacity(data_capacity),
42            offsets_builder,
43            null_buffer_builder: NullBufferBuilder::new(item_capacity),
44        }
45    }
46
47    /// Creates a new  [`GenericByteBuilder`] from buffers.
48    ///
49    /// Any bytes in `value_buffer` after the last offset are dropped. The first
50    /// bit of `null_buffer` is for the first row.
51    ///
52    /// # Safety
53    ///
54    /// This doesn't verify buffer contents as it assumes the buffers are from
55    /// existing and valid [`GenericByteArray`].
56    pub unsafe fn new_from_buffer(
57        offsets_buffer: MutableBuffer,
58        mut value_buffer: MutableBuffer,
59        null_buffer: Option<MutableBuffer>,
60    ) -> Self {
61        let offsets_builder: Vec<T::Offset> =
62            ScalarBuffer::<T::Offset>::from(offsets_buffer).into();
63
64        // New values are appended at the end of the values buffer, so it must end
65        // at the last offset
66        value_buffer.truncate(offsets_builder.last().unwrap().as_usize());
67        let value_builder: Vec<u8> = ScalarBuffer::<u8>::from(value_buffer).into();
68
69        let null_buffer_builder = null_buffer
70            .map(|buffer| NullBufferBuilder::new_from_buffer(buffer, offsets_builder.len() - 1))
71            .unwrap_or_else(|| NullBufferBuilder::new_with_len(offsets_builder.len() - 1));
72
73        Self {
74            value_builder,
75            offsets_builder,
76            null_buffer_builder,
77        }
78    }
79
80    #[inline]
81    fn next_offset(&self) -> T::Offset {
82        T::Offset::from_usize(self.value_builder.len()).expect("byte array offset overflow")
83    }
84
85    /// Appends a value into the builder.
86    ///
87    /// See the [GenericStringBuilder] documentation for examples of
88    /// incrementally building string values with multiple `write!` calls.
89    ///
90    /// # Panics
91    ///
92    /// Panics if the resulting length of [`Self::values_slice`] would exceed
93    /// `T::Offset::MAX` bytes.
94    ///
95    /// For example, this can happen with [`StringArray`] or [`BinaryArray`]
96    /// where the total length of all values exceeds 2GB
97    ///
98    /// [`StringArray`]: crate::StringArray
99    /// [`BinaryArray`]: crate::BinaryArray
100    #[inline]
101    pub fn append_value(&mut self, value: impl AsRef<T::Native>) {
102        self.value_builder
103            .extend_from_slice(value.as_ref().as_ref());
104        self.null_buffer_builder.append(true);
105        self.offsets_builder.push(self.next_offset());
106    }
107
108    /// Appends a value of type `T` into the builder `n` times.
109    ///
110    /// See [`Self::append_value`] for more panic information.
111    ///
112    /// # Panics
113    ///
114    /// Panics for the same reasons as [`Self::append_value`]
115    #[inline]
116    pub fn append_value_n(&mut self, value: impl AsRef<T::Native>, n: usize) {
117        let bytes: &[u8] = value.as_ref().as_ref();
118        self.value_builder.reserve(bytes.len() * n);
119        self.offsets_builder.reserve(n);
120        for _ in 0..n {
121            self.value_builder.extend_from_slice(bytes);
122            self.offsets_builder.push(self.next_offset());
123        }
124        self.null_buffer_builder.append_n_non_nulls(n);
125    }
126
127    /// Append an `Option` value into the builder.
128    ///
129    /// - A `None` value will append a null value.
130    /// - A `Some` value will append the value.
131    ///
132    /// See [`Self::append_value`] for more panic information.
133    ///
134    /// # Panics
135    ///
136    /// Panics for the same reasons as [`Self::append_value`]
137    #[inline]
138    pub fn append_option(&mut self, value: Option<impl AsRef<T::Native>>) {
139        match value {
140            None => self.append_null(),
141            Some(v) => self.append_value(v),
142        }
143    }
144
145    /// Append a null value into the builder.
146    #[inline]
147    pub fn append_null(&mut self) {
148        self.null_buffer_builder.append(false);
149        self.offsets_builder.push(self.next_offset());
150    }
151
152    /// Appends `n` `null`s into the builder.
153    #[inline]
154    pub fn append_nulls(&mut self, n: usize) {
155        self.null_buffer_builder.append_n_nulls(n);
156        let next_offset = self.next_offset();
157        self.offsets_builder
158            .extend(std::iter::repeat_n(next_offset, n));
159    }
160
161    /// Appends array values and null to this builder as is
162    /// (this means that underlying null values are copied as is).
163    #[inline]
164    pub fn append_array(&mut self, array: &GenericByteArray<T>) -> Result<(), ArrowError> {
165        use num_traits::CheckedAdd;
166        if array.len() == 0 {
167            return Ok(());
168        }
169
170        let offsets = array.offsets();
171
172        // If the offsets are contiguous, we can append them directly avoiding the need to align
173        // for example, when the first appended array is not sliced (starts at offset 0)
174        if self.next_offset() == offsets[0] {
175            self.offsets_builder.extend_from_slice(&offsets[1..]);
176        } else {
177            // Shifting all the offsets
178            let shift: T::Offset = self.next_offset() - offsets[0];
179
180            if shift.checked_add(&offsets[offsets.len() - 1]).is_none() {
181                return Err(ArrowError::OffsetOverflowError(
182                    shift.as_usize() + offsets[offsets.len() - 1].as_usize(),
183                ));
184            }
185
186            self.offsets_builder
187                .extend(offsets[1..].iter().map(|&offset| offset + shift));
188        }
189
190        // Append underlying values, starting from the first offset and ending at the last offset
191        self.value_builder.extend_from_slice(
192            &array.values().as_slice()[offsets[0].as_usize()..offsets[array.len()].as_usize()],
193        );
194
195        if let Some(null_buffer) = array.nulls() {
196            self.null_buffer_builder.append_buffer(null_buffer);
197        } else {
198            self.null_buffer_builder.append_n_non_nulls(array.len());
199        }
200        Ok(())
201    }
202
203    /// Builds the [`GenericByteArray`] and reset this builder.
204    pub fn finish(&mut self) -> GenericByteArray<T> {
205        debug_assert!(self.offsets_builder.is_sorted());
206        // SAFETY: offsets start at 0 and every append pushes the current end of
207        // `value_builder`, so they are non-empty and monotonically increasing.
208        let offsets = unsafe {
209            OffsetBuffer::new_unchecked(std::mem::take(&mut self.offsets_builder).into())
210        };
211        let values = std::mem::take(&mut self.value_builder).into();
212        let nulls = self.null_buffer_builder.finish();
213
214        self.offsets_builder.push(T::Offset::usize_as(0));
215
216        // SAFETY: `try_new` would not error. No append pushes an offset past
217        // the end of `value_builder`, which only grows, so the last offset is
218        // at most `values.len()` (`write!` can leave bytes after it). `nulls`
219        // has one entry per value. For strings, every append, `write!`
220        // included, copies whole UTF-8 sequences, so each offset pair bounds
221        // valid UTF-8.
222        unsafe { GenericByteArray::new_unchecked(offsets, values, nulls) }
223    }
224
225    /// Builds the [`GenericByteArray`] without resetting the builder.
226    pub fn finish_cloned(&self) -> GenericByteArray<T> {
227        debug_assert!(self.offsets_builder.is_sorted());
228        // SAFETY: offsets start at 0 and every append pushes the current end of
229        // `value_builder`, so they are non-empty and monotonically increasing.
230        let offsets = unsafe {
231            OffsetBuffer::new_unchecked(
232                Buffer::from_slice_ref(self.offsets_builder.as_slice()).into(),
233            )
234        };
235        let values = Buffer::from_slice_ref(self.value_builder.as_slice());
236        let nulls = self.null_buffer_builder.finish_cloned();
237
238        // SAFETY: `try_new` would not error. No append pushes an offset past
239        // the end of `value_builder`, which only grows, so the last offset is
240        // at most `values.len()` (`write!` can leave bytes after it). `nulls`
241        // has one entry per value. For strings, every append, `write!`
242        // included, copies whole UTF-8 sequences, so each offset pair bounds
243        // valid UTF-8.
244        unsafe { GenericByteArray::new_unchecked(offsets, values, nulls) }
245    }
246
247    /// Returns the current values buffer as a slice
248    pub fn values_slice(&self) -> &[u8] {
249        self.value_builder.as_slice()
250    }
251
252    /// Returns the current values buffer capacity, in bytes.
253    pub fn values_capacity(&self) -> usize {
254        self.value_builder.capacity()
255    }
256
257    /// Returns the current offsets buffer as a slice
258    pub fn offsets_slice(&self) -> &[T::Offset] {
259        self.offsets_builder.as_slice()
260    }
261
262    /// Returns the current offsets buffer capacity, in offsets.
263    pub fn offsets_capacity(&self) -> usize {
264        self.offsets_builder.capacity()
265    }
266
267    /// Returns the current null buffer as a slice
268    pub fn validity_slice(&self) -> Option<&[u8]> {
269        self.null_buffer_builder.as_slice()
270    }
271
272    /// Returns the current null buffer allocated capacity, in bytes.
273    pub fn validity_capacity(&self) -> usize {
274        self.null_buffer_builder.allocated_size()
275    }
276
277    /// Returns the current null buffer as a mutable slice
278    pub fn validity_slice_mut(&mut self) -> Option<&mut [u8]> {
279        self.null_buffer_builder.as_slice_mut()
280    }
281}
282
283impl<T: ByteArrayType> std::fmt::Debug for GenericByteBuilder<T> {
284    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
285        write!(f, "{}{}Builder", T::Offset::PREFIX, T::PREFIX)?;
286        f.debug_struct("")
287            .field("value_builder", &self.value_builder)
288            .field("offsets_builder", &self.offsets_builder)
289            .field("null_buffer_builder", &self.null_buffer_builder)
290            .finish()
291    }
292}
293
294impl<T: ByteArrayType> Default for GenericByteBuilder<T> {
295    fn default() -> Self {
296        Self::new()
297    }
298}
299
300impl<T: ByteArrayType> ArrayBuilder for GenericByteBuilder<T> {
301    /// Returns the number of binary slots in the builder
302    fn len(&self) -> usize {
303        self.null_buffer_builder.len()
304    }
305
306    /// Builds the array and reset this builder.
307    fn finish(&mut self) -> ArrayRef {
308        Arc::new(self.finish())
309    }
310
311    /// Builds the array without resetting the builder.
312    fn finish_cloned(&self) -> ArrayRef {
313        Arc::new(self.finish_cloned())
314    }
315
316    /// Returns the builder as a non-mutable `Any` reference.
317    fn as_any(&self) -> &dyn Any {
318        self
319    }
320
321    /// Returns the builder as a mutable `Any` reference.
322    fn as_any_mut(&mut self) -> &mut dyn Any {
323        self
324    }
325
326    /// Returns the boxed builder as a box of `Any`.
327    fn into_box_any(self: Box<Self>) -> Box<dyn Any> {
328        self
329    }
330}
331
332impl<T: ByteArrayType, V: AsRef<T::Native>> Extend<Option<V>> for GenericByteBuilder<T> {
333    #[inline]
334    fn extend<I: IntoIterator<Item = Option<V>>>(&mut self, iter: I) {
335        for v in iter {
336            self.append_option(v)
337        }
338    }
339}
340
341/// Array builder for [`GenericStringArray`][crate::GenericStringArray]
342///
343/// Values can be appended using [`GenericByteBuilder::append_value`], and nulls with
344/// [`GenericByteBuilder::append_null`].
345///
346/// This builder also implements [`std::fmt::Write`] with any written data
347/// included in the next appended value. This allows using [`std::fmt::Display`]
348/// with standard Rust idioms like `write!` and `writeln!` to write data
349/// directly to the builder without intermediate allocations.
350///
351/// # Example writing strings with `append_value`
352/// ```
353/// # use arrow_array::builder::GenericStringBuilder;
354/// let mut builder = GenericStringBuilder::<i32>::new();
355///
356/// // Write one string value
357/// builder.append_value("foobarbaz");
358///
359/// // Write a second string
360/// builder.append_value("v2");
361///
362/// let array = builder.finish();
363/// assert_eq!(array.value(0), "foobarbaz");
364/// assert_eq!(array.value(1), "v2");
365/// ```
366///
367/// # Example incrementally writing strings with `std::fmt::Write`
368///
369/// ```
370/// # use std::fmt::Write;
371/// # use arrow_array::builder::GenericStringBuilder;
372/// let mut builder = GenericStringBuilder::<i32>::new();
373///
374/// // Write data in multiple `write!` calls
375/// write!(builder, "foo").unwrap();
376/// write!(builder, "bar").unwrap();
377/// // The next call to append_value finishes the current string
378/// // including all previously written strings.
379/// builder.append_value("baz");
380///
381/// // Write second value with a single write call
382/// write!(builder, "v2").unwrap();
383/// // finish the value by calling append_value with an empty string
384/// builder.append_value("");
385///
386/// let array = builder.finish();
387/// assert_eq!(array.value(0), "foobarbaz");
388/// assert_eq!(array.value(1), "v2");
389/// ```
390pub type GenericStringBuilder<O> = GenericByteBuilder<GenericStringType<O>>;
391
392impl<O: OffsetSizeTrait> std::fmt::Write for GenericStringBuilder<O> {
393    fn write_str(&mut self, s: &str) -> std::fmt::Result {
394        self.value_builder.extend_from_slice(s.as_bytes());
395        Ok(())
396    }
397}
398
399/// A byte size value representing the number of bytes to allocate per string in [`GenericStringBuilder`]
400///
401/// To create a [`GenericStringBuilder`] using `.with_capacity` we are required to provide: \
402/// - `item_capacity` - the row count \
403/// - `data_capacity` - total string byte count \
404///
405/// We will use the `AVERAGE_STRING_LENGTH` * row_count for `data_capacity`. \
406///
407/// These capacities are preallocation hints used to improve performance,
408/// but consequences of passing a hint too large or too small should be negligible.
409const AVERAGE_STRING_LENGTH: usize = 16;
410/// Trait for string-like array builders
411///
412/// This trait provides unified interface for builders that append string-like data
413/// such as [`GenericStringBuilder<O>`] and [`crate::builder::StringViewBuilder`]
414pub trait StringLikeArrayBuilder: ArrayBuilder {
415    /// Returns a human-readable type name for the builder.
416    fn type_name() -> &'static str;
417
418    /// Creates a new builder with the given row capacity.
419    fn with_capacity(capacity: usize) -> Self;
420
421    /// Appends a non-null string value to the builder.
422    fn append_value(&mut self, value: &str);
423
424    /// Appends a null value to the builder.
425    fn append_null(&mut self);
426}
427
428impl<O: OffsetSizeTrait> StringLikeArrayBuilder for GenericStringBuilder<O> {
429    fn type_name() -> &'static str {
430        std::any::type_name::<Self>()
431    }
432    fn with_capacity(capacity: usize) -> Self {
433        Self::with_capacity(capacity, capacity * AVERAGE_STRING_LENGTH)
434    }
435    fn append_value(&mut self, value: &str) {
436        Self::append_value(self, value);
437    }
438    fn append_null(&mut self) {
439        Self::append_null(self);
440    }
441}
442
443/// A byte size value representing the number of bytes to allocate per binary in [`GenericBinaryBuilder`]
444///
445/// To create a [`GenericBinaryBuilder`] using `.with_capacity` we are required to provide: \
446/// - `item_capacity` - the row count \
447/// - `data_capacity` - total binary byte count \
448///
449/// We will use the `AVERAGE_BINARY_LENGTH` * row_count for `data_capacity`. \
450///
451/// These capacities are preallocation hints used to improve performance,
452/// but consequences of passing a hint too large or too small should be negligible.
453const AVERAGE_BINARY_LENGTH: usize = 128;
454/// Trait for binary-like array builders
455///
456/// This trait provides unified interface for builders that append binary-like data
457/// such as [`GenericBinaryBuilder<O>`] and [`crate::builder::BinaryViewBuilder`]
458pub trait BinaryLikeArrayBuilder: ArrayBuilder {
459    /// Returns a human-readable type name for the builder.
460    fn type_name() -> &'static str;
461
462    /// Creates a new builder with the given row capacity.
463    fn with_capacity(capacity: usize) -> Self;
464
465    /// Appends a non-null string value to the builder.
466    fn append_value(&mut self, value: &[u8]);
467
468    /// Appends a null value to the builder.
469    fn append_null(&mut self);
470}
471
472impl<O: OffsetSizeTrait> BinaryLikeArrayBuilder for GenericBinaryBuilder<O> {
473    fn type_name() -> &'static str {
474        std::any::type_name::<Self>()
475    }
476    fn with_capacity(capacity: usize) -> Self {
477        Self::with_capacity(capacity, capacity * AVERAGE_BINARY_LENGTH)
478    }
479    fn append_value(&mut self, value: &[u8]) {
480        Self::append_value(self, value);
481    }
482    fn append_null(&mut self) {
483        Self::append_null(self);
484    }
485}
486
487///  Array builder for [`GenericBinaryArray`][crate::GenericBinaryArray]
488///
489/// Values can be appended using [`GenericByteBuilder::append_value`], and nulls with
490/// [`GenericByteBuilder::append_null`].
491///
492/// # Example
493/// ```
494/// # use arrow_array::builder::GenericBinaryBuilder;
495/// let mut builder = GenericBinaryBuilder::<i32>::new();
496///
497/// // Write data
498/// builder.append_value("foo");
499///
500/// // Write second value
501/// builder.append_value(&[0,1,2]);
502///
503/// let array = builder.finish();
504/// // binary values
505/// assert_eq!(array.value(0), b"foo");
506/// assert_eq!(array.value(1), b"\x00\x01\x02");
507/// ```
508///
509/// # Example incrementally writing bytes with `write_bytes`
510///
511/// ```
512/// # use std::io::Write;
513/// # use arrow_array::builder::GenericBinaryBuilder;
514/// let mut builder = GenericBinaryBuilder::<i32>::new();
515///
516/// // Write data in multiple `write_bytes` calls
517/// write!(builder, "foo").unwrap();
518/// write!(builder, "bar").unwrap();
519/// // The next call to append_value finishes the current string
520/// // including all previously written strings.
521/// builder.append_value("baz");
522///
523/// // Write second value with a single write call
524/// write!(builder, "v2").unwrap();
525/// // finish the value by calling append_value with an empty string
526/// builder.append_value("");
527///
528/// let array = builder.finish();
529/// assert_eq!(array.value(0), "foobarbaz".as_bytes());
530/// assert_eq!(array.value(1), "v2".as_bytes());
531/// ```
532pub type GenericBinaryBuilder<O> = GenericByteBuilder<GenericBinaryType<O>>;
533
534impl<O: OffsetSizeTrait> std::io::Write for GenericBinaryBuilder<O> {
535    fn write(&mut self, bs: &[u8]) -> std::io::Result<usize> {
536        self.value_builder.extend_from_slice(bs);
537        Ok(bs.len())
538    }
539
540    fn flush(&mut self) -> std::io::Result<()> {
541        Ok(())
542    }
543}
544
545#[cfg(test)]
546mod tests {
547    use super::*;
548    use crate::GenericStringArray;
549    use crate::array::Array;
550    use arrow_buffer::NullBuffer;
551    use std::fmt::Write as _;
552    use std::io::Write as _;
553
554    #[test]
555    fn test_finish_materialized_without_nulls() {
556        let mut builder = GenericStringBuilder::<i32>::new();
557        builder.append_nulls(0);
558        builder.append_value("a");
559        assert!(builder.finish_cloned().nulls().is_none());
560        assert!(builder.finish().nulls().is_none());
561    }
562
563    fn _test_generic_binary_builder<O: OffsetSizeTrait>() {
564        let mut builder = GenericBinaryBuilder::<O>::new();
565
566        builder.append_value(b"hello");
567        builder.append_value(b"");
568        builder.append_null();
569        builder.append_value(b"rust");
570
571        let array = builder.finish();
572
573        assert_eq!(4, array.len());
574        assert_eq!(1, array.null_count());
575        assert_eq!(b"hello", array.value(0));
576        assert_eq!([] as [u8; 0], array.value(1));
577        assert!(array.is_null(2));
578        assert_eq!(b"rust", array.value(3));
579        assert_eq!(O::from_usize(5).unwrap(), array.value_offsets()[2]);
580        assert_eq!(O::from_usize(4).unwrap(), array.value_length(3));
581    }
582
583    #[test]
584    fn test_binary_builder() {
585        _test_generic_binary_builder::<i32>()
586    }
587
588    #[test]
589    fn test_large_binary_builder() {
590        _test_generic_binary_builder::<i64>()
591    }
592
593    fn _test_generic_binary_builder_all_nulls<O: OffsetSizeTrait>() {
594        let mut builder = GenericBinaryBuilder::<O>::new();
595        builder.append_null();
596        builder.append_null();
597        builder.append_null();
598        builder.append_nulls(2);
599        assert_eq!(5, builder.len());
600        assert!(!builder.is_empty());
601
602        let array = builder.finish();
603        assert_eq!(5, array.null_count());
604        assert_eq!(5, array.len());
605        assert!(array.is_null(0));
606        assert!(array.is_null(1));
607        assert!(array.is_null(2));
608        assert!(array.is_null(3));
609        assert!(array.is_null(4));
610    }
611
612    #[test]
613    fn test_binary_builder_all_nulls() {
614        _test_generic_binary_builder_all_nulls::<i32>()
615    }
616
617    #[test]
618    fn test_large_binary_builder_all_nulls() {
619        _test_generic_binary_builder_all_nulls::<i64>()
620    }
621
622    fn _test_generic_binary_builder_reset<O: OffsetSizeTrait>() {
623        let mut builder = GenericBinaryBuilder::<O>::new();
624
625        builder.append_value(b"hello");
626        builder.append_value(b"");
627        builder.append_null();
628        builder.append_value(b"rust");
629        builder.finish();
630
631        assert!(builder.is_empty());
632
633        builder.append_value(b"parquet");
634        builder.append_null();
635        builder.append_value(b"arrow");
636        builder.append_value(b"");
637        builder.append_nulls(2);
638        builder.append_value(b"hi");
639        let array = builder.finish();
640
641        assert_eq!(7, array.len());
642        assert_eq!(3, array.null_count());
643        assert_eq!(b"parquet", array.value(0));
644        assert!(array.is_null(1));
645        assert!(array.is_null(4));
646        assert!(array.is_null(5));
647        assert_eq!(b"arrow", array.value(2));
648        assert_eq!(b"", array.value(1));
649        assert_eq!(b"hi", array.value(6));
650
651        assert_eq!(O::zero(), array.value_offsets()[0]);
652        assert_eq!(O::from_usize(7).unwrap(), array.value_offsets()[2]);
653        assert_eq!(O::from_usize(14).unwrap(), array.value_offsets()[7]);
654        assert_eq!(O::from_usize(5).unwrap(), array.value_length(2));
655    }
656
657    #[test]
658    fn test_binary_builder_reset() {
659        _test_generic_binary_builder_reset::<i32>()
660    }
661
662    #[test]
663    fn test_large_binary_builder_reset() {
664        _test_generic_binary_builder_reset::<i64>()
665    }
666
667    fn _test_generic_string_array_builder<O: OffsetSizeTrait>() {
668        let mut builder = GenericStringBuilder::<O>::new();
669        let owned = "arrow".to_owned();
670
671        builder.append_value("hello");
672        builder.append_value("");
673        builder.append_value(&owned);
674        builder.append_null();
675        builder.append_option(Some("rust"));
676        builder.append_option(None::<&str>);
677        builder.append_option(None::<String>);
678        builder.append_nulls(2);
679        builder.append_value("parquet");
680        assert_eq!(10, builder.len());
681
682        assert_eq!(
683            GenericStringArray::<O>::from(vec![
684                Some("hello"),
685                Some(""),
686                Some("arrow"),
687                None,
688                Some("rust"),
689                None,
690                None,
691                None,
692                None,
693                Some("parquet")
694            ]),
695            builder.finish()
696        );
697    }
698
699    #[test]
700    fn test_string_array_builder() {
701        _test_generic_string_array_builder::<i32>()
702    }
703
704    #[test]
705    fn test_large_string_array_builder() {
706        _test_generic_string_array_builder::<i64>()
707    }
708
709    fn _test_generic_string_array_builder_finish<O: OffsetSizeTrait>() {
710        let mut builder = GenericStringBuilder::<O>::with_capacity(3, 11);
711
712        builder.append_value("hello");
713        builder.append_value("rust");
714        builder.append_null();
715
716        builder.finish();
717        assert!(builder.is_empty());
718        assert_eq!(&[O::zero()], builder.offsets_slice());
719
720        builder.append_value("arrow");
721        builder.append_value("parquet");
722        let arr = builder.finish();
723        // array should not have null buffer because there is not `null` value.
724        assert!(arr.nulls().is_none());
725        assert_eq!(GenericStringArray::<O>::from(vec!["arrow", "parquet"]), arr,)
726    }
727
728    #[test]
729    fn test_string_array_builder_finish() {
730        _test_generic_string_array_builder_finish::<i32>()
731    }
732
733    #[test]
734    fn test_large_string_array_builder_finish() {
735        _test_generic_string_array_builder_finish::<i64>()
736    }
737
738    fn _test_generic_string_array_builder_finish_cloned<O: OffsetSizeTrait>() {
739        let mut builder = GenericStringBuilder::<O>::with_capacity(3, 11);
740
741        builder.append_value("hello");
742        builder.append_value("rust");
743        builder.append_null();
744
745        let mut arr = builder.finish_cloned();
746        assert!(!builder.is_empty());
747        assert_eq!(3, arr.len());
748
749        builder.append_value("arrow");
750        builder.append_value("parquet");
751        arr = builder.finish();
752
753        assert!(arr.nulls().is_some());
754        assert_eq!(&[O::zero()], builder.offsets_slice());
755        assert_eq!(5, arr.len());
756    }
757
758    #[test]
759    fn test_string_array_builder_finish_cloned() {
760        _test_generic_string_array_builder_finish_cloned::<i32>()
761    }
762
763    #[test]
764    fn test_large_string_array_builder_finish_cloned() {
765        _test_generic_string_array_builder_finish_cloned::<i64>()
766    }
767
768    #[test]
769    fn test_extend() {
770        let mut builder = GenericStringBuilder::<i32>::new();
771        builder.extend(["a", "b", "c", "", "a", "b", "c"].into_iter().map(Some));
772        builder.extend(["d", "cupcakes", "hello"].into_iter().map(Some));
773        let array = builder.finish();
774        assert_eq!(array.value_offsets(), &[0, 1, 2, 3, 3, 4, 5, 6, 7, 15, 20]);
775        assert_eq!(array.value_data(), b"abcabcdcupcakeshello");
776    }
777
778    #[test]
779    fn test_write_str() {
780        let mut builder = GenericStringBuilder::<i32>::new();
781        write!(builder, "foo").unwrap();
782        builder.append_value("");
783        writeln!(builder, "bar").unwrap();
784        builder.append_value("");
785        write!(builder, "fiz").unwrap();
786        write!(builder, "buz").unwrap();
787        builder.append_value("");
788        let a = builder.finish();
789        let r: Vec<_> = a.iter().flatten().collect();
790        assert_eq!(r, &["foo", "bar\n", "fizbuz"])
791    }
792
793    #[test]
794    fn test_write_bytes() {
795        let mut builder = GenericBinaryBuilder::<i32>::new();
796        write!(builder, "foo").unwrap();
797        builder.append_value("");
798        writeln!(builder, "bar").unwrap();
799        builder.append_value("");
800        write!(builder, "fiz").unwrap();
801        write!(builder, "buz").unwrap();
802        builder.append_value("");
803        let a = builder.finish();
804        let r: Vec<_> = a.iter().flatten().collect();
805        assert_eq!(
806            r,
807            &[b"foo".as_slice(), b"bar\n".as_slice(), b"fizbuz".as_slice()]
808        )
809    }
810
811    #[test]
812    fn test_append_array_without_nulls() {
813        let input = vec![
814            "hello", "world", "how", "are", "you", "doing", "today", "I", "am", "doing", "well",
815            "thank", "you", "for", "asking",
816        ];
817        let arr1 = GenericStringArray::<i32>::from(input[..3].to_vec());
818        let arr2 = GenericStringArray::<i32>::from(input[3..7].to_vec());
819        let arr3 = GenericStringArray::<i32>::from(input[7..].to_vec());
820
821        let mut builder = GenericStringBuilder::<i32>::new();
822        builder.append_array(&arr1).unwrap();
823        builder.append_array(&arr2).unwrap();
824        builder.append_array(&arr3).unwrap();
825
826        let actual = builder.finish();
827        let expected = GenericStringArray::<i32>::from(input);
828
829        assert_eq!(actual, expected);
830    }
831
832    #[test]
833    fn test_append_array_with_nulls() {
834        let input = vec![
835            Some("hello"),
836            None,
837            Some("how"),
838            None,
839            None,
840            None,
841            None,
842            Some("I"),
843            Some("am"),
844            Some("doing"),
845            Some("well"),
846        ];
847        let arr1 = GenericStringArray::<i32>::from(input[..3].to_vec());
848        let arr2 = GenericStringArray::<i32>::from(input[3..7].to_vec());
849        let arr3 = GenericStringArray::<i32>::from(input[7..].to_vec());
850
851        let mut builder = GenericStringBuilder::<i32>::new();
852        builder.append_array(&arr1).unwrap();
853        builder.append_array(&arr2).unwrap();
854        builder.append_array(&arr3).unwrap();
855
856        let actual = builder.finish();
857        let expected = GenericStringArray::<i32>::from(input);
858
859        assert_eq!(actual, expected);
860    }
861
862    #[test]
863    fn test_append_empty_array() {
864        let arr = GenericStringArray::<i32>::from(Vec::<&str>::new());
865        let mut builder = GenericStringBuilder::<i32>::new();
866        builder.append_array(&arr).unwrap();
867        let result = builder.finish();
868        assert_eq!(result.len(), 0);
869    }
870
871    #[test]
872    fn test_append_array_with_offset_not_starting_at_0() {
873        let input = vec![
874            Some("hello"),
875            None,
876            Some("how"),
877            None,
878            None,
879            None,
880            None,
881            Some("I"),
882            Some("am"),
883            Some("doing"),
884            Some("well"),
885        ];
886        let full_array = GenericStringArray::<i32>::from(input);
887        let sliced = full_array.slice(1, 4);
888
889        assert_ne!(sliced.offsets()[0].as_usize(), 0);
890        assert_ne!(sliced.offsets().last(), full_array.offsets().last());
891
892        let mut builder = GenericStringBuilder::<i32>::new();
893        builder.append_array(&sliced).unwrap();
894        let actual = builder.finish();
895
896        let expected = GenericStringArray::<i32>::from(vec![None, Some("how"), None, None]);
897
898        assert_eq!(actual, expected);
899    }
900
901    #[test]
902    fn test_append_underlying_null_values_added_as_is() {
903        let input_1_array_with_nulls = {
904            let input = vec![
905                "hello", "world", "how", "are", "you", "doing", "today", "I", "am",
906            ];
907            let (offsets, buffer, _) = GenericStringArray::<i32>::from(input).into_parts();
908
909            GenericStringArray::<i32>::new(
910                offsets,
911                buffer,
912                Some(NullBuffer::from(&[
913                    true, false, true, false, false, true, true, true, false,
914                ])),
915            )
916        };
917        let input_2_array_with_nulls = {
918            let input = vec!["doing", "well", "thank", "you", "for", "asking"];
919            let (offsets, buffer, _) = GenericStringArray::<i32>::from(input).into_parts();
920
921            GenericStringArray::<i32>::new(
922                offsets,
923                buffer,
924                Some(NullBuffer::from(&[false, false, true, false, true, true])),
925            )
926        };
927
928        let mut builder = GenericStringBuilder::<i32>::new();
929        builder.append_array(&input_1_array_with_nulls).unwrap();
930        builder.append_array(&input_2_array_with_nulls).unwrap();
931
932        let actual = builder.finish();
933        let expected = GenericStringArray::<i32>::from(vec![
934            Some("hello"),
935            None, // world
936            Some("how"),
937            None, // are
938            None, // you
939            Some("doing"),
940            Some("today"),
941            Some("I"),
942            None, // am
943            None, // doing
944            None, // well
945            Some("thank"),
946            None, // "you",
947            Some("for"),
948            Some("asking"),
949        ]);
950
951        assert_eq!(actual, expected);
952
953        let expected_underlying_buffer = Buffer::from(
954            [
955                "hello", "world", "how", "are", "you", "doing", "today", "I", "am", "doing",
956                "well", "thank", "you", "for", "asking",
957            ]
958            .join("")
959            .as_bytes(),
960        );
961        assert_eq!(actual.values(), &expected_underlying_buffer);
962    }
963
964    #[test]
965    fn append_array_with_continues_indices() {
966        let input = vec![
967            "hello", "world", "how", "are", "you", "doing", "today", "I", "am", "doing", "well",
968            "thank", "you", "for", "asking",
969        ];
970        let full_array = GenericStringArray::<i32>::from(input);
971        let slice1 = full_array.slice(0, 3);
972        let slice2 = full_array.slice(3, 4);
973        let slice3 = full_array.slice(7, full_array.len() - 7);
974
975        let mut builder = GenericStringBuilder::<i32>::new();
976        builder.append_array(&slice1).unwrap();
977        builder.append_array(&slice2).unwrap();
978        builder.append_array(&slice3).unwrap();
979
980        let actual = builder.finish();
981
982        assert_eq!(actual, full_array);
983    }
984
985    #[test]
986    fn test_append_array_offset_overflow_precise() {
987        let mut builder = GenericStringBuilder::<i32>::new();
988
989        let initial_string = "x".repeat(i32::MAX as usize - 100);
990        builder.append_value(&initial_string);
991
992        let overflow_string = "y".repeat(200);
993        let overflow_array = GenericStringArray::<i32>::from(vec![overflow_string.as_str()]);
994
995        let result = builder.append_array(&overflow_array);
996
997        assert!(matches!(result, Err(ArrowError::OffsetOverflowError(_))));
998    }
999
1000    #[test]
1001    fn test_append_value_n() {
1002        let mut builder = GenericStringBuilder::<i32>::new();
1003        builder.append_value("hello");
1004        builder.append_value_n("world", 3);
1005        builder.append_null();
1006        let array = builder.finish();
1007
1008        assert_eq!(5, array.len());
1009        assert_eq!(1, array.null_count());
1010        assert_eq!("hello", array.value(0));
1011        assert_eq!("world", array.value(1));
1012        assert_eq!("world", array.value(2));
1013        assert_eq!("world", array.value(3));
1014        assert!(array.is_null(4));
1015    }
1016}