arrow_array/builder/
generic_bytes_builder.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use crate::builder::ArrayBuilder;
19use crate::types::{ByteArrayType, GenericBinaryType, GenericStringType};
20use crate::{Array, ArrayRef, GenericByteArray, OffsetSizeTrait};
21use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer, NullBufferBuilder, ScalarBuffer};
22use arrow_data::ArrayDataBuilder;
23use arrow_schema::ArrowError;
24use std::any::Any;
25use std::sync::Arc;
26
27/// Builder for [`GenericByteArray`]
28///
29/// For building strings, see docs on [`GenericStringBuilder`].
30/// For building binary, see docs on [`GenericBinaryBuilder`].
31pub struct GenericByteBuilder<T: ByteArrayType> {
32    value_builder: Vec<u8>,
33    offsets_builder: Vec<T::Offset>,
34    null_buffer_builder: NullBufferBuilder,
35}
36
37impl<T: ByteArrayType> GenericByteBuilder<T> {
38    /// Creates a new [`GenericByteBuilder`].
39    pub fn new() -> Self {
40        Self::with_capacity(1024, 1024)
41    }
42
43    /// Creates a new [`GenericByteBuilder`].
44    ///
45    /// - `item_capacity` is the number of items to pre-allocate.
46    ///   The size of the preallocated buffer of offsets is the number of items plus one.
47    /// - `data_capacity` is the total number of bytes of data to pre-allocate
48    ///   (for all items, not per item).
49    pub fn with_capacity(item_capacity: usize, data_capacity: usize) -> Self {
50        let mut offsets_builder = Vec::with_capacity(item_capacity + 1);
51        offsets_builder.push(T::Offset::from_usize(0).unwrap());
52        Self {
53            value_builder: Vec::with_capacity(data_capacity),
54            offsets_builder,
55            null_buffer_builder: NullBufferBuilder::new(item_capacity),
56        }
57    }
58
59    /// Creates a new  [`GenericByteBuilder`] from buffers.
60    ///
61    /// # Safety
62    ///
63    /// This doesn't verify buffer contents as it assumes the buffers are from
64    /// existing and valid [`GenericByteArray`].
65    pub unsafe fn new_from_buffer(
66        offsets_buffer: MutableBuffer,
67        value_buffer: MutableBuffer,
68        null_buffer: Option<MutableBuffer>,
69    ) -> Self {
70        let offsets_builder: Vec<T::Offset> =
71            ScalarBuffer::<T::Offset>::from(offsets_buffer).into();
72        let value_builder: Vec<u8> = ScalarBuffer::<u8>::from(value_buffer).into();
73
74        let null_buffer_builder = null_buffer
75            .map(|buffer| NullBufferBuilder::new_from_buffer(buffer, offsets_builder.len() - 1))
76            .unwrap_or_else(|| NullBufferBuilder::new_with_len(offsets_builder.len() - 1));
77
78        Self {
79            value_builder,
80            offsets_builder,
81            null_buffer_builder,
82        }
83    }
84
85    #[inline]
86    fn next_offset(&self) -> T::Offset {
87        T::Offset::from_usize(self.value_builder.len()).expect("byte array offset overflow")
88    }
89
90    /// Appends a value into the builder.
91    ///
92    /// See the [GenericStringBuilder] documentation for examples of
93    /// incrementally building string values with multiple `write!` calls.
94    ///
95    /// # Panics
96    ///
97    /// Panics if the resulting length of [`Self::values_slice`] would exceed
98    /// `T::Offset::MAX` bytes.
99    ///
100    /// For example, this can happen with [`StringArray`] or [`BinaryArray`]
101    /// where the total length of all values exceeds 2GB
102    ///
103    /// [`StringArray`]: crate::StringArray
104    /// [`BinaryArray`]: crate::BinaryArray
105    #[inline]
106    pub fn append_value(&mut self, value: impl AsRef<T::Native>) {
107        self.value_builder
108            .extend_from_slice(value.as_ref().as_ref());
109        self.null_buffer_builder.append(true);
110        self.offsets_builder.push(self.next_offset());
111    }
112
113    /// Appends a value of type `T` into the builder `n` times.
114    ///
115    /// See [`Self::append_value`] for more panic information.
116    ///
117    /// # Panics
118    ///
119    /// Panics for the same reasons as [`Self::append_value`]
120    #[inline]
121    pub fn append_value_n(&mut self, value: impl AsRef<T::Native>, n: usize) {
122        let bytes: &[u8] = value.as_ref().as_ref();
123        self.value_builder.reserve(bytes.len() * n);
124        self.offsets_builder.reserve(n);
125        for _ in 0..n {
126            self.value_builder.extend_from_slice(bytes);
127            self.offsets_builder.push(self.next_offset());
128        }
129        self.null_buffer_builder.append_n_non_nulls(n);
130    }
131
132    /// Append an `Option` value into the builder.
133    ///
134    /// - A `None` value will append a null value.
135    /// - A `Some` value will append the value.
136    ///
137    /// See [`Self::append_value`] for more panic information.
138    ///
139    /// # Panics
140    ///
141    /// Panics for the same reasons as [`Self::append_value`]
142    #[inline]
143    pub fn append_option(&mut self, value: Option<impl AsRef<T::Native>>) {
144        match value {
145            None => self.append_null(),
146            Some(v) => self.append_value(v),
147        }
148    }
149
150    /// Append a null value into the builder.
151    #[inline]
152    pub fn append_null(&mut self) {
153        self.null_buffer_builder.append(false);
154        self.offsets_builder.push(self.next_offset());
155    }
156
157    /// Appends `n` `null`s into the builder.
158    #[inline]
159    pub fn append_nulls(&mut self, n: usize) {
160        self.null_buffer_builder.append_n_nulls(n);
161        let next_offset = self.next_offset();
162        self.offsets_builder
163            .extend(std::iter::repeat_n(next_offset, n));
164    }
165
166    /// Appends array values and null to this builder as is
167    /// (this means that underlying null values are copied as is).
168    #[inline]
169    pub fn append_array(&mut self, array: &GenericByteArray<T>) -> Result<(), ArrowError> {
170        use num_traits::CheckedAdd;
171        if array.len() == 0 {
172            return Ok(());
173        }
174
175        let offsets = array.offsets();
176
177        // If the offsets are contiguous, we can append them directly avoiding the need to align
178        // for example, when the first appended array is not sliced (starts at offset 0)
179        if self.next_offset() == offsets[0] {
180            self.offsets_builder.extend_from_slice(&offsets[1..]);
181        } else {
182            // Shifting all the offsets
183            let shift: T::Offset = self.next_offset() - offsets[0];
184
185            if shift.checked_add(&offsets[offsets.len() - 1]).is_none() {
186                return Err(ArrowError::OffsetOverflowError(
187                    shift.as_usize() + offsets[offsets.len() - 1].as_usize(),
188                ));
189            }
190
191            self.offsets_builder
192                .extend(offsets[1..].iter().map(|&offset| offset + shift));
193        }
194
195        // Append underlying values, starting from the first offset and ending at the last offset
196        self.value_builder.extend_from_slice(
197            &array.values().as_slice()[offsets[0].as_usize()..offsets[array.len()].as_usize()],
198        );
199
200        if let Some(null_buffer) = array.nulls() {
201            self.null_buffer_builder.append_buffer(null_buffer);
202        } else {
203            self.null_buffer_builder.append_n_non_nulls(array.len());
204        }
205        Ok(())
206    }
207
208    /// Builds the [`GenericByteArray`] and reset this builder.
209    pub fn finish(&mut self) -> GenericByteArray<T> {
210        let array_type = T::DATA_TYPE;
211        let array_builder = ArrayDataBuilder::new(array_type)
212            .len(self.len())
213            .add_buffer(std::mem::take(&mut self.offsets_builder).into())
214            .add_buffer(std::mem::take(&mut self.value_builder).into())
215            .nulls(self.null_buffer_builder.finish());
216
217        self.offsets_builder.push(self.next_offset());
218        // SAFETY: builder is constructed from valid offset and value buffers maintained by the builder
219        let array_data = unsafe { array_builder.build_unchecked() };
220        GenericByteArray::from(array_data)
221    }
222
223    /// Builds the [`GenericByteArray`] without resetting the builder.
224    pub fn finish_cloned(&self) -> GenericByteArray<T> {
225        let array_type = T::DATA_TYPE;
226        let offset_buffer = Buffer::from_slice_ref(self.offsets_builder.as_slice());
227        let value_buffer = Buffer::from_slice_ref(self.value_builder.as_slice());
228        let array_builder = ArrayDataBuilder::new(array_type)
229            .len(self.len())
230            .add_buffer(offset_buffer)
231            .add_buffer(value_buffer)
232            .nulls(self.null_buffer_builder.finish_cloned());
233
234        // SAFETY: builder is constructed from valid offset and value buffers maintained by the builder
235        let array_data = unsafe { array_builder.build_unchecked() };
236        GenericByteArray::from(array_data)
237    }
238
239    /// Returns the current values buffer as a slice
240    pub fn values_slice(&self) -> &[u8] {
241        self.value_builder.as_slice()
242    }
243
244    /// Returns the current values buffer capacity, in bytes.
245    pub fn values_capacity(&self) -> usize {
246        self.value_builder.capacity()
247    }
248
249    /// Returns the current offsets buffer as a slice
250    pub fn offsets_slice(&self) -> &[T::Offset] {
251        self.offsets_builder.as_slice()
252    }
253
254    /// Returns the current offsets buffer capacity, in offsets.
255    pub fn offsets_capacity(&self) -> usize {
256        self.offsets_builder.capacity()
257    }
258
259    /// Returns the current null buffer as a slice
260    pub fn validity_slice(&self) -> Option<&[u8]> {
261        self.null_buffer_builder.as_slice()
262    }
263
264    /// Returns the current null buffer allocated capacity, in bytes.
265    pub fn validity_capacity(&self) -> usize {
266        self.null_buffer_builder.allocated_size()
267    }
268
269    /// Returns the current null buffer as a mutable slice
270    pub fn validity_slice_mut(&mut self) -> Option<&mut [u8]> {
271        self.null_buffer_builder.as_slice_mut()
272    }
273}
274
275impl<T: ByteArrayType> std::fmt::Debug for GenericByteBuilder<T> {
276    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
277        write!(f, "{}{}Builder", T::Offset::PREFIX, T::PREFIX)?;
278        f.debug_struct("")
279            .field("value_builder", &self.value_builder)
280            .field("offsets_builder", &self.offsets_builder)
281            .field("null_buffer_builder", &self.null_buffer_builder)
282            .finish()
283    }
284}
285
286impl<T: ByteArrayType> Default for GenericByteBuilder<T> {
287    fn default() -> Self {
288        Self::new()
289    }
290}
291
292impl<T: ByteArrayType> ArrayBuilder for GenericByteBuilder<T> {
293    /// Returns the number of binary slots in the builder
294    fn len(&self) -> usize {
295        self.null_buffer_builder.len()
296    }
297
298    /// Builds the array and reset this builder.
299    fn finish(&mut self) -> ArrayRef {
300        Arc::new(self.finish())
301    }
302
303    /// Builds the array without resetting the builder.
304    fn finish_cloned(&self) -> ArrayRef {
305        Arc::new(self.finish_cloned())
306    }
307
308    /// Returns the builder as a non-mutable `Any` reference.
309    fn as_any(&self) -> &dyn Any {
310        self
311    }
312
313    /// Returns the builder as a mutable `Any` reference.
314    fn as_any_mut(&mut self) -> &mut dyn Any {
315        self
316    }
317
318    /// Returns the boxed builder as a box of `Any`.
319    fn into_box_any(self: Box<Self>) -> Box<dyn Any> {
320        self
321    }
322}
323
324impl<T: ByteArrayType, V: AsRef<T::Native>> Extend<Option<V>> for GenericByteBuilder<T> {
325    #[inline]
326    fn extend<I: IntoIterator<Item = Option<V>>>(&mut self, iter: I) {
327        for v in iter {
328            self.append_option(v)
329        }
330    }
331}
332
333/// Array builder for [`GenericStringArray`][crate::GenericStringArray]
334///
335/// Values can be appended using [`GenericByteBuilder::append_value`], and nulls with
336/// [`GenericByteBuilder::append_null`].
337///
338/// This builder also implements [`std::fmt::Write`] with any written data
339/// included in the next appended value. This allows using [`std::fmt::Display`]
340/// with standard Rust idioms like `write!` and `writeln!` to write data
341/// directly to the builder without intermediate allocations.
342///
343/// # Example writing strings with `append_value`
344/// ```
345/// # use arrow_array::builder::GenericStringBuilder;
346/// let mut builder = GenericStringBuilder::<i32>::new();
347///
348/// // Write one string value
349/// builder.append_value("foobarbaz");
350///
351/// // Write a second string
352/// builder.append_value("v2");
353///
354/// let array = builder.finish();
355/// assert_eq!(array.value(0), "foobarbaz");
356/// assert_eq!(array.value(1), "v2");
357/// ```
358///
359/// # Example incrementally writing strings with `std::fmt::Write`
360///
361/// ```
362/// # use std::fmt::Write;
363/// # use arrow_array::builder::GenericStringBuilder;
364/// let mut builder = GenericStringBuilder::<i32>::new();
365///
366/// // Write data in multiple `write!` calls
367/// write!(builder, "foo").unwrap();
368/// write!(builder, "bar").unwrap();
369/// // The next call to append_value finishes the current string
370/// // including all previously written strings.
371/// builder.append_value("baz");
372///
373/// // Write second value with a single write call
374/// write!(builder, "v2").unwrap();
375/// // finish the value by calling append_value with an empty string
376/// builder.append_value("");
377///
378/// let array = builder.finish();
379/// assert_eq!(array.value(0), "foobarbaz");
380/// assert_eq!(array.value(1), "v2");
381/// ```
382pub type GenericStringBuilder<O> = GenericByteBuilder<GenericStringType<O>>;
383
384impl<O: OffsetSizeTrait> std::fmt::Write for GenericStringBuilder<O> {
385    fn write_str(&mut self, s: &str) -> std::fmt::Result {
386        self.value_builder.extend_from_slice(s.as_bytes());
387        Ok(())
388    }
389}
390
391/// A byte size value representing the number of bytes to allocate per string in [`GenericStringBuilder`]
392///
393/// To create a [`GenericStringBuilder`] using `.with_capacity` we are required to provide: \
394/// - `item_capacity` - the row count \
395/// - `data_capacity` - total string byte count \
396///
397/// We will use the `AVERAGE_STRING_LENGTH` * row_count for `data_capacity`. \
398///
399/// These capacities are preallocation hints used to improve performance,
400/// but consequences of passing a hint too large or too small should be negligible.
401const AVERAGE_STRING_LENGTH: usize = 16;
402/// Trait for string-like array builders
403///
404/// This trait provides unified interface for builders that append string-like data
405/// such as [`GenericStringBuilder<O>`] and [`crate::builder::StringViewBuilder`]
406pub trait StringLikeArrayBuilder: ArrayBuilder {
407    /// Returns a human-readable type name for the builder.
408    fn type_name() -> &'static str;
409
410    /// Creates a new builder with the given row capacity.
411    fn with_capacity(capacity: usize) -> Self;
412
413    /// Appends a non-null string value to the builder.
414    fn append_value(&mut self, value: &str);
415
416    /// Appends a null value to the builder.
417    fn append_null(&mut self);
418}
419
420impl<O: OffsetSizeTrait> StringLikeArrayBuilder for GenericStringBuilder<O> {
421    fn type_name() -> &'static str {
422        std::any::type_name::<Self>()
423    }
424    fn with_capacity(capacity: usize) -> Self {
425        Self::with_capacity(capacity, capacity * AVERAGE_STRING_LENGTH)
426    }
427    fn append_value(&mut self, value: &str) {
428        Self::append_value(self, value);
429    }
430    fn append_null(&mut self) {
431        Self::append_null(self);
432    }
433}
434
435/// A byte size value representing the number of bytes to allocate per binary in [`GenericBinaryBuilder`]
436///
437/// To create a [`GenericBinaryBuilder`] using `.with_capacity` we are required to provide: \
438/// - `item_capacity` - the row count \
439/// - `data_capacity` - total binary byte count \
440///
441/// We will use the `AVERAGE_BINARY_LENGTH` * row_count for `data_capacity`. \
442///
443/// These capacities are preallocation hints used to improve performance,
444/// but consequences of passing a hint too large or too small should be negligible.
445const AVERAGE_BINARY_LENGTH: usize = 128;
446/// Trait for binary-like array builders
447///
448/// This trait provides unified interface for builders that append binary-like data
449/// such as [`GenericBinaryBuilder<O>`] and [`crate::builder::BinaryViewBuilder`]
450pub trait BinaryLikeArrayBuilder: ArrayBuilder {
451    /// Returns a human-readable type name for the builder.
452    fn type_name() -> &'static str;
453
454    /// Creates a new builder with the given row capacity.
455    fn with_capacity(capacity: usize) -> Self;
456
457    /// Appends a non-null string value to the builder.
458    fn append_value(&mut self, value: &[u8]);
459
460    /// Appends a null value to the builder.
461    fn append_null(&mut self);
462}
463
464impl<O: OffsetSizeTrait> BinaryLikeArrayBuilder for GenericBinaryBuilder<O> {
465    fn type_name() -> &'static str {
466        std::any::type_name::<Self>()
467    }
468    fn with_capacity(capacity: usize) -> Self {
469        Self::with_capacity(capacity, capacity * AVERAGE_BINARY_LENGTH)
470    }
471    fn append_value(&mut self, value: &[u8]) {
472        Self::append_value(self, value);
473    }
474    fn append_null(&mut self) {
475        Self::append_null(self);
476    }
477}
478
479///  Array builder for [`GenericBinaryArray`][crate::GenericBinaryArray]
480///
481/// Values can be appended using [`GenericByteBuilder::append_value`], and nulls with
482/// [`GenericByteBuilder::append_null`].
483///
484/// # Example
485/// ```
486/// # use arrow_array::builder::GenericBinaryBuilder;
487/// let mut builder = GenericBinaryBuilder::<i32>::new();
488///
489/// // Write data
490/// builder.append_value("foo");
491///
492/// // Write second value
493/// builder.append_value(&[0,1,2]);
494///
495/// let array = builder.finish();
496/// // binary values
497/// assert_eq!(array.value(0), b"foo");
498/// assert_eq!(array.value(1), b"\x00\x01\x02");
499/// ```
500///
501/// # Example incrementally writing bytes with `write_bytes`
502///
503/// ```
504/// # use std::io::Write;
505/// # use arrow_array::builder::GenericBinaryBuilder;
506/// let mut builder = GenericBinaryBuilder::<i32>::new();
507///
508/// // Write data in multiple `write_bytes` calls
509/// write!(builder, "foo").unwrap();
510/// write!(builder, "bar").unwrap();
511/// // The next call to append_value finishes the current string
512/// // including all previously written strings.
513/// builder.append_value("baz");
514///
515/// // Write second value with a single write call
516/// write!(builder, "v2").unwrap();
517/// // finish the value by calling append_value with an empty string
518/// builder.append_value("");
519///
520/// let array = builder.finish();
521/// assert_eq!(array.value(0), "foobarbaz".as_bytes());
522/// assert_eq!(array.value(1), "v2".as_bytes());
523/// ```
524pub type GenericBinaryBuilder<O> = GenericByteBuilder<GenericBinaryType<O>>;
525
526impl<O: OffsetSizeTrait> std::io::Write for GenericBinaryBuilder<O> {
527    fn write(&mut self, bs: &[u8]) -> std::io::Result<usize> {
528        self.value_builder.extend_from_slice(bs);
529        Ok(bs.len())
530    }
531
532    fn flush(&mut self) -> std::io::Result<()> {
533        Ok(())
534    }
535}
536
537#[cfg(test)]
538mod tests {
539    use super::*;
540    use crate::GenericStringArray;
541    use crate::array::Array;
542    use arrow_buffer::NullBuffer;
543    use std::fmt::Write as _;
544    use std::io::Write as _;
545
546    fn _test_generic_binary_builder<O: OffsetSizeTrait>() {
547        let mut builder = GenericBinaryBuilder::<O>::new();
548
549        builder.append_value(b"hello");
550        builder.append_value(b"");
551        builder.append_null();
552        builder.append_value(b"rust");
553
554        let array = builder.finish();
555
556        assert_eq!(4, array.len());
557        assert_eq!(1, array.null_count());
558        assert_eq!(b"hello", array.value(0));
559        assert_eq!([] as [u8; 0], array.value(1));
560        assert!(array.is_null(2));
561        assert_eq!(b"rust", array.value(3));
562        assert_eq!(O::from_usize(5).unwrap(), array.value_offsets()[2]);
563        assert_eq!(O::from_usize(4).unwrap(), array.value_length(3));
564    }
565
566    #[test]
567    fn test_binary_builder() {
568        _test_generic_binary_builder::<i32>()
569    }
570
571    #[test]
572    fn test_large_binary_builder() {
573        _test_generic_binary_builder::<i64>()
574    }
575
576    fn _test_generic_binary_builder_all_nulls<O: OffsetSizeTrait>() {
577        let mut builder = GenericBinaryBuilder::<O>::new();
578        builder.append_null();
579        builder.append_null();
580        builder.append_null();
581        builder.append_nulls(2);
582        assert_eq!(5, builder.len());
583        assert!(!builder.is_empty());
584
585        let array = builder.finish();
586        assert_eq!(5, array.null_count());
587        assert_eq!(5, array.len());
588        assert!(array.is_null(0));
589        assert!(array.is_null(1));
590        assert!(array.is_null(2));
591        assert!(array.is_null(3));
592        assert!(array.is_null(4));
593    }
594
595    #[test]
596    fn test_binary_builder_all_nulls() {
597        _test_generic_binary_builder_all_nulls::<i32>()
598    }
599
600    #[test]
601    fn test_large_binary_builder_all_nulls() {
602        _test_generic_binary_builder_all_nulls::<i64>()
603    }
604
605    fn _test_generic_binary_builder_reset<O: OffsetSizeTrait>() {
606        let mut builder = GenericBinaryBuilder::<O>::new();
607
608        builder.append_value(b"hello");
609        builder.append_value(b"");
610        builder.append_null();
611        builder.append_value(b"rust");
612        builder.finish();
613
614        assert!(builder.is_empty());
615
616        builder.append_value(b"parquet");
617        builder.append_null();
618        builder.append_value(b"arrow");
619        builder.append_value(b"");
620        builder.append_nulls(2);
621        builder.append_value(b"hi");
622        let array = builder.finish();
623
624        assert_eq!(7, array.len());
625        assert_eq!(3, array.null_count());
626        assert_eq!(b"parquet", array.value(0));
627        assert!(array.is_null(1));
628        assert!(array.is_null(4));
629        assert!(array.is_null(5));
630        assert_eq!(b"arrow", array.value(2));
631        assert_eq!(b"", array.value(1));
632        assert_eq!(b"hi", array.value(6));
633
634        assert_eq!(O::zero(), array.value_offsets()[0]);
635        assert_eq!(O::from_usize(7).unwrap(), array.value_offsets()[2]);
636        assert_eq!(O::from_usize(14).unwrap(), array.value_offsets()[7]);
637        assert_eq!(O::from_usize(5).unwrap(), array.value_length(2));
638    }
639
640    #[test]
641    fn test_binary_builder_reset() {
642        _test_generic_binary_builder_reset::<i32>()
643    }
644
645    #[test]
646    fn test_large_binary_builder_reset() {
647        _test_generic_binary_builder_reset::<i64>()
648    }
649
650    fn _test_generic_string_array_builder<O: OffsetSizeTrait>() {
651        let mut builder = GenericStringBuilder::<O>::new();
652        let owned = "arrow".to_owned();
653
654        builder.append_value("hello");
655        builder.append_value("");
656        builder.append_value(&owned);
657        builder.append_null();
658        builder.append_option(Some("rust"));
659        builder.append_option(None::<&str>);
660        builder.append_option(None::<String>);
661        builder.append_nulls(2);
662        builder.append_value("parquet");
663        assert_eq!(10, builder.len());
664
665        assert_eq!(
666            GenericStringArray::<O>::from(vec![
667                Some("hello"),
668                Some(""),
669                Some("arrow"),
670                None,
671                Some("rust"),
672                None,
673                None,
674                None,
675                None,
676                Some("parquet")
677            ]),
678            builder.finish()
679        );
680    }
681
682    #[test]
683    fn test_string_array_builder() {
684        _test_generic_string_array_builder::<i32>()
685    }
686
687    #[test]
688    fn test_large_string_array_builder() {
689        _test_generic_string_array_builder::<i64>()
690    }
691
692    fn _test_generic_string_array_builder_finish<O: OffsetSizeTrait>() {
693        let mut builder = GenericStringBuilder::<O>::with_capacity(3, 11);
694
695        builder.append_value("hello");
696        builder.append_value("rust");
697        builder.append_null();
698
699        builder.finish();
700        assert!(builder.is_empty());
701        assert_eq!(&[O::zero()], builder.offsets_slice());
702
703        builder.append_value("arrow");
704        builder.append_value("parquet");
705        let arr = builder.finish();
706        // array should not have null buffer because there is not `null` value.
707        assert!(arr.nulls().is_none());
708        assert_eq!(GenericStringArray::<O>::from(vec!["arrow", "parquet"]), arr,)
709    }
710
711    #[test]
712    fn test_string_array_builder_finish() {
713        _test_generic_string_array_builder_finish::<i32>()
714    }
715
716    #[test]
717    fn test_large_string_array_builder_finish() {
718        _test_generic_string_array_builder_finish::<i64>()
719    }
720
721    fn _test_generic_string_array_builder_finish_cloned<O: OffsetSizeTrait>() {
722        let mut builder = GenericStringBuilder::<O>::with_capacity(3, 11);
723
724        builder.append_value("hello");
725        builder.append_value("rust");
726        builder.append_null();
727
728        let mut arr = builder.finish_cloned();
729        assert!(!builder.is_empty());
730        assert_eq!(3, arr.len());
731
732        builder.append_value("arrow");
733        builder.append_value("parquet");
734        arr = builder.finish();
735
736        assert!(arr.nulls().is_some());
737        assert_eq!(&[O::zero()], builder.offsets_slice());
738        assert_eq!(5, arr.len());
739    }
740
741    #[test]
742    fn test_string_array_builder_finish_cloned() {
743        _test_generic_string_array_builder_finish_cloned::<i32>()
744    }
745
746    #[test]
747    fn test_large_string_array_builder_finish_cloned() {
748        _test_generic_string_array_builder_finish_cloned::<i64>()
749    }
750
751    #[test]
752    fn test_extend() {
753        let mut builder = GenericStringBuilder::<i32>::new();
754        builder.extend(["a", "b", "c", "", "a", "b", "c"].into_iter().map(Some));
755        builder.extend(["d", "cupcakes", "hello"].into_iter().map(Some));
756        let array = builder.finish();
757        assert_eq!(array.value_offsets(), &[0, 1, 2, 3, 3, 4, 5, 6, 7, 15, 20]);
758        assert_eq!(array.value_data(), b"abcabcdcupcakeshello");
759    }
760
761    #[test]
762    fn test_write_str() {
763        let mut builder = GenericStringBuilder::<i32>::new();
764        write!(builder, "foo").unwrap();
765        builder.append_value("");
766        writeln!(builder, "bar").unwrap();
767        builder.append_value("");
768        write!(builder, "fiz").unwrap();
769        write!(builder, "buz").unwrap();
770        builder.append_value("");
771        let a = builder.finish();
772        let r: Vec<_> = a.iter().flatten().collect();
773        assert_eq!(r, &["foo", "bar\n", "fizbuz"])
774    }
775
776    #[test]
777    fn test_write_bytes() {
778        let mut builder = GenericBinaryBuilder::<i32>::new();
779        write!(builder, "foo").unwrap();
780        builder.append_value("");
781        writeln!(builder, "bar").unwrap();
782        builder.append_value("");
783        write!(builder, "fiz").unwrap();
784        write!(builder, "buz").unwrap();
785        builder.append_value("");
786        let a = builder.finish();
787        let r: Vec<_> = a.iter().flatten().collect();
788        assert_eq!(
789            r,
790            &[b"foo".as_slice(), b"bar\n".as_slice(), b"fizbuz".as_slice()]
791        )
792    }
793
794    #[test]
795    fn test_append_array_without_nulls() {
796        let input = vec![
797            "hello", "world", "how", "are", "you", "doing", "today", "I", "am", "doing", "well",
798            "thank", "you", "for", "asking",
799        ];
800        let arr1 = GenericStringArray::<i32>::from(input[..3].to_vec());
801        let arr2 = GenericStringArray::<i32>::from(input[3..7].to_vec());
802        let arr3 = GenericStringArray::<i32>::from(input[7..].to_vec());
803
804        let mut builder = GenericStringBuilder::<i32>::new();
805        builder.append_array(&arr1).unwrap();
806        builder.append_array(&arr2).unwrap();
807        builder.append_array(&arr3).unwrap();
808
809        let actual = builder.finish();
810        let expected = GenericStringArray::<i32>::from(input);
811
812        assert_eq!(actual, expected);
813    }
814
815    #[test]
816    fn test_append_array_with_nulls() {
817        let input = vec![
818            Some("hello"),
819            None,
820            Some("how"),
821            None,
822            None,
823            None,
824            None,
825            Some("I"),
826            Some("am"),
827            Some("doing"),
828            Some("well"),
829        ];
830        let arr1 = GenericStringArray::<i32>::from(input[..3].to_vec());
831        let arr2 = GenericStringArray::<i32>::from(input[3..7].to_vec());
832        let arr3 = GenericStringArray::<i32>::from(input[7..].to_vec());
833
834        let mut builder = GenericStringBuilder::<i32>::new();
835        builder.append_array(&arr1).unwrap();
836        builder.append_array(&arr2).unwrap();
837        builder.append_array(&arr3).unwrap();
838
839        let actual = builder.finish();
840        let expected = GenericStringArray::<i32>::from(input);
841
842        assert_eq!(actual, expected);
843    }
844
845    #[test]
846    fn test_append_empty_array() {
847        let arr = GenericStringArray::<i32>::from(Vec::<&str>::new());
848        let mut builder = GenericStringBuilder::<i32>::new();
849        builder.append_array(&arr).unwrap();
850        let result = builder.finish();
851        assert_eq!(result.len(), 0);
852    }
853
854    #[test]
855    fn test_append_array_with_offset_not_starting_at_0() {
856        let input = vec![
857            Some("hello"),
858            None,
859            Some("how"),
860            None,
861            None,
862            None,
863            None,
864            Some("I"),
865            Some("am"),
866            Some("doing"),
867            Some("well"),
868        ];
869        let full_array = GenericStringArray::<i32>::from(input);
870        let sliced = full_array.slice(1, 4);
871
872        assert_ne!(sliced.offsets()[0].as_usize(), 0);
873        assert_ne!(sliced.offsets().last(), full_array.offsets().last());
874
875        let mut builder = GenericStringBuilder::<i32>::new();
876        builder.append_array(&sliced).unwrap();
877        let actual = builder.finish();
878
879        let expected = GenericStringArray::<i32>::from(vec![None, Some("how"), None, None]);
880
881        assert_eq!(actual, expected);
882    }
883
884    #[test]
885    fn test_append_underlying_null_values_added_as_is() {
886        let input_1_array_with_nulls = {
887            let input = vec![
888                "hello", "world", "how", "are", "you", "doing", "today", "I", "am",
889            ];
890            let (offsets, buffer, _) = GenericStringArray::<i32>::from(input).into_parts();
891
892            GenericStringArray::<i32>::new(
893                offsets,
894                buffer,
895                Some(NullBuffer::from(&[
896                    true, false, true, false, false, true, true, true, false,
897                ])),
898            )
899        };
900        let input_2_array_with_nulls = {
901            let input = vec!["doing", "well", "thank", "you", "for", "asking"];
902            let (offsets, buffer, _) = GenericStringArray::<i32>::from(input).into_parts();
903
904            GenericStringArray::<i32>::new(
905                offsets,
906                buffer,
907                Some(NullBuffer::from(&[false, false, true, false, true, true])),
908            )
909        };
910
911        let mut builder = GenericStringBuilder::<i32>::new();
912        builder.append_array(&input_1_array_with_nulls).unwrap();
913        builder.append_array(&input_2_array_with_nulls).unwrap();
914
915        let actual = builder.finish();
916        let expected = GenericStringArray::<i32>::from(vec![
917            Some("hello"),
918            None, // world
919            Some("how"),
920            None, // are
921            None, // you
922            Some("doing"),
923            Some("today"),
924            Some("I"),
925            None, // am
926            None, // doing
927            None, // well
928            Some("thank"),
929            None, // "you",
930            Some("for"),
931            Some("asking"),
932        ]);
933
934        assert_eq!(actual, expected);
935
936        let expected_underlying_buffer = Buffer::from(
937            [
938                "hello", "world", "how", "are", "you", "doing", "today", "I", "am", "doing",
939                "well", "thank", "you", "for", "asking",
940            ]
941            .join("")
942            .as_bytes(),
943        );
944        assert_eq!(actual.values(), &expected_underlying_buffer);
945    }
946
947    #[test]
948    fn append_array_with_continues_indices() {
949        let input = vec![
950            "hello", "world", "how", "are", "you", "doing", "today", "I", "am", "doing", "well",
951            "thank", "you", "for", "asking",
952        ];
953        let full_array = GenericStringArray::<i32>::from(input);
954        let slice1 = full_array.slice(0, 3);
955        let slice2 = full_array.slice(3, 4);
956        let slice3 = full_array.slice(7, full_array.len() - 7);
957
958        let mut builder = GenericStringBuilder::<i32>::new();
959        builder.append_array(&slice1).unwrap();
960        builder.append_array(&slice2).unwrap();
961        builder.append_array(&slice3).unwrap();
962
963        let actual = builder.finish();
964
965        assert_eq!(actual, full_array);
966    }
967
968    #[test]
969    fn test_append_array_offset_overflow_precise() {
970        let mut builder = GenericStringBuilder::<i32>::new();
971
972        let initial_string = "x".repeat(i32::MAX as usize - 100);
973        builder.append_value(&initial_string);
974
975        let overflow_string = "y".repeat(200);
976        let overflow_array = GenericStringArray::<i32>::from(vec![overflow_string.as_str()]);
977
978        let result = builder.append_array(&overflow_array);
979
980        assert!(matches!(result, Err(ArrowError::OffsetOverflowError(_))));
981    }
982
983    #[test]
984    fn test_append_value_n() {
985        let mut builder = GenericStringBuilder::<i32>::new();
986        builder.append_value("hello");
987        builder.append_value_n("world", 3);
988        builder.append_null();
989        let array = builder.finish();
990
991        assert_eq!(5, array.len());
992        assert_eq!(1, array.null_count());
993        assert_eq!("hello", array.value(0));
994        assert_eq!("world", array.value(1));
995        assert_eq!("world", array.value(2));
996        assert_eq!("world", array.value(3));
997        assert!(array.is_null(4));
998    }
999}
arrow_array/builder/generic_bytes_builder.rs

arrow_array/builder/
generic_bytes_builder.rs
