arrow_array/builder/
generic_bytes_builder.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use crate::builder::ArrayBuilder;
19use crate::types::{ByteArrayType, GenericBinaryType, GenericStringType};
20use crate::{Array, ArrayRef, GenericByteArray, OffsetSizeTrait};
21use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer, NullBufferBuilder, ScalarBuffer};
22use arrow_data::ArrayDataBuilder;
23use arrow_schema::ArrowError;
24use std::any::Any;
25use std::sync::Arc;
26
27/// Builder for [`GenericByteArray`]
28///
29/// For building strings, see docs on [`GenericStringBuilder`].
30/// For building binary, see docs on [`GenericBinaryBuilder`].
31pub struct GenericByteBuilder<T: ByteArrayType> {
32    value_builder: Vec<u8>,
33    offsets_builder: Vec<T::Offset>,
34    null_buffer_builder: NullBufferBuilder,
35}
36
37impl<T: ByteArrayType> GenericByteBuilder<T> {
38    /// Creates a new [`GenericByteBuilder`].
39    pub fn new() -> Self {
40        Self::with_capacity(1024, 1024)
41    }
42
43    /// Creates a new [`GenericByteBuilder`].
44    ///
45    /// - `item_capacity` is the number of items to pre-allocate.
46    ///   The size of the preallocated buffer of offsets is the number of items plus one.
47    /// - `data_capacity` is the total number of bytes of data to pre-allocate
48    ///   (for all items, not per item).
49    pub fn with_capacity(item_capacity: usize, data_capacity: usize) -> Self {
50        let mut offsets_builder = Vec::with_capacity(item_capacity + 1);
51        offsets_builder.push(T::Offset::from_usize(0).unwrap());
52        Self {
53            value_builder: Vec::with_capacity(data_capacity),
54            offsets_builder,
55            null_buffer_builder: NullBufferBuilder::new(item_capacity),
56        }
57    }
58
59    /// Creates a new  [`GenericByteBuilder`] from buffers.
60    ///
61    /// Any bytes in `value_buffer` after the last offset are dropped. The first
62    /// bit of `null_buffer` is for the first row.
63    ///
64    /// # Safety
65    ///
66    /// This doesn't verify buffer contents as it assumes the buffers are from
67    /// existing and valid [`GenericByteArray`].
68    pub unsafe fn new_from_buffer(
69        offsets_buffer: MutableBuffer,
70        mut value_buffer: MutableBuffer,
71        null_buffer: Option<MutableBuffer>,
72    ) -> Self {
73        let offsets_builder: Vec<T::Offset> =
74            ScalarBuffer::<T::Offset>::from(offsets_buffer).into();
75
76        // New values are appended at the end of the values buffer, so it must end
77        // at the last offset
78        value_buffer.truncate(offsets_builder.last().unwrap().as_usize());
79        let value_builder: Vec<u8> = ScalarBuffer::<u8>::from(value_buffer).into();
80
81        let null_buffer_builder = null_buffer
82            .map(|buffer| NullBufferBuilder::new_from_buffer(buffer, offsets_builder.len() - 1))
83            .unwrap_or_else(|| NullBufferBuilder::new_with_len(offsets_builder.len() - 1));
84
85        Self {
86            value_builder,
87            offsets_builder,
88            null_buffer_builder,
89        }
90    }
91
92    #[inline]
93    fn next_offset(&self) -> T::Offset {
94        T::Offset::from_usize(self.value_builder.len()).expect("byte array offset overflow")
95    }
96
97    /// Appends a value into the builder.
98    ///
99    /// See the [GenericStringBuilder] documentation for examples of
100    /// incrementally building string values with multiple `write!` calls.
101    ///
102    /// # Panics
103    ///
104    /// Panics if the resulting length of [`Self::values_slice`] would exceed
105    /// `T::Offset::MAX` bytes.
106    ///
107    /// For example, this can happen with [`StringArray`] or [`BinaryArray`]
108    /// where the total length of all values exceeds 2GB
109    ///
110    /// [`StringArray`]: crate::StringArray
111    /// [`BinaryArray`]: crate::BinaryArray
112    #[inline]
113    pub fn append_value(&mut self, value: impl AsRef<T::Native>) {
114        self.value_builder
115            .extend_from_slice(value.as_ref().as_ref());
116        self.null_buffer_builder.append(true);
117        self.offsets_builder.push(self.next_offset());
118    }
119
120    /// Appends a value of type `T` into the builder `n` times.
121    ///
122    /// See [`Self::append_value`] for more panic information.
123    ///
124    /// # Panics
125    ///
126    /// Panics for the same reasons as [`Self::append_value`]
127    #[inline]
128    pub fn append_value_n(&mut self, value: impl AsRef<T::Native>, n: usize) {
129        let bytes: &[u8] = value.as_ref().as_ref();
130        self.value_builder.reserve(bytes.len() * n);
131        self.offsets_builder.reserve(n);
132        for _ in 0..n {
133            self.value_builder.extend_from_slice(bytes);
134            self.offsets_builder.push(self.next_offset());
135        }
136        self.null_buffer_builder.append_n_non_nulls(n);
137    }
138
139    /// Append an `Option` value into the builder.
140    ///
141    /// - A `None` value will append a null value.
142    /// - A `Some` value will append the value.
143    ///
144    /// See [`Self::append_value`] for more panic information.
145    ///
146    /// # Panics
147    ///
148    /// Panics for the same reasons as [`Self::append_value`]
149    #[inline]
150    pub fn append_option(&mut self, value: Option<impl AsRef<T::Native>>) {
151        match value {
152            None => self.append_null(),
153            Some(v) => self.append_value(v),
154        }
155    }
156
157    /// Append a null value into the builder.
158    #[inline]
159    pub fn append_null(&mut self) {
160        self.null_buffer_builder.append(false);
161        self.offsets_builder.push(self.next_offset());
162    }
163
164    /// Appends `n` `null`s into the builder.
165    #[inline]
166    pub fn append_nulls(&mut self, n: usize) {
167        self.null_buffer_builder.append_n_nulls(n);
168        let next_offset = self.next_offset();
169        self.offsets_builder
170            .extend(std::iter::repeat_n(next_offset, n));
171    }
172
173    /// Appends array values and null to this builder as is
174    /// (this means that underlying null values are copied as is).
175    #[inline]
176    pub fn append_array(&mut self, array: &GenericByteArray<T>) -> Result<(), ArrowError> {
177        use num_traits::CheckedAdd;
178        if array.len() == 0 {
179            return Ok(());
180        }
181
182        let offsets = array.offsets();
183
184        // If the offsets are contiguous, we can append them directly avoiding the need to align
185        // for example, when the first appended array is not sliced (starts at offset 0)
186        if self.next_offset() == offsets[0] {
187            self.offsets_builder.extend_from_slice(&offsets[1..]);
188        } else {
189            // Shifting all the offsets
190            let shift: T::Offset = self.next_offset() - offsets[0];
191
192            if shift.checked_add(&offsets[offsets.len() - 1]).is_none() {
193                return Err(ArrowError::OffsetOverflowError(
194                    shift.as_usize() + offsets[offsets.len() - 1].as_usize(),
195                ));
196            }
197
198            self.offsets_builder
199                .extend(offsets[1..].iter().map(|&offset| offset + shift));
200        }
201
202        // Append underlying values, starting from the first offset and ending at the last offset
203        self.value_builder.extend_from_slice(
204            &array.values().as_slice()[offsets[0].as_usize()..offsets[array.len()].as_usize()],
205        );
206
207        if let Some(null_buffer) = array.nulls() {
208            self.null_buffer_builder.append_buffer(null_buffer);
209        } else {
210            self.null_buffer_builder.append_n_non_nulls(array.len());
211        }
212        Ok(())
213    }
214
215    /// Builds the [`GenericByteArray`] and reset this builder.
216    pub fn finish(&mut self) -> GenericByteArray<T> {
217        let array_type = T::DATA_TYPE;
218        let array_builder = ArrayDataBuilder::new(array_type)
219            .len(self.len())
220            .add_buffer(std::mem::take(&mut self.offsets_builder).into())
221            .add_buffer(std::mem::take(&mut self.value_builder).into())
222            .nulls(self.null_buffer_builder.finish());
223
224        self.offsets_builder.push(self.next_offset());
225        // SAFETY: builder is constructed from valid offset and value buffers maintained by the builder
226        let array_data = unsafe { array_builder.build_unchecked() };
227        GenericByteArray::from(array_data)
228    }
229
230    /// Builds the [`GenericByteArray`] without resetting the builder.
231    pub fn finish_cloned(&self) -> GenericByteArray<T> {
232        let array_type = T::DATA_TYPE;
233        let offset_buffer = Buffer::from_slice_ref(self.offsets_builder.as_slice());
234        let value_buffer = Buffer::from_slice_ref(self.value_builder.as_slice());
235        let array_builder = ArrayDataBuilder::new(array_type)
236            .len(self.len())
237            .add_buffer(offset_buffer)
238            .add_buffer(value_buffer)
239            .nulls(self.null_buffer_builder.finish_cloned());
240
241        // SAFETY: builder is constructed from valid offset and value buffers maintained by the builder
242        let array_data = unsafe { array_builder.build_unchecked() };
243        GenericByteArray::from(array_data)
244    }
245
246    /// Returns the current values buffer as a slice
247    pub fn values_slice(&self) -> &[u8] {
248        self.value_builder.as_slice()
249    }
250
251    /// Returns the current values buffer capacity, in bytes.
252    pub fn values_capacity(&self) -> usize {
253        self.value_builder.capacity()
254    }
255
256    /// Returns the current offsets buffer as a slice
257    pub fn offsets_slice(&self) -> &[T::Offset] {
258        self.offsets_builder.as_slice()
259    }
260
261    /// Returns the current offsets buffer capacity, in offsets.
262    pub fn offsets_capacity(&self) -> usize {
263        self.offsets_builder.capacity()
264    }
265
266    /// Returns the current null buffer as a slice
267    pub fn validity_slice(&self) -> Option<&[u8]> {
268        self.null_buffer_builder.as_slice()
269    }
270
271    /// Returns the current null buffer allocated capacity, in bytes.
272    pub fn validity_capacity(&self) -> usize {
273        self.null_buffer_builder.allocated_size()
274    }
275
276    /// Returns the current null buffer as a mutable slice
277    pub fn validity_slice_mut(&mut self) -> Option<&mut [u8]> {
278        self.null_buffer_builder.as_slice_mut()
279    }
280}
281
282impl<T: ByteArrayType> std::fmt::Debug for GenericByteBuilder<T> {
283    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
284        write!(f, "{}{}Builder", T::Offset::PREFIX, T::PREFIX)?;
285        f.debug_struct("")
286            .field("value_builder", &self.value_builder)
287            .field("offsets_builder", &self.offsets_builder)
288            .field("null_buffer_builder", &self.null_buffer_builder)
289            .finish()
290    }
291}
292
293impl<T: ByteArrayType> Default for GenericByteBuilder<T> {
294    fn default() -> Self {
295        Self::new()
296    }
297}
298
299impl<T: ByteArrayType> ArrayBuilder for GenericByteBuilder<T> {
300    /// Returns the number of binary slots in the builder
301    fn len(&self) -> usize {
302        self.null_buffer_builder.len()
303    }
304
305    /// Builds the array and reset this builder.
306    fn finish(&mut self) -> ArrayRef {
307        Arc::new(self.finish())
308    }
309
310    /// Builds the array without resetting the builder.
311    fn finish_cloned(&self) -> ArrayRef {
312        Arc::new(self.finish_cloned())
313    }
314
315    /// Returns the builder as a non-mutable `Any` reference.
316    fn as_any(&self) -> &dyn Any {
317        self
318    }
319
320    /// Returns the builder as a mutable `Any` reference.
321    fn as_any_mut(&mut self) -> &mut dyn Any {
322        self
323    }
324
325    /// Returns the boxed builder as a box of `Any`.
326    fn into_box_any(self: Box<Self>) -> Box<dyn Any> {
327        self
328    }
329}
330
331impl<T: ByteArrayType, V: AsRef<T::Native>> Extend<Option<V>> for GenericByteBuilder<T> {
332    #[inline]
333    fn extend<I: IntoIterator<Item = Option<V>>>(&mut self, iter: I) {
334        for v in iter {
335            self.append_option(v)
336        }
337    }
338}
339
340/// Array builder for [`GenericStringArray`][crate::GenericStringArray]
341///
342/// Values can be appended using [`GenericByteBuilder::append_value`], and nulls with
343/// [`GenericByteBuilder::append_null`].
344///
345/// This builder also implements [`std::fmt::Write`] with any written data
346/// included in the next appended value. This allows using [`std::fmt::Display`]
347/// with standard Rust idioms like `write!` and `writeln!` to write data
348/// directly to the builder without intermediate allocations.
349///
350/// # Example writing strings with `append_value`
351/// ```
352/// # use arrow_array::builder::GenericStringBuilder;
353/// let mut builder = GenericStringBuilder::<i32>::new();
354///
355/// // Write one string value
356/// builder.append_value("foobarbaz");
357///
358/// // Write a second string
359/// builder.append_value("v2");
360///
361/// let array = builder.finish();
362/// assert_eq!(array.value(0), "foobarbaz");
363/// assert_eq!(array.value(1), "v2");
364/// ```
365///
366/// # Example incrementally writing strings with `std::fmt::Write`
367///
368/// ```
369/// # use std::fmt::Write;
370/// # use arrow_array::builder::GenericStringBuilder;
371/// let mut builder = GenericStringBuilder::<i32>::new();
372///
373/// // Write data in multiple `write!` calls
374/// write!(builder, "foo").unwrap();
375/// write!(builder, "bar").unwrap();
376/// // The next call to append_value finishes the current string
377/// // including all previously written strings.
378/// builder.append_value("baz");
379///
380/// // Write second value with a single write call
381/// write!(builder, "v2").unwrap();
382/// // finish the value by calling append_value with an empty string
383/// builder.append_value("");
384///
385/// let array = builder.finish();
386/// assert_eq!(array.value(0), "foobarbaz");
387/// assert_eq!(array.value(1), "v2");
388/// ```
389pub type GenericStringBuilder<O> = GenericByteBuilder<GenericStringType<O>>;
390
391impl<O: OffsetSizeTrait> std::fmt::Write for GenericStringBuilder<O> {
392    fn write_str(&mut self, s: &str) -> std::fmt::Result {
393        self.value_builder.extend_from_slice(s.as_bytes());
394        Ok(())
395    }
396}
397
398/// A byte size value representing the number of bytes to allocate per string in [`GenericStringBuilder`]
399///
400/// To create a [`GenericStringBuilder`] using `.with_capacity` we are required to provide: \
401/// - `item_capacity` - the row count \
402/// - `data_capacity` - total string byte count \
403///
404/// We will use the `AVERAGE_STRING_LENGTH` * row_count for `data_capacity`. \
405///
406/// These capacities are preallocation hints used to improve performance,
407/// but consequences of passing a hint too large or too small should be negligible.
408const AVERAGE_STRING_LENGTH: usize = 16;
409/// Trait for string-like array builders
410///
411/// This trait provides unified interface for builders that append string-like data
412/// such as [`GenericStringBuilder<O>`] and [`crate::builder::StringViewBuilder`]
413pub trait StringLikeArrayBuilder: ArrayBuilder {
414    /// Returns a human-readable type name for the builder.
415    fn type_name() -> &'static str;
416
417    /// Creates a new builder with the given row capacity.
418    fn with_capacity(capacity: usize) -> Self;
419
420    /// Appends a non-null string value to the builder.
421    fn append_value(&mut self, value: &str);
422
423    /// Appends a null value to the builder.
424    fn append_null(&mut self);
425}
426
427impl<O: OffsetSizeTrait> StringLikeArrayBuilder for GenericStringBuilder<O> {
428    fn type_name() -> &'static str {
429        std::any::type_name::<Self>()
430    }
431    fn with_capacity(capacity: usize) -> Self {
432        Self::with_capacity(capacity, capacity * AVERAGE_STRING_LENGTH)
433    }
434    fn append_value(&mut self, value: &str) {
435        Self::append_value(self, value);
436    }
437    fn append_null(&mut self) {
438        Self::append_null(self);
439    }
440}
441
442/// A byte size value representing the number of bytes to allocate per binary in [`GenericBinaryBuilder`]
443///
444/// To create a [`GenericBinaryBuilder`] using `.with_capacity` we are required to provide: \
445/// - `item_capacity` - the row count \
446/// - `data_capacity` - total binary byte count \
447///
448/// We will use the `AVERAGE_BINARY_LENGTH` * row_count for `data_capacity`. \
449///
450/// These capacities are preallocation hints used to improve performance,
451/// but consequences of passing a hint too large or too small should be negligible.
452const AVERAGE_BINARY_LENGTH: usize = 128;
453/// Trait for binary-like array builders
454///
455/// This trait provides unified interface for builders that append binary-like data
456/// such as [`GenericBinaryBuilder<O>`] and [`crate::builder::BinaryViewBuilder`]
457pub trait BinaryLikeArrayBuilder: ArrayBuilder {
458    /// Returns a human-readable type name for the builder.
459    fn type_name() -> &'static str;
460
461    /// Creates a new builder with the given row capacity.
462    fn with_capacity(capacity: usize) -> Self;
463
464    /// Appends a non-null string value to the builder.
465    fn append_value(&mut self, value: &[u8]);
466
467    /// Appends a null value to the builder.
468    fn append_null(&mut self);
469}
470
471impl<O: OffsetSizeTrait> BinaryLikeArrayBuilder for GenericBinaryBuilder<O> {
472    fn type_name() -> &'static str {
473        std::any::type_name::<Self>()
474    }
475    fn with_capacity(capacity: usize) -> Self {
476        Self::with_capacity(capacity, capacity * AVERAGE_BINARY_LENGTH)
477    }
478    fn append_value(&mut self, value: &[u8]) {
479        Self::append_value(self, value);
480    }
481    fn append_null(&mut self) {
482        Self::append_null(self);
483    }
484}
485
486///  Array builder for [`GenericBinaryArray`][crate::GenericBinaryArray]
487///
488/// Values can be appended using [`GenericByteBuilder::append_value`], and nulls with
489/// [`GenericByteBuilder::append_null`].
490///
491/// # Example
492/// ```
493/// # use arrow_array::builder::GenericBinaryBuilder;
494/// let mut builder = GenericBinaryBuilder::<i32>::new();
495///
496/// // Write data
497/// builder.append_value("foo");
498///
499/// // Write second value
500/// builder.append_value(&[0,1,2]);
501///
502/// let array = builder.finish();
503/// // binary values
504/// assert_eq!(array.value(0), b"foo");
505/// assert_eq!(array.value(1), b"\x00\x01\x02");
506/// ```
507///
508/// # Example incrementally writing bytes with `write_bytes`
509///
510/// ```
511/// # use std::io::Write;
512/// # use arrow_array::builder::GenericBinaryBuilder;
513/// let mut builder = GenericBinaryBuilder::<i32>::new();
514///
515/// // Write data in multiple `write_bytes` calls
516/// write!(builder, "foo").unwrap();
517/// write!(builder, "bar").unwrap();
518/// // The next call to append_value finishes the current string
519/// // including all previously written strings.
520/// builder.append_value("baz");
521///
522/// // Write second value with a single write call
523/// write!(builder, "v2").unwrap();
524/// // finish the value by calling append_value with an empty string
525/// builder.append_value("");
526///
527/// let array = builder.finish();
528/// assert_eq!(array.value(0), "foobarbaz".as_bytes());
529/// assert_eq!(array.value(1), "v2".as_bytes());
530/// ```
531pub type GenericBinaryBuilder<O> = GenericByteBuilder<GenericBinaryType<O>>;
532
533impl<O: OffsetSizeTrait> std::io::Write for GenericBinaryBuilder<O> {
534    fn write(&mut self, bs: &[u8]) -> std::io::Result<usize> {
535        self.value_builder.extend_from_slice(bs);
536        Ok(bs.len())
537    }
538
539    fn flush(&mut self) -> std::io::Result<()> {
540        Ok(())
541    }
542}
543
544#[cfg(test)]
545mod tests {
546    use super::*;
547    use crate::GenericStringArray;
548    use crate::array::Array;
549    use arrow_buffer::NullBuffer;
550    use std::fmt::Write as _;
551    use std::io::Write as _;
552
553    fn _test_generic_binary_builder<O: OffsetSizeTrait>() {
554        let mut builder = GenericBinaryBuilder::<O>::new();
555
556        builder.append_value(b"hello");
557        builder.append_value(b"");
558        builder.append_null();
559        builder.append_value(b"rust");
560
561        let array = builder.finish();
562
563        assert_eq!(4, array.len());
564        assert_eq!(1, array.null_count());
565        assert_eq!(b"hello", array.value(0));
566        assert_eq!([] as [u8; 0], array.value(1));
567        assert!(array.is_null(2));
568        assert_eq!(b"rust", array.value(3));
569        assert_eq!(O::from_usize(5).unwrap(), array.value_offsets()[2]);
570        assert_eq!(O::from_usize(4).unwrap(), array.value_length(3));
571    }
572
573    #[test]
574    fn test_binary_builder() {
575        _test_generic_binary_builder::<i32>()
576    }
577
578    #[test]
579    fn test_large_binary_builder() {
580        _test_generic_binary_builder::<i64>()
581    }
582
583    fn _test_generic_binary_builder_all_nulls<O: OffsetSizeTrait>() {
584        let mut builder = GenericBinaryBuilder::<O>::new();
585        builder.append_null();
586        builder.append_null();
587        builder.append_null();
588        builder.append_nulls(2);
589        assert_eq!(5, builder.len());
590        assert!(!builder.is_empty());
591
592        let array = builder.finish();
593        assert_eq!(5, array.null_count());
594        assert_eq!(5, array.len());
595        assert!(array.is_null(0));
596        assert!(array.is_null(1));
597        assert!(array.is_null(2));
598        assert!(array.is_null(3));
599        assert!(array.is_null(4));
600    }
601
602    #[test]
603    fn test_binary_builder_all_nulls() {
604        _test_generic_binary_builder_all_nulls::<i32>()
605    }
606
607    #[test]
608    fn test_large_binary_builder_all_nulls() {
609        _test_generic_binary_builder_all_nulls::<i64>()
610    }
611
612    fn _test_generic_binary_builder_reset<O: OffsetSizeTrait>() {
613        let mut builder = GenericBinaryBuilder::<O>::new();
614
615        builder.append_value(b"hello");
616        builder.append_value(b"");
617        builder.append_null();
618        builder.append_value(b"rust");
619        builder.finish();
620
621        assert!(builder.is_empty());
622
623        builder.append_value(b"parquet");
624        builder.append_null();
625        builder.append_value(b"arrow");
626        builder.append_value(b"");
627        builder.append_nulls(2);
628        builder.append_value(b"hi");
629        let array = builder.finish();
630
631        assert_eq!(7, array.len());
632        assert_eq!(3, array.null_count());
633        assert_eq!(b"parquet", array.value(0));
634        assert!(array.is_null(1));
635        assert!(array.is_null(4));
636        assert!(array.is_null(5));
637        assert_eq!(b"arrow", array.value(2));
638        assert_eq!(b"", array.value(1));
639        assert_eq!(b"hi", array.value(6));
640
641        assert_eq!(O::zero(), array.value_offsets()[0]);
642        assert_eq!(O::from_usize(7).unwrap(), array.value_offsets()[2]);
643        assert_eq!(O::from_usize(14).unwrap(), array.value_offsets()[7]);
644        assert_eq!(O::from_usize(5).unwrap(), array.value_length(2));
645    }
646
647    #[test]
648    fn test_binary_builder_reset() {
649        _test_generic_binary_builder_reset::<i32>()
650    }
651
652    #[test]
653    fn test_large_binary_builder_reset() {
654        _test_generic_binary_builder_reset::<i64>()
655    }
656
657    fn _test_generic_string_array_builder<O: OffsetSizeTrait>() {
658        let mut builder = GenericStringBuilder::<O>::new();
659        let owned = "arrow".to_owned();
660
661        builder.append_value("hello");
662        builder.append_value("");
663        builder.append_value(&owned);
664        builder.append_null();
665        builder.append_option(Some("rust"));
666        builder.append_option(None::<&str>);
667        builder.append_option(None::<String>);
668        builder.append_nulls(2);
669        builder.append_value("parquet");
670        assert_eq!(10, builder.len());
671
672        assert_eq!(
673            GenericStringArray::<O>::from(vec![
674                Some("hello"),
675                Some(""),
676                Some("arrow"),
677                None,
678                Some("rust"),
679                None,
680                None,
681                None,
682                None,
683                Some("parquet")
684            ]),
685            builder.finish()
686        );
687    }
688
689    #[test]
690    fn test_string_array_builder() {
691        _test_generic_string_array_builder::<i32>()
692    }
693
694    #[test]
695    fn test_large_string_array_builder() {
696        _test_generic_string_array_builder::<i64>()
697    }
698
699    fn _test_generic_string_array_builder_finish<O: OffsetSizeTrait>() {
700        let mut builder = GenericStringBuilder::<O>::with_capacity(3, 11);
701
702        builder.append_value("hello");
703        builder.append_value("rust");
704        builder.append_null();
705
706        builder.finish();
707        assert!(builder.is_empty());
708        assert_eq!(&[O::zero()], builder.offsets_slice());
709
710        builder.append_value("arrow");
711        builder.append_value("parquet");
712        let arr = builder.finish();
713        // array should not have null buffer because there is not `null` value.
714        assert!(arr.nulls().is_none());
715        assert_eq!(GenericStringArray::<O>::from(vec!["arrow", "parquet"]), arr,)
716    }
717
718    #[test]
719    fn test_string_array_builder_finish() {
720        _test_generic_string_array_builder_finish::<i32>()
721    }
722
723    #[test]
724    fn test_large_string_array_builder_finish() {
725        _test_generic_string_array_builder_finish::<i64>()
726    }
727
728    fn _test_generic_string_array_builder_finish_cloned<O: OffsetSizeTrait>() {
729        let mut builder = GenericStringBuilder::<O>::with_capacity(3, 11);
730
731        builder.append_value("hello");
732        builder.append_value("rust");
733        builder.append_null();
734
735        let mut arr = builder.finish_cloned();
736        assert!(!builder.is_empty());
737        assert_eq!(3, arr.len());
738
739        builder.append_value("arrow");
740        builder.append_value("parquet");
741        arr = builder.finish();
742
743        assert!(arr.nulls().is_some());
744        assert_eq!(&[O::zero()], builder.offsets_slice());
745        assert_eq!(5, arr.len());
746    }
747
748    #[test]
749    fn test_string_array_builder_finish_cloned() {
750        _test_generic_string_array_builder_finish_cloned::<i32>()
751    }
752
753    #[test]
754    fn test_large_string_array_builder_finish_cloned() {
755        _test_generic_string_array_builder_finish_cloned::<i64>()
756    }
757
758    #[test]
759    fn test_extend() {
760        let mut builder = GenericStringBuilder::<i32>::new();
761        builder.extend(["a", "b", "c", "", "a", "b", "c"].into_iter().map(Some));
762        builder.extend(["d", "cupcakes", "hello"].into_iter().map(Some));
763        let array = builder.finish();
764        assert_eq!(array.value_offsets(), &[0, 1, 2, 3, 3, 4, 5, 6, 7, 15, 20]);
765        assert_eq!(array.value_data(), b"abcabcdcupcakeshello");
766    }
767
768    #[test]
769    fn test_write_str() {
770        let mut builder = GenericStringBuilder::<i32>::new();
771        write!(builder, "foo").unwrap();
772        builder.append_value("");
773        writeln!(builder, "bar").unwrap();
774        builder.append_value("");
775        write!(builder, "fiz").unwrap();
776        write!(builder, "buz").unwrap();
777        builder.append_value("");
778        let a = builder.finish();
779        let r: Vec<_> = a.iter().flatten().collect();
780        assert_eq!(r, &["foo", "bar\n", "fizbuz"])
781    }
782
783    #[test]
784    fn test_write_bytes() {
785        let mut builder = GenericBinaryBuilder::<i32>::new();
786        write!(builder, "foo").unwrap();
787        builder.append_value("");
788        writeln!(builder, "bar").unwrap();
789        builder.append_value("");
790        write!(builder, "fiz").unwrap();
791        write!(builder, "buz").unwrap();
792        builder.append_value("");
793        let a = builder.finish();
794        let r: Vec<_> = a.iter().flatten().collect();
795        assert_eq!(
796            r,
797            &[b"foo".as_slice(), b"bar\n".as_slice(), b"fizbuz".as_slice()]
798        )
799    }
800
801    #[test]
802    fn test_append_array_without_nulls() {
803        let input = vec![
804            "hello", "world", "how", "are", "you", "doing", "today", "I", "am", "doing", "well",
805            "thank", "you", "for", "asking",
806        ];
807        let arr1 = GenericStringArray::<i32>::from(input[..3].to_vec());
808        let arr2 = GenericStringArray::<i32>::from(input[3..7].to_vec());
809        let arr3 = GenericStringArray::<i32>::from(input[7..].to_vec());
810
811        let mut builder = GenericStringBuilder::<i32>::new();
812        builder.append_array(&arr1).unwrap();
813        builder.append_array(&arr2).unwrap();
814        builder.append_array(&arr3).unwrap();
815
816        let actual = builder.finish();
817        let expected = GenericStringArray::<i32>::from(input);
818
819        assert_eq!(actual, expected);
820    }
821
822    #[test]
823    fn test_append_array_with_nulls() {
824        let input = vec![
825            Some("hello"),
826            None,
827            Some("how"),
828            None,
829            None,
830            None,
831            None,
832            Some("I"),
833            Some("am"),
834            Some("doing"),
835            Some("well"),
836        ];
837        let arr1 = GenericStringArray::<i32>::from(input[..3].to_vec());
838        let arr2 = GenericStringArray::<i32>::from(input[3..7].to_vec());
839        let arr3 = GenericStringArray::<i32>::from(input[7..].to_vec());
840
841        let mut builder = GenericStringBuilder::<i32>::new();
842        builder.append_array(&arr1).unwrap();
843        builder.append_array(&arr2).unwrap();
844        builder.append_array(&arr3).unwrap();
845
846        let actual = builder.finish();
847        let expected = GenericStringArray::<i32>::from(input);
848
849        assert_eq!(actual, expected);
850    }
851
852    #[test]
853    fn test_append_empty_array() {
854        let arr = GenericStringArray::<i32>::from(Vec::<&str>::new());
855        let mut builder = GenericStringBuilder::<i32>::new();
856        builder.append_array(&arr).unwrap();
857        let result = builder.finish();
858        assert_eq!(result.len(), 0);
859    }
860
861    #[test]
862    fn test_append_array_with_offset_not_starting_at_0() {
863        let input = vec![
864            Some("hello"),
865            None,
866            Some("how"),
867            None,
868            None,
869            None,
870            None,
871            Some("I"),
872            Some("am"),
873            Some("doing"),
874            Some("well"),
875        ];
876        let full_array = GenericStringArray::<i32>::from(input);
877        let sliced = full_array.slice(1, 4);
878
879        assert_ne!(sliced.offsets()[0].as_usize(), 0);
880        assert_ne!(sliced.offsets().last(), full_array.offsets().last());
881
882        let mut builder = GenericStringBuilder::<i32>::new();
883        builder.append_array(&sliced).unwrap();
884        let actual = builder.finish();
885
886        let expected = GenericStringArray::<i32>::from(vec![None, Some("how"), None, None]);
887
888        assert_eq!(actual, expected);
889    }
890
891    #[test]
892    fn test_append_underlying_null_values_added_as_is() {
893        let input_1_array_with_nulls = {
894            let input = vec![
895                "hello", "world", "how", "are", "you", "doing", "today", "I", "am",
896            ];
897            let (offsets, buffer, _) = GenericStringArray::<i32>::from(input).into_parts();
898
899            GenericStringArray::<i32>::new(
900                offsets,
901                buffer,
902                Some(NullBuffer::from(&[
903                    true, false, true, false, false, true, true, true, false,
904                ])),
905            )
906        };
907        let input_2_array_with_nulls = {
908            let input = vec!["doing", "well", "thank", "you", "for", "asking"];
909            let (offsets, buffer, _) = GenericStringArray::<i32>::from(input).into_parts();
910
911            GenericStringArray::<i32>::new(
912                offsets,
913                buffer,
914                Some(NullBuffer::from(&[false, false, true, false, true, true])),
915            )
916        };
917
918        let mut builder = GenericStringBuilder::<i32>::new();
919        builder.append_array(&input_1_array_with_nulls).unwrap();
920        builder.append_array(&input_2_array_with_nulls).unwrap();
921
922        let actual = builder.finish();
923        let expected = GenericStringArray::<i32>::from(vec![
924            Some("hello"),
925            None, // world
926            Some("how"),
927            None, // are
928            None, // you
929            Some("doing"),
930            Some("today"),
931            Some("I"),
932            None, // am
933            None, // doing
934            None, // well
935            Some("thank"),
936            None, // "you",
937            Some("for"),
938            Some("asking"),
939        ]);
940
941        assert_eq!(actual, expected);
942
943        let expected_underlying_buffer = Buffer::from(
944            [
945                "hello", "world", "how", "are", "you", "doing", "today", "I", "am", "doing",
946                "well", "thank", "you", "for", "asking",
947            ]
948            .join("")
949            .as_bytes(),
950        );
951        assert_eq!(actual.values(), &expected_underlying_buffer);
952    }
953
954    #[test]
955    fn append_array_with_continues_indices() {
956        let input = vec![
957            "hello", "world", "how", "are", "you", "doing", "today", "I", "am", "doing", "well",
958            "thank", "you", "for", "asking",
959        ];
960        let full_array = GenericStringArray::<i32>::from(input);
961        let slice1 = full_array.slice(0, 3);
962        let slice2 = full_array.slice(3, 4);
963        let slice3 = full_array.slice(7, full_array.len() - 7);
964
965        let mut builder = GenericStringBuilder::<i32>::new();
966        builder.append_array(&slice1).unwrap();
967        builder.append_array(&slice2).unwrap();
968        builder.append_array(&slice3).unwrap();
969
970        let actual = builder.finish();
971
972        assert_eq!(actual, full_array);
973    }
974
975    #[test]
976    fn test_append_array_offset_overflow_precise() {
977        let mut builder = GenericStringBuilder::<i32>::new();
978
979        let initial_string = "x".repeat(i32::MAX as usize - 100);
980        builder.append_value(&initial_string);
981
982        let overflow_string = "y".repeat(200);
983        let overflow_array = GenericStringArray::<i32>::from(vec![overflow_string.as_str()]);
984
985        let result = builder.append_array(&overflow_array);
986
987        assert!(matches!(result, Err(ArrowError::OffsetOverflowError(_))));
988    }
989
990    #[test]
991    fn test_append_value_n() {
992        let mut builder = GenericStringBuilder::<i32>::new();
993        builder.append_value("hello");
994        builder.append_value_n("world", 3);
995        builder.append_null();
996        let array = builder.finish();
997
998        assert_eq!(5, array.len());
999        assert_eq!(1, array.null_count());
1000        assert_eq!("hello", array.value(0));
1001        assert_eq!("world", array.value(1));
1002        assert_eq!("world", array.value(2));
1003        assert_eq!("world", array.value(3));
1004        assert!(array.is_null(4));
1005    }
1006}
arrow_array/builder/generic_bytes_builder.rs

arrow_array/builder/
generic_bytes_builder.rs
