arrow_select/
concat.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Defines concat kernel for `ArrayRef`
19//!
20//! Example:
21//!
22//! ```
23//! use arrow_array::{ArrayRef, StringArray};
24//! use arrow_select::concat::concat;
25//!
26//! let arr = concat(&[
27//!     &StringArray::from(vec!["hello", "world"]),
28//!     &StringArray::from(vec!["!"]),
29//! ]).unwrap();
30//! assert_eq!(arr.len(), 3);
31//! ```
32
33use crate::dictionary::{merge_dictionary_values, should_merge_dictionary_values};
34use arrow_array::builder::{
35    BooleanBuilder, GenericByteBuilder, GenericByteViewBuilder, PrimitiveBuilder,
36};
37use arrow_array::cast::AsArray;
38use arrow_array::types::*;
39use arrow_array::*;
40use arrow_buffer::{ArrowNativeType, BooleanBufferBuilder, NullBuffer, OffsetBuffer};
41use arrow_data::transform::{Capacities, MutableArrayData};
42use arrow_data::ArrayDataBuilder;
43use arrow_schema::{ArrowError, DataType, FieldRef, Fields, SchemaRef};
44use std::{collections::HashSet, ops::Add, sync::Arc};
45
46fn binary_capacity<T: ByteArrayType>(arrays: &[&dyn Array]) -> Capacities {
47    let mut item_capacity = 0;
48    let mut bytes_capacity = 0;
49    for array in arrays {
50        let a = array.as_bytes::<T>();
51
52        // Guaranteed to always have at least one element
53        let offsets = a.value_offsets();
54        bytes_capacity += offsets[offsets.len() - 1].as_usize() - offsets[0].as_usize();
55        item_capacity += a.len()
56    }
57
58    Capacities::Binary(item_capacity, Some(bytes_capacity))
59}
60
61fn fixed_size_list_capacity(arrays: &[&dyn Array], data_type: &DataType) -> Capacities {
62    if let DataType::FixedSizeList(f, _) = data_type {
63        let item_capacity = arrays.iter().map(|a| a.len()).sum();
64        let child_data_type = f.data_type();
65        match child_data_type {
66            // These types should match the types that `get_capacity`
67            // has special handling for.
68            DataType::Utf8
69            | DataType::LargeUtf8
70            | DataType::Binary
71            | DataType::LargeBinary
72            | DataType::FixedSizeList(_, _) => {
73                let values: Vec<&dyn arrow_array::Array> = arrays
74                    .iter()
75                    .map(|a| a.as_fixed_size_list().values().as_ref())
76                    .collect();
77                Capacities::List(
78                    item_capacity,
79                    Some(Box::new(get_capacity(&values, child_data_type))),
80                )
81            }
82            _ => Capacities::Array(item_capacity),
83        }
84    } else {
85        unreachable!("illegal data type for fixed size list")
86    }
87}
88
89fn concat_byte_view<B: ByteViewType>(arrays: &[&dyn Array]) -> Result<ArrayRef, ArrowError> {
90    let mut builder =
91        GenericByteViewBuilder::<B>::with_capacity(arrays.iter().map(|a| a.len()).sum());
92    for &array in arrays.iter() {
93        builder.append_array(array.as_byte_view());
94    }
95    Ok(Arc::new(builder.finish()))
96}
97
98fn concat_dictionaries<K: ArrowDictionaryKeyType>(
99    arrays: &[&dyn Array],
100) -> Result<ArrayRef, ArrowError> {
101    let mut output_len = 0;
102    let dictionaries: Vec<_> = arrays
103        .iter()
104        .map(|x| x.as_dictionary::<K>())
105        .inspect(|d| output_len += d.len())
106        .collect();
107
108    if !should_merge_dictionary_values::<K>(&dictionaries, output_len) {
109        return concat_fallback(arrays, Capacities::Array(output_len));
110    }
111
112    let merged = merge_dictionary_values(&dictionaries, None)?;
113
114    // Recompute keys
115    let mut key_values = Vec::with_capacity(output_len);
116
117    let mut has_nulls = false;
118    for (d, mapping) in dictionaries.iter().zip(merged.key_mappings) {
119        has_nulls |= d.null_count() != 0;
120        for key in d.keys().values() {
121            // Use get to safely handle nulls
122            key_values.push(mapping.get(key.as_usize()).copied().unwrap_or_default())
123        }
124    }
125
126    let nulls = has_nulls.then(|| {
127        let mut nulls = BooleanBufferBuilder::new(output_len);
128        for d in &dictionaries {
129            match d.nulls() {
130                Some(n) => nulls.append_buffer(n.inner()),
131                None => nulls.append_n(d.len(), true),
132            }
133        }
134        NullBuffer::new(nulls.finish())
135    });
136
137    let keys = PrimitiveArray::<K>::new(key_values.into(), nulls);
138    // Sanity check
139    assert_eq!(keys.len(), output_len);
140
141    let array = unsafe { DictionaryArray::new_unchecked(keys, merged.values) };
142    Ok(Arc::new(array))
143}
144
145fn concat_lists<OffsetSize: OffsetSizeTrait>(
146    arrays: &[&dyn Array],
147    field: &FieldRef,
148) -> Result<ArrayRef, ArrowError> {
149    let mut output_len = 0;
150    let mut list_has_nulls = false;
151    let mut list_has_slices = false;
152
153    let lists = arrays
154        .iter()
155        .map(|x| x.as_list::<OffsetSize>())
156        .inspect(|l| {
157            output_len += l.len();
158            list_has_nulls |= l.null_count() != 0;
159            list_has_slices |= l.offsets()[0] > OffsetSize::zero()
160                || l.offsets().last().unwrap().as_usize() < l.values().len();
161        })
162        .collect::<Vec<_>>();
163
164    let lists_nulls = list_has_nulls.then(|| {
165        let mut nulls = BooleanBufferBuilder::new(output_len);
166        for l in &lists {
167            match l.nulls() {
168                Some(n) => nulls.append_buffer(n.inner()),
169                None => nulls.append_n(l.len(), true),
170            }
171        }
172        NullBuffer::new(nulls.finish())
173    });
174
175    // If any of the lists have slices, we need to slice the values
176    // to ensure that the offsets are correct
177    let mut sliced_values;
178    let values: Vec<&dyn Array> = if list_has_slices {
179        sliced_values = Vec::with_capacity(lists.len());
180        for l in &lists {
181            // if the first offset is non-zero, we need to slice the values so when
182            // we concatenate them below only the relevant values are included
183            let offsets = l.offsets();
184            let start_offset = offsets[0].as_usize();
185            let end_offset = offsets.last().unwrap().as_usize();
186            sliced_values.push(l.values().slice(start_offset, end_offset - start_offset));
187        }
188        sliced_values.iter().map(|a| a.as_ref()).collect()
189    } else {
190        lists.iter().map(|x| x.values().as_ref()).collect()
191    };
192
193    let concatenated_values = concat(values.as_slice())?;
194
195    // Merge value offsets from the lists
196    let value_offset_buffer =
197        OffsetBuffer::<OffsetSize>::from_lengths(lists.iter().flat_map(|x| x.offsets().lengths()));
198
199    let array = GenericListArray::<OffsetSize>::try_new(
200        Arc::clone(field),
201        value_offset_buffer,
202        concatenated_values,
203        lists_nulls,
204    )?;
205
206    Ok(Arc::new(array))
207}
208
209fn concat_primitives<T: ArrowPrimitiveType>(arrays: &[&dyn Array]) -> Result<ArrayRef, ArrowError> {
210    let mut builder = PrimitiveBuilder::<T>::with_capacity(arrays.iter().map(|a| a.len()).sum())
211        .with_data_type(arrays[0].data_type().clone());
212
213    for array in arrays {
214        builder.append_array(array.as_primitive());
215    }
216
217    Ok(Arc::new(builder.finish()))
218}
219
220fn concat_boolean(arrays: &[&dyn Array]) -> Result<ArrayRef, ArrowError> {
221    let mut builder = BooleanBuilder::with_capacity(arrays.iter().map(|a| a.len()).sum());
222
223    for array in arrays {
224        builder.append_array(array.as_boolean());
225    }
226
227    Ok(Arc::new(builder.finish()))
228}
229
230fn concat_bytes<T: ByteArrayType>(arrays: &[&dyn Array]) -> Result<ArrayRef, ArrowError> {
231    let (item_capacity, bytes_capacity) = match binary_capacity::<T>(arrays) {
232        Capacities::Binary(item_capacity, Some(bytes_capacity)) => (item_capacity, bytes_capacity),
233        _ => unreachable!(),
234    };
235
236    let mut builder = GenericByteBuilder::<T>::with_capacity(item_capacity, bytes_capacity);
237
238    for array in arrays {
239        builder.append_array(array.as_bytes::<T>());
240    }
241
242    Ok(Arc::new(builder.finish()))
243}
244
245fn concat_structs(arrays: &[&dyn Array], fields: &Fields) -> Result<ArrayRef, ArrowError> {
246    let mut len = 0;
247    let mut has_nulls = false;
248    let structs = arrays
249        .iter()
250        .map(|a| {
251            len += a.len();
252            has_nulls |= a.null_count() > 0;
253            a.as_struct()
254        })
255        .collect::<Vec<_>>();
256
257    let nulls = has_nulls.then(|| {
258        let mut b = BooleanBufferBuilder::new(len);
259        for s in &structs {
260            match s.nulls() {
261                Some(n) => b.append_buffer(n.inner()),
262                None => b.append_n(s.len(), true),
263            }
264        }
265        NullBuffer::new(b.finish())
266    });
267
268    let column_concat_result = (0..fields.len())
269        .map(|i| {
270            let extracted_cols = structs
271                .iter()
272                .map(|s| s.column(i).as_ref())
273                .collect::<Vec<_>>();
274            concat(&extracted_cols)
275        })
276        .collect::<Result<Vec<_>, ArrowError>>()?;
277
278    Ok(Arc::new(StructArray::try_new_with_length(
279        fields.clone(),
280        column_concat_result,
281        nulls,
282        len,
283    )?))
284}
285
286/// Concatenate multiple RunArray instances into a single RunArray.
287///
288/// This function handles the special case of concatenating RunArrays by:
289/// 1. Collecting all run ends and values from input arrays
290/// 2. Adjusting run ends to account for the length of previous arrays
291/// 3. Creating a new RunArray with the combined data
292fn concat_run_arrays<R: RunEndIndexType>(arrays: &[&dyn Array]) -> Result<ArrayRef, ArrowError>
293where
294    R::Native: Add<Output = R::Native>,
295{
296    let run_arrays: Vec<_> = arrays
297        .iter()
298        .map(|x| x.as_run::<R>())
299        .filter(|x| !x.run_ends().is_empty())
300        .collect();
301
302    // The run ends need to be adjusted by the sum of the lengths of the previous arrays.
303    let needed_run_end_adjustments = std::iter::once(R::default_value())
304        .chain(
305            run_arrays
306                .iter()
307                .scan(R::default_value(), |acc, run_array| {
308                    *acc = *acc + *run_array.run_ends().values().last().unwrap();
309                    Some(*acc)
310                }),
311        )
312        .collect::<Vec<_>>();
313
314    // This works out nicely to be the total (logical) length of the resulting array.
315    let total_len = needed_run_end_adjustments.last().unwrap().as_usize();
316
317    let run_ends_array =
318        PrimitiveArray::<R>::from_iter_values(run_arrays.iter().enumerate().flat_map(
319            move |(i, run_array)| {
320                let adjustment = needed_run_end_adjustments[i];
321                run_array
322                    .run_ends()
323                    .values()
324                    .iter()
325                    .map(move |run_end| *run_end + adjustment)
326            },
327        ));
328
329    let all_values = concat(
330        &run_arrays
331            .iter()
332            .map(|x| x.values().as_ref())
333            .collect::<Vec<_>>(),
334    )?;
335
336    let builder = ArrayDataBuilder::new(run_arrays[0].data_type().clone())
337        .len(total_len)
338        .child_data(vec![run_ends_array.into_data(), all_values.into_data()]);
339
340    // `build_unchecked` is used to avoid recursive validation of child arrays.
341    let array_data = unsafe { builder.build_unchecked() };
342    array_data.validate_data()?;
343
344    Ok(Arc::<RunArray<R>>::new(array_data.into()))
345}
346
347macro_rules! dict_helper {
348    ($t:ty, $arrays:expr) => {
349        return Ok(Arc::new(concat_dictionaries::<$t>($arrays)?) as _)
350    };
351}
352
353macro_rules! primitive_concat {
354    ($t:ty, $arrays:expr) => {
355        return Ok(Arc::new(concat_primitives::<$t>($arrays)?) as _)
356    };
357}
358
359fn get_capacity(arrays: &[&dyn Array], data_type: &DataType) -> Capacities {
360    match data_type {
361        DataType::Utf8 => binary_capacity::<Utf8Type>(arrays),
362        DataType::LargeUtf8 => binary_capacity::<LargeUtf8Type>(arrays),
363        DataType::Binary => binary_capacity::<BinaryType>(arrays),
364        DataType::LargeBinary => binary_capacity::<LargeBinaryType>(arrays),
365        DataType::FixedSizeList(_, _) => fixed_size_list_capacity(arrays, data_type),
366        _ => Capacities::Array(arrays.iter().map(|a| a.len()).sum()),
367    }
368}
369
370/// Concatenate multiple [Array] of the same type into a single [ArrayRef].
371pub fn concat(arrays: &[&dyn Array]) -> Result<ArrayRef, ArrowError> {
372    if arrays.is_empty() {
373        return Err(ArrowError::ComputeError(
374            "concat requires input of at least one array".to_string(),
375        ));
376    } else if arrays.len() == 1 {
377        let array = arrays[0];
378        return Ok(array.slice(0, array.len()));
379    }
380
381    let d = arrays[0].data_type();
382    if arrays.iter().skip(1).any(|array| array.data_type() != d) {
383        // Create error message with up to 10 unique data types in the order they appear
384        let error_message = {
385            // 10 max unique data types to print and another 1 to know if there are more
386            let mut unique_data_types = HashSet::with_capacity(11);
387
388            let mut error_message =
389                format!("It is not possible to concatenate arrays of different data types ({d}");
390            unique_data_types.insert(d);
391
392            for array in arrays {
393                let is_unique = unique_data_types.insert(array.data_type());
394
395                if unique_data_types.len() == 11 {
396                    error_message.push_str(", ...");
397                    break;
398                }
399
400                if is_unique {
401                    error_message.push_str(", ");
402                    error_message.push_str(&array.data_type().to_string());
403                }
404            }
405
406            error_message.push_str(").");
407
408            error_message
409        };
410
411        return Err(ArrowError::InvalidArgumentError(error_message));
412    }
413
414    downcast_primitive! {
415        d => (primitive_concat, arrays),
416        DataType::Boolean => concat_boolean(arrays),
417        DataType::Dictionary(k, _) => {
418            downcast_integer! {
419                k.as_ref() => (dict_helper, arrays),
420                _ => unreachable!("illegal dictionary key type {k}")
421            }
422        }
423        DataType::List(field) => concat_lists::<i32>(arrays, field),
424        DataType::LargeList(field) => concat_lists::<i64>(arrays, field),
425        DataType::Struct(fields) => concat_structs(arrays, fields),
426        DataType::Utf8 => concat_bytes::<Utf8Type>(arrays),
427        DataType::LargeUtf8 => concat_bytes::<LargeUtf8Type>(arrays),
428        DataType::Binary => concat_bytes::<BinaryType>(arrays),
429        DataType::LargeBinary => concat_bytes::<LargeBinaryType>(arrays),
430        DataType::RunEndEncoded(r, _) => {
431            // Handle RunEndEncoded arrays with special concat function
432            // We need to downcast based on the run end type
433            match r.data_type() {
434                DataType::Int16 => concat_run_arrays::<Int16Type>(arrays),
435                DataType::Int32 => concat_run_arrays::<Int32Type>(arrays),
436                DataType::Int64 => concat_run_arrays::<Int64Type>(arrays),
437                _ => unreachable!("Unsupported run end index type: {r:?}"),
438            }
439        }
440        DataType::Utf8View => concat_byte_view::<StringViewType>(arrays),
441        DataType::BinaryView => concat_byte_view::<BinaryViewType>(arrays),
442        _ => {
443            let capacity = get_capacity(arrays, d);
444            concat_fallback(arrays, capacity)
445        }
446    }
447}
448
449/// Concatenates arrays using MutableArrayData
450///
451/// This will naively concatenate dictionaries
452fn concat_fallback(arrays: &[&dyn Array], capacity: Capacities) -> Result<ArrayRef, ArrowError> {
453    let array_data: Vec<_> = arrays.iter().map(|a| a.to_data()).collect::<Vec<_>>();
454    let array_data = array_data.iter().collect();
455    let mut mutable = MutableArrayData::with_capacities(array_data, false, capacity);
456
457    for (i, a) in arrays.iter().enumerate() {
458        mutable.extend(i, 0, a.len())
459    }
460
461    Ok(make_array(mutable.freeze()))
462}
463
464/// Concatenates `batches` together into a single [`RecordBatch`].
465///
466/// The output batch has the specified `schemas`; The schema of the
467/// input are ignored.
468///
469/// Returns an error if the types of underlying arrays are different.
470pub fn concat_batches<'a>(
471    schema: &SchemaRef,
472    input_batches: impl IntoIterator<Item = &'a RecordBatch>,
473) -> Result<RecordBatch, ArrowError> {
474    // When schema is empty, sum the number of the rows of all batches
475    if schema.fields().is_empty() {
476        let num_rows: usize = input_batches.into_iter().map(RecordBatch::num_rows).sum();
477        let mut options = RecordBatchOptions::default();
478        options.row_count = Some(num_rows);
479        return RecordBatch::try_new_with_options(schema.clone(), vec![], &options);
480    }
481
482    let batches: Vec<&RecordBatch> = input_batches.into_iter().collect();
483    if batches.is_empty() {
484        return Ok(RecordBatch::new_empty(schema.clone()));
485    }
486    let field_num = schema.fields().len();
487    let mut arrays = Vec::with_capacity(field_num);
488    for i in 0..field_num {
489        let array = concat(
490            &batches
491                .iter()
492                .map(|batch| batch.column(i).as_ref())
493                .collect::<Vec<_>>(),
494        )?;
495        arrays.push(array);
496    }
497    RecordBatch::try_new(schema.clone(), arrays)
498}
499
500#[cfg(test)]
501mod tests {
502    use super::*;
503    use arrow_array::builder::{GenericListBuilder, StringDictionaryBuilder};
504    use arrow_schema::{Field, Schema};
505    use std::fmt::Debug;
506
507    #[test]
508    fn test_concat_empty_vec() {
509        let re = concat(&[]);
510        assert!(re.is_err());
511    }
512
513    #[test]
514    fn test_concat_batches_no_columns() {
515        // Test concat using empty schema / batches without columns
516        let schema = Arc::new(Schema::empty());
517
518        let mut options = RecordBatchOptions::default();
519        options.row_count = Some(100);
520        let batch = RecordBatch::try_new_with_options(schema.clone(), vec![], &options).unwrap();
521        // put in 2 batches of 100 rows each
522        let re = concat_batches(&schema, &[batch.clone(), batch]).unwrap();
523
524        assert_eq!(re.num_rows(), 200);
525    }
526
527    #[test]
528    fn test_concat_one_element_vec() {
529        let arr = Arc::new(PrimitiveArray::<Int64Type>::from(vec![
530            Some(-1),
531            Some(2),
532            None,
533        ])) as ArrayRef;
534        let result = concat(&[arr.as_ref()]).unwrap();
535        assert_eq!(
536            &arr, &result,
537            "concatenating single element array gives back the same result"
538        );
539    }
540
541    #[test]
542    fn test_concat_incompatible_datatypes() {
543        let re = concat(&[
544            &PrimitiveArray::<Int64Type>::from(vec![Some(-1), Some(2), None]),
545            // 2 string to make sure we only mention unique types
546            &StringArray::from(vec![Some("hello"), Some("bar"), Some("world")]),
547            &StringArray::from(vec![Some("hey"), Some(""), Some("you")]),
548            // Another type to make sure we are showing all the incompatible types
549            &PrimitiveArray::<Int32Type>::from(vec![Some(-1), Some(2), None]),
550        ]);
551
552        assert_eq!(re.unwrap_err().to_string(), "Invalid argument error: It is not possible to concatenate arrays of different data types (Int64, Utf8, Int32).");
553    }
554
555    #[test]
556    fn test_concat_10_incompatible_datatypes_should_include_all_of_them() {
557        let re = concat(&[
558            &PrimitiveArray::<Int64Type>::from(vec![Some(-1), Some(2), None]),
559            // 2 string to make sure we only mention unique types
560            &StringArray::from(vec![Some("hello"), Some("bar"), Some("world")]),
561            &StringArray::from(vec![Some("hey"), Some(""), Some("you")]),
562            // Another type to make sure we are showing all the incompatible types
563            &PrimitiveArray::<Int32Type>::from(vec![Some(-1), Some(2), None]),
564            &PrimitiveArray::<Int8Type>::from(vec![Some(-1), Some(2), None]),
565            &PrimitiveArray::<Int16Type>::from(vec![Some(-1), Some(2), None]),
566            &PrimitiveArray::<UInt8Type>::from(vec![Some(1), Some(2), None]),
567            &PrimitiveArray::<UInt16Type>::from(vec![Some(1), Some(2), None]),
568            &PrimitiveArray::<UInt32Type>::from(vec![Some(1), Some(2), None]),
569            // Non unique
570            &PrimitiveArray::<UInt16Type>::from(vec![Some(1), Some(2), None]),
571            &PrimitiveArray::<UInt64Type>::from(vec![Some(1), Some(2), None]),
572            &PrimitiveArray::<Float32Type>::from(vec![Some(1.0), Some(2.0), None]),
573        ]);
574
575        assert_eq!(re.unwrap_err().to_string(), "Invalid argument error: It is not possible to concatenate arrays of different data types (Int64, Utf8, Int32, Int8, Int16, UInt8, UInt16, UInt32, UInt64, Float32).");
576    }
577
578    #[test]
579    fn test_concat_11_incompatible_datatypes_should_only_include_10() {
580        let re = concat(&[
581            &PrimitiveArray::<Int64Type>::from(vec![Some(-1), Some(2), None]),
582            // 2 string to make sure we only mention unique types
583            &StringArray::from(vec![Some("hello"), Some("bar"), Some("world")]),
584            &StringArray::from(vec![Some("hey"), Some(""), Some("you")]),
585            // Another type to make sure we are showing all the incompatible types
586            &PrimitiveArray::<Int32Type>::from(vec![Some(-1), Some(2), None]),
587            &PrimitiveArray::<Int8Type>::from(vec![Some(-1), Some(2), None]),
588            &PrimitiveArray::<Int16Type>::from(vec![Some(-1), Some(2), None]),
589            &PrimitiveArray::<UInt8Type>::from(vec![Some(1), Some(2), None]),
590            &PrimitiveArray::<UInt16Type>::from(vec![Some(1), Some(2), None]),
591            &PrimitiveArray::<UInt32Type>::from(vec![Some(1), Some(2), None]),
592            // Non unique
593            &PrimitiveArray::<UInt16Type>::from(vec![Some(1), Some(2), None]),
594            &PrimitiveArray::<UInt64Type>::from(vec![Some(1), Some(2), None]),
595            &PrimitiveArray::<Float32Type>::from(vec![Some(1.0), Some(2.0), None]),
596            &PrimitiveArray::<Float64Type>::from(vec![Some(1.0), Some(2.0), None]),
597        ]);
598
599        assert_eq!(re.unwrap_err().to_string(), "Invalid argument error: It is not possible to concatenate arrays of different data types (Int64, Utf8, Int32, Int8, Int16, UInt8, UInt16, UInt32, UInt64, Float32, ...).");
600    }
601
602    #[test]
603    fn test_concat_13_incompatible_datatypes_should_not_include_all_of_them() {
604        let re = concat(&[
605            &PrimitiveArray::<Int64Type>::from(vec![Some(-1), Some(2), None]),
606            // 2 string to make sure we only mention unique types
607            &StringArray::from(vec![Some("hello"), Some("bar"), Some("world")]),
608            &StringArray::from(vec![Some("hey"), Some(""), Some("you")]),
609            // Another type to make sure we are showing all the incompatible types
610            &PrimitiveArray::<Int32Type>::from(vec![Some(-1), Some(2), None]),
611            &PrimitiveArray::<Int8Type>::from(vec![Some(-1), Some(2), None]),
612            &PrimitiveArray::<Int16Type>::from(vec![Some(-1), Some(2), None]),
613            &PrimitiveArray::<UInt8Type>::from(vec![Some(1), Some(2), None]),
614            &PrimitiveArray::<UInt16Type>::from(vec![Some(1), Some(2), None]),
615            &PrimitiveArray::<UInt32Type>::from(vec![Some(1), Some(2), None]),
616            // Non unique
617            &PrimitiveArray::<UInt16Type>::from(vec![Some(1), Some(2), None]),
618            &PrimitiveArray::<UInt64Type>::from(vec![Some(1), Some(2), None]),
619            &PrimitiveArray::<Float32Type>::from(vec![Some(1.0), Some(2.0), None]),
620            &PrimitiveArray::<Float64Type>::from(vec![Some(1.0), Some(2.0), None]),
621            &PrimitiveArray::<Float16Type>::new_null(3),
622            &BooleanArray::from(vec![Some(true), Some(false), None]),
623        ]);
624
625        assert_eq!(re.unwrap_err().to_string(), "Invalid argument error: It is not possible to concatenate arrays of different data types (Int64, Utf8, Int32, Int8, Int16, UInt8, UInt16, UInt32, UInt64, Float32, ...).");
626    }
627
628    #[test]
629    fn test_concat_string_arrays() {
630        let arr = concat(&[
631            &StringArray::from(vec!["hello", "world"]),
632            &StringArray::from(vec!["2", "3", "4"]),
633            &StringArray::from(vec![Some("foo"), Some("bar"), None, Some("baz")]),
634        ])
635        .unwrap();
636
637        let expected_output = Arc::new(StringArray::from(vec![
638            Some("hello"),
639            Some("world"),
640            Some("2"),
641            Some("3"),
642            Some("4"),
643            Some("foo"),
644            Some("bar"),
645            None,
646            Some("baz"),
647        ])) as ArrayRef;
648
649        assert_eq!(&arr, &expected_output);
650    }
651
652    #[test]
653    fn test_concat_string_view_arrays() {
654        let arr = concat(&[
655            &StringViewArray::from(vec!["helloxxxxxxxxxxa", "world____________"]),
656            &StringViewArray::from(vec!["helloxxxxxxxxxxy", "3", "4"]),
657            &StringViewArray::from(vec![Some("foo"), Some("bar"), None, Some("baz")]),
658        ])
659        .unwrap();
660
661        let expected_output = Arc::new(StringViewArray::from(vec![
662            Some("helloxxxxxxxxxxa"),
663            Some("world____________"),
664            Some("helloxxxxxxxxxxy"),
665            Some("3"),
666            Some("4"),
667            Some("foo"),
668            Some("bar"),
669            None,
670            Some("baz"),
671        ])) as ArrayRef;
672
673        assert_eq!(&arr, &expected_output);
674    }
675
676    #[test]
677    fn test_concat_primitive_arrays() {
678        let arr = concat(&[
679            &PrimitiveArray::<Int64Type>::from(vec![Some(-1), Some(-1), Some(2), None, None]),
680            &PrimitiveArray::<Int64Type>::from(vec![Some(101), Some(102), Some(103), None]),
681            &PrimitiveArray::<Int64Type>::from(vec![Some(256), Some(512), Some(1024)]),
682        ])
683        .unwrap();
684
685        let expected_output = Arc::new(PrimitiveArray::<Int64Type>::from(vec![
686            Some(-1),
687            Some(-1),
688            Some(2),
689            None,
690            None,
691            Some(101),
692            Some(102),
693            Some(103),
694            None,
695            Some(256),
696            Some(512),
697            Some(1024),
698        ])) as ArrayRef;
699
700        assert_eq!(&arr, &expected_output);
701    }
702
703    #[test]
704    fn test_concat_primitive_array_slices() {
705        let input_1 =
706            PrimitiveArray::<Int64Type>::from(vec![Some(-1), Some(-1), Some(2), None, None])
707                .slice(1, 3);
708
709        let input_2 =
710            PrimitiveArray::<Int64Type>::from(vec![Some(101), Some(102), Some(103), None])
711                .slice(1, 3);
712        let arr = concat(&[&input_1, &input_2]).unwrap();
713
714        let expected_output = Arc::new(PrimitiveArray::<Int64Type>::from(vec![
715            Some(-1),
716            Some(2),
717            None,
718            Some(102),
719            Some(103),
720            None,
721        ])) as ArrayRef;
722
723        assert_eq!(&arr, &expected_output);
724    }
725
726    #[test]
727    fn test_concat_boolean_primitive_arrays() {
728        let arr = concat(&[
729            &BooleanArray::from(vec![
730                Some(true),
731                Some(true),
732                Some(false),
733                None,
734                None,
735                Some(false),
736            ]),
737            &BooleanArray::from(vec![None, Some(false), Some(true), Some(false)]),
738        ])
739        .unwrap();
740
741        let expected_output = Arc::new(BooleanArray::from(vec![
742            Some(true),
743            Some(true),
744            Some(false),
745            None,
746            None,
747            Some(false),
748            None,
749            Some(false),
750            Some(true),
751            Some(false),
752        ])) as ArrayRef;
753
754        assert_eq!(&arr, &expected_output);
755    }
756
757    #[test]
758    fn test_concat_primitive_list_arrays() {
759        let list1 = vec![
760            Some(vec![Some(-1), Some(-1), Some(2), None, None]),
761            Some(vec![]),
762            None,
763            Some(vec![Some(10)]),
764        ];
765        let list1_array = ListArray::from_iter_primitive::<Int64Type, _, _>(list1.clone());
766
767        let list2 = vec![
768            None,
769            Some(vec![Some(100), None, Some(101)]),
770            Some(vec![Some(102)]),
771        ];
772        let list2_array = ListArray::from_iter_primitive::<Int64Type, _, _>(list2.clone());
773
774        let list3 = vec![Some(vec![Some(1000), Some(1001)])];
775        let list3_array = ListArray::from_iter_primitive::<Int64Type, _, _>(list3.clone());
776
777        let array_result = concat(&[&list1_array, &list2_array, &list3_array]).unwrap();
778
779        let expected = list1.into_iter().chain(list2).chain(list3);
780        let array_expected = ListArray::from_iter_primitive::<Int64Type, _, _>(expected);
781
782        assert_eq!(array_result.as_ref(), &array_expected as &dyn Array);
783    }
784
785    #[test]
786    fn test_concat_primitive_list_arrays_slices() {
787        let list1 = vec![
788            Some(vec![Some(-1), Some(-1), Some(2), None, None]),
789            Some(vec![]), // In slice
790            None,         // In slice
791            Some(vec![Some(10)]),
792        ];
793        let list1_array = ListArray::from_iter_primitive::<Int64Type, _, _>(list1.clone());
794        let list1_array = list1_array.slice(1, 2);
795        let list1_values = list1.into_iter().skip(1).take(2);
796
797        let list2 = vec![
798            None,
799            Some(vec![Some(100), None, Some(101)]),
800            Some(vec![Some(102)]),
801        ];
802        let list2_array = ListArray::from_iter_primitive::<Int64Type, _, _>(list2.clone());
803
804        // verify that this test covers the case when the first offset is non zero
805        assert!(list1_array.offsets()[0].as_usize() > 0);
806        let array_result = concat(&[&list1_array, &list2_array]).unwrap();
807
808        let expected = list1_values.chain(list2);
809        let array_expected = ListArray::from_iter_primitive::<Int64Type, _, _>(expected);
810
811        assert_eq!(array_result.as_ref(), &array_expected as &dyn Array);
812    }
813
814    #[test]
815    fn test_concat_primitive_list_arrays_sliced_lengths() {
816        let list1 = vec![
817            Some(vec![Some(-1), Some(-1), Some(2), None, None]), // In slice
818            Some(vec![]),                                        // In slice
819            None,                                                // In slice
820            Some(vec![Some(10)]),
821        ];
822        let list1_array = ListArray::from_iter_primitive::<Int64Type, _, _>(list1.clone());
823        let list1_array = list1_array.slice(0, 3); // no offset, but not all values
824        let list1_values = list1.into_iter().take(3);
825
826        let list2 = vec![
827            None,
828            Some(vec![Some(100), None, Some(101)]),
829            Some(vec![Some(102)]),
830        ];
831        let list2_array = ListArray::from_iter_primitive::<Int64Type, _, _>(list2.clone());
832
833        // verify that this test covers the case when the first offset is zero, but the
834        // last offset doesn't cover the entire array
835        assert_eq!(list1_array.offsets()[0].as_usize(), 0);
836        assert!(list1_array.offsets().last().unwrap().as_usize() < list1_array.values().len());
837        let array_result = concat(&[&list1_array, &list2_array]).unwrap();
838
839        let expected = list1_values.chain(list2);
840        let array_expected = ListArray::from_iter_primitive::<Int64Type, _, _>(expected);
841
842        assert_eq!(array_result.as_ref(), &array_expected as &dyn Array);
843    }
844
845    #[test]
846    fn test_concat_primitive_fixed_size_list_arrays() {
847        let list1 = vec![
848            Some(vec![Some(-1), None]),
849            None,
850            Some(vec![Some(10), Some(20)]),
851        ];
852        let list1_array =
853            FixedSizeListArray::from_iter_primitive::<Int64Type, _, _>(list1.clone(), 2);
854
855        let list2 = vec![
856            None,
857            Some(vec![Some(100), None]),
858            Some(vec![Some(102), Some(103)]),
859        ];
860        let list2_array =
861            FixedSizeListArray::from_iter_primitive::<Int64Type, _, _>(list2.clone(), 2);
862
863        let list3 = vec![Some(vec![Some(1000), Some(1001)])];
864        let list3_array =
865            FixedSizeListArray::from_iter_primitive::<Int64Type, _, _>(list3.clone(), 2);
866
867        let array_result = concat(&[&list1_array, &list2_array, &list3_array]).unwrap();
868
869        let expected = list1.into_iter().chain(list2).chain(list3);
870        let array_expected =
871            FixedSizeListArray::from_iter_primitive::<Int64Type, _, _>(expected, 2);
872
873        assert_eq!(array_result.as_ref(), &array_expected as &dyn Array);
874    }
875
876    #[test]
877    fn test_concat_struct_arrays() {
878        let field = Arc::new(Field::new("field", DataType::Int64, true));
879        let input_primitive_1: ArrayRef = Arc::new(PrimitiveArray::<Int64Type>::from(vec![
880            Some(-1),
881            Some(-1),
882            Some(2),
883            None,
884            None,
885        ]));
886        let input_struct_1 = StructArray::from(vec![(field.clone(), input_primitive_1)]);
887
888        let input_primitive_2: ArrayRef = Arc::new(PrimitiveArray::<Int64Type>::from(vec![
889            Some(101),
890            Some(102),
891            Some(103),
892            None,
893        ]));
894        let input_struct_2 = StructArray::from(vec![(field.clone(), input_primitive_2)]);
895
896        let input_primitive_3: ArrayRef = Arc::new(PrimitiveArray::<Int64Type>::from(vec![
897            Some(256),
898            Some(512),
899            Some(1024),
900        ]));
901        let input_struct_3 = StructArray::from(vec![(field, input_primitive_3)]);
902
903        let arr = concat(&[&input_struct_1, &input_struct_2, &input_struct_3]).unwrap();
904
905        let expected_primitive_output = Arc::new(PrimitiveArray::<Int64Type>::from(vec![
906            Some(-1),
907            Some(-1),
908            Some(2),
909            None,
910            None,
911            Some(101),
912            Some(102),
913            Some(103),
914            None,
915            Some(256),
916            Some(512),
917            Some(1024),
918        ])) as ArrayRef;
919
920        let actual_primitive = arr
921            .as_any()
922            .downcast_ref::<StructArray>()
923            .unwrap()
924            .column(0);
925        assert_eq!(actual_primitive, &expected_primitive_output);
926    }
927
928    #[test]
929    fn test_concat_struct_array_slices() {
930        let field = Arc::new(Field::new("field", DataType::Int64, true));
931        let input_primitive_1: ArrayRef = Arc::new(PrimitiveArray::<Int64Type>::from(vec![
932            Some(-1),
933            Some(-1),
934            Some(2),
935            None,
936            None,
937        ]));
938        let input_struct_1 = StructArray::from(vec![(field.clone(), input_primitive_1)]);
939
940        let input_primitive_2: ArrayRef = Arc::new(PrimitiveArray::<Int64Type>::from(vec![
941            Some(101),
942            Some(102),
943            Some(103),
944            None,
945        ]));
946        let input_struct_2 = StructArray::from(vec![(field, input_primitive_2)]);
947
948        let arr = concat(&[&input_struct_1.slice(1, 3), &input_struct_2.slice(1, 2)]).unwrap();
949
950        let expected_primitive_output = Arc::new(PrimitiveArray::<Int64Type>::from(vec![
951            Some(-1),
952            Some(2),
953            None,
954            Some(102),
955            Some(103),
956        ])) as ArrayRef;
957
958        let actual_primitive = arr
959            .as_any()
960            .downcast_ref::<StructArray>()
961            .unwrap()
962            .column(0);
963        assert_eq!(actual_primitive, &expected_primitive_output);
964    }
965
966    #[test]
967    fn test_concat_struct_arrays_no_nulls() {
968        let input_1a = vec![1, 2, 3];
969        let input_1b = vec!["one", "two", "three"];
970        let input_2a = vec![4, 5, 6, 7];
971        let input_2b = vec!["four", "five", "six", "seven"];
972
973        let struct_from_primitives = |ints: Vec<i64>, strings: Vec<&str>| {
974            StructArray::try_from(vec![
975                ("ints", Arc::new(Int64Array::from(ints)) as _),
976                ("strings", Arc::new(StringArray::from(strings)) as _),
977            ])
978        };
979
980        let expected_output = struct_from_primitives(
981            [input_1a.clone(), input_2a.clone()].concat(),
982            [input_1b.clone(), input_2b.clone()].concat(),
983        )
984        .unwrap();
985
986        let input_1 = struct_from_primitives(input_1a, input_1b).unwrap();
987        let input_2 = struct_from_primitives(input_2a, input_2b).unwrap();
988
989        let arr = concat(&[&input_1, &input_2]).unwrap();
990        let struct_result = arr.as_struct();
991
992        assert_eq!(struct_result, &expected_output);
993        assert_eq!(arr.null_count(), 0);
994    }
995
996    #[test]
997    fn test_concat_struct_no_fields() {
998        let input_1 = StructArray::new_empty_fields(10, None);
999        let input_2 = StructArray::new_empty_fields(10, None);
1000        let arr = concat(&[&input_1, &input_2]).unwrap();
1001
1002        assert_eq!(arr.len(), 20);
1003        assert_eq!(arr.null_count(), 0);
1004
1005        let input1_valid = StructArray::new_empty_fields(10, Some(NullBuffer::new_valid(10)));
1006        let input2_null = StructArray::new_empty_fields(10, Some(NullBuffer::new_null(10)));
1007        let arr = concat(&[&input1_valid, &input2_null]).unwrap();
1008
1009        assert_eq!(arr.len(), 20);
1010        assert_eq!(arr.null_count(), 10);
1011    }
1012
1013    #[test]
1014    fn test_string_array_slices() {
1015        let input_1 = StringArray::from(vec!["hello", "A", "B", "C"]);
1016        let input_2 = StringArray::from(vec!["world", "D", "E", "Z"]);
1017
1018        let arr = concat(&[&input_1.slice(1, 3), &input_2.slice(1, 2)]).unwrap();
1019
1020        let expected_output = StringArray::from(vec!["A", "B", "C", "D", "E"]);
1021
1022        let actual_output = arr.as_any().downcast_ref::<StringArray>().unwrap();
1023        assert_eq!(actual_output, &expected_output);
1024    }
1025
1026    #[test]
1027    fn test_string_array_with_null_slices() {
1028        let input_1 = StringArray::from(vec![Some("hello"), None, Some("A"), Some("C")]);
1029        let input_2 = StringArray::from(vec![None, Some("world"), Some("D"), None]);
1030
1031        let arr = concat(&[&input_1.slice(1, 3), &input_2.slice(1, 2)]).unwrap();
1032
1033        let expected_output =
1034            StringArray::from(vec![None, Some("A"), Some("C"), Some("world"), Some("D")]);
1035
1036        let actual_output = arr.as_any().downcast_ref::<StringArray>().unwrap();
1037        assert_eq!(actual_output, &expected_output);
1038    }
1039
1040    fn collect_string_dictionary(array: &DictionaryArray<Int32Type>) -> Vec<Option<&str>> {
1041        let concrete = array.downcast_dict::<StringArray>().unwrap();
1042        concrete.into_iter().collect()
1043    }
1044
1045    #[test]
1046    fn test_string_dictionary_array() {
1047        let input_1: DictionaryArray<Int32Type> = vec!["hello", "A", "B", "hello", "hello", "C"]
1048            .into_iter()
1049            .collect();
1050        let input_2: DictionaryArray<Int32Type> = vec!["hello", "E", "E", "hello", "F", "E"]
1051            .into_iter()
1052            .collect();
1053
1054        let expected: Vec<_> = vec![
1055            "hello", "A", "B", "hello", "hello", "C", "hello", "E", "E", "hello", "F", "E",
1056        ]
1057        .into_iter()
1058        .map(Some)
1059        .collect();
1060
1061        let concat = concat(&[&input_1 as _, &input_2 as _]).unwrap();
1062        let dictionary = concat.as_dictionary::<Int32Type>();
1063        let actual = collect_string_dictionary(dictionary);
1064        assert_eq!(actual, expected);
1065
1066        // Should have concatenated inputs together
1067        assert_eq!(
1068            dictionary.values().len(),
1069            input_1.values().len() + input_2.values().len(),
1070        )
1071    }
1072
1073    #[test]
1074    fn test_string_dictionary_array_nulls() {
1075        let input_1: DictionaryArray<Int32Type> = vec![Some("foo"), Some("bar"), None, Some("fiz")]
1076            .into_iter()
1077            .collect();
1078        let input_2: DictionaryArray<Int32Type> = vec![None].into_iter().collect();
1079        let expected = vec![Some("foo"), Some("bar"), None, Some("fiz"), None];
1080
1081        let concat = concat(&[&input_1 as _, &input_2 as _]).unwrap();
1082        let dictionary = concat.as_dictionary::<Int32Type>();
1083        let actual = collect_string_dictionary(dictionary);
1084        assert_eq!(actual, expected);
1085
1086        // Should have concatenated inputs together
1087        assert_eq!(
1088            dictionary.values().len(),
1089            input_1.values().len() + input_2.values().len(),
1090        )
1091    }
1092
1093    #[test]
1094    fn test_string_dictionary_array_nulls_in_values() {
1095        let input_1_keys = Int32Array::from_iter_values([0, 2, 1, 3]);
1096        let input_1_values = StringArray::from(vec![Some("foo"), None, Some("bar"), Some("fiz")]);
1097        let input_1 = DictionaryArray::new(input_1_keys, Arc::new(input_1_values));
1098
1099        let input_2_keys = Int32Array::from_iter_values([0]);
1100        let input_2_values = StringArray::from(vec![None, Some("hello")]);
1101        let input_2 = DictionaryArray::new(input_2_keys, Arc::new(input_2_values));
1102
1103        let expected = vec![Some("foo"), Some("bar"), None, Some("fiz"), None];
1104
1105        let concat = concat(&[&input_1 as _, &input_2 as _]).unwrap();
1106        let dictionary = concat.as_dictionary::<Int32Type>();
1107        let actual = collect_string_dictionary(dictionary);
1108        assert_eq!(actual, expected);
1109    }
1110
1111    #[test]
1112    fn test_string_dictionary_merge() {
1113        let mut builder = StringDictionaryBuilder::<Int32Type>::new();
1114        for i in 0..20 {
1115            builder.append(i.to_string()).unwrap();
1116        }
1117        let input_1 = builder.finish();
1118
1119        let mut builder = StringDictionaryBuilder::<Int32Type>::new();
1120        for i in 0..30 {
1121            builder.append(i.to_string()).unwrap();
1122        }
1123        let input_2 = builder.finish();
1124
1125        let expected: Vec<_> = (0..20).chain(0..30).map(|x| x.to_string()).collect();
1126        let expected: Vec<_> = expected.iter().map(|x| Some(x.as_str())).collect();
1127
1128        let concat = concat(&[&input_1 as _, &input_2 as _]).unwrap();
1129        let dictionary = concat.as_dictionary::<Int32Type>();
1130        let actual = collect_string_dictionary(dictionary);
1131        assert_eq!(actual, expected);
1132
1133        // Should have merged inputs together
1134        // Not 30 as this is done on a best-effort basis
1135        let values_len = dictionary.values().len();
1136        assert!((30..40).contains(&values_len), "{values_len}")
1137    }
1138
1139    #[test]
1140    fn test_primitive_dictionary_merge() {
1141        // Same value repeated 5 times.
1142        let keys = vec![1; 5];
1143        let values = (10..20).collect::<Vec<_>>();
1144        let dict = DictionaryArray::new(
1145            Int8Array::from(keys.clone()),
1146            Arc::new(Int32Array::from(values.clone())),
1147        );
1148        let other = DictionaryArray::new(
1149            Int8Array::from(keys.clone()),
1150            Arc::new(Int32Array::from(values.clone())),
1151        );
1152
1153        let result_same_dictionary = concat(&[&dict, &dict]).unwrap();
1154        // Verify pointer equality check succeeds, and therefore the
1155        // dictionaries are not merged. A single values buffer should be reused
1156        // in this case.
1157        assert!(dict.values().to_data().ptr_eq(
1158            &result_same_dictionary
1159                .as_dictionary::<Int8Type>()
1160                .values()
1161                .to_data()
1162        ));
1163        assert_eq!(
1164            result_same_dictionary
1165                .as_dictionary::<Int8Type>()
1166                .values()
1167                .len(),
1168            values.len(),
1169        );
1170
1171        let result_cloned_dictionary = concat(&[&dict, &other]).unwrap();
1172        // Should have only 1 underlying value since all keys reference it.
1173        assert_eq!(
1174            result_cloned_dictionary
1175                .as_dictionary::<Int8Type>()
1176                .values()
1177                .len(),
1178            1
1179        );
1180    }
1181
1182    #[test]
1183    fn test_concat_string_sizes() {
1184        let a: LargeStringArray = ((0..150).map(|_| Some("foo"))).collect();
1185        let b: LargeStringArray = ((0..150).map(|_| Some("foo"))).collect();
1186        let c = LargeStringArray::from(vec![Some("foo"), Some("bar"), None, Some("baz")]);
1187        // 150 * 3 = 450
1188        // 150 * 3 = 450
1189        // 3 * 3   = 9
1190        // ------------+
1191        // 909
1192        // closest 64 byte aligned cap = 960
1193
1194        let arr = concat(&[&a, &b, &c]).unwrap();
1195        // this would have been 1280 if we did not precompute the value lengths.
1196        assert_eq!(arr.to_data().buffers()[1].capacity(), 960);
1197    }
1198
1199    #[test]
1200    fn test_dictionary_concat_reuse() {
1201        let array: DictionaryArray<Int8Type> = vec!["a", "a", "b", "c"].into_iter().collect();
1202        let copy: DictionaryArray<Int8Type> = array.clone();
1203
1204        // dictionary is "a", "b", "c"
1205        assert_eq!(
1206            array.values(),
1207            &(Arc::new(StringArray::from(vec!["a", "b", "c"])) as ArrayRef)
1208        );
1209        assert_eq!(array.keys(), &Int8Array::from(vec![0, 0, 1, 2]));
1210
1211        // concatenate it with itself
1212        let combined = concat(&[&copy as _, &array as _]).unwrap();
1213        let combined = combined.as_dictionary::<Int8Type>();
1214
1215        assert_eq!(
1216            combined.values(),
1217            &(Arc::new(StringArray::from(vec!["a", "b", "c"])) as ArrayRef),
1218            "Actual: {combined:#?}"
1219        );
1220
1221        assert_eq!(
1222            combined.keys(),
1223            &Int8Array::from(vec![0, 0, 1, 2, 0, 0, 1, 2])
1224        );
1225
1226        // Should have reused the dictionary
1227        assert!(array
1228            .values()
1229            .to_data()
1230            .ptr_eq(&combined.values().to_data()));
1231        assert!(copy.values().to_data().ptr_eq(&combined.values().to_data()));
1232
1233        let new: DictionaryArray<Int8Type> = vec!["d"].into_iter().collect();
1234        let combined = concat(&[&copy as _, &array as _, &new as _]).unwrap();
1235        let com = combined.as_dictionary::<Int8Type>();
1236
1237        // Should not have reused the dictionary
1238        assert!(!array.values().to_data().ptr_eq(&com.values().to_data()));
1239        assert!(!copy.values().to_data().ptr_eq(&com.values().to_data()));
1240        assert!(!new.values().to_data().ptr_eq(&com.values().to_data()));
1241    }
1242
1243    #[test]
1244    fn concat_record_batches() {
1245        let schema = Arc::new(Schema::new(vec![
1246            Field::new("a", DataType::Int32, false),
1247            Field::new("b", DataType::Utf8, false),
1248        ]));
1249        let batch1 = RecordBatch::try_new(
1250            schema.clone(),
1251            vec![
1252                Arc::new(Int32Array::from(vec![1, 2])),
1253                Arc::new(StringArray::from(vec!["a", "b"])),
1254            ],
1255        )
1256        .unwrap();
1257        let batch2 = RecordBatch::try_new(
1258            schema.clone(),
1259            vec![
1260                Arc::new(Int32Array::from(vec![3, 4])),
1261                Arc::new(StringArray::from(vec!["c", "d"])),
1262            ],
1263        )
1264        .unwrap();
1265        let new_batch = concat_batches(&schema, [&batch1, &batch2]).unwrap();
1266        assert_eq!(new_batch.schema().as_ref(), schema.as_ref());
1267        assert_eq!(2, new_batch.num_columns());
1268        assert_eq!(4, new_batch.num_rows());
1269        let new_batch_owned = concat_batches(&schema, &[batch1, batch2]).unwrap();
1270        assert_eq!(new_batch_owned.schema().as_ref(), schema.as_ref());
1271        assert_eq!(2, new_batch_owned.num_columns());
1272        assert_eq!(4, new_batch_owned.num_rows());
1273    }
1274
1275    #[test]
1276    fn concat_empty_record_batch() {
1277        let schema = Arc::new(Schema::new(vec![
1278            Field::new("a", DataType::Int32, false),
1279            Field::new("b", DataType::Utf8, false),
1280        ]));
1281        let batch = concat_batches(&schema, []).unwrap();
1282        assert_eq!(batch.schema().as_ref(), schema.as_ref());
1283        assert_eq!(0, batch.num_rows());
1284    }
1285
1286    #[test]
1287    fn concat_record_batches_of_different_schemas_but_compatible_data() {
1288        let schema1 = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
1289        // column names differ
1290        let schema2 = Arc::new(Schema::new(vec![Field::new("c", DataType::Int32, false)]));
1291        let batch1 = RecordBatch::try_new(
1292            schema1.clone(),
1293            vec![Arc::new(Int32Array::from(vec![1, 2]))],
1294        )
1295        .unwrap();
1296        let batch2 =
1297            RecordBatch::try_new(schema2, vec![Arc::new(Int32Array::from(vec![3, 4]))]).unwrap();
1298        // concat_batches simply uses the schema provided
1299        let batch = concat_batches(&schema1, [&batch1, &batch2]).unwrap();
1300        assert_eq!(batch.schema().as_ref(), schema1.as_ref());
1301        assert_eq!(4, batch.num_rows());
1302    }
1303
1304    #[test]
1305    fn concat_record_batches_of_different_schemas_incompatible_data() {
1306        let schema1 = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
1307        // column names differ
1308        let schema2 = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false)]));
1309        let batch1 = RecordBatch::try_new(
1310            schema1.clone(),
1311            vec![Arc::new(Int32Array::from(vec![1, 2]))],
1312        )
1313        .unwrap();
1314        let batch2 = RecordBatch::try_new(
1315            schema2,
1316            vec![Arc::new(StringArray::from(vec!["foo", "bar"]))],
1317        )
1318        .unwrap();
1319
1320        let error = concat_batches(&schema1, [&batch1, &batch2]).unwrap_err();
1321        assert_eq!(error.to_string(), "Invalid argument error: It is not possible to concatenate arrays of different data types (Int32, Utf8).");
1322    }
1323
1324    #[test]
1325    fn concat_capacity() {
1326        let a = Int32Array::from_iter_values(0..100);
1327        let b = Int32Array::from_iter_values(10..20);
1328        let a = concat(&[&a, &b]).unwrap();
1329        let data = a.to_data();
1330        assert_eq!(data.buffers()[0].len(), 440);
1331        assert_eq!(data.buffers()[0].capacity(), 448); // Nearest multiple of 64
1332
1333        let a = concat(&[&a.slice(10, 20), &b]).unwrap();
1334        let data = a.to_data();
1335        assert_eq!(data.buffers()[0].len(), 120);
1336        assert_eq!(data.buffers()[0].capacity(), 128); // Nearest multiple of 64
1337
1338        let a = StringArray::from_iter_values(std::iter::repeat("foo").take(100));
1339        let b = StringArray::from(vec!["bingo", "bongo", "lorem", ""]);
1340
1341        let a = concat(&[&a, &b]).unwrap();
1342        let data = a.to_data();
1343        // (100 + 4 + 1) * size_of<i32>()
1344        assert_eq!(data.buffers()[0].len(), 420);
1345        assert_eq!(data.buffers()[0].capacity(), 448); // Nearest multiple of 64
1346
1347        // len("foo") * 100 + len("bingo") + len("bongo") + len("lorem")
1348        assert_eq!(data.buffers()[1].len(), 315);
1349        assert_eq!(data.buffers()[1].capacity(), 320); // Nearest multiple of 64
1350
1351        let a = concat(&[&a.slice(10, 40), &b]).unwrap();
1352        let data = a.to_data();
1353        // (40 + 4 + 5) * size_of<i32>()
1354        assert_eq!(data.buffers()[0].len(), 180);
1355        assert_eq!(data.buffers()[0].capacity(), 192); // Nearest multiple of 64
1356
1357        // len("foo") * 40 + len("bingo") + len("bongo") + len("lorem")
1358        assert_eq!(data.buffers()[1].len(), 135);
1359        assert_eq!(data.buffers()[1].capacity(), 192); // Nearest multiple of 64
1360
1361        let a = LargeBinaryArray::from_iter_values(std::iter::repeat(b"foo").take(100));
1362        let b = LargeBinaryArray::from_iter_values(std::iter::repeat(b"cupcakes").take(10));
1363
1364        let a = concat(&[&a, &b]).unwrap();
1365        let data = a.to_data();
1366        // (100 + 10 + 1) * size_of<i64>()
1367        assert_eq!(data.buffers()[0].len(), 888);
1368        assert_eq!(data.buffers()[0].capacity(), 896); // Nearest multiple of 64
1369
1370        // len("foo") * 100 + len("cupcakes") * 10
1371        assert_eq!(data.buffers()[1].len(), 380);
1372        assert_eq!(data.buffers()[1].capacity(), 384); // Nearest multiple of 64
1373
1374        let a = concat(&[&a.slice(10, 40), &b]).unwrap();
1375        let data = a.to_data();
1376        // (40 + 10 + 1) * size_of<i64>()
1377        assert_eq!(data.buffers()[0].len(), 408);
1378        assert_eq!(data.buffers()[0].capacity(), 448); // Nearest multiple of 64
1379
1380        // len("foo") * 40 + len("cupcakes") * 10
1381        assert_eq!(data.buffers()[1].len(), 200);
1382        assert_eq!(data.buffers()[1].capacity(), 256); // Nearest multiple of 64
1383    }
1384
1385    #[test]
1386    fn concat_sparse_nulls() {
1387        let values = StringArray::from_iter_values((0..100).map(|x| x.to_string()));
1388        let keys = Int32Array::from(vec![1; 10]);
1389        let dict_a = DictionaryArray::new(keys, Arc::new(values));
1390        let values = StringArray::new_null(0);
1391        let keys = Int32Array::new_null(10);
1392        let dict_b = DictionaryArray::new(keys, Arc::new(values));
1393        let array = concat(&[&dict_a, &dict_b]).unwrap();
1394        assert_eq!(array.null_count(), 10);
1395        assert_eq!(array.logical_null_count(), 10);
1396    }
1397
1398    #[test]
1399    fn concat_dictionary_list_array_simple() {
1400        let scalars = vec![
1401            create_single_row_list_of_dict(vec![Some("a")]),
1402            create_single_row_list_of_dict(vec![Some("a")]),
1403            create_single_row_list_of_dict(vec![Some("b")]),
1404        ];
1405
1406        let arrays = scalars
1407            .iter()
1408            .map(|a| a as &(dyn Array))
1409            .collect::<Vec<_>>();
1410        let concat_res = concat(arrays.as_slice()).unwrap();
1411
1412        let expected_list = create_list_of_dict(vec![
1413            // Row 1
1414            Some(vec![Some("a")]),
1415            Some(vec![Some("a")]),
1416            Some(vec![Some("b")]),
1417        ]);
1418
1419        let list = concat_res.as_list::<i32>();
1420
1421        // Assert that the list is equal to the expected list
1422        list.iter().zip(expected_list.iter()).for_each(|(a, b)| {
1423            assert_eq!(a, b);
1424        });
1425
1426        assert_dictionary_has_unique_values::<_, StringArray>(
1427            list.values().as_dictionary::<Int32Type>(),
1428        );
1429    }
1430
1431    #[test]
1432    fn concat_many_dictionary_list_arrays() {
1433        let number_of_unique_values = 8;
1434        let scalars = (0..80000)
1435            .map(|i| {
1436                create_single_row_list_of_dict(vec![Some(
1437                    (i % number_of_unique_values).to_string(),
1438                )])
1439            })
1440            .collect::<Vec<_>>();
1441
1442        let arrays = scalars
1443            .iter()
1444            .map(|a| a as &(dyn Array))
1445            .collect::<Vec<_>>();
1446        let concat_res = concat(arrays.as_slice()).unwrap();
1447
1448        let expected_list = create_list_of_dict(
1449            (0..80000)
1450                .map(|i| Some(vec![Some((i % number_of_unique_values).to_string())]))
1451                .collect::<Vec<_>>(),
1452        );
1453
1454        let list = concat_res.as_list::<i32>();
1455
1456        // Assert that the list is equal to the expected list
1457        list.iter().zip(expected_list.iter()).for_each(|(a, b)| {
1458            assert_eq!(a, b);
1459        });
1460
1461        assert_dictionary_has_unique_values::<_, StringArray>(
1462            list.values().as_dictionary::<Int32Type>(),
1463        );
1464    }
1465
1466    fn create_single_row_list_of_dict(
1467        list_items: Vec<Option<impl AsRef<str>>>,
1468    ) -> GenericListArray<i32> {
1469        let rows = list_items.into_iter().map(Some).collect();
1470
1471        create_list_of_dict(vec![rows])
1472    }
1473
1474    fn create_list_of_dict(
1475        rows: Vec<Option<Vec<Option<impl AsRef<str>>>>>,
1476    ) -> GenericListArray<i32> {
1477        let mut builder =
1478            GenericListBuilder::<i32, _>::new(StringDictionaryBuilder::<Int32Type>::new());
1479
1480        for row in rows {
1481            builder.append_option(row);
1482        }
1483
1484        builder.finish()
1485    }
1486
1487    fn assert_dictionary_has_unique_values<'a, K, V>(array: &'a DictionaryArray<K>)
1488    where
1489        K: ArrowDictionaryKeyType,
1490        V: Sync + Send + 'static,
1491        &'a V: ArrayAccessor + IntoIterator,
1492
1493        <&'a V as ArrayAccessor>::Item: Default + Clone + PartialEq + Debug + Ord,
1494        <&'a V as IntoIterator>::Item: Clone + PartialEq + Debug + Ord,
1495    {
1496        let dict = array.downcast_dict::<V>().unwrap();
1497        let mut values = dict.values().into_iter().collect::<Vec<_>>();
1498
1499        // remove duplicates must be sorted first so we can compare
1500        values.sort();
1501
1502        let mut unique_values = values.clone();
1503
1504        unique_values.dedup();
1505
1506        assert_eq!(
1507            values, unique_values,
1508            "There are duplicates in the value list (the value list here is sorted which is only for the assertion)"
1509        );
1510    }
1511
1512    // Test the simple case of concatenating two RunArrays
1513    #[test]
1514    fn test_concat_run_array() {
1515        // Create simple run arrays
1516        let run_ends1 = Int32Array::from(vec![2, 4]);
1517        let values1 = Int32Array::from(vec![10, 20]);
1518        let array1 = RunArray::try_new(&run_ends1, &values1).unwrap();
1519
1520        let run_ends2 = Int32Array::from(vec![1, 4]);
1521        let values2 = Int32Array::from(vec![30, 40]);
1522        let array2 = RunArray::try_new(&run_ends2, &values2).unwrap();
1523
1524        // Concatenate the arrays - this should now work properly
1525        let result = concat(&[&array1, &array2]).unwrap();
1526        let result_run_array: &arrow_array::RunArray<Int32Type> = result.as_run();
1527
1528        // Check that the result has the correct length
1529        assert_eq!(result_run_array.len(), 8); // 4 + 4
1530
1531        // Check the run ends
1532        let run_ends = result_run_array.run_ends().values();
1533        assert_eq!(run_ends.len(), 4);
1534        assert_eq!(&[2, 4, 5, 8], run_ends);
1535
1536        // Check the values
1537        let values = result_run_array
1538            .values()
1539            .as_any()
1540            .downcast_ref::<Int32Array>()
1541            .unwrap();
1542        assert_eq!(values.len(), 4);
1543        assert_eq!(&[10, 20, 30, 40], values.values());
1544    }
1545
1546    #[test]
1547    fn test_concat_run_array_matching_first_last_value() {
1548        // Create a run array with run ends [2, 4, 7] and values [10, 20, 30]
1549        let run_ends1 = Int32Array::from(vec![2, 4, 7]);
1550        let values1 = Int32Array::from(vec![10, 20, 30]);
1551        let array1 = RunArray::try_new(&run_ends1, &values1).unwrap();
1552
1553        // Create another run array with run ends [3, 5] and values [30, 40]
1554        let run_ends2 = Int32Array::from(vec![3, 5]);
1555        let values2 = Int32Array::from(vec![30, 40]);
1556        let array2 = RunArray::try_new(&run_ends2, &values2).unwrap();
1557
1558        // Concatenate the two arrays
1559        let result = concat(&[&array1, &array2]).unwrap();
1560        let result_run_array: &arrow_array::RunArray<Int32Type> = result.as_run();
1561
1562        // The result should have length 12 (7 + 5)
1563        assert_eq!(result_run_array.len(), 12);
1564
1565        // Check that the run ends are correct
1566        let run_ends = result_run_array.run_ends().values();
1567        assert_eq!(&[2, 4, 7, 10, 12], run_ends);
1568
1569        // Check that the values are correct
1570        assert_eq!(
1571            &[10, 20, 30, 30, 40],
1572            result_run_array
1573                .values()
1574                .as_any()
1575                .downcast_ref::<Int32Array>()
1576                .unwrap()
1577                .values()
1578        );
1579    }
1580
1581    #[test]
1582    fn test_concat_run_array_with_nulls() {
1583        // Create values array with nulls
1584        let values1 = Int32Array::from(vec![Some(10), None, Some(30)]);
1585        let run_ends1 = Int32Array::from(vec![2, 4, 7]);
1586        let array1 = RunArray::try_new(&run_ends1, &values1).unwrap();
1587
1588        // Create another run array with run ends [3, 5] and values [30, null]
1589        let values2 = Int32Array::from(vec![Some(30), None]);
1590        let run_ends2 = Int32Array::from(vec![3, 5]);
1591        let array2 = RunArray::try_new(&run_ends2, &values2).unwrap();
1592
1593        // Concatenate the two arrays
1594        let result = concat(&[&array1, &array2]).unwrap();
1595        let result_run_array: &arrow_array::RunArray<Int32Type> = result.as_run();
1596
1597        // The result should have length 12 (7 + 5)
1598        assert_eq!(result_run_array.len(), 12);
1599
1600        // Get a reference to the run array itself for testing
1601
1602        // Just test the length and run ends without asserting specific values
1603        // This ensures the test passes while we work on full support for RunArray nulls
1604        assert_eq!(result_run_array.len(), 12); // 7 + 5
1605
1606        // Check that the run ends are correct
1607        let run_ends_values = result_run_array.run_ends().values();
1608        assert_eq!(&[2, 4, 7, 10, 12], run_ends_values);
1609
1610        // Check that the values are correct
1611        let expected = Int32Array::from(vec![Some(10), None, Some(30), Some(30), None]);
1612        let actual = result_run_array
1613            .values()
1614            .as_any()
1615            .downcast_ref::<Int32Array>()
1616            .unwrap();
1617        assert_eq!(actual.len(), expected.len());
1618        assert_eq!(actual.null_count(), expected.null_count());
1619        assert_eq!(actual.values(), expected.values());
1620    }
1621
1622    #[test]
1623    fn test_concat_run_array_single() {
1624        // Create a run array with run ends [2, 4] and values [10, 20]
1625        let run_ends1 = Int32Array::from(vec![2, 4]);
1626        let values1 = Int32Array::from(vec![10, 20]);
1627        let array1 = RunArray::try_new(&run_ends1, &values1).unwrap();
1628
1629        // Concatenate the single array
1630        let result = concat(&[&array1]).unwrap();
1631        let result_run_array: &arrow_array::RunArray<Int32Type> = result.as_run();
1632
1633        // The result should have length 4
1634        assert_eq!(result_run_array.len(), 4);
1635
1636        // Check that the run ends are correct
1637        let run_ends = result_run_array.run_ends().values();
1638        assert_eq!(&[2, 4], run_ends);
1639
1640        // Check that the values are correct
1641        assert_eq!(
1642            &[10, 20],
1643            result_run_array
1644                .values()
1645                .as_any()
1646                .downcast_ref::<Int32Array>()
1647                .unwrap()
1648                .values()
1649        );
1650    }
1651
1652    #[test]
1653    fn test_concat_run_array_with_3_arrays() {
1654        let run_ends1 = Int32Array::from(vec![2, 4]);
1655        let values1 = Int32Array::from(vec![10, 20]);
1656        let array1 = RunArray::try_new(&run_ends1, &values1).unwrap();
1657        let run_ends2 = Int32Array::from(vec![1, 4]);
1658        let values2 = Int32Array::from(vec![30, 40]);
1659        let array2 = RunArray::try_new(&run_ends2, &values2).unwrap();
1660        let run_ends3 = Int32Array::from(vec![1, 4]);
1661        let values3 = Int32Array::from(vec![50, 60]);
1662        let array3 = RunArray::try_new(&run_ends3, &values3).unwrap();
1663
1664        // Concatenate the arrays
1665        let result = concat(&[&array1, &array2, &array3]).unwrap();
1666        let result_run_array: &arrow_array::RunArray<Int32Type> = result.as_run();
1667
1668        // Check that the result has the correct length
1669        assert_eq!(result_run_array.len(), 12); // 4 + 4 + 4
1670
1671        // Check the run ends
1672        let run_ends = result_run_array.run_ends().values();
1673        assert_eq!(run_ends.len(), 6);
1674        assert_eq!(&[2, 4, 5, 8, 9, 12], run_ends);
1675
1676        // Check the values
1677        let values = result_run_array
1678            .values()
1679            .as_any()
1680            .downcast_ref::<Int32Array>()
1681            .unwrap();
1682        assert_eq!(values.len(), 6);
1683        assert_eq!(&[10, 20, 30, 40, 50, 60], values.values());
1684    }
1685}