Skip to main content

arrow/util/
data_gen.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Utilities to generate random arrays and batches
6
7use std::sync::Arc;
8
9use rand::{
10    Rng, RngExt,
11    distr::uniform::{SampleRange, SampleUniform},
12};
13
14use crate::array::*;
15use crate::error::{ArrowError, Result};
16use crate::{
17    buffer::{Buffer, MutableBuffer},
18    datatypes::*,
19};
20
21use super::{bench_util::*, bit_util, test_util::seedable_rng};
22
23/// Create a random [RecordBatch] from a schema
24pub fn create_random_batch(
25    schema: SchemaRef,
26    size: usize,
27    null_density: f32,
28    true_density: f32,
29) -> Result<RecordBatch> {
30    let columns = schema
31        .fields()
32        .iter()
33        .map(|field| create_random_array(field, size, null_density, true_density))
34        .collect::<Result<Vec<ArrayRef>>>()?;
35
36    RecordBatch::try_new_with_options(
37        schema,
38        columns,
39        &RecordBatchOptions::new().with_match_field_names(false),
40    )
41}
42
43/// Create a random [ArrayRef] from a [DataType] with a length,
44/// null density and true density (for [BooleanArray]).
45///
46/// # Arguments
47///
48/// * `field` - The field containing the data type for which to create a random array
49/// * `size` - The number of elements in the generated array
50/// * `null_density` - The approximate fraction of null values in the resulting array (0.0 to 1.0)
51/// * `true_density` - The approximate fraction of true values in boolean arrays (0.0 to 1.0)
52///
53pub fn create_random_array(
54    field: &Field,
55    size: usize,
56    mut null_density: f32,
57    true_density: f32,
58) -> Result<ArrayRef> {
59    // Override nullability in case of not nested and not dictionary
60    // For nested we don't want to override as we want to keep the nullability for the children
61    // For dictionary it handle the nullability internally
62    if !field.data_type().is_nested() && !matches!(field.data_type(), Dictionary(_, _)) {
63        // Override null density with 0.0 if the array is non-nullable
64        null_density = match field.is_nullable() {
65            true => null_density,
66            false => 0.0,
67        };
68    }
69
70    use DataType::*;
71    let array = match field.data_type() {
72        Null => Arc::new(NullArray::new(size)) as ArrayRef,
73        Boolean => Arc::new(create_boolean_array(size, null_density, true_density)),
74        Int8 => Arc::new(create_primitive_array::<Int8Type>(size, null_density)),
75        Int16 => Arc::new(create_primitive_array::<Int16Type>(size, null_density)),
76        Int32 => Arc::new(create_primitive_array::<Int32Type>(size, null_density)),
77        Int64 => Arc::new(create_primitive_array::<Int64Type>(size, null_density)),
78        UInt8 => Arc::new(create_primitive_array::<UInt8Type>(size, null_density)),
79        UInt16 => Arc::new(create_primitive_array::<UInt16Type>(size, null_density)),
80        UInt32 => Arc::new(create_primitive_array::<UInt32Type>(size, null_density)),
81        UInt64 => Arc::new(create_primitive_array::<UInt64Type>(size, null_density)),
82        Float16 => Arc::new(create_nullable_f16_array(size, null_density)),
83        Float32 => Arc::new(create_primitive_array::<Float32Type>(size, null_density)),
84        Float64 => Arc::new(create_primitive_array::<Float64Type>(size, null_density)),
85        Timestamp(unit, tz) => match unit {
86            TimeUnit::Second => Arc::new(
87                create_random_temporal_array::<TimestampSecondType>(size, null_density)
88                    .with_timezone_opt(tz.clone()),
89            ) as ArrayRef,
90            TimeUnit::Millisecond => Arc::new(
91                create_random_temporal_array::<TimestampMillisecondType>(size, null_density)
92                    .with_timezone_opt(tz.clone()),
93            ),
94            TimeUnit::Microsecond => Arc::new(
95                create_random_temporal_array::<TimestampMicrosecondType>(size, null_density)
96                    .with_timezone_opt(tz.clone()),
97            ),
98            TimeUnit::Nanosecond => Arc::new(
99                create_random_temporal_array::<TimestampNanosecondType>(size, null_density)
100                    .with_timezone_opt(tz.clone()),
101            ),
102        },
103        Date32 => Arc::new(create_random_temporal_array::<Date32Type>(
104            size,
105            null_density,
106        )),
107        Date64 => Arc::new(create_random_temporal_array::<Date64Type>(
108            size,
109            null_density,
110        )),
111        Time32(unit) => match unit {
112            TimeUnit::Second => Arc::new(create_random_temporal_array::<Time32SecondType>(
113                size,
114                null_density,
115            )) as ArrayRef,
116            TimeUnit::Millisecond => Arc::new(
117                create_random_temporal_array::<Time32MillisecondType>(size, null_density),
118            ),
119            _ => {
120                return Err(ArrowError::InvalidArgumentError(format!(
121                    "Unsupported unit {unit:?} for Time32"
122                )));
123            }
124        },
125        Time64(unit) => match unit {
126            TimeUnit::Microsecond => Arc::new(
127                create_random_temporal_array::<Time64MicrosecondType>(size, null_density),
128            ) as ArrayRef,
129            TimeUnit::Nanosecond => Arc::new(create_random_temporal_array::<Time64NanosecondType>(
130                size,
131                null_density,
132            )),
133            _ => {
134                return Err(ArrowError::InvalidArgumentError(format!(
135                    "Unsupported unit {unit:?} for Time64"
136                )));
137            }
138        },
139        Utf8 => Arc::new(create_string_array::<i32>(size, null_density)),
140        LargeUtf8 => Arc::new(create_string_array::<i64>(size, null_density)),
141        Utf8View => Arc::new(create_string_view_array_with_len(
142            size,
143            null_density,
144            4,
145            false,
146        )),
147        Binary => Arc::new(create_binary_array::<i32>(size, null_density)),
148        LargeBinary => Arc::new(create_binary_array::<i64>(size, null_density)),
149        FixedSizeBinary(len) => {
150            let len = TryInto::<usize>::try_into(*len).map_err(|_| {
151                ArrowError::InvalidArgumentError(format!("cannot use FixedSizeBinary({len})"))
152            })?;
153            Arc::new(create_fsb_array(size, null_density, len))
154        }
155        BinaryView => Arc::new(
156            create_string_view_array_with_len(size, null_density, 4, false).to_binary_view(),
157        ),
158        List(_) => create_random_list_array(field, size, null_density, true_density)?,
159        LargeList(_) => create_random_list_array(field, size, null_density, true_density)?,
160        Struct(_) => create_random_struct_array(field, size, null_density, true_density)?,
161        d @ Dictionary(_, value_type) if crate::compute::can_cast_types(value_type, d) => {
162            let f = Field::new(
163                field.name(),
164                value_type.as_ref().clone(),
165                field.is_nullable(),
166            );
167            let v = create_random_array(&f, size, null_density, true_density)?;
168            crate::compute::cast(&v, d)?
169        }
170        Map(_, _) => create_random_map_array(field, size, null_density, true_density)?,
171        Decimal32(_, _) => create_random_decimal_array(field, size, null_density)?,
172        Decimal64(_, _) => create_random_decimal_array(field, size, null_density)?,
173        Decimal128(_, _) => create_random_decimal_array(field, size, null_density)?,
174        Decimal256(_, _) => create_random_decimal_array(field, size, null_density)?,
175        RunEndEncoded(index, value) => {
176            create_random_run_end_encoded_array(index, value, size, null_density, true_density)?
177        }
178        other => {
179            return Err(ArrowError::NotYetImplemented(format!(
180                "Generating random arrays not yet implemented for {other:?}"
181            )));
182        }
183    };
184
185    if !field.is_nullable() {
186        assert_eq!(array.null_count(), 0);
187    }
188
189    Ok(array)
190}
191
192#[inline]
193fn create_random_decimal_array(field: &Field, size: usize, null_density: f32) -> Result<ArrayRef> {
194    let mut rng = seedable_rng();
195
196    match field.data_type() {
197        DataType::Decimal32(precision, scale) => {
198            let values = (0..size)
199                .map(|_| {
200                    if rng.random::<f32>() < null_density {
201                        None
202                    } else {
203                        Some(rng.random::<i32>())
204                    }
205                })
206                .collect::<Vec<_>>();
207            Ok(Arc::new(
208                Decimal32Array::from(values).with_precision_and_scale(*precision, *scale)?,
209            ))
210        }
211        DataType::Decimal64(precision, scale) => {
212            let values = (0..size)
213                .map(|_| {
214                    if rng.random::<f32>() < null_density {
215                        None
216                    } else {
217                        Some(rng.random::<i64>())
218                    }
219                })
220                .collect::<Vec<_>>();
221            Ok(Arc::new(
222                Decimal64Array::from(values).with_precision_and_scale(*precision, *scale)?,
223            ))
224        }
225        DataType::Decimal128(precision, scale) => {
226            let values = (0..size)
227                .map(|_| {
228                    if rng.random::<f32>() < null_density {
229                        None
230                    } else {
231                        Some(rng.random::<i128>())
232                    }
233                })
234                .collect::<Vec<_>>();
235            Ok(Arc::new(
236                Decimal128Array::from(values).with_precision_and_scale(*precision, *scale)?,
237            ))
238        }
239        DataType::Decimal256(precision, scale) => {
240            let values = (0..size)
241                .map(|_| {
242                    if rng.random::<f32>() < null_density {
243                        None
244                    } else {
245                        Some(i256::from_parts(rng.random::<u128>(), rng.random::<i128>()))
246                    }
247                })
248                .collect::<Vec<_>>();
249            Ok(Arc::new(
250                Decimal256Array::from(values).with_precision_and_scale(*precision, *scale)?,
251            ))
252        }
253        _ => Err(ArrowError::InvalidArgumentError(format!(
254            "Cannot create decimal array for field {field}"
255        ))),
256    }
257}
258#[inline]
259fn create_random_run_end_encoded_array(
260    index: &Field,
261    value: &Field,
262    size: usize,
263    null_density: f32,
264    true_density: f32,
265) -> Result<ArrayRef> {
266    const MIN_RUN: usize = 8;
267    const MAX_RUN: usize = 32;
268
269    let mut rng = seedable_rng();
270    let mut run_lengths: Vec<usize> = Vec::new();
271    let mut remaining = size;
272    while remaining > 0 {
273        let len = rng.random_range(MIN_RUN..=MAX_RUN).min(remaining);
274        run_lengths.push(len);
275        remaining -= len;
276    }
277    let num_runs = run_lengths.len();
278
279    let mut cumulative: i64 = 0;
280    let run_ends_i64: Vec<i64> = run_lengths
281        .iter()
282        .map(|&l| {
283            cumulative += l as i64;
284            cumulative
285        })
286        .collect();
287
288    let values = create_random_array(value, num_runs, null_density, true_density)?;
289
290    match index.data_type() {
291        DataType::Int16 => {
292            let run_ends: Int16Array = run_ends_i64.iter().map(|&v| v as i16).collect();
293            Ok(Arc::new(RunArray::<Int16Type>::try_new(
294                &run_ends, &values,
295            )?))
296        }
297        DataType::Int32 => {
298            let run_ends: Int32Array = run_ends_i64.iter().map(|&v| v as i32).collect();
299            Ok(Arc::new(RunArray::<Int32Type>::try_new(
300                &run_ends, &values,
301            )?))
302        }
303        DataType::Int64 => {
304            let run_ends: Int64Array = run_ends_i64.iter().copied().collect();
305            Ok(Arc::new(RunArray::<Int64Type>::try_new(
306                &run_ends, &values,
307            )?))
308        }
309        other => Err(ArrowError::InvalidArgumentError(format!(
310            "Unsupported run-ends type for REE: {other:?}"
311        ))),
312    }
313}
314
315#[inline]
316fn create_random_list_array(
317    field: &Field,
318    size: usize,
319    null_density: f32,
320    true_density: f32,
321) -> Result<ArrayRef> {
322    // Override null density with 0.0 if the array is non-nullable
323    let list_null_density = match field.is_nullable() {
324        true => null_density,
325        false => 0.0,
326    };
327    let list_field;
328    let (offsets, child_len) = match field.data_type() {
329        DataType::List(f) => {
330            let (offsets, child_len) = create_random_offsets::<i32>(size, 0, 5);
331            list_field = f;
332            (Buffer::from(offsets.to_byte_slice()), child_len as usize)
333        }
334        DataType::LargeList(f) => {
335            let (offsets, child_len) = create_random_offsets::<i64>(size, 0, 5);
336            list_field = f;
337            (Buffer::from(offsets.to_byte_slice()), child_len as usize)
338        }
339        _ => {
340            return Err(ArrowError::InvalidArgumentError(format!(
341                "Cannot create list array for field {field}"
342            )));
343        }
344    };
345
346    // Create list's child data
347    let child_array = create_random_array(list_field, child_len, null_density, true_density)?;
348    let child_data = child_array.to_data();
349    // Create list's null buffers, if it is nullable
350    let null_buffer = match field.is_nullable() {
351        true => Some(create_random_null_buffer(size, list_null_density)),
352        false => None,
353    };
354    let list_data = unsafe {
355        ArrayData::new_unchecked(
356            field.data_type().clone(),
357            size,
358            None,
359            null_buffer,
360            0,
361            vec![offsets],
362            vec![child_data],
363        )
364    };
365    Ok(make_array(list_data))
366}
367
368#[inline]
369fn create_random_struct_array(
370    field: &Field,
371    size: usize,
372    null_density: f32,
373    true_density: f32,
374) -> Result<ArrayRef> {
375    let DataType::Struct(struct_fields) = field.data_type() else {
376        return Err(ArrowError::InvalidArgumentError(format!(
377            "Cannot create struct array for field {field}"
378        )));
379    };
380
381    let child_arrays = struct_fields
382        .iter()
383        .map(|struct_field| create_random_array(struct_field, size, null_density, true_density))
384        .collect::<Result<Vec<_>>>()?;
385
386    let null_buffer = match field.is_nullable() {
387        true => {
388            let nulls = arrow_buffer::BooleanBuffer::new(
389                create_random_null_buffer(size, null_density),
390                0,
391                size,
392            );
393            Some(nulls.into())
394        }
395        false => None,
396    };
397
398    Ok(Arc::new(StructArray::try_new(
399        struct_fields.clone(),
400        child_arrays,
401        null_buffer,
402    )?))
403}
404
405#[inline]
406fn create_random_map_array(
407    field: &Field,
408    size: usize,
409    null_density: f32,
410    true_density: f32,
411) -> Result<ArrayRef> {
412    // Override null density with 0.0 if the array is non-nullable
413    let map_null_density = match field.is_nullable() {
414        true => null_density,
415        false => 0.0,
416    };
417
418    let DataType::Map(entries_field, _) = field.data_type() else {
419        return Err(ArrowError::InvalidArgumentError(format!(
420            "Cannot create map array for field {field:?}"
421        )));
422    };
423
424    let (offsets, child_len) = create_random_offsets::<i32>(size, 0, 5);
425    let offsets = Buffer::from(offsets.to_byte_slice());
426
427    let entries = create_random_array(
428        entries_field,
429        child_len as usize,
430        null_density,
431        true_density,
432    )?
433    .to_data();
434
435    let null_buffer = match field.is_nullable() {
436        true => Some(create_random_null_buffer(size, map_null_density)),
437        false => None,
438    };
439
440    let map_data = unsafe {
441        ArrayData::new_unchecked(
442            field.data_type().clone(),
443            size,
444            None,
445            null_buffer,
446            0,
447            vec![offsets],
448            vec![entries],
449        )
450    };
451    Ok(make_array(map_data))
452}
453
454/// Generate random offsets for list arrays
455fn create_random_offsets<T: OffsetSizeTrait + SampleUniform>(
456    size: usize,
457    min: T,
458    max: T,
459) -> (Vec<T>, T) {
460    let rng = &mut seedable_rng();
461
462    let mut current_offset = T::zero();
463
464    let mut offsets = Vec::with_capacity(size + 1);
465    offsets.push(current_offset);
466
467    (0..size).for_each(|_| {
468        current_offset += rng.random_range(min..max);
469        offsets.push(current_offset);
470    });
471
472    (offsets, current_offset)
473}
474
475fn create_random_null_buffer(size: usize, null_density: f32) -> Buffer {
476    let mut rng = seedable_rng();
477    let mut mut_buf = MutableBuffer::new_null(size);
478    {
479        let mut_slice = mut_buf.as_slice_mut();
480        (0..size).for_each(|i| {
481            if rng.random::<f32>() >= null_density {
482                bit_util::set_bit(mut_slice, i)
483            }
484        })
485    };
486    mut_buf.into()
487}
488
489/// Useful for testing. The range of values are not likely to be representative of the
490/// actual bounds.
491pub trait RandomTemporalValue: ArrowTemporalType {
492    /// Returns the range of values for `impl`'d type
493    fn value_range() -> impl SampleRange<Self::Native>;
494
495    /// Generate a random value within the range of the type
496    fn gen_range<R: Rng>(rng: &mut R) -> Self::Native
497    where
498        Self::Native: SampleUniform,
499    {
500        rng.random_range(Self::value_range())
501    }
502
503    /// Generate a random value of the type
504    fn random<R: Rng>(rng: &mut R) -> Self::Native
505    where
506        Self::Native: SampleUniform,
507    {
508        Self::gen_range(rng)
509    }
510}
511
512impl RandomTemporalValue for TimestampSecondType {
513    /// Range of values for a timestamp in seconds. The range begins at the start
514    /// of the unix epoch and continues for 100 years.
515    fn value_range() -> impl SampleRange<Self::Native> {
516        0..60 * 60 * 24 * 365 * 100
517    }
518}
519
520impl RandomTemporalValue for TimestampMillisecondType {
521    /// Range of values for a timestamp in milliseconds. The range begins at the start
522    /// of the unix epoch and continues for 100 years.
523    fn value_range() -> impl SampleRange<Self::Native> {
524        0..1_000 * 60 * 60 * 24 * 365 * 100
525    }
526}
527
528impl RandomTemporalValue for TimestampMicrosecondType {
529    /// Range of values for a timestamp in microseconds. The range begins at the start
530    /// of the unix epoch and continues for 100 years.
531    fn value_range() -> impl SampleRange<Self::Native> {
532        0..1_000 * 1_000 * 60 * 60 * 24 * 365 * 100
533    }
534}
535
536impl RandomTemporalValue for TimestampNanosecondType {
537    /// Range of values for a timestamp in nanoseconds. The range begins at the start
538    /// of the unix epoch and continues for 100 years.
539    fn value_range() -> impl SampleRange<Self::Native> {
540        0..1_000 * 1_000 * 1_000 * 60 * 60 * 24 * 365 * 100
541    }
542}
543
544impl RandomTemporalValue for Date32Type {
545    /// Range of values representing the elapsed time since UNIX epoch in days. The
546    /// range begins at the start of the unix epoch and continues for 100 years.
547    fn value_range() -> impl SampleRange<Self::Native> {
548        0..365 * 100
549    }
550}
551
552impl RandomTemporalValue for Date64Type {
553    /// Range of values  representing the elapsed time since UNIX epoch in milliseconds.
554    /// The range begins at the start of the unix epoch and continues for 100 years.
555    fn value_range() -> impl SampleRange<Self::Native> {
556        0..1_000 * 60 * 60 * 24 * 365 * 100
557    }
558}
559
560impl RandomTemporalValue for Time32SecondType {
561    /// Range of values representing the elapsed time since midnight in seconds. The
562    /// range is from 0 to 24 hours.
563    fn value_range() -> impl SampleRange<Self::Native> {
564        0..60 * 60 * 24
565    }
566}
567
568impl RandomTemporalValue for Time32MillisecondType {
569    /// Range of values representing the elapsed time since midnight in milliseconds. The
570    /// range is from 0 to 24 hours.
571    fn value_range() -> impl SampleRange<Self::Native> {
572        0..1_000 * 60 * 60 * 24
573    }
574}
575
576impl RandomTemporalValue for Time64MicrosecondType {
577    /// Range of values representing the elapsed time since midnight in microseconds. The
578    /// range is from 0 to 24 hours.
579    fn value_range() -> impl SampleRange<Self::Native> {
580        0..1_000 * 1_000 * 60 * 60 * 24
581    }
582}
583
584impl RandomTemporalValue for Time64NanosecondType {
585    /// Range of values representing the elapsed time since midnight in nanoseconds. The
586    /// range is from 0 to 24 hours.
587    fn value_range() -> impl SampleRange<Self::Native> {
588        0..1_000 * 1_000 * 1_000 * 60 * 60 * 24
589    }
590}
591
592fn create_random_temporal_array<T>(size: usize, null_density: f32) -> PrimitiveArray<T>
593where
594    T: RandomTemporalValue,
595    <T as ArrowPrimitiveType>::Native: SampleUniform,
596{
597    let mut rng = seedable_rng();
598
599    (0..size)
600        .map(|_| {
601            if rng.random::<f32>() < null_density {
602                None
603            } else {
604                Some(T::random(&mut rng))
605            }
606        })
607        .collect()
608}
609
610#[cfg(test)]
611mod tests {
612    use super::*;
613
614    #[test]
615    fn test_create_batch() {
616        let size = 32;
617        let fields = vec![
618            Field::new("a", DataType::Int32, true),
619            Field::new("f16", DataType::Float16, true),
620            Field::new(
621                "timestamp_without_timezone",
622                DataType::Timestamp(TimeUnit::Nanosecond, None),
623                true,
624            ),
625            Field::new(
626                "timestamp_with_timezone",
627                DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),
628                true,
629            ),
630        ];
631        let schema = Schema::new(fields);
632        let schema_ref = Arc::new(schema);
633        let batch = create_random_batch(schema_ref.clone(), size, 0.35, 0.7).unwrap();
634
635        assert_eq!(batch.schema(), schema_ref);
636        assert_eq!(batch.num_columns(), schema_ref.fields().len());
637        for array in batch.columns() {
638            assert_eq!(array.len(), size);
639        }
640    }
641
642    #[test]
643    fn test_create_batch_non_null() {
644        let size = 32;
645        let fields = vec![
646            Field::new("a", DataType::Int32, false),
647            Field::new(
648                "b",
649                DataType::List(Arc::new(Field::new_list_field(DataType::LargeUtf8, false))),
650                false,
651            ),
652            Field::new("a", DataType::Int32, false),
653        ];
654        let schema = Schema::new(fields);
655        let schema_ref = Arc::new(schema);
656        let batch = create_random_batch(schema_ref.clone(), size, 0.35, 0.7).unwrap();
657
658        assert_eq!(batch.schema(), schema_ref);
659        assert_eq!(batch.num_columns(), schema_ref.fields().len());
660        for array in batch.columns() {
661            assert_eq!(array.null_count(), 0);
662            assert_eq!(array.logical_null_count(), 0);
663        }
664        // Test that the list's child values are non-null
665        let b_array = batch.column(1);
666        let list_array = b_array.as_list::<i32>();
667        let child_array = list_array.values();
668        assert_eq!(child_array.null_count(), 0);
669        // There should be more values than the list, to show that it's a list
670        assert!(child_array.len() > list_array.len());
671    }
672
673    #[test]
674    fn test_create_struct_array() {
675        let size = 32;
676        let struct_fields = Fields::from(vec![
677            Field::new("b", DataType::Boolean, true),
678            Field::new(
679                "c",
680                DataType::LargeList(Arc::new(Field::new_list_field(
681                    DataType::List(Arc::new(Field::new_list_field(
682                        DataType::FixedSizeBinary(6),
683                        true,
684                    ))),
685                    false,
686                ))),
687                true,
688            ),
689            Field::new(
690                "d",
691                DataType::Struct(Fields::from(vec![
692                    Field::new("d_x", DataType::Int32, true),
693                    Field::new("d_y", DataType::Float32, false),
694                    Field::new("d_z", DataType::Binary, true),
695                ])),
696                true,
697            ),
698        ]);
699        let field = Field::new("struct", DataType::Struct(struct_fields), true);
700        let array = create_random_array(&field, size, 0.2, 0.5).unwrap();
701
702        assert_eq!(array.len(), 32);
703        let struct_array = array.as_any().downcast_ref::<StructArray>().unwrap();
704        assert_eq!(struct_array.columns().len(), 3);
705
706        // Test that the nested list makes sense,
707        // i.e. its children's values are more than the parent, to show repetition
708        let col_c = struct_array.column_by_name("c").unwrap();
709        let col_c = col_c.as_any().downcast_ref::<LargeListArray>().unwrap();
710        assert_eq!(col_c.len(), size);
711        let col_c_list = col_c.values().as_list::<i32>();
712        assert!(col_c_list.len() > size);
713        // Its values should be FixedSizeBinary(6)
714        let fsb = col_c_list.values();
715        assert_eq!(fsb.data_type(), &DataType::FixedSizeBinary(6));
716        assert!(fsb.len() > col_c_list.len());
717
718        // Test nested struct
719        let col_d = struct_array.column_by_name("d").unwrap();
720        let col_d = col_d.as_any().downcast_ref::<StructArray>().unwrap();
721        let col_d_y = col_d.column_by_name("d_y").unwrap();
722        assert_eq!(col_d_y.data_type(), &DataType::Float32);
723        assert_eq!(col_d_y.null_count(), 0);
724    }
725
726    #[test]
727    fn test_create_run_end_encoded_array() {
728        let size = 1000;
729        let ree_field = Field::new(
730            "ree",
731            DataType::RunEndEncoded(
732                Arc::new(Field::new(
733                    Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
734                    DataType::Int32,
735                    false,
736                )),
737                Arc::new(Field::new(
738                    Field::REE_VALUES_FIELD_DEFAULT_NAME,
739                    DataType::Utf8,
740                    true,
741                )),
742            ),
743            false,
744        );
745
746        let array = create_random_array(&ree_field, size, 0.25, 0.0).unwrap();
747        assert_eq!(array.len(), size);
748
749        let ree = array.as_run::<Int32Type>();
750        let run_ends = ree.run_ends().values();
751        let num_runs = run_ends.len();
752
753        assert_eq!(*run_ends.last().unwrap() as usize, size);
754
755        assert_eq!(ree.values().len(), num_runs);
756    }
757
758    #[test]
759    fn test_create_list_array_nested_nullability() {
760        let list_field = Field::new_list(
761            "not_null_list",
762            Field::new_list_field(DataType::Boolean, true),
763            false,
764        );
765
766        let list_array = create_random_array(&list_field, 100, 0.95, 0.5).unwrap();
767
768        assert_eq!(list_array.null_count(), 0);
769        assert!(list_array.as_list::<i32>().values().null_count() > 0);
770    }
771
772    #[test]
773    fn test_create_struct_array_nested_nullability() {
774        let struct_child_fields = vec![
775            Field::new("null_int", DataType::Int32, true),
776            Field::new("int", DataType::Int32, false),
777        ];
778        let struct_field = Field::new_struct("not_null_struct", struct_child_fields, false);
779
780        let struct_array = create_random_array(&struct_field, 100, 0.95, 0.5).unwrap();
781
782        assert_eq!(struct_array.null_count(), 0);
783        assert!(
784            struct_array
785                .as_struct()
786                .column_by_name("null_int")
787                .unwrap()
788                .null_count()
789                > 0
790        );
791        assert_eq!(
792            struct_array
793                .as_struct()
794                .column_by_name("int")
795                .unwrap()
796                .null_count(),
797            0
798        );
799    }
800
801    #[test]
802    fn test_create_list_array_nested_struct_nullability() {
803        let struct_child_fields = vec![
804            Field::new("null_int", DataType::Int32, true),
805            Field::new("int", DataType::Int32, false),
806        ];
807        let list_item_field =
808            Field::new_list_field(DataType::Struct(struct_child_fields.into()), true);
809        let list_field = Field::new_list("not_null_list", list_item_field, false);
810
811        let list_array = create_random_array(&list_field, 100, 0.95, 0.5).unwrap();
812
813        assert_eq!(list_array.null_count(), 0);
814        assert!(list_array.as_list::<i32>().values().null_count() > 0);
815        assert!(
816            list_array
817                .as_list::<i32>()
818                .values()
819                .as_struct()
820                .column_by_name("null_int")
821                .unwrap()
822                .null_count()
823                > 0
824        );
825        assert_eq!(
826            list_array
827                .as_list::<i32>()
828                .values()
829                .as_struct()
830                .column_by_name("int")
831                .unwrap()
832                .null_count(),
833            0
834        );
835    }
836
837    #[test]
838    fn test_create_map_array() {
839        let map_field = Field::new_map(
840            "map",
841            Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
842            Field::new(Field::MAP_KEY_FIELD_DEFAULT_NAME, DataType::Utf8, false),
843            Field::new(Field::MAP_VALUE_FIELD_DEFAULT_NAME, DataType::Utf8, true),
844            false,
845            false,
846        );
847        let array = create_random_array(&map_field, 100, 0.8, 0.5).unwrap();
848
849        assert_eq!(array.len(), 100);
850        // Map field is not null
851        assert_eq!(array.null_count(), 0);
852        assert_eq!(array.logical_null_count(), 0);
853        // Maps have multiple values like a list, so internal arrays are longer
854        assert!(array.as_map().keys().len() > array.len());
855        assert!(array.as_map().values().len() > array.len());
856        // Keys are not nullable
857        assert_eq!(array.as_map().keys().null_count(), 0);
858        // Values are nullable
859        assert!(array.as_map().values().null_count() > 0);
860
861        assert_eq!(array.as_map().keys().data_type(), &DataType::Utf8);
862        assert_eq!(array.as_map().values().data_type(), &DataType::Utf8);
863    }
864
865    #[test]
866    fn test_create_decimal_array() {
867        let size = 10;
868        let fields = vec![
869            Field::new("a", DataType::Decimal128(10, -2), true),
870            Field::new("b", DataType::Decimal256(10, -2), true),
871        ];
872        let schema = Schema::new(fields);
873        let schema_ref = Arc::new(schema);
874        let batch = create_random_batch(schema_ref.clone(), size, 0.35, 0.7).unwrap();
875
876        assert_eq!(batch.schema(), schema_ref);
877        assert_eq!(batch.num_columns(), schema_ref.fields().len());
878        for array in batch.columns() {
879            assert_eq!(array.len(), size);
880        }
881    }
882
883    #[test]
884    fn create_non_nullable_decimal_array_with_null_density() {
885        let size = 10;
886        let fields = vec![
887            Field::new("a", DataType::Decimal128(10, -2), false),
888            Field::new("b", DataType::Decimal256(10, -2), false),
889        ];
890        let schema = Schema::new(fields);
891        let schema_ref = Arc::new(schema);
892        let batch = create_random_batch(schema_ref.clone(), size, 0.35, 0.7).unwrap();
893
894        assert_eq!(batch.schema(), schema_ref);
895        assert_eq!(batch.num_columns(), schema_ref.fields().len());
896        for array in batch.columns() {
897            assert_eq!(array.len(), size);
898            assert_eq!(array.null_count(), 0);
899        }
900    }
901}