1use std::sync::Arc;
8
9use rand::{
10 Rng, RngExt,
11 distr::uniform::{SampleRange, SampleUniform},
12};
13
14use crate::array::*;
15use crate::error::{ArrowError, Result};
16use crate::{
17 buffer::{Buffer, MutableBuffer},
18 datatypes::*,
19};
20
21use super::{bench_util::*, bit_util, test_util::seedable_rng};
22
23pub fn create_random_batch(
25 schema: SchemaRef,
26 size: usize,
27 null_density: f32,
28 true_density: f32,
29) -> Result<RecordBatch> {
30 let columns = schema
31 .fields()
32 .iter()
33 .map(|field| create_random_array(field, size, null_density, true_density))
34 .collect::<Result<Vec<ArrayRef>>>()?;
35
36 RecordBatch::try_new_with_options(
37 schema,
38 columns,
39 &RecordBatchOptions::new().with_match_field_names(false),
40 )
41}
42
43pub fn create_random_array(
54 field: &Field,
55 size: usize,
56 mut null_density: f32,
57 true_density: f32,
58) -> Result<ArrayRef> {
59 if !field.data_type().is_nested() && !matches!(field.data_type(), Dictionary(_, _)) {
63 null_density = match field.is_nullable() {
65 true => null_density,
66 false => 0.0,
67 };
68 }
69
70 use DataType::*;
71 let array = match field.data_type() {
72 Null => Arc::new(NullArray::new(size)) as ArrayRef,
73 Boolean => Arc::new(create_boolean_array(size, null_density, true_density)),
74 Int8 => Arc::new(create_primitive_array::<Int8Type>(size, null_density)),
75 Int16 => Arc::new(create_primitive_array::<Int16Type>(size, null_density)),
76 Int32 => Arc::new(create_primitive_array::<Int32Type>(size, null_density)),
77 Int64 => Arc::new(create_primitive_array::<Int64Type>(size, null_density)),
78 UInt8 => Arc::new(create_primitive_array::<UInt8Type>(size, null_density)),
79 UInt16 => Arc::new(create_primitive_array::<UInt16Type>(size, null_density)),
80 UInt32 => Arc::new(create_primitive_array::<UInt32Type>(size, null_density)),
81 UInt64 => Arc::new(create_primitive_array::<UInt64Type>(size, null_density)),
82 Float16 => Arc::new(create_nullable_f16_array(size, null_density)),
83 Float32 => Arc::new(create_primitive_array::<Float32Type>(size, null_density)),
84 Float64 => Arc::new(create_primitive_array::<Float64Type>(size, null_density)),
85 Timestamp(unit, tz) => match unit {
86 TimeUnit::Second => Arc::new(
87 create_random_temporal_array::<TimestampSecondType>(size, null_density)
88 .with_timezone_opt(tz.clone()),
89 ) as ArrayRef,
90 TimeUnit::Millisecond => Arc::new(
91 create_random_temporal_array::<TimestampMillisecondType>(size, null_density)
92 .with_timezone_opt(tz.clone()),
93 ),
94 TimeUnit::Microsecond => Arc::new(
95 create_random_temporal_array::<TimestampMicrosecondType>(size, null_density)
96 .with_timezone_opt(tz.clone()),
97 ),
98 TimeUnit::Nanosecond => Arc::new(
99 create_random_temporal_array::<TimestampNanosecondType>(size, null_density)
100 .with_timezone_opt(tz.clone()),
101 ),
102 },
103 Date32 => Arc::new(create_random_temporal_array::<Date32Type>(
104 size,
105 null_density,
106 )),
107 Date64 => Arc::new(create_random_temporal_array::<Date64Type>(
108 size,
109 null_density,
110 )),
111 Time32(unit) => match unit {
112 TimeUnit::Second => Arc::new(create_random_temporal_array::<Time32SecondType>(
113 size,
114 null_density,
115 )) as ArrayRef,
116 TimeUnit::Millisecond => Arc::new(
117 create_random_temporal_array::<Time32MillisecondType>(size, null_density),
118 ),
119 _ => {
120 return Err(ArrowError::InvalidArgumentError(format!(
121 "Unsupported unit {unit:?} for Time32"
122 )));
123 }
124 },
125 Time64(unit) => match unit {
126 TimeUnit::Microsecond => Arc::new(
127 create_random_temporal_array::<Time64MicrosecondType>(size, null_density),
128 ) as ArrayRef,
129 TimeUnit::Nanosecond => Arc::new(create_random_temporal_array::<Time64NanosecondType>(
130 size,
131 null_density,
132 )),
133 _ => {
134 return Err(ArrowError::InvalidArgumentError(format!(
135 "Unsupported unit {unit:?} for Time64"
136 )));
137 }
138 },
139 Utf8 => Arc::new(create_string_array::<i32>(size, null_density)),
140 LargeUtf8 => Arc::new(create_string_array::<i64>(size, null_density)),
141 Utf8View => Arc::new(create_string_view_array_with_len(
142 size,
143 null_density,
144 4,
145 false,
146 )),
147 Binary => Arc::new(create_binary_array::<i32>(size, null_density)),
148 LargeBinary => Arc::new(create_binary_array::<i64>(size, null_density)),
149 FixedSizeBinary(len) => {
150 let len = TryInto::<usize>::try_into(*len).map_err(|_| {
151 ArrowError::InvalidArgumentError(format!("cannot use FixedSizeBinary({len})"))
152 })?;
153 Arc::new(create_fsb_array(size, null_density, len))
154 }
155 BinaryView => Arc::new(
156 create_string_view_array_with_len(size, null_density, 4, false).to_binary_view(),
157 ),
158 List(_) => create_random_list_array(field, size, null_density, true_density)?,
159 LargeList(_) => create_random_list_array(field, size, null_density, true_density)?,
160 Struct(_) => create_random_struct_array(field, size, null_density, true_density)?,
161 d @ Dictionary(_, value_type) if crate::compute::can_cast_types(value_type, d) => {
162 let f = Field::new(
163 field.name(),
164 value_type.as_ref().clone(),
165 field.is_nullable(),
166 );
167 let v = create_random_array(&f, size, null_density, true_density)?;
168 crate::compute::cast(&v, d)?
169 }
170 Map(_, _) => create_random_map_array(field, size, null_density, true_density)?,
171 Decimal32(_, _) => create_random_decimal_array(field, size, null_density)?,
172 Decimal64(_, _) => create_random_decimal_array(field, size, null_density)?,
173 Decimal128(_, _) => create_random_decimal_array(field, size, null_density)?,
174 Decimal256(_, _) => create_random_decimal_array(field, size, null_density)?,
175 RunEndEncoded(index, value) => {
176 create_random_run_end_encoded_array(index, value, size, null_density, true_density)?
177 }
178 other => {
179 return Err(ArrowError::NotYetImplemented(format!(
180 "Generating random arrays not yet implemented for {other:?}"
181 )));
182 }
183 };
184
185 if !field.is_nullable() {
186 assert_eq!(array.null_count(), 0);
187 }
188
189 Ok(array)
190}
191
192#[inline]
193fn create_random_decimal_array(field: &Field, size: usize, null_density: f32) -> Result<ArrayRef> {
194 let mut rng = seedable_rng();
195
196 match field.data_type() {
197 DataType::Decimal32(precision, scale) => {
198 let values = (0..size)
199 .map(|_| {
200 if rng.random::<f32>() < null_density {
201 None
202 } else {
203 Some(rng.random::<i32>())
204 }
205 })
206 .collect::<Vec<_>>();
207 Ok(Arc::new(
208 Decimal32Array::from(values).with_precision_and_scale(*precision, *scale)?,
209 ))
210 }
211 DataType::Decimal64(precision, scale) => {
212 let values = (0..size)
213 .map(|_| {
214 if rng.random::<f32>() < null_density {
215 None
216 } else {
217 Some(rng.random::<i64>())
218 }
219 })
220 .collect::<Vec<_>>();
221 Ok(Arc::new(
222 Decimal64Array::from(values).with_precision_and_scale(*precision, *scale)?,
223 ))
224 }
225 DataType::Decimal128(precision, scale) => {
226 let values = (0..size)
227 .map(|_| {
228 if rng.random::<f32>() < null_density {
229 None
230 } else {
231 Some(rng.random::<i128>())
232 }
233 })
234 .collect::<Vec<_>>();
235 Ok(Arc::new(
236 Decimal128Array::from(values).with_precision_and_scale(*precision, *scale)?,
237 ))
238 }
239 DataType::Decimal256(precision, scale) => {
240 let values = (0..size)
241 .map(|_| {
242 if rng.random::<f32>() < null_density {
243 None
244 } else {
245 Some(i256::from_parts(rng.random::<u128>(), rng.random::<i128>()))
246 }
247 })
248 .collect::<Vec<_>>();
249 Ok(Arc::new(
250 Decimal256Array::from(values).with_precision_and_scale(*precision, *scale)?,
251 ))
252 }
253 _ => Err(ArrowError::InvalidArgumentError(format!(
254 "Cannot create decimal array for field {field}"
255 ))),
256 }
257}
258#[inline]
259fn create_random_run_end_encoded_array(
260 index: &Field,
261 value: &Field,
262 size: usize,
263 null_density: f32,
264 true_density: f32,
265) -> Result<ArrayRef> {
266 const MIN_RUN: usize = 8;
267 const MAX_RUN: usize = 32;
268
269 let mut rng = seedable_rng();
270 let mut run_lengths: Vec<usize> = Vec::new();
271 let mut remaining = size;
272 while remaining > 0 {
273 let len = rng.random_range(MIN_RUN..=MAX_RUN).min(remaining);
274 run_lengths.push(len);
275 remaining -= len;
276 }
277 let num_runs = run_lengths.len();
278
279 let mut cumulative: i64 = 0;
280 let run_ends_i64: Vec<i64> = run_lengths
281 .iter()
282 .map(|&l| {
283 cumulative += l as i64;
284 cumulative
285 })
286 .collect();
287
288 let values = create_random_array(value, num_runs, null_density, true_density)?;
289
290 match index.data_type() {
291 DataType::Int16 => {
292 let run_ends: Int16Array = run_ends_i64.iter().map(|&v| v as i16).collect();
293 Ok(Arc::new(RunArray::<Int16Type>::try_new(
294 &run_ends, &values,
295 )?))
296 }
297 DataType::Int32 => {
298 let run_ends: Int32Array = run_ends_i64.iter().map(|&v| v as i32).collect();
299 Ok(Arc::new(RunArray::<Int32Type>::try_new(
300 &run_ends, &values,
301 )?))
302 }
303 DataType::Int64 => {
304 let run_ends: Int64Array = run_ends_i64.iter().copied().collect();
305 Ok(Arc::new(RunArray::<Int64Type>::try_new(
306 &run_ends, &values,
307 )?))
308 }
309 other => Err(ArrowError::InvalidArgumentError(format!(
310 "Unsupported run-ends type for REE: {other:?}"
311 ))),
312 }
313}
314
315#[inline]
316fn create_random_list_array(
317 field: &Field,
318 size: usize,
319 null_density: f32,
320 true_density: f32,
321) -> Result<ArrayRef> {
322 let list_null_density = match field.is_nullable() {
324 true => null_density,
325 false => 0.0,
326 };
327 let list_field;
328 let (offsets, child_len) = match field.data_type() {
329 DataType::List(f) => {
330 let (offsets, child_len) = create_random_offsets::<i32>(size, 0, 5);
331 list_field = f;
332 (Buffer::from(offsets.to_byte_slice()), child_len as usize)
333 }
334 DataType::LargeList(f) => {
335 let (offsets, child_len) = create_random_offsets::<i64>(size, 0, 5);
336 list_field = f;
337 (Buffer::from(offsets.to_byte_slice()), child_len as usize)
338 }
339 _ => {
340 return Err(ArrowError::InvalidArgumentError(format!(
341 "Cannot create list array for field {field}"
342 )));
343 }
344 };
345
346 let child_array = create_random_array(list_field, child_len, null_density, true_density)?;
348 let child_data = child_array.to_data();
349 let null_buffer = match field.is_nullable() {
351 true => Some(create_random_null_buffer(size, list_null_density)),
352 false => None,
353 };
354 let list_data = unsafe {
355 ArrayData::new_unchecked(
356 field.data_type().clone(),
357 size,
358 None,
359 null_buffer,
360 0,
361 vec![offsets],
362 vec![child_data],
363 )
364 };
365 Ok(make_array(list_data))
366}
367
368#[inline]
369fn create_random_struct_array(
370 field: &Field,
371 size: usize,
372 null_density: f32,
373 true_density: f32,
374) -> Result<ArrayRef> {
375 let DataType::Struct(struct_fields) = field.data_type() else {
376 return Err(ArrowError::InvalidArgumentError(format!(
377 "Cannot create struct array for field {field}"
378 )));
379 };
380
381 let child_arrays = struct_fields
382 .iter()
383 .map(|struct_field| create_random_array(struct_field, size, null_density, true_density))
384 .collect::<Result<Vec<_>>>()?;
385
386 let null_buffer = match field.is_nullable() {
387 true => {
388 let nulls = arrow_buffer::BooleanBuffer::new(
389 create_random_null_buffer(size, null_density),
390 0,
391 size,
392 );
393 Some(nulls.into())
394 }
395 false => None,
396 };
397
398 Ok(Arc::new(StructArray::try_new(
399 struct_fields.clone(),
400 child_arrays,
401 null_buffer,
402 )?))
403}
404
405#[inline]
406fn create_random_map_array(
407 field: &Field,
408 size: usize,
409 null_density: f32,
410 true_density: f32,
411) -> Result<ArrayRef> {
412 let map_null_density = match field.is_nullable() {
414 true => null_density,
415 false => 0.0,
416 };
417
418 let DataType::Map(entries_field, _) = field.data_type() else {
419 return Err(ArrowError::InvalidArgumentError(format!(
420 "Cannot create map array for field {field:?}"
421 )));
422 };
423
424 let (offsets, child_len) = create_random_offsets::<i32>(size, 0, 5);
425 let offsets = Buffer::from(offsets.to_byte_slice());
426
427 let entries = create_random_array(
428 entries_field,
429 child_len as usize,
430 null_density,
431 true_density,
432 )?
433 .to_data();
434
435 let null_buffer = match field.is_nullable() {
436 true => Some(create_random_null_buffer(size, map_null_density)),
437 false => None,
438 };
439
440 let map_data = unsafe {
441 ArrayData::new_unchecked(
442 field.data_type().clone(),
443 size,
444 None,
445 null_buffer,
446 0,
447 vec![offsets],
448 vec![entries],
449 )
450 };
451 Ok(make_array(map_data))
452}
453
454fn create_random_offsets<T: OffsetSizeTrait + SampleUniform>(
456 size: usize,
457 min: T,
458 max: T,
459) -> (Vec<T>, T) {
460 let rng = &mut seedable_rng();
461
462 let mut current_offset = T::zero();
463
464 let mut offsets = Vec::with_capacity(size + 1);
465 offsets.push(current_offset);
466
467 (0..size).for_each(|_| {
468 current_offset += rng.random_range(min..max);
469 offsets.push(current_offset);
470 });
471
472 (offsets, current_offset)
473}
474
475fn create_random_null_buffer(size: usize, null_density: f32) -> Buffer {
476 let mut rng = seedable_rng();
477 let mut mut_buf = MutableBuffer::new_null(size);
478 {
479 let mut_slice = mut_buf.as_slice_mut();
480 (0..size).for_each(|i| {
481 if rng.random::<f32>() >= null_density {
482 bit_util::set_bit(mut_slice, i)
483 }
484 })
485 };
486 mut_buf.into()
487}
488
489pub trait RandomTemporalValue: ArrowTemporalType {
492 fn value_range() -> impl SampleRange<Self::Native>;
494
495 fn gen_range<R: Rng>(rng: &mut R) -> Self::Native
497 where
498 Self::Native: SampleUniform,
499 {
500 rng.random_range(Self::value_range())
501 }
502
503 fn random<R: Rng>(rng: &mut R) -> Self::Native
505 where
506 Self::Native: SampleUniform,
507 {
508 Self::gen_range(rng)
509 }
510}
511
512impl RandomTemporalValue for TimestampSecondType {
513 fn value_range() -> impl SampleRange<Self::Native> {
516 0..60 * 60 * 24 * 365 * 100
517 }
518}
519
520impl RandomTemporalValue for TimestampMillisecondType {
521 fn value_range() -> impl SampleRange<Self::Native> {
524 0..1_000 * 60 * 60 * 24 * 365 * 100
525 }
526}
527
528impl RandomTemporalValue for TimestampMicrosecondType {
529 fn value_range() -> impl SampleRange<Self::Native> {
532 0..1_000 * 1_000 * 60 * 60 * 24 * 365 * 100
533 }
534}
535
536impl RandomTemporalValue for TimestampNanosecondType {
537 fn value_range() -> impl SampleRange<Self::Native> {
540 0..1_000 * 1_000 * 1_000 * 60 * 60 * 24 * 365 * 100
541 }
542}
543
544impl RandomTemporalValue for Date32Type {
545 fn value_range() -> impl SampleRange<Self::Native> {
548 0..365 * 100
549 }
550}
551
552impl RandomTemporalValue for Date64Type {
553 fn value_range() -> impl SampleRange<Self::Native> {
556 0..1_000 * 60 * 60 * 24 * 365 * 100
557 }
558}
559
560impl RandomTemporalValue for Time32SecondType {
561 fn value_range() -> impl SampleRange<Self::Native> {
564 0..60 * 60 * 24
565 }
566}
567
568impl RandomTemporalValue for Time32MillisecondType {
569 fn value_range() -> impl SampleRange<Self::Native> {
572 0..1_000 * 60 * 60 * 24
573 }
574}
575
576impl RandomTemporalValue for Time64MicrosecondType {
577 fn value_range() -> impl SampleRange<Self::Native> {
580 0..1_000 * 1_000 * 60 * 60 * 24
581 }
582}
583
584impl RandomTemporalValue for Time64NanosecondType {
585 fn value_range() -> impl SampleRange<Self::Native> {
588 0..1_000 * 1_000 * 1_000 * 60 * 60 * 24
589 }
590}
591
592fn create_random_temporal_array<T>(size: usize, null_density: f32) -> PrimitiveArray<T>
593where
594 T: RandomTemporalValue,
595 <T as ArrowPrimitiveType>::Native: SampleUniform,
596{
597 let mut rng = seedable_rng();
598
599 (0..size)
600 .map(|_| {
601 if rng.random::<f32>() < null_density {
602 None
603 } else {
604 Some(T::random(&mut rng))
605 }
606 })
607 .collect()
608}
609
610#[cfg(test)]
611mod tests {
612 use super::*;
613
614 #[test]
615 fn test_create_batch() {
616 let size = 32;
617 let fields = vec![
618 Field::new("a", DataType::Int32, true),
619 Field::new("f16", DataType::Float16, true),
620 Field::new(
621 "timestamp_without_timezone",
622 DataType::Timestamp(TimeUnit::Nanosecond, None),
623 true,
624 ),
625 Field::new(
626 "timestamp_with_timezone",
627 DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),
628 true,
629 ),
630 ];
631 let schema = Schema::new(fields);
632 let schema_ref = Arc::new(schema);
633 let batch = create_random_batch(schema_ref.clone(), size, 0.35, 0.7).unwrap();
634
635 assert_eq!(batch.schema(), schema_ref);
636 assert_eq!(batch.num_columns(), schema_ref.fields().len());
637 for array in batch.columns() {
638 assert_eq!(array.len(), size);
639 }
640 }
641
642 #[test]
643 fn test_create_batch_non_null() {
644 let size = 32;
645 let fields = vec![
646 Field::new("a", DataType::Int32, false),
647 Field::new(
648 "b",
649 DataType::List(Arc::new(Field::new_list_field(DataType::LargeUtf8, false))),
650 false,
651 ),
652 Field::new("a", DataType::Int32, false),
653 ];
654 let schema = Schema::new(fields);
655 let schema_ref = Arc::new(schema);
656 let batch = create_random_batch(schema_ref.clone(), size, 0.35, 0.7).unwrap();
657
658 assert_eq!(batch.schema(), schema_ref);
659 assert_eq!(batch.num_columns(), schema_ref.fields().len());
660 for array in batch.columns() {
661 assert_eq!(array.null_count(), 0);
662 assert_eq!(array.logical_null_count(), 0);
663 }
664 let b_array = batch.column(1);
666 let list_array = b_array.as_list::<i32>();
667 let child_array = list_array.values();
668 assert_eq!(child_array.null_count(), 0);
669 assert!(child_array.len() > list_array.len());
671 }
672
673 #[test]
674 fn test_create_struct_array() {
675 let size = 32;
676 let struct_fields = Fields::from(vec![
677 Field::new("b", DataType::Boolean, true),
678 Field::new(
679 "c",
680 DataType::LargeList(Arc::new(Field::new_list_field(
681 DataType::List(Arc::new(Field::new_list_field(
682 DataType::FixedSizeBinary(6),
683 true,
684 ))),
685 false,
686 ))),
687 true,
688 ),
689 Field::new(
690 "d",
691 DataType::Struct(Fields::from(vec![
692 Field::new("d_x", DataType::Int32, true),
693 Field::new("d_y", DataType::Float32, false),
694 Field::new("d_z", DataType::Binary, true),
695 ])),
696 true,
697 ),
698 ]);
699 let field = Field::new("struct", DataType::Struct(struct_fields), true);
700 let array = create_random_array(&field, size, 0.2, 0.5).unwrap();
701
702 assert_eq!(array.len(), 32);
703 let struct_array = array.as_any().downcast_ref::<StructArray>().unwrap();
704 assert_eq!(struct_array.columns().len(), 3);
705
706 let col_c = struct_array.column_by_name("c").unwrap();
709 let col_c = col_c.as_any().downcast_ref::<LargeListArray>().unwrap();
710 assert_eq!(col_c.len(), size);
711 let col_c_list = col_c.values().as_list::<i32>();
712 assert!(col_c_list.len() > size);
713 let fsb = col_c_list.values();
715 assert_eq!(fsb.data_type(), &DataType::FixedSizeBinary(6));
716 assert!(fsb.len() > col_c_list.len());
717
718 let col_d = struct_array.column_by_name("d").unwrap();
720 let col_d = col_d.as_any().downcast_ref::<StructArray>().unwrap();
721 let col_d_y = col_d.column_by_name("d_y").unwrap();
722 assert_eq!(col_d_y.data_type(), &DataType::Float32);
723 assert_eq!(col_d_y.null_count(), 0);
724 }
725
726 #[test]
727 fn test_create_run_end_encoded_array() {
728 let size = 1000;
729 let ree_field = Field::new(
730 "ree",
731 DataType::RunEndEncoded(
732 Arc::new(Field::new(
733 Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
734 DataType::Int32,
735 false,
736 )),
737 Arc::new(Field::new(
738 Field::REE_VALUES_FIELD_DEFAULT_NAME,
739 DataType::Utf8,
740 true,
741 )),
742 ),
743 false,
744 );
745
746 let array = create_random_array(&ree_field, size, 0.25, 0.0).unwrap();
747 assert_eq!(array.len(), size);
748
749 let ree = array.as_run::<Int32Type>();
750 let run_ends = ree.run_ends().values();
751 let num_runs = run_ends.len();
752
753 assert_eq!(*run_ends.last().unwrap() as usize, size);
754
755 assert_eq!(ree.values().len(), num_runs);
756 }
757
758 #[test]
759 fn test_create_list_array_nested_nullability() {
760 let list_field = Field::new_list(
761 "not_null_list",
762 Field::new_list_field(DataType::Boolean, true),
763 false,
764 );
765
766 let list_array = create_random_array(&list_field, 100, 0.95, 0.5).unwrap();
767
768 assert_eq!(list_array.null_count(), 0);
769 assert!(list_array.as_list::<i32>().values().null_count() > 0);
770 }
771
772 #[test]
773 fn test_create_struct_array_nested_nullability() {
774 let struct_child_fields = vec![
775 Field::new("null_int", DataType::Int32, true),
776 Field::new("int", DataType::Int32, false),
777 ];
778 let struct_field = Field::new_struct("not_null_struct", struct_child_fields, false);
779
780 let struct_array = create_random_array(&struct_field, 100, 0.95, 0.5).unwrap();
781
782 assert_eq!(struct_array.null_count(), 0);
783 assert!(
784 struct_array
785 .as_struct()
786 .column_by_name("null_int")
787 .unwrap()
788 .null_count()
789 > 0
790 );
791 assert_eq!(
792 struct_array
793 .as_struct()
794 .column_by_name("int")
795 .unwrap()
796 .null_count(),
797 0
798 );
799 }
800
801 #[test]
802 fn test_create_list_array_nested_struct_nullability() {
803 let struct_child_fields = vec![
804 Field::new("null_int", DataType::Int32, true),
805 Field::new("int", DataType::Int32, false),
806 ];
807 let list_item_field =
808 Field::new_list_field(DataType::Struct(struct_child_fields.into()), true);
809 let list_field = Field::new_list("not_null_list", list_item_field, false);
810
811 let list_array = create_random_array(&list_field, 100, 0.95, 0.5).unwrap();
812
813 assert_eq!(list_array.null_count(), 0);
814 assert!(list_array.as_list::<i32>().values().null_count() > 0);
815 assert!(
816 list_array
817 .as_list::<i32>()
818 .values()
819 .as_struct()
820 .column_by_name("null_int")
821 .unwrap()
822 .null_count()
823 > 0
824 );
825 assert_eq!(
826 list_array
827 .as_list::<i32>()
828 .values()
829 .as_struct()
830 .column_by_name("int")
831 .unwrap()
832 .null_count(),
833 0
834 );
835 }
836
837 #[test]
838 fn test_create_map_array() {
839 let map_field = Field::new_map(
840 "map",
841 Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
842 Field::new(Field::MAP_KEY_FIELD_DEFAULT_NAME, DataType::Utf8, false),
843 Field::new(Field::MAP_VALUE_FIELD_DEFAULT_NAME, DataType::Utf8, true),
844 false,
845 false,
846 );
847 let array = create_random_array(&map_field, 100, 0.8, 0.5).unwrap();
848
849 assert_eq!(array.len(), 100);
850 assert_eq!(array.null_count(), 0);
852 assert_eq!(array.logical_null_count(), 0);
853 assert!(array.as_map().keys().len() > array.len());
855 assert!(array.as_map().values().len() > array.len());
856 assert_eq!(array.as_map().keys().null_count(), 0);
858 assert!(array.as_map().values().null_count() > 0);
860
861 assert_eq!(array.as_map().keys().data_type(), &DataType::Utf8);
862 assert_eq!(array.as_map().values().data_type(), &DataType::Utf8);
863 }
864
865 #[test]
866 fn test_create_decimal_array() {
867 let size = 10;
868 let fields = vec![
869 Field::new("a", DataType::Decimal128(10, -2), true),
870 Field::new("b", DataType::Decimal256(10, -2), true),
871 ];
872 let schema = Schema::new(fields);
873 let schema_ref = Arc::new(schema);
874 let batch = create_random_batch(schema_ref.clone(), size, 0.35, 0.7).unwrap();
875
876 assert_eq!(batch.schema(), schema_ref);
877 assert_eq!(batch.num_columns(), schema_ref.fields().len());
878 for array in batch.columns() {
879 assert_eq!(array.len(), size);
880 }
881 }
882
883 #[test]
884 fn create_non_nullable_decimal_array_with_null_density() {
885 let size = 10;
886 let fields = vec![
887 Field::new("a", DataType::Decimal128(10, -2), false),
888 Field::new("b", DataType::Decimal256(10, -2), false),
889 ];
890 let schema = Schema::new(fields);
891 let schema_ref = Arc::new(schema);
892 let batch = create_random_batch(schema_ref.clone(), size, 0.35, 0.7).unwrap();
893
894 assert_eq!(batch.schema(), schema_ref);
895 assert_eq!(batch.num_columns(), schema_ref.fields().len());
896 for array in batch.columns() {
897 assert_eq!(array.len(), size);
898 assert_eq!(array.null_count(), 0);
899 }
900 }
901}