1use arrow_array::*;
8use arrow_array::{cast::AsArray, types::*};
9use arrow_buffer::{ArrowNativeType, NullBuffer, OffsetBuffer};
10use arrow_schema::{ArrowError, DataType};
11use std::sync::Arc;
12
13fn length_impl<P: ArrowPrimitiveType>(
14 offsets: &OffsetBuffer<P::Native>,
15 nulls: Option<&NullBuffer>,
16) -> ArrayRef {
17 let v: Vec<_> = offsets
18 .windows(2)
19 .map(|w| w[1].sub_wrapping(w[0]))
20 .collect();
21 Arc::new(PrimitiveArray::<P>::new(v.into(), nulls.cloned()))
22}
23
24fn bit_length_impl<P: ArrowPrimitiveType>(
25 offsets: &OffsetBuffer<P::Native>,
26 nulls: Option<&NullBuffer>,
27) -> ArrayRef {
28 let bits = P::Native::usize_as(8);
29 let c = |w: &[P::Native]| w[1].sub_wrapping(w[0]).mul_wrapping(bits);
30 let v: Vec<_> = offsets.windows(2).map(c).collect();
31 Arc::new(PrimitiveArray::<P>::new(v.into(), nulls.cloned()))
32}
33
34pub fn length(array: &dyn Array) -> Result<ArrayRef, ArrowError> {
45 if let Some(d) = array.as_any_dictionary_opt() {
46 let lengths = length(d.values().as_ref())?;
47 return Ok(d.with_values(lengths));
48 }
49 if let Some(ree) = array.as_any_ree_opt() {
50 let lengths = length(ree.values())?;
51 return Ok(ree.with_values(lengths));
52 }
53 match array.data_type() {
54 DataType::List(_) => {
55 let list = array.as_list::<i32>();
56 Ok(length_impl::<Int32Type>(list.offsets(), list.nulls()))
57 }
58 DataType::LargeList(_) => {
59 let list = array.as_list::<i64>();
60 Ok(length_impl::<Int64Type>(list.offsets(), list.nulls()))
61 }
62 DataType::ListView(_) => {
63 let list = array.as_list_view::<i32>();
64 Ok(Arc::new(Int32Array::new(
65 list.sizes().clone(),
66 list.nulls().cloned(),
67 )))
68 }
69 DataType::LargeListView(_) => {
70 let list = array.as_list_view::<i64>();
71 Ok(Arc::new(Int64Array::new(
72 list.sizes().clone(),
73 list.nulls().cloned(),
74 )))
75 }
76 DataType::Map(_, _) => {
77 let map = array.as_map();
78 Ok(length_impl::<Int32Type>(map.offsets(), map.nulls()))
79 }
80 DataType::Utf8 => {
81 let list = array.as_string::<i32>();
82 Ok(length_impl::<Int32Type>(list.offsets(), list.nulls()))
83 }
84 DataType::LargeUtf8 => {
85 let list = array.as_string::<i64>();
86 Ok(length_impl::<Int64Type>(list.offsets(), list.nulls()))
87 }
88 DataType::Utf8View => {
89 let list = array.as_string_view();
90 let v = list.views().iter().map(|v| *v as i32).collect::<Vec<_>>();
91 Ok(Arc::new(PrimitiveArray::<Int32Type>::try_new(
92 v.into(),
93 list.nulls().cloned(),
94 )?))
95 }
96 DataType::Binary => {
97 let list = array.as_binary::<i32>();
98 Ok(length_impl::<Int32Type>(list.offsets(), list.nulls()))
99 }
100 DataType::LargeBinary => {
101 let list = array.as_binary::<i64>();
102 Ok(length_impl::<Int64Type>(list.offsets(), list.nulls()))
103 }
104 DataType::FixedSizeBinary(len) | DataType::FixedSizeList(_, len) => Ok(Arc::new(
105 Int32Array::try_new(vec![*len; array.len()].into(), array.nulls().cloned())?,
106 )),
107 DataType::BinaryView => {
108 let list = array.as_binary_view();
109 let v = list.views().iter().map(|v| *v as i32).collect::<Vec<_>>();
110 Ok(Arc::new(PrimitiveArray::<Int32Type>::try_new(
111 v.into(),
112 list.nulls().cloned(),
113 )?))
114 }
115 other => Err(ArrowError::ComputeError(format!(
116 "length not supported for {other:?}"
117 ))),
118 }
119}
120
121pub fn bit_length(array: &dyn Array) -> Result<ArrayRef, ArrowError> {
129 if let Some(d) = array.as_any_dictionary_opt() {
130 let lengths = bit_length(d.values().as_ref())?;
131 return Ok(d.with_values(lengths));
132 }
133 if let Some(ree) = array.as_any_ree_opt() {
134 let lengths = bit_length(ree.values())?;
135 return Ok(ree.with_values(lengths));
136 }
137
138 match array.data_type() {
139 DataType::Utf8 => {
140 let list = array.as_string::<i32>();
141 Ok(bit_length_impl::<Int32Type>(list.offsets(), list.nulls()))
142 }
143 DataType::LargeUtf8 => {
144 let list = array.as_string::<i64>();
145 Ok(bit_length_impl::<Int64Type>(list.offsets(), list.nulls()))
146 }
147 DataType::Utf8View => {
148 let list = array.as_string_view();
149 let values = list
150 .views()
151 .iter()
152 .map(|view| (*view as i32).wrapping_mul(8))
153 .collect();
154 Ok(Arc::new(Int32Array::try_new(
155 values,
156 array.nulls().cloned(),
157 )?))
158 }
159 DataType::Binary => {
160 let list = array.as_binary::<i32>();
161 Ok(bit_length_impl::<Int32Type>(list.offsets(), list.nulls()))
162 }
163 DataType::LargeBinary => {
164 let list = array.as_binary::<i64>();
165 Ok(bit_length_impl::<Int64Type>(list.offsets(), list.nulls()))
166 }
167 DataType::FixedSizeBinary(len) => Ok(Arc::new(Int32Array::try_new(
168 vec![*len * 8; array.len()].into(),
169 array.nulls().cloned(),
170 )?)),
171 DataType::BinaryView => {
172 let list = array.as_binary_view();
173 let values = list
174 .views()
175 .iter()
176 .map(|view| (*view as i32).wrapping_mul(8))
177 .collect();
178 Ok(Arc::new(Int32Array::try_new(
179 values,
180 array.nulls().cloned(),
181 )?))
182 }
183 other => Err(ArrowError::ComputeError(format!(
184 "bit_length not supported for {other:?}"
185 ))),
186 }
187}
188
189#[cfg(test)]
190mod tests {
191 use super::*;
192 use arrow_array::builder::{Int32Builder, MapBuilder, StringBuilder};
193 use arrow_buffer::{Buffer, ScalarBuffer};
194 use arrow_data::ArrayData;
195 use arrow_schema::Field;
196
197 fn length_cases_string() -> Vec<(Vec<&'static str>, usize, Vec<i32>)> {
198 let values = [
200 "one",
201 "on",
202 "o",
203 "",
204 "this is a longer string to test string array with",
205 ];
206 let values = values.into_iter().cycle().take(4096).collect();
207 let expected = [3, 2, 1, 0, 49].into_iter().cycle().take(4096).collect();
208
209 vec![
210 (vec!["hello", " ", "world"], 3, vec![5, 1, 5]),
211 (vec!["hello", " ", "world", "!"], 4, vec![5, 1, 5, 1]),
212 (vec!["💖"], 1, vec![4]),
213 (values, 4096, expected),
214 ]
215 }
216
217 macro_rules! length_binary_helper {
218 ($offset_ty: ty, $result_ty: ty, $kernel: ident, $value: expr, $expected: expr) => {{
219 let array = GenericBinaryArray::<$offset_ty>::from($value);
220 let result = $kernel(&array).unwrap();
221 let result = result.as_any().downcast_ref::<$result_ty>().unwrap();
222 let expected: $result_ty = $expected.into();
223 assert_eq!(&expected, result);
224 }};
225 }
226
227 macro_rules! length_list_helper {
228 ($offset_ty: ty, $result_ty: ty, $element_ty: ty, $value: expr, $expected: expr) => {{
229 let array =
230 GenericListArray::<$offset_ty>::from_iter_primitive::<$element_ty, _, _>($value);
231 let result = length(&array).unwrap();
232 let result = result.as_any().downcast_ref::<$result_ty>().unwrap();
233 let expected: $result_ty = $expected.into();
234 assert_eq!(&expected, result);
235 }};
236 }
237
238 #[test]
239 fn length_test_string() {
240 length_cases_string()
241 .into_iter()
242 .for_each(|(input, len, expected)| {
243 let array = StringArray::from(input);
244 let result = length(&array).unwrap();
245 assert_eq!(len, result.len());
246 let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
247 expected.iter().enumerate().for_each(|(i, value)| {
248 assert_eq!(*value, result.value(i));
249 });
250 })
251 }
252
253 #[test]
254 fn length_test_large_string() {
255 length_cases_string()
256 .into_iter()
257 .for_each(|(input, len, expected)| {
258 let array = LargeStringArray::from(input);
259 let result = length(&array).unwrap();
260 assert_eq!(len, result.len());
261 let result = result.as_any().downcast_ref::<Int64Array>().unwrap();
262 expected.iter().enumerate().for_each(|(i, value)| {
263 assert_eq!(*value as i64, result.value(i));
264 });
265 })
266 }
267
268 #[test]
269 fn length_test_string_view() {
270 length_cases_string()
271 .into_iter()
272 .for_each(|(input, len, expected)| {
273 let array = StringViewArray::from(input);
274 let result = length(&array).unwrap();
275 assert_eq!(len, result.len());
276 let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
277 expected.iter().enumerate().for_each(|(i, value)| {
278 assert_eq!(*value, result.value(i));
279 });
280 })
281 }
282
283 #[test]
284 fn length_test_binary() {
285 let value: Vec<&[u8]> = vec![b"zero", b"one", &[0xff, 0xf8]];
286 let result: Vec<i32> = vec![4, 3, 2];
287 length_binary_helper!(i32, Int32Array, length, value, result)
288 }
289
290 #[test]
291 fn length_test_large_binary() {
292 let value: Vec<&[u8]> = vec![b"zero", &[0xff, 0xf8], b"two"];
293 let result: Vec<i64> = vec![4, 2, 3];
294 length_binary_helper!(i64, Int64Array, length, value, result)
295 }
296
297 #[test]
298 fn length_test_binary_view() {
299 let value: Vec<&[u8]> = vec![
300 b"zero",
301 &[0xff, 0xf8],
302 b"two",
303 b"this is a longer string to test binary array with",
304 ];
305 let expected: Vec<i32> = vec![4, 2, 3, 49];
306
307 let array = BinaryViewArray::from(value);
308 let result = length(&array).unwrap();
309 let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
310 let expected: Int32Array = expected.into();
311 assert_eq!(&expected, result);
312 }
313
314 #[test]
315 fn length_test_list() {
316 let value = vec![
317 Some(vec![]),
318 Some(vec![Some(1), Some(2), Some(4)]),
319 Some(vec![Some(0)]),
320 ];
321 let result: Vec<i32> = vec![0, 3, 1];
322 length_list_helper!(i32, Int32Array, Int32Type, value, result)
323 }
324
325 #[test]
326 fn length_test_large_list() {
327 let value = vec![
328 Some(vec![]),
329 Some(vec![Some(1.1), Some(2.2), Some(3.3)]),
330 Some(vec![None]),
331 ];
332 let result: Vec<i64> = vec![0, 3, 1];
333 length_list_helper!(i64, Int64Array, Float32Type, value, result)
334 }
335
336 #[test]
337 fn length_test_map() {
338 let mut map_builder = MapBuilder::new(None, StringBuilder::new(), Int32Builder::default());
339 map_builder.append(true).unwrap();
341
342 map_builder.keys().extend(["a", "b", "cd"].map(Some));
344 map_builder.values().extend([1, 2, 4].map(Some));
345 map_builder.append(true).unwrap();
346
347 map_builder.keys().append_value("e");
349 map_builder.values().append_value(0);
350 map_builder.append(true).unwrap();
351
352 let map_array = map_builder.finish();
353
354 let lengths = length(&map_array).unwrap();
355 let lengths = lengths.as_primitive::<Int32Type>();
356 assert_eq!(lengths, &Int32Array::from(vec![0, 3, 1]));
357 }
358
359 type OptionStr = Option<&'static str>;
360
361 fn length_null_cases_string() -> Vec<(Vec<OptionStr>, usize, Vec<Option<i32>>)> {
362 vec![(
363 vec![Some("one"), None, Some("three"), Some("four")],
364 4,
365 vec![Some(3), None, Some(5), Some(4)],
366 )]
367 }
368
369 #[test]
370 fn length_null_string() {
371 length_null_cases_string()
372 .into_iter()
373 .for_each(|(input, len, expected)| {
374 let array = StringArray::from(input);
375 let result = length(&array).unwrap();
376 assert_eq!(len, result.len());
377 let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
378
379 let expected: Int32Array = expected.into();
380 assert_eq!(&expected, result);
381 })
382 }
383
384 #[test]
385 fn length_null_large_string() {
386 length_null_cases_string()
387 .into_iter()
388 .for_each(|(input, len, expected)| {
389 let array = LargeStringArray::from(input);
390 let result = length(&array).unwrap();
391 assert_eq!(len, result.len());
392 let result = result.as_any().downcast_ref::<Int64Array>().unwrap();
393
394 let expected: Int64Array = expected
396 .iter()
397 .map(|e| e.map(|e| e as i64))
398 .collect::<Vec<_>>()
399 .into();
400 assert_eq!(&expected, result);
401 })
402 }
403
404 #[test]
405 fn length_null_binary() {
406 let value: Vec<Option<&[u8]>> =
407 vec![Some(b"zero"), None, Some(&[0xff, 0xf8]), Some(b"three")];
408 let result: Vec<Option<i32>> = vec![Some(4), None, Some(2), Some(5)];
409 length_binary_helper!(i32, Int32Array, length, value, result)
410 }
411
412 #[test]
413 fn length_null_large_binary() {
414 let value: Vec<Option<&[u8]>> =
415 vec![Some(&[0xff, 0xf8]), None, Some(b"two"), Some(b"three")];
416 let result: Vec<Option<i64>> = vec![Some(2), None, Some(3), Some(5)];
417 length_binary_helper!(i64, Int64Array, length, value, result)
418 }
419
420 #[test]
421 fn length_null_list() {
422 let value = vec![
423 Some(vec![]),
424 None,
425 Some(vec![Some(1), None, Some(2), Some(4)]),
426 Some(vec![Some(0)]),
427 ];
428 let result: Vec<Option<i32>> = vec![Some(0), None, Some(4), Some(1)];
429 length_list_helper!(i32, Int32Array, Int8Type, value, result)
430 }
431
432 #[test]
433 fn length_null_large_list() {
434 let value = vec![
435 Some(vec![]),
436 None,
437 Some(vec![Some(1.1), None, Some(4.0)]),
438 Some(vec![Some(0.1)]),
439 ];
440 let result: Vec<Option<i64>> = vec![Some(0), None, Some(3), Some(1)];
441 length_list_helper!(i64, Int64Array, Float32Type, value, result)
442 }
443
444 #[test]
445 fn length_test_null_map() {
446 let mut map_builder = MapBuilder::new(None, StringBuilder::new(), Int32Builder::default());
447 map_builder.append(true).unwrap();
449
450 map_builder.append_nulls(1).unwrap();
452
453 map_builder.keys().extend(["a", "b", "cd"].map(Some));
455 map_builder.values().extend([1, 2, 4].map(Some));
456 map_builder.append(true).unwrap();
457
458 map_builder.keys().append_value("e");
460 map_builder.values().append_value(0);
461 map_builder.append(true).unwrap();
462
463 let map_array = map_builder.finish();
464 let lengths = length(&map_array).unwrap();
465 let lengths = lengths.as_primitive::<Int32Type>();
466 assert_eq!(
467 lengths,
468 &Int32Array::from(vec![Some(0), None, Some(3), Some(1)])
469 );
470 }
471
472 #[test]
473 fn length_test_list_view() {
474 let field = Arc::new(Field::new_list_field(DataType::Int32, true));
476 let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7]);
477 let offsets = ScalarBuffer::from(vec![0i32, 3, 6]);
478 let sizes = ScalarBuffer::from(vec![3i32, 3, 2]);
479 let list_array = ListViewArray::new(field, offsets, sizes, Arc::new(values), None);
480
481 let result = length(&list_array).unwrap();
482 let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
483 let expected: Int32Array = vec![3, 3, 2].into();
484 assert_eq!(&expected, result);
485 }
486
487 #[test]
488 fn length_test_large_list_view() {
489 let field = Arc::new(Field::new_list_field(DataType::Int32, true));
491 let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7]);
492 let offsets = ScalarBuffer::from(vec![0i64, 3, 6]);
493 let sizes = ScalarBuffer::from(vec![3i64, 3, 2]);
494 let list_array = LargeListViewArray::new(field, offsets, sizes, Arc::new(values), None);
495
496 let result = length(&list_array).unwrap();
497 let result = result.as_any().downcast_ref::<Int64Array>().unwrap();
498 let expected: Int64Array = vec![3i64, 3, 2].into();
499 assert_eq!(&expected, result);
500 }
501
502 #[test]
503 fn length_null_list_view() {
504 let field = Arc::new(Field::new_list_field(DataType::Int32, true));
506 let values = Int32Array::from(vec![1, 2, 3, 4, 0]);
507 let offsets = ScalarBuffer::from(vec![0i32, 0, 0, 4]);
508 let sizes = ScalarBuffer::from(vec![0i32, 0, 4, 1]);
509 let nulls = NullBuffer::from(vec![true, false, true, true]);
510 let list_array = ListViewArray::new(field, offsets, sizes, Arc::new(values), Some(nulls));
511
512 let result = length(&list_array).unwrap();
513 let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
514 let expected: Int32Array = vec![Some(0), None, Some(4), Some(1)].into();
515 assert_eq!(&expected, result);
516 }
517
518 #[test]
519 fn length_null_large_list_view() {
520 let field = Arc::new(Field::new_list_field(DataType::Float32, true));
522 let values = Float32Array::from(vec![1.0, 2.0, 3.0, 0.1]);
523 let offsets = ScalarBuffer::from(vec![0i64, 0, 0, 3]);
524 let sizes = ScalarBuffer::from(vec![0i64, 0, 3, 1]);
525 let nulls = NullBuffer::from(vec![true, false, true, true]);
526 let list_array =
527 LargeListViewArray::new(field, offsets, sizes, Arc::new(values), Some(nulls));
528
529 let result = length(&list_array).unwrap();
530 let result = result.as_any().downcast_ref::<Int64Array>().unwrap();
531 let expected: Int64Array = vec![Some(0i64), None, Some(3), Some(1)].into();
532 assert_eq!(&expected, result);
533 }
534
535 #[test]
537 fn length_wrong_type() {
538 let array: UInt64Array = vec![1u64].into();
539
540 assert!(length(&array).is_err());
541 }
542
543 #[test]
545 fn length_offsets_string() {
546 let a = StringArray::from(vec![Some("hello"), Some(" "), Some("world"), None]);
547 let b = a.slice(1, 3);
548 let result = length(&b).unwrap();
549 let result: &Int32Array = result.as_primitive();
550
551 let expected = Int32Array::from(vec![Some(1), Some(5), None]);
552 assert_eq!(&expected, result);
553 }
554
555 #[test]
556 fn length_offsets_binary() {
557 let value: Vec<Option<&[u8]>> = vec![Some(b"hello"), Some(b" "), Some(&[0xff, 0xf8]), None];
558 let a = BinaryArray::from(value);
559 let b = a.slice(1, 3);
560 let result = length(&b).unwrap();
561 let result: &Int32Array = result.as_primitive();
562
563 let expected = Int32Array::from(vec![Some(1), Some(2), None]);
564 assert_eq!(&expected, result);
565 }
566
567 fn bit_length_cases() -> Vec<(Vec<&'static str>, usize, Vec<i32>)> {
568 let values = ["one", "on", "o", ""];
570 let values = values.into_iter().cycle().take(4096).collect();
571 let expected = [24, 16, 8, 0].into_iter().cycle().take(4096).collect();
572
573 vec![
574 (vec!["hello", " ", "world", "!"], 4, vec![40, 8, 40, 8]),
575 (vec!["💖"], 1, vec![32]),
576 (vec!["josé"], 1, vec![40]),
577 (values, 4096, expected),
578 ]
579 }
580
581 #[test]
582 fn bit_length_test_string() {
583 bit_length_cases()
584 .into_iter()
585 .for_each(|(input, len, expected)| {
586 let array = StringArray::from(input);
587 let result = bit_length(&array).unwrap();
588 assert_eq!(len, result.len());
589 let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
590 expected.iter().enumerate().for_each(|(i, value)| {
591 assert_eq!(*value, result.value(i));
592 });
593 })
594 }
595
596 #[test]
597 fn bit_length_test_large_string() {
598 bit_length_cases()
599 .into_iter()
600 .for_each(|(input, len, expected)| {
601 let array = LargeStringArray::from(input);
602 let result = bit_length(&array).unwrap();
603 assert_eq!(len, result.len());
604 let result = result.as_any().downcast_ref::<Int64Array>().unwrap();
605 expected.iter().enumerate().for_each(|(i, value)| {
606 assert_eq!(*value as i64, result.value(i));
607 });
608 })
609 }
610
611 #[test]
612 fn bit_length_test_utf8view() {
613 bit_length_cases()
614 .into_iter()
615 .for_each(|(input, len, expected)| {
616 let string_array = StringViewArray::from(input);
617 let result = bit_length(&string_array).unwrap();
618 assert_eq!(len, result.len());
619 let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
620 expected.iter().enumerate().for_each(|(i, value)| {
621 assert_eq!(*value, result.value(i));
622 });
623 })
624 }
625
626 #[test]
627 fn bit_length_null_utf8view() {
628 bit_length_null_cases()
629 .into_iter()
630 .for_each(|(input, len, expected)| {
631 let array = StringArray::from(input);
632 let result = bit_length(&array).unwrap();
633 assert_eq!(len, result.len());
634 let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
635
636 let expected: Int32Array = expected.into();
637 assert_eq!(&expected, result);
638 })
639 }
640 #[test]
641 fn bit_length_binary() {
642 let value: Vec<&[u8]> = vec![b"one", &[0xff, 0xf8], b"three"];
643 let expected: Vec<i32> = vec![24, 16, 40];
644 length_binary_helper!(i32, Int32Array, bit_length, value, expected)
645 }
646
647 #[test]
648 fn bit_length_large_binary() {
649 let value: Vec<&[u8]> = vec![b"zero", b" ", &[0xff, 0xf8]];
650 let expected: Vec<i64> = vec![32, 8, 16];
651 length_binary_helper!(i64, Int64Array, bit_length, value, expected)
652 }
653
654 #[test]
655 fn bit_length_binary_view() {
656 let value: Vec<&[u8]> = vec![
657 b"zero",
658 &[0xff, 0xf8],
659 b"two",
660 b"this is a longer string to test binary array with",
661 ];
662 let expected: Vec<i32> = vec![32, 16, 24, 392];
663
664 let array = BinaryViewArray::from(value);
665 let result = bit_length(&array).unwrap();
666 let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
667 let expected: Int32Array = expected.into();
668 assert_eq!(&expected, result);
669 }
670
671 #[test]
672 fn bit_length_null_binary_view() {
673 let value: Vec<Option<&[u8]>> =
674 vec![Some(b"one"), None, Some(b"three"), Some(&[0xff, 0xf8])];
675 let expected: Vec<Option<i32>> = vec![Some(24), None, Some(40), Some(16)];
676
677 let array = BinaryViewArray::from(value);
678 let result = bit_length(&array).unwrap();
679 let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
680 let expected: Int32Array = expected.into();
681 assert_eq!(&expected, result);
682 }
683
684 fn bit_length_null_cases() -> Vec<(Vec<OptionStr>, usize, Vec<Option<i32>>)> {
685 vec![(
686 vec![Some("one"), None, Some("three"), Some("four")],
687 4,
688 vec![Some(24), None, Some(40), Some(32)],
689 )]
690 }
691
692 #[test]
693 fn bit_length_null_string() {
694 bit_length_null_cases()
695 .into_iter()
696 .for_each(|(input, len, expected)| {
697 let array = StringArray::from(input);
698 let result = bit_length(&array).unwrap();
699 assert_eq!(len, result.len());
700 let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
701
702 let expected: Int32Array = expected.into();
703 assert_eq!(&expected, result);
704 })
705 }
706
707 #[test]
708 fn bit_length_null_large_string() {
709 bit_length_null_cases()
710 .into_iter()
711 .for_each(|(input, len, expected)| {
712 let array = LargeStringArray::from(input);
713 let result = bit_length(&array).unwrap();
714 assert_eq!(len, result.len());
715 let result = result.as_any().downcast_ref::<Int64Array>().unwrap();
716
717 let expected: Int64Array = expected
719 .iter()
720 .map(|e| e.map(|e| e as i64))
721 .collect::<Vec<_>>()
722 .into();
723 assert_eq!(&expected, result);
724 })
725 }
726
727 #[test]
728 fn bit_length_null_binary() {
729 let value: Vec<Option<&[u8]>> =
730 vec![Some(b"one"), None, Some(b"three"), Some(&[0xff, 0xf8])];
731 let expected: Vec<Option<i32>> = vec![Some(24), None, Some(40), Some(16)];
732 length_binary_helper!(i32, Int32Array, bit_length, value, expected)
733 }
734
735 #[test]
736 fn bit_length_null_large_binary() {
737 let value: Vec<Option<&[u8]>> =
738 vec![Some(b"one"), None, Some(&[0xff, 0xf8]), Some(b"four")];
739 let expected: Vec<Option<i64>> = vec![Some(24), None, Some(16), Some(32)];
740 length_binary_helper!(i64, Int64Array, bit_length, value, expected)
741 }
742
743 #[test]
745 fn bit_length_wrong_type() {
746 let array: UInt64Array = vec![1u64].into();
747
748 assert!(bit_length(&array).is_err());
749 }
750
751 #[test]
753 fn bit_length_offsets_string() {
754 let a = StringArray::from(vec![Some("hello"), Some(" "), Some("world"), None]);
755 let b = a.slice(1, 3);
756 let result = bit_length(&b).unwrap();
757 let result: &Int32Array = result.as_primitive();
758
759 let expected = Int32Array::from(vec![Some(8), Some(40), None]);
760 assert_eq!(&expected, result);
761 }
762
763 #[test]
764 fn bit_length_offsets_binary() {
765 let value: Vec<Option<&[u8]>> = vec![Some(b"hello"), Some(&[]), Some(b"world"), None];
766 let a = BinaryArray::from(value);
767 let b = a.slice(1, 3);
768 let result = bit_length(&b).unwrap();
769 let result: &Int32Array = result.as_primitive();
770
771 let expected = Int32Array::from(vec![Some(0), Some(40), None]);
772 assert_eq!(&expected, result);
773 }
774
775 #[test]
776 fn length_dictionary() {
777 _length_dictionary::<Int8Type>();
778 _length_dictionary::<Int16Type>();
779 _length_dictionary::<Int32Type>();
780 _length_dictionary::<Int64Type>();
781 _length_dictionary::<UInt8Type>();
782 _length_dictionary::<UInt16Type>();
783 _length_dictionary::<UInt32Type>();
784 _length_dictionary::<UInt64Type>();
785 }
786
787 fn _length_dictionary<K: ArrowDictionaryKeyType>() {
788 const TOTAL: i32 = 100;
789
790 let v = ["aaaa", "bb", "ccccc", "ddd", "eeeeee"];
791 let data: Vec<Option<&str>> = (0..TOTAL)
792 .map(|n| {
793 let i = n % 5;
794 if i == 3 { None } else { Some(v[i as usize]) }
795 })
796 .collect();
797
798 let dict_array: DictionaryArray<K> = data.clone().into_iter().collect();
799
800 let expected: Vec<Option<i32>> =
801 data.iter().map(|opt| opt.map(|s| s.len() as i32)).collect();
802
803 let res = length(&dict_array).unwrap();
804 let actual = res.as_any().downcast_ref::<DictionaryArray<K>>().unwrap();
805 let actual: Vec<Option<i32>> = actual
806 .values()
807 .as_any()
808 .downcast_ref::<Int32Array>()
809 .unwrap()
810 .take_iter(dict_array.keys_iter())
811 .collect();
812
813 for i in 0..TOTAL as usize {
814 assert_eq!(expected[i], actual[i],);
815 }
816 }
817
818 #[test]
819 fn bit_length_dictionary() {
820 _bit_length_dictionary::<Int8Type>();
821 _bit_length_dictionary::<Int16Type>();
822 _bit_length_dictionary::<Int32Type>();
823 _bit_length_dictionary::<Int64Type>();
824 _bit_length_dictionary::<UInt8Type>();
825 _bit_length_dictionary::<UInt16Type>();
826 _bit_length_dictionary::<UInt32Type>();
827 _bit_length_dictionary::<UInt64Type>();
828 }
829
830 fn _bit_length_dictionary<K: ArrowDictionaryKeyType>() {
831 const TOTAL: i32 = 100;
832
833 let v = ["aaaa", "bb", "ccccc", "ddd", "eeeeee"];
834 let data: Vec<Option<&str>> = (0..TOTAL)
835 .map(|n| {
836 let i = n % 5;
837 if i == 3 { None } else { Some(v[i as usize]) }
838 })
839 .collect();
840
841 let dict_array: DictionaryArray<K> = data.clone().into_iter().collect();
842
843 let expected: Vec<Option<i32>> = data
844 .iter()
845 .map(|opt| opt.map(|s| (s.chars().count() * 8) as i32))
846 .collect();
847
848 let res = bit_length(&dict_array).unwrap();
849 let actual = res.as_any().downcast_ref::<DictionaryArray<K>>().unwrap();
850 let actual: Vec<Option<i32>> = actual
851 .values()
852 .as_any()
853 .downcast_ref::<Int32Array>()
854 .unwrap()
855 .take_iter(dict_array.keys_iter())
856 .collect();
857
858 for i in 0..TOTAL as usize {
859 assert_eq!(expected[i], actual[i],);
860 }
861 }
862
863 #[test]
864 fn test_fixed_size_list_length() {
865 let value_data = ArrayData::builder(DataType::Int32)
867 .len(9)
868 .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7, 8]))
869 .build()
870 .unwrap();
871 let list_data_type =
872 DataType::FixedSizeList(Arc::new(Field::new_list_field(DataType::Int32, false)), 3);
873 let nulls = NullBuffer::from(vec![true, false, true]);
874 let list_data = ArrayData::builder(list_data_type)
875 .len(3)
876 .add_child_data(value_data)
877 .nulls(Some(nulls))
878 .build()
879 .unwrap();
880 let list_array = FixedSizeListArray::from(list_data);
881
882 let lengths = length(&list_array).unwrap();
883 let lengths = lengths.as_primitive::<Int32Type>();
884
885 assert_eq!(lengths.len(), 3);
886 assert_eq!(lengths.value(0), 3);
887 assert!(lengths.is_null(1));
888 assert_eq!(lengths.value(2), 3);
889 }
890
891 #[test]
892 fn test_fixed_size_binary() {
893 let array = FixedSizeBinaryArray::new(4, [0; 16].into(), None);
894 let result = length(&array).unwrap();
895 assert_eq!(result.as_ref(), &Int32Array::from(vec![4; 4]));
896
897 let result = bit_length(&array).unwrap();
898 assert_eq!(result.as_ref(), &Int32Array::from(vec![32; 4]));
899 }
900 #[test]
901 fn length_test_ree_string_values() {
902 use arrow_array::RunArray;
903 use arrow_array::types::Int32Type;
904
905 let string_values = StringArray::from(vec!["hello", "owl", "test", "arrow", "a"]);
906 let run_ends = PrimitiveArray::<Int32Type>::from(vec![2i32, 5, 9, 11, 14]);
907 let ree_array = RunArray::<Int32Type>::try_new(&run_ends, &string_values).unwrap();
908
909 let result = length(&ree_array).unwrap();
910 let result = result
911 .as_any()
912 .downcast_ref::<RunArray<Int32Type>>()
913 .unwrap();
914
915 let result_values = result
916 .values()
917 .as_any()
918 .downcast_ref::<Int32Array>()
919 .unwrap();
920
921 let expected: Int32Array = vec![5, 3, 4, 5, 1].into();
922 assert_eq!(&expected, result_values);
923 }
924 #[test]
925 fn length_test_ree_invalid_type_early_fail() {
926 use arrow_array::RunArray;
927 use arrow_array::types::Int32Type;
928
929 let uint64_values = UInt64Array::from(vec![1u64, 2, 3]);
930 let run_ends = PrimitiveArray::<Int32Type>::from(vec![1i32, 2, 3]);
931 let ree_array = RunArray::<Int32Type>::try_new(&run_ends, &uint64_values).unwrap();
932
933 assert!(length(&ree_array).is_err());
934 }
935
936 #[test]
937 fn bit_length_test_ree_utf8() {
938 use arrow_array::RunArray;
939 use arrow_array::types::Int32Type;
940
941 let strings = StringArray::from(vec!["hello", "world", "test"]);
942 let run_ends = PrimitiveArray::<Int32Type>::from(vec![1i32, 2, 3]);
943 let ree_array = RunArray::<Int32Type>::try_new(&run_ends, &strings).unwrap();
944
945 let result = bit_length(&ree_array).unwrap();
946 let result_values = result
947 .as_any()
948 .downcast_ref::<RunArray<Int32Type>>()
949 .unwrap()
950 .values()
951 .as_any()
952 .downcast_ref::<Int32Array>()
953 .unwrap();
954
955 let expected: Int32Array = vec![40, 40, 32].into();
956 assert_eq!(&expected, result_values);
957 }
958}