Skip to main content

arrow_string/
length.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Defines kernel for length of string arrays and binary arrays
6
7use arrow_array::*;
8use arrow_array::{cast::AsArray, types::*};
9use arrow_buffer::{ArrowNativeType, NullBuffer, OffsetBuffer};
10use arrow_schema::{ArrowError, DataType};
11use std::sync::Arc;
12
13fn length_impl<P: ArrowPrimitiveType>(
14    offsets: &OffsetBuffer<P::Native>,
15    nulls: Option<&NullBuffer>,
16) -> ArrayRef {
17    let v: Vec<_> = offsets
18        .windows(2)
19        .map(|w| w[1].sub_wrapping(w[0]))
20        .collect();
21    Arc::new(PrimitiveArray::<P>::new(v.into(), nulls.cloned()))
22}
23
24fn bit_length_impl<P: ArrowPrimitiveType>(
25    offsets: &OffsetBuffer<P::Native>,
26    nulls: Option<&NullBuffer>,
27) -> ArrayRef {
28    let bits = P::Native::usize_as(8);
29    let c = |w: &[P::Native]| w[1].sub_wrapping(w[0]).mul_wrapping(bits);
30    let v: Vec<_> = offsets.windows(2).map(c).collect();
31    Arc::new(PrimitiveArray::<P>::new(v.into(), nulls.cloned()))
32}
33
34/// Returns an array of Int32/Int64 denoting the length of each value in the array.
35///
36/// For list array, length is the number of elements in each list.
37/// For map array, length is the number of entries in each map.
38/// For string array and binary array, length is the number of bytes of each value.
39///
40/// * this only accepts ListArray/LargeListArray, MapArray, StringArray/LargeStringArray/StringViewArray, BinaryArray/LargeBinaryArray, FixedSizeListArray,
41///   and ListViewArray/LargeListViewArray, or DictionaryArray with above Arrays as values, or
42///   RunEndEncoded arrays with above arrays as values
43/// * length of null is null.
44pub fn length(array: &dyn Array) -> Result<ArrayRef, ArrowError> {
45    if let Some(d) = array.as_any_dictionary_opt() {
46        let lengths = length(d.values().as_ref())?;
47        return Ok(d.with_values(lengths));
48    }
49    if let Some(ree) = array.as_any_ree_opt() {
50        let lengths = length(ree.values())?;
51        return Ok(ree.with_values(lengths));
52    }
53    match array.data_type() {
54        DataType::List(_) => {
55            let list = array.as_list::<i32>();
56            Ok(length_impl::<Int32Type>(list.offsets(), list.nulls()))
57        }
58        DataType::LargeList(_) => {
59            let list = array.as_list::<i64>();
60            Ok(length_impl::<Int64Type>(list.offsets(), list.nulls()))
61        }
62        DataType::ListView(_) => {
63            let list = array.as_list_view::<i32>();
64            Ok(Arc::new(Int32Array::new(
65                list.sizes().clone(),
66                list.nulls().cloned(),
67            )))
68        }
69        DataType::LargeListView(_) => {
70            let list = array.as_list_view::<i64>();
71            Ok(Arc::new(Int64Array::new(
72                list.sizes().clone(),
73                list.nulls().cloned(),
74            )))
75        }
76        DataType::Map(_, _) => {
77            let map = array.as_map();
78            Ok(length_impl::<Int32Type>(map.offsets(), map.nulls()))
79        }
80        DataType::Utf8 => {
81            let list = array.as_string::<i32>();
82            Ok(length_impl::<Int32Type>(list.offsets(), list.nulls()))
83        }
84        DataType::LargeUtf8 => {
85            let list = array.as_string::<i64>();
86            Ok(length_impl::<Int64Type>(list.offsets(), list.nulls()))
87        }
88        DataType::Utf8View => {
89            let list = array.as_string_view();
90            let v = list.views().iter().map(|v| *v as i32).collect::<Vec<_>>();
91            Ok(Arc::new(PrimitiveArray::<Int32Type>::try_new(
92                v.into(),
93                list.nulls().cloned(),
94            )?))
95        }
96        DataType::Binary => {
97            let list = array.as_binary::<i32>();
98            Ok(length_impl::<Int32Type>(list.offsets(), list.nulls()))
99        }
100        DataType::LargeBinary => {
101            let list = array.as_binary::<i64>();
102            Ok(length_impl::<Int64Type>(list.offsets(), list.nulls()))
103        }
104        DataType::FixedSizeBinary(len) | DataType::FixedSizeList(_, len) => Ok(Arc::new(
105            Int32Array::try_new(vec![*len; array.len()].into(), array.nulls().cloned())?,
106        )),
107        DataType::BinaryView => {
108            let list = array.as_binary_view();
109            let v = list.views().iter().map(|v| *v as i32).collect::<Vec<_>>();
110            Ok(Arc::new(PrimitiveArray::<Int32Type>::try_new(
111                v.into(),
112                list.nulls().cloned(),
113            )?))
114        }
115        other => Err(ArrowError::ComputeError(format!(
116            "length not supported for {other:?}"
117        ))),
118    }
119}
120
121/// Returns an array of Int32/Int64 denoting the number of bits in each value in the array.
122///
123/// * this only accepts StringArray/Utf8, LargeString/LargeUtf8, StringViewArray/Utf8View,
124///   BinaryArray, LargeBinaryArray, BinaryViewArray, and FixedSizeBinaryArray,
125///   or DictionaryArray/REE with above Arrays as values
126/// * bit_length of null is null.
127/// * bit_length is in number of bits
128pub fn bit_length(array: &dyn Array) -> Result<ArrayRef, ArrowError> {
129    if let Some(d) = array.as_any_dictionary_opt() {
130        let lengths = bit_length(d.values().as_ref())?;
131        return Ok(d.with_values(lengths));
132    }
133    if let Some(ree) = array.as_any_ree_opt() {
134        let lengths = bit_length(ree.values())?;
135        return Ok(ree.with_values(lengths));
136    }
137
138    match array.data_type() {
139        DataType::Utf8 => {
140            let list = array.as_string::<i32>();
141            Ok(bit_length_impl::<Int32Type>(list.offsets(), list.nulls()))
142        }
143        DataType::LargeUtf8 => {
144            let list = array.as_string::<i64>();
145            Ok(bit_length_impl::<Int64Type>(list.offsets(), list.nulls()))
146        }
147        DataType::Utf8View => {
148            let list = array.as_string_view();
149            let values = list
150                .views()
151                .iter()
152                .map(|view| (*view as i32).wrapping_mul(8))
153                .collect();
154            Ok(Arc::new(Int32Array::try_new(
155                values,
156                array.nulls().cloned(),
157            )?))
158        }
159        DataType::Binary => {
160            let list = array.as_binary::<i32>();
161            Ok(bit_length_impl::<Int32Type>(list.offsets(), list.nulls()))
162        }
163        DataType::LargeBinary => {
164            let list = array.as_binary::<i64>();
165            Ok(bit_length_impl::<Int64Type>(list.offsets(), list.nulls()))
166        }
167        DataType::FixedSizeBinary(len) => Ok(Arc::new(Int32Array::try_new(
168            vec![*len * 8; array.len()].into(),
169            array.nulls().cloned(),
170        )?)),
171        DataType::BinaryView => {
172            let list = array.as_binary_view();
173            let values = list
174                .views()
175                .iter()
176                .map(|view| (*view as i32).wrapping_mul(8))
177                .collect();
178            Ok(Arc::new(Int32Array::try_new(
179                values,
180                array.nulls().cloned(),
181            )?))
182        }
183        other => Err(ArrowError::ComputeError(format!(
184            "bit_length not supported for {other:?}"
185        ))),
186    }
187}
188
189#[cfg(test)]
190mod tests {
191    use super::*;
192    use arrow_array::builder::{Int32Builder, MapBuilder, StringBuilder};
193    use arrow_buffer::{Buffer, ScalarBuffer};
194    use arrow_data::ArrayData;
195    use arrow_schema::Field;
196
197    fn length_cases_string() -> Vec<(Vec<&'static str>, usize, Vec<i32>)> {
198        // a large array
199        let values = [
200            "one",
201            "on",
202            "o",
203            "",
204            "this is a longer string to test string array with",
205        ];
206        let values = values.into_iter().cycle().take(4096).collect();
207        let expected = [3, 2, 1, 0, 49].into_iter().cycle().take(4096).collect();
208
209        vec![
210            (vec!["hello", " ", "world"], 3, vec![5, 1, 5]),
211            (vec!["hello", " ", "world", "!"], 4, vec![5, 1, 5, 1]),
212            (vec!["💖"], 1, vec![4]),
213            (values, 4096, expected),
214        ]
215    }
216
217    macro_rules! length_binary_helper {
218        ($offset_ty: ty, $result_ty: ty, $kernel: ident, $value: expr, $expected: expr) => {{
219            let array = GenericBinaryArray::<$offset_ty>::from($value);
220            let result = $kernel(&array).unwrap();
221            let result = result.as_any().downcast_ref::<$result_ty>().unwrap();
222            let expected: $result_ty = $expected.into();
223            assert_eq!(&expected, result);
224        }};
225    }
226
227    macro_rules! length_list_helper {
228        ($offset_ty: ty, $result_ty: ty, $element_ty: ty, $value: expr, $expected: expr) => {{
229            let array =
230                GenericListArray::<$offset_ty>::from_iter_primitive::<$element_ty, _, _>($value);
231            let result = length(&array).unwrap();
232            let result = result.as_any().downcast_ref::<$result_ty>().unwrap();
233            let expected: $result_ty = $expected.into();
234            assert_eq!(&expected, result);
235        }};
236    }
237
238    #[test]
239    fn length_test_string() {
240        length_cases_string()
241            .into_iter()
242            .for_each(|(input, len, expected)| {
243                let array = StringArray::from(input);
244                let result = length(&array).unwrap();
245                assert_eq!(len, result.len());
246                let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
247                expected.iter().enumerate().for_each(|(i, value)| {
248                    assert_eq!(*value, result.value(i));
249                });
250            })
251    }
252
253    #[test]
254    fn length_test_large_string() {
255        length_cases_string()
256            .into_iter()
257            .for_each(|(input, len, expected)| {
258                let array = LargeStringArray::from(input);
259                let result = length(&array).unwrap();
260                assert_eq!(len, result.len());
261                let result = result.as_any().downcast_ref::<Int64Array>().unwrap();
262                expected.iter().enumerate().for_each(|(i, value)| {
263                    assert_eq!(*value as i64, result.value(i));
264                });
265            })
266    }
267
268    #[test]
269    fn length_test_string_view() {
270        length_cases_string()
271            .into_iter()
272            .for_each(|(input, len, expected)| {
273                let array = StringViewArray::from(input);
274                let result = length(&array).unwrap();
275                assert_eq!(len, result.len());
276                let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
277                expected.iter().enumerate().for_each(|(i, value)| {
278                    assert_eq!(*value, result.value(i));
279                });
280            })
281    }
282
283    #[test]
284    fn length_test_binary() {
285        let value: Vec<&[u8]> = vec![b"zero", b"one", &[0xff, 0xf8]];
286        let result: Vec<i32> = vec![4, 3, 2];
287        length_binary_helper!(i32, Int32Array, length, value, result)
288    }
289
290    #[test]
291    fn length_test_large_binary() {
292        let value: Vec<&[u8]> = vec![b"zero", &[0xff, 0xf8], b"two"];
293        let result: Vec<i64> = vec![4, 2, 3];
294        length_binary_helper!(i64, Int64Array, length, value, result)
295    }
296
297    #[test]
298    fn length_test_binary_view() {
299        let value: Vec<&[u8]> = vec![
300            b"zero",
301            &[0xff, 0xf8],
302            b"two",
303            b"this is a longer string to test binary array with",
304        ];
305        let expected: Vec<i32> = vec![4, 2, 3, 49];
306
307        let array = BinaryViewArray::from(value);
308        let result = length(&array).unwrap();
309        let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
310        let expected: Int32Array = expected.into();
311        assert_eq!(&expected, result);
312    }
313
314    #[test]
315    fn length_test_list() {
316        let value = vec![
317            Some(vec![]),
318            Some(vec![Some(1), Some(2), Some(4)]),
319            Some(vec![Some(0)]),
320        ];
321        let result: Vec<i32> = vec![0, 3, 1];
322        length_list_helper!(i32, Int32Array, Int32Type, value, result)
323    }
324
325    #[test]
326    fn length_test_large_list() {
327        let value = vec![
328            Some(vec![]),
329            Some(vec![Some(1.1), Some(2.2), Some(3.3)]),
330            Some(vec![None]),
331        ];
332        let result: Vec<i64> = vec![0, 3, 1];
333        length_list_helper!(i64, Int64Array, Float32Type, value, result)
334    }
335
336    #[test]
337    fn length_test_map() {
338        let mut map_builder = MapBuilder::new(None, StringBuilder::new(), Int32Builder::default());
339        // {}
340        map_builder.append(true).unwrap();
341
342        // {"a": 1, "b": 2, "cd": 4}
343        map_builder.keys().extend(["a", "b", "cd"].map(Some));
344        map_builder.values().extend([1, 2, 4].map(Some));
345        map_builder.append(true).unwrap();
346
347        // {"e": 0}
348        map_builder.keys().append_value("e");
349        map_builder.values().append_value(0);
350        map_builder.append(true).unwrap();
351
352        let map_array = map_builder.finish();
353
354        let lengths = length(&map_array).unwrap();
355        let lengths = lengths.as_primitive::<Int32Type>();
356        assert_eq!(lengths, &Int32Array::from(vec![0, 3, 1]));
357    }
358
359    type OptionStr = Option<&'static str>;
360
361    fn length_null_cases_string() -> Vec<(Vec<OptionStr>, usize, Vec<Option<i32>>)> {
362        vec![(
363            vec![Some("one"), None, Some("three"), Some("four")],
364            4,
365            vec![Some(3), None, Some(5), Some(4)],
366        )]
367    }
368
369    #[test]
370    fn length_null_string() {
371        length_null_cases_string()
372            .into_iter()
373            .for_each(|(input, len, expected)| {
374                let array = StringArray::from(input);
375                let result = length(&array).unwrap();
376                assert_eq!(len, result.len());
377                let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
378
379                let expected: Int32Array = expected.into();
380                assert_eq!(&expected, result);
381            })
382    }
383
384    #[test]
385    fn length_null_large_string() {
386        length_null_cases_string()
387            .into_iter()
388            .for_each(|(input, len, expected)| {
389                let array = LargeStringArray::from(input);
390                let result = length(&array).unwrap();
391                assert_eq!(len, result.len());
392                let result = result.as_any().downcast_ref::<Int64Array>().unwrap();
393
394                // convert to i64
395                let expected: Int64Array = expected
396                    .iter()
397                    .map(|e| e.map(|e| e as i64))
398                    .collect::<Vec<_>>()
399                    .into();
400                assert_eq!(&expected, result);
401            })
402    }
403
404    #[test]
405    fn length_null_binary() {
406        let value: Vec<Option<&[u8]>> =
407            vec![Some(b"zero"), None, Some(&[0xff, 0xf8]), Some(b"three")];
408        let result: Vec<Option<i32>> = vec![Some(4), None, Some(2), Some(5)];
409        length_binary_helper!(i32, Int32Array, length, value, result)
410    }
411
412    #[test]
413    fn length_null_large_binary() {
414        let value: Vec<Option<&[u8]>> =
415            vec![Some(&[0xff, 0xf8]), None, Some(b"two"), Some(b"three")];
416        let result: Vec<Option<i64>> = vec![Some(2), None, Some(3), Some(5)];
417        length_binary_helper!(i64, Int64Array, length, value, result)
418    }
419
420    #[test]
421    fn length_null_list() {
422        let value = vec![
423            Some(vec![]),
424            None,
425            Some(vec![Some(1), None, Some(2), Some(4)]),
426            Some(vec![Some(0)]),
427        ];
428        let result: Vec<Option<i32>> = vec![Some(0), None, Some(4), Some(1)];
429        length_list_helper!(i32, Int32Array, Int8Type, value, result)
430    }
431
432    #[test]
433    fn length_null_large_list() {
434        let value = vec![
435            Some(vec![]),
436            None,
437            Some(vec![Some(1.1), None, Some(4.0)]),
438            Some(vec![Some(0.1)]),
439        ];
440        let result: Vec<Option<i64>> = vec![Some(0), None, Some(3), Some(1)];
441        length_list_helper!(i64, Int64Array, Float32Type, value, result)
442    }
443
444    #[test]
445    fn length_test_null_map() {
446        let mut map_builder = MapBuilder::new(None, StringBuilder::new(), Int32Builder::default());
447        // {}
448        map_builder.append(true).unwrap();
449
450        // null
451        map_builder.append_nulls(1).unwrap();
452
453        // {"a": 1, "b": 2, "cd": 4}
454        map_builder.keys().extend(["a", "b", "cd"].map(Some));
455        map_builder.values().extend([1, 2, 4].map(Some));
456        map_builder.append(true).unwrap();
457
458        // {"e": 0}
459        map_builder.keys().append_value("e");
460        map_builder.values().append_value(0);
461        map_builder.append(true).unwrap();
462
463        let map_array = map_builder.finish();
464        let lengths = length(&map_array).unwrap();
465        let lengths = lengths.as_primitive::<Int32Type>();
466        assert_eq!(
467            lengths,
468            &Int32Array::from(vec![Some(0), None, Some(3), Some(1)])
469        );
470    }
471
472    #[test]
473    fn length_test_list_view() {
474        // Create a ListViewArray with values [0, 1, 2], [3, 4, 5], [6, 7]
475        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
476        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7]);
477        let offsets = ScalarBuffer::from(vec![0i32, 3, 6]);
478        let sizes = ScalarBuffer::from(vec![3i32, 3, 2]);
479        let list_array = ListViewArray::new(field, offsets, sizes, Arc::new(values), None);
480
481        let result = length(&list_array).unwrap();
482        let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
483        let expected: Int32Array = vec![3, 3, 2].into();
484        assert_eq!(&expected, result);
485    }
486
487    #[test]
488    fn length_test_large_list_view() {
489        // Create a LargeListViewArray with values [0, 1, 2], [3, 4, 5], [6, 7]
490        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
491        let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7]);
492        let offsets = ScalarBuffer::from(vec![0i64, 3, 6]);
493        let sizes = ScalarBuffer::from(vec![3i64, 3, 2]);
494        let list_array = LargeListViewArray::new(field, offsets, sizes, Arc::new(values), None);
495
496        let result = length(&list_array).unwrap();
497        let result = result.as_any().downcast_ref::<Int64Array>().unwrap();
498        let expected: Int64Array = vec![3i64, 3, 2].into();
499        assert_eq!(&expected, result);
500    }
501
502    #[test]
503    fn length_null_list_view() {
504        // Create a ListViewArray with nulls: [], null, [1, 2, 3, 4], [0]
505        let field = Arc::new(Field::new_list_field(DataType::Int32, true));
506        let values = Int32Array::from(vec![1, 2, 3, 4, 0]);
507        let offsets = ScalarBuffer::from(vec![0i32, 0, 0, 4]);
508        let sizes = ScalarBuffer::from(vec![0i32, 0, 4, 1]);
509        let nulls = NullBuffer::from(vec![true, false, true, true]);
510        let list_array = ListViewArray::new(field, offsets, sizes, Arc::new(values), Some(nulls));
511
512        let result = length(&list_array).unwrap();
513        let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
514        let expected: Int32Array = vec![Some(0), None, Some(4), Some(1)].into();
515        assert_eq!(&expected, result);
516    }
517
518    #[test]
519    fn length_null_large_list_view() {
520        // Create a LargeListViewArray with nulls: [], null, [1.0, 2.0, 3.0], [0.1]
521        let field = Arc::new(Field::new_list_field(DataType::Float32, true));
522        let values = Float32Array::from(vec![1.0, 2.0, 3.0, 0.1]);
523        let offsets = ScalarBuffer::from(vec![0i64, 0, 0, 3]);
524        let sizes = ScalarBuffer::from(vec![0i64, 0, 3, 1]);
525        let nulls = NullBuffer::from(vec![true, false, true, true]);
526        let list_array =
527            LargeListViewArray::new(field, offsets, sizes, Arc::new(values), Some(nulls));
528
529        let result = length(&list_array).unwrap();
530        let result = result.as_any().downcast_ref::<Int64Array>().unwrap();
531        let expected: Int64Array = vec![Some(0i64), None, Some(3), Some(1)].into();
532        assert_eq!(&expected, result);
533    }
534
535    /// Tests that length is not valid for u64.
536    #[test]
537    fn length_wrong_type() {
538        let array: UInt64Array = vec![1u64].into();
539
540        assert!(length(&array).is_err());
541    }
542
543    /// Tests with an offset
544    #[test]
545    fn length_offsets_string() {
546        let a = StringArray::from(vec![Some("hello"), Some(" "), Some("world"), None]);
547        let b = a.slice(1, 3);
548        let result = length(&b).unwrap();
549        let result: &Int32Array = result.as_primitive();
550
551        let expected = Int32Array::from(vec![Some(1), Some(5), None]);
552        assert_eq!(&expected, result);
553    }
554
555    #[test]
556    fn length_offsets_binary() {
557        let value: Vec<Option<&[u8]>> = vec![Some(b"hello"), Some(b" "), Some(&[0xff, 0xf8]), None];
558        let a = BinaryArray::from(value);
559        let b = a.slice(1, 3);
560        let result = length(&b).unwrap();
561        let result: &Int32Array = result.as_primitive();
562
563        let expected = Int32Array::from(vec![Some(1), Some(2), None]);
564        assert_eq!(&expected, result);
565    }
566
567    fn bit_length_cases() -> Vec<(Vec<&'static str>, usize, Vec<i32>)> {
568        // a large array
569        let values = ["one", "on", "o", ""];
570        let values = values.into_iter().cycle().take(4096).collect();
571        let expected = [24, 16, 8, 0].into_iter().cycle().take(4096).collect();
572
573        vec![
574            (vec!["hello", " ", "world", "!"], 4, vec![40, 8, 40, 8]),
575            (vec!["💖"], 1, vec![32]),
576            (vec!["josé"], 1, vec![40]),
577            (values, 4096, expected),
578        ]
579    }
580
581    #[test]
582    fn bit_length_test_string() {
583        bit_length_cases()
584            .into_iter()
585            .for_each(|(input, len, expected)| {
586                let array = StringArray::from(input);
587                let result = bit_length(&array).unwrap();
588                assert_eq!(len, result.len());
589                let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
590                expected.iter().enumerate().for_each(|(i, value)| {
591                    assert_eq!(*value, result.value(i));
592                });
593            })
594    }
595
596    #[test]
597    fn bit_length_test_large_string() {
598        bit_length_cases()
599            .into_iter()
600            .for_each(|(input, len, expected)| {
601                let array = LargeStringArray::from(input);
602                let result = bit_length(&array).unwrap();
603                assert_eq!(len, result.len());
604                let result = result.as_any().downcast_ref::<Int64Array>().unwrap();
605                expected.iter().enumerate().for_each(|(i, value)| {
606                    assert_eq!(*value as i64, result.value(i));
607                });
608            })
609    }
610
611    #[test]
612    fn bit_length_test_utf8view() {
613        bit_length_cases()
614            .into_iter()
615            .for_each(|(input, len, expected)| {
616                let string_array = StringViewArray::from(input);
617                let result = bit_length(&string_array).unwrap();
618                assert_eq!(len, result.len());
619                let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
620                expected.iter().enumerate().for_each(|(i, value)| {
621                    assert_eq!(*value, result.value(i));
622                });
623            })
624    }
625
626    #[test]
627    fn bit_length_null_utf8view() {
628        bit_length_null_cases()
629            .into_iter()
630            .for_each(|(input, len, expected)| {
631                let array = StringArray::from(input);
632                let result = bit_length(&array).unwrap();
633                assert_eq!(len, result.len());
634                let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
635
636                let expected: Int32Array = expected.into();
637                assert_eq!(&expected, result);
638            })
639    }
640    #[test]
641    fn bit_length_binary() {
642        let value: Vec<&[u8]> = vec![b"one", &[0xff, 0xf8], b"three"];
643        let expected: Vec<i32> = vec![24, 16, 40];
644        length_binary_helper!(i32, Int32Array, bit_length, value, expected)
645    }
646
647    #[test]
648    fn bit_length_large_binary() {
649        let value: Vec<&[u8]> = vec![b"zero", b" ", &[0xff, 0xf8]];
650        let expected: Vec<i64> = vec![32, 8, 16];
651        length_binary_helper!(i64, Int64Array, bit_length, value, expected)
652    }
653
654    #[test]
655    fn bit_length_binary_view() {
656        let value: Vec<&[u8]> = vec![
657            b"zero",
658            &[0xff, 0xf8],
659            b"two",
660            b"this is a longer string to test binary array with",
661        ];
662        let expected: Vec<i32> = vec![32, 16, 24, 392];
663
664        let array = BinaryViewArray::from(value);
665        let result = bit_length(&array).unwrap();
666        let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
667        let expected: Int32Array = expected.into();
668        assert_eq!(&expected, result);
669    }
670
671    #[test]
672    fn bit_length_null_binary_view() {
673        let value: Vec<Option<&[u8]>> =
674            vec![Some(b"one"), None, Some(b"three"), Some(&[0xff, 0xf8])];
675        let expected: Vec<Option<i32>> = vec![Some(24), None, Some(40), Some(16)];
676
677        let array = BinaryViewArray::from(value);
678        let result = bit_length(&array).unwrap();
679        let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
680        let expected: Int32Array = expected.into();
681        assert_eq!(&expected, result);
682    }
683
684    fn bit_length_null_cases() -> Vec<(Vec<OptionStr>, usize, Vec<Option<i32>>)> {
685        vec![(
686            vec![Some("one"), None, Some("three"), Some("four")],
687            4,
688            vec![Some(24), None, Some(40), Some(32)],
689        )]
690    }
691
692    #[test]
693    fn bit_length_null_string() {
694        bit_length_null_cases()
695            .into_iter()
696            .for_each(|(input, len, expected)| {
697                let array = StringArray::from(input);
698                let result = bit_length(&array).unwrap();
699                assert_eq!(len, result.len());
700                let result = result.as_any().downcast_ref::<Int32Array>().unwrap();
701
702                let expected: Int32Array = expected.into();
703                assert_eq!(&expected, result);
704            })
705    }
706
707    #[test]
708    fn bit_length_null_large_string() {
709        bit_length_null_cases()
710            .into_iter()
711            .for_each(|(input, len, expected)| {
712                let array = LargeStringArray::from(input);
713                let result = bit_length(&array).unwrap();
714                assert_eq!(len, result.len());
715                let result = result.as_any().downcast_ref::<Int64Array>().unwrap();
716
717                // convert to i64
718                let expected: Int64Array = expected
719                    .iter()
720                    .map(|e| e.map(|e| e as i64))
721                    .collect::<Vec<_>>()
722                    .into();
723                assert_eq!(&expected, result);
724            })
725    }
726
727    #[test]
728    fn bit_length_null_binary() {
729        let value: Vec<Option<&[u8]>> =
730            vec![Some(b"one"), None, Some(b"three"), Some(&[0xff, 0xf8])];
731        let expected: Vec<Option<i32>> = vec![Some(24), None, Some(40), Some(16)];
732        length_binary_helper!(i32, Int32Array, bit_length, value, expected)
733    }
734
735    #[test]
736    fn bit_length_null_large_binary() {
737        let value: Vec<Option<&[u8]>> =
738            vec![Some(b"one"), None, Some(&[0xff, 0xf8]), Some(b"four")];
739        let expected: Vec<Option<i64>> = vec![Some(24), None, Some(16), Some(32)];
740        length_binary_helper!(i64, Int64Array, bit_length, value, expected)
741    }
742
743    /// Tests that bit_length is not valid for u64.
744    #[test]
745    fn bit_length_wrong_type() {
746        let array: UInt64Array = vec![1u64].into();
747
748        assert!(bit_length(&array).is_err());
749    }
750
751    /// Tests with an offset
752    #[test]
753    fn bit_length_offsets_string() {
754        let a = StringArray::from(vec![Some("hello"), Some(" "), Some("world"), None]);
755        let b = a.slice(1, 3);
756        let result = bit_length(&b).unwrap();
757        let result: &Int32Array = result.as_primitive();
758
759        let expected = Int32Array::from(vec![Some(8), Some(40), None]);
760        assert_eq!(&expected, result);
761    }
762
763    #[test]
764    fn bit_length_offsets_binary() {
765        let value: Vec<Option<&[u8]>> = vec![Some(b"hello"), Some(&[]), Some(b"world"), None];
766        let a = BinaryArray::from(value);
767        let b = a.slice(1, 3);
768        let result = bit_length(&b).unwrap();
769        let result: &Int32Array = result.as_primitive();
770
771        let expected = Int32Array::from(vec![Some(0), Some(40), None]);
772        assert_eq!(&expected, result);
773    }
774
775    #[test]
776    fn length_dictionary() {
777        _length_dictionary::<Int8Type>();
778        _length_dictionary::<Int16Type>();
779        _length_dictionary::<Int32Type>();
780        _length_dictionary::<Int64Type>();
781        _length_dictionary::<UInt8Type>();
782        _length_dictionary::<UInt16Type>();
783        _length_dictionary::<UInt32Type>();
784        _length_dictionary::<UInt64Type>();
785    }
786
787    fn _length_dictionary<K: ArrowDictionaryKeyType>() {
788        const TOTAL: i32 = 100;
789
790        let v = ["aaaa", "bb", "ccccc", "ddd", "eeeeee"];
791        let data: Vec<Option<&str>> = (0..TOTAL)
792            .map(|n| {
793                let i = n % 5;
794                if i == 3 { None } else { Some(v[i as usize]) }
795            })
796            .collect();
797
798        let dict_array: DictionaryArray<K> = data.clone().into_iter().collect();
799
800        let expected: Vec<Option<i32>> =
801            data.iter().map(|opt| opt.map(|s| s.len() as i32)).collect();
802
803        let res = length(&dict_array).unwrap();
804        let actual = res.as_any().downcast_ref::<DictionaryArray<K>>().unwrap();
805        let actual: Vec<Option<i32>> = actual
806            .values()
807            .as_any()
808            .downcast_ref::<Int32Array>()
809            .unwrap()
810            .take_iter(dict_array.keys_iter())
811            .collect();
812
813        for i in 0..TOTAL as usize {
814            assert_eq!(expected[i], actual[i],);
815        }
816    }
817
818    #[test]
819    fn bit_length_dictionary() {
820        _bit_length_dictionary::<Int8Type>();
821        _bit_length_dictionary::<Int16Type>();
822        _bit_length_dictionary::<Int32Type>();
823        _bit_length_dictionary::<Int64Type>();
824        _bit_length_dictionary::<UInt8Type>();
825        _bit_length_dictionary::<UInt16Type>();
826        _bit_length_dictionary::<UInt32Type>();
827        _bit_length_dictionary::<UInt64Type>();
828    }
829
830    fn _bit_length_dictionary<K: ArrowDictionaryKeyType>() {
831        const TOTAL: i32 = 100;
832
833        let v = ["aaaa", "bb", "ccccc", "ddd", "eeeeee"];
834        let data: Vec<Option<&str>> = (0..TOTAL)
835            .map(|n| {
836                let i = n % 5;
837                if i == 3 { None } else { Some(v[i as usize]) }
838            })
839            .collect();
840
841        let dict_array: DictionaryArray<K> = data.clone().into_iter().collect();
842
843        let expected: Vec<Option<i32>> = data
844            .iter()
845            .map(|opt| opt.map(|s| (s.chars().count() * 8) as i32))
846            .collect();
847
848        let res = bit_length(&dict_array).unwrap();
849        let actual = res.as_any().downcast_ref::<DictionaryArray<K>>().unwrap();
850        let actual: Vec<Option<i32>> = actual
851            .values()
852            .as_any()
853            .downcast_ref::<Int32Array>()
854            .unwrap()
855            .take_iter(dict_array.keys_iter())
856            .collect();
857
858        for i in 0..TOTAL as usize {
859            assert_eq!(expected[i], actual[i],);
860        }
861    }
862
863    #[test]
864    fn test_fixed_size_list_length() {
865        // Construct a value array
866        let value_data = ArrayData::builder(DataType::Int32)
867            .len(9)
868            .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7, 8]))
869            .build()
870            .unwrap();
871        let list_data_type =
872            DataType::FixedSizeList(Arc::new(Field::new_list_field(DataType::Int32, false)), 3);
873        let nulls = NullBuffer::from(vec![true, false, true]);
874        let list_data = ArrayData::builder(list_data_type)
875            .len(3)
876            .add_child_data(value_data)
877            .nulls(Some(nulls))
878            .build()
879            .unwrap();
880        let list_array = FixedSizeListArray::from(list_data);
881
882        let lengths = length(&list_array).unwrap();
883        let lengths = lengths.as_primitive::<Int32Type>();
884
885        assert_eq!(lengths.len(), 3);
886        assert_eq!(lengths.value(0), 3);
887        assert!(lengths.is_null(1));
888        assert_eq!(lengths.value(2), 3);
889    }
890
891    #[test]
892    fn test_fixed_size_binary() {
893        let array = FixedSizeBinaryArray::new(4, [0; 16].into(), None);
894        let result = length(&array).unwrap();
895        assert_eq!(result.as_ref(), &Int32Array::from(vec![4; 4]));
896
897        let result = bit_length(&array).unwrap();
898        assert_eq!(result.as_ref(), &Int32Array::from(vec![32; 4]));
899    }
900    #[test]
901    fn length_test_ree_string_values() {
902        use arrow_array::RunArray;
903        use arrow_array::types::Int32Type;
904
905        let string_values = StringArray::from(vec!["hello", "owl", "test", "arrow", "a"]);
906        let run_ends = PrimitiveArray::<Int32Type>::from(vec![2i32, 5, 9, 11, 14]);
907        let ree_array = RunArray::<Int32Type>::try_new(&run_ends, &string_values).unwrap();
908
909        let result = length(&ree_array).unwrap();
910        let result = result
911            .as_any()
912            .downcast_ref::<RunArray<Int32Type>>()
913            .unwrap();
914
915        let result_values = result
916            .values()
917            .as_any()
918            .downcast_ref::<Int32Array>()
919            .unwrap();
920
921        let expected: Int32Array = vec![5, 3, 4, 5, 1].into();
922        assert_eq!(&expected, result_values);
923    }
924    #[test]
925    fn length_test_ree_invalid_type_early_fail() {
926        use arrow_array::RunArray;
927        use arrow_array::types::Int32Type;
928
929        let uint64_values = UInt64Array::from(vec![1u64, 2, 3]);
930        let run_ends = PrimitiveArray::<Int32Type>::from(vec![1i32, 2, 3]);
931        let ree_array = RunArray::<Int32Type>::try_new(&run_ends, &uint64_values).unwrap();
932
933        assert!(length(&ree_array).is_err());
934    }
935
936    #[test]
937    fn bit_length_test_ree_utf8() {
938        use arrow_array::RunArray;
939        use arrow_array::types::Int32Type;
940
941        let strings = StringArray::from(vec!["hello", "world", "test"]);
942        let run_ends = PrimitiveArray::<Int32Type>::from(vec![1i32, 2, 3]);
943        let ree_array = RunArray::<Int32Type>::try_new(&run_ends, &strings).unwrap();
944
945        let result = bit_length(&ree_array).unwrap();
946        let result_values = result
947            .as_any()
948            .downcast_ref::<RunArray<Int32Type>>()
949            .unwrap()
950            .values()
951            .as_any()
952            .downcast_ref::<Int32Array>()
953            .unwrap();
954
955        let expected: Int32Array = vec![40, 40, 32].into();
956        assert_eq!(&expected, result_values);
957    }
958}