1use crate::types::GenericStringType;
6use crate::{GenericBinaryArray, GenericByteArray, GenericListArray, OffsetSizeTrait};
7use arrow_schema::ArrowError;
8
9pub type GenericStringArray<OffsetSize> = GenericByteArray<GenericStringType<OffsetSize>>;
11
12impl<OffsetSize: OffsetSizeTrait> GenericStringArray<OffsetSize> {
13 pub fn num_chars(&self, i: usize) -> usize {
19 self.value(i).chars().count()
20 }
21
22 pub fn take_iter<'a>(
24 &'a self,
25 indexes: impl Iterator<Item = Option<usize>> + 'a,
26 ) -> impl Iterator<Item = Option<&'a str>> {
27 indexes.map(|opt_index| opt_index.map(|index| self.value(index)))
28 }
29
30 pub unsafe fn take_iter_unchecked<'a>(
35 &'a self,
36 indexes: impl Iterator<Item = Option<usize>> + 'a,
37 ) -> impl Iterator<Item = Option<&'a str>> {
38 indexes.map(|opt_index| opt_index.map(|index| unsafe { self.value_unchecked(index) }))
39 }
40
41 pub fn try_from_binary(v: GenericBinaryArray<OffsetSize>) -> Result<Self, ArrowError> {
44 let (offsets, values, nulls) = v.into_parts();
45 Self::try_new(offsets, values, nulls)
46 }
47}
48
49impl<OffsetSize: OffsetSizeTrait> From<GenericListArray<OffsetSize>>
50 for GenericStringArray<OffsetSize>
51{
52 fn from(v: GenericListArray<OffsetSize>) -> Self {
53 GenericBinaryArray::<OffsetSize>::from(v).into()
54 }
55}
56
57impl<OffsetSize: OffsetSizeTrait> From<GenericBinaryArray<OffsetSize>>
58 for GenericStringArray<OffsetSize>
59{
60 fn from(v: GenericBinaryArray<OffsetSize>) -> Self {
61 Self::try_from_binary(v).unwrap()
62 }
63}
64
65impl<OffsetSize: OffsetSizeTrait> From<Vec<Option<&str>>> for GenericStringArray<OffsetSize> {
66 fn from(v: Vec<Option<&str>>) -> Self {
67 v.into_iter().collect()
68 }
69}
70
71impl<OffsetSize: OffsetSizeTrait> From<Vec<&str>> for GenericStringArray<OffsetSize> {
72 fn from(v: Vec<&str>) -> Self {
73 Self::from_iter_values(v)
74 }
75}
76
77impl<OffsetSize: OffsetSizeTrait> From<Vec<Option<String>>> for GenericStringArray<OffsetSize> {
78 fn from(v: Vec<Option<String>>) -> Self {
79 v.into_iter().collect()
80 }
81}
82
83impl<OffsetSize: OffsetSizeTrait> From<Vec<String>> for GenericStringArray<OffsetSize> {
84 fn from(v: Vec<String>) -> Self {
85 Self::from_iter_values(v)
86 }
87}
88
89pub type StringArray = GenericStringArray<i32>;
115
116pub type LargeStringArray = GenericStringArray<i64>;
142
143#[cfg(test)]
144mod tests {
145 use super::*;
146 use crate::Array;
147 use crate::builder::{ListBuilder, PrimitiveBuilder, StringBuilder};
148 use crate::types::UInt8Type;
149 use arrow_buffer::{Buffer, NullBuffer, OffsetBuffer};
150 use arrow_data::ArrayData;
151 use arrow_schema::{DataType, Field};
152 use std::sync::Arc;
153
154 #[test]
155 fn test_string_array_from_u8_slice() {
156 let values: Vec<&str> = vec!["hello", "", "A£ऀ𖼚𝌆৩ƐZ"];
157
158 let string_array = StringArray::from(values);
160
161 assert_eq!(3, string_array.len());
162 assert_eq!(0, string_array.null_count());
163 assert_eq!("hello", string_array.value(0));
164 assert_eq!("hello", unsafe { string_array.value_unchecked(0) });
165 assert_eq!("", string_array.value(1));
166 assert_eq!("", unsafe { string_array.value_unchecked(1) });
167 assert_eq!("A£ऀ𖼚𝌆৩ƐZ", string_array.value(2));
168 assert_eq!("A£ऀ𖼚𝌆৩ƐZ", unsafe {
169 string_array.value_unchecked(2)
170 });
171 assert_eq!(20, string_array.value_length(2)); assert_eq!(8, string_array.num_chars(2));
173 for i in 0..3 {
174 assert!(string_array.is_valid(i));
175 assert!(!string_array.is_null(i));
176 }
177 }
178
179 #[test]
180 #[should_panic(expected = "StringArray expects DataType::Utf8")]
181 fn test_string_array_from_int() {
182 let array = LargeStringArray::from(vec!["a", "b"]);
183 drop(StringArray::from(array.into_data()));
184 }
185
186 #[test]
187 fn test_large_string_array_from_u8_slice() {
188 let values: Vec<&str> = vec!["hello", "", "A£ऀ𖼚𝌆৩ƐZ"];
189
190 let string_array = LargeStringArray::from(values);
192
193 assert_eq!(3, string_array.len());
194 assert_eq!(0, string_array.null_count());
195 assert_eq!("hello", string_array.value(0));
196 assert_eq!("hello", unsafe { string_array.value_unchecked(0) });
197 assert_eq!("", string_array.value(1));
198 assert_eq!("", unsafe { string_array.value_unchecked(1) });
199 assert_eq!("A£ऀ𖼚𝌆৩ƐZ", string_array.value(2));
200 assert_eq!("A£ऀ𖼚𝌆৩ƐZ", unsafe {
201 string_array.value_unchecked(2)
202 });
203 assert_eq!(5, string_array.value_offsets()[2]);
204 assert_eq!(20, string_array.value_length(2)); assert_eq!(8, string_array.num_chars(2));
206 for i in 0..3 {
207 assert!(string_array.is_valid(i));
208 assert!(!string_array.is_null(i));
209 }
210 }
211
212 #[test]
213 fn test_nested_string_array() {
214 let string_builder = StringBuilder::with_capacity(3, 10);
215 let mut list_of_string_builder = ListBuilder::new(string_builder);
216
217 list_of_string_builder.values().append_value("foo");
218 list_of_string_builder.values().append_value("bar");
219 list_of_string_builder.append(true);
220
221 list_of_string_builder.values().append_value("foobar");
222 list_of_string_builder.append(true);
223 let list_of_strings = list_of_string_builder.finish();
224
225 assert_eq!(list_of_strings.len(), 2);
226
227 let first_slot = list_of_strings.value(0);
228 let first_list = first_slot.as_any().downcast_ref::<StringArray>().unwrap();
229 assert_eq!(first_list.len(), 2);
230 assert_eq!(first_list.value(0), "foo");
231 assert_eq!(unsafe { first_list.value_unchecked(0) }, "foo");
232 assert_eq!(first_list.value(1), "bar");
233 assert_eq!(unsafe { first_list.value_unchecked(1) }, "bar");
234
235 let second_slot = list_of_strings.value(1);
236 let second_list = second_slot.as_any().downcast_ref::<StringArray>().unwrap();
237 assert_eq!(second_list.len(), 1);
238 assert_eq!(second_list.value(0), "foobar");
239 assert_eq!(unsafe { second_list.value_unchecked(0) }, "foobar");
240 }
241
242 #[test]
243 #[should_panic(
244 expected = "Trying to access an element at index 4 from a StringArray of length 3"
245 )]
246 fn test_string_array_get_value_index_out_of_bound() {
247 let values = b"helloparquet";
248 let offsets: [i32; 4] = [0, 5, 5, 12];
249 let array_data = ArrayData::builder(DataType::Utf8)
250 .len(3)
251 .add_buffer(Buffer::from_slice_ref(offsets))
252 .add_buffer(Buffer::from_slice_ref(values))
253 .build()
254 .unwrap();
255 let string_array = StringArray::from(array_data);
256 string_array.value(4);
257 }
258
259 #[test]
260 fn test_string_array_fmt_debug() {
261 let arr: StringArray = vec!["hello", "arrow"].into();
262 assert_eq!(
263 "StringArray\n[\n \"hello\",\n \"arrow\",\n]",
264 format!("{arr:?}")
265 );
266 }
267
268 #[test]
269 fn test_large_string_array_fmt_debug() {
270 let arr: LargeStringArray = vec!["hello", "arrow"].into();
271 assert_eq!(
272 "LargeStringArray\n[\n \"hello\",\n \"arrow\",\n]",
273 format!("{arr:?}")
274 );
275 }
276
277 #[test]
278 fn test_string_array_from_iter() {
279 let data = [Some("hello"), None, Some("arrow")];
280 let data_vec = data.to_vec();
281 let array1 = StringArray::from(data_vec.clone());
283 let array2: StringArray = data_vec.clone().into_iter().collect();
285 let array3: StringArray = data_vec
287 .into_iter()
288 .map(|x| x.map(|s| s.to_string()))
289 .collect();
290 let array4: StringArray = data.iter().collect::<StringArray>();
292
293 assert_eq!(array1, array2);
294 assert_eq!(array2, array3);
295 assert_eq!(array3, array4);
296 }
297
298 #[test]
299 fn test_string_array_from_iter_values() {
300 let data = ["hello", "hello2"];
301 let array1 = StringArray::from_iter_values(data.iter());
302
303 assert_eq!(array1.value(0), "hello");
304 assert_eq!(array1.value(1), "hello2");
305
306 let data2 = ["goodbye".to_string(), "goodbye2".to_string()];
308 let array2 = StringArray::from_iter_values(data2.iter());
309
310 assert_eq!(array2.value(0), "goodbye");
311 assert_eq!(array2.value(1), "goodbye2");
312 }
313
314 #[test]
315 fn test_string_array_from_unbound_iter() {
316 let string_iter = (0..)
318 .scan(0usize, |pos, i| {
319 if *pos < 10 {
320 *pos += 1;
321 Some(Some(format!("value {i}")))
322 } else {
323 None
325 }
326 })
327 .take(100);
329
330 let (_, upper_size_bound) = string_iter.size_hint();
331 assert_eq!(upper_size_bound, Some(100));
333 let string_array: StringArray = string_iter.collect();
334 assert_eq!(string_array.len(), 10);
336 }
337
338 #[test]
339 fn test_string_array_all_null() {
340 let data: Vec<Option<&str>> = vec![None];
341 let array = StringArray::from(data);
342 array
343 .into_data()
344 .validate_full()
345 .expect("All null array has valid array data");
346 }
347
348 #[test]
349 fn test_large_string_array_all_null() {
350 let data: Vec<Option<&str>> = vec![None];
351 let array = LargeStringArray::from(data);
352 array
353 .into_data()
354 .validate_full()
355 .expect("All null array has valid array data");
356 }
357
358 fn _test_generic_string_array_from_list_array<O: OffsetSizeTrait>() {
359 let values = b"HelloArrowAndParquet";
360 let child_data = ArrayData::builder(DataType::UInt8)
362 .len(15)
363 .offset(5)
364 .add_buffer(Buffer::from(values))
365 .build()
366 .unwrap();
367
368 let offsets = [0, 5, 8, 15].map(|n| O::from_usize(n).unwrap());
369 let null_buffer = Buffer::from_slice_ref([0b101]);
370 let data_type = GenericListArray::<O>::DATA_TYPE_CONSTRUCTOR(Arc::new(
371 Field::new_list_field(DataType::UInt8, false),
372 ));
373
374 let array_data = ArrayData::builder(data_type)
376 .len(2)
377 .offset(1)
378 .add_buffer(Buffer::from_slice_ref(offsets))
379 .null_bit_buffer(Some(null_buffer))
380 .add_child_data(child_data)
381 .build()
382 .unwrap();
383 let list_array = GenericListArray::<O>::from(array_data);
384 let string_array = GenericStringArray::<O>::from(list_array);
385
386 assert_eq!(2, string_array.len());
387 assert_eq!(1, string_array.null_count());
388 assert!(string_array.is_null(0));
389 assert!(string_array.is_valid(1));
390 assert_eq!("Parquet", string_array.value(1));
391 }
392
393 #[test]
394 fn test_string_array_from_list_array() {
395 _test_generic_string_array_from_list_array::<i32>();
396 }
397
398 #[test]
399 fn test_large_string_array_from_list_array() {
400 _test_generic_string_array_from_list_array::<i64>();
401 }
402
403 fn _test_generic_string_array_from_list_array_with_child_nulls_failed<O: OffsetSizeTrait>() {
404 let values = b"HelloArrow";
405 let child_data = ArrayData::builder(DataType::UInt8)
406 .len(10)
407 .add_buffer(Buffer::from(values))
408 .null_bit_buffer(Some(Buffer::from_slice_ref([0b1010101010])))
409 .build()
410 .unwrap();
411
412 let offsets = [0, 5, 10].map(|n| O::from_usize(n).unwrap());
413
414 let data_type = GenericListArray::<O>::DATA_TYPE_CONSTRUCTOR(Arc::new(
417 Field::new_list_field(DataType::UInt8, true),
418 ));
419
420 let array_data = ArrayData::builder(data_type)
422 .len(2)
423 .add_buffer(Buffer::from_slice_ref(offsets))
424 .add_child_data(child_data)
425 .build()
426 .unwrap();
427 let list_array = GenericListArray::<O>::from(array_data);
428 drop(GenericStringArray::<O>::from(list_array));
429 }
430
431 #[test]
432 #[should_panic(expected = "The child array cannot contain null values.")]
433 fn test_string_array_from_list_array_with_child_nulls_failed() {
434 _test_generic_string_array_from_list_array_with_child_nulls_failed::<i32>();
435 }
436
437 #[test]
438 #[should_panic(expected = "The child array cannot contain null values.")]
439 fn test_large_string_array_from_list_array_with_child_nulls_failed() {
440 _test_generic_string_array_from_list_array_with_child_nulls_failed::<i64>();
441 }
442
443 fn _test_generic_string_array_from_list_array_wrong_type<O: OffsetSizeTrait>() {
444 let values = b"HelloArrow";
445 let child_data = ArrayData::builder(DataType::UInt16)
446 .len(5)
447 .add_buffer(Buffer::from(values))
448 .build()
449 .unwrap();
450
451 let offsets = [0, 2, 3].map(|n| O::from_usize(n).unwrap());
452 let data_type = GenericListArray::<O>::DATA_TYPE_CONSTRUCTOR(Arc::new(
453 Field::new_list_field(DataType::UInt16, false),
454 ));
455
456 let array_data = ArrayData::builder(data_type)
457 .len(2)
458 .add_buffer(Buffer::from_slice_ref(offsets))
459 .add_child_data(child_data)
460 .build()
461 .unwrap();
462 let list_array = GenericListArray::<O>::from(array_data);
463 drop(GenericStringArray::<O>::from(list_array));
464 }
465
466 #[test]
467 #[should_panic(
468 expected = "BinaryArray can only be created from List<u8> arrays, mismatched data types."
469 )]
470 fn test_string_array_from_list_array_wrong_type() {
471 _test_generic_string_array_from_list_array_wrong_type::<i32>();
472 }
473
474 #[test]
475 #[should_panic(
476 expected = "BinaryArray can only be created from List<u8> arrays, mismatched data types."
477 )]
478 fn test_large_string_array_from_list_array_wrong_type() {
479 _test_generic_string_array_from_list_array_wrong_type::<i64>();
480 }
481
482 #[test]
483 #[should_panic(
484 expected = "Encountered non UTF-8 data: invalid utf-8 sequence of 1 bytes from index 0"
485 )]
486 fn test_list_array_utf8_validation() {
487 let mut builder = ListBuilder::new(PrimitiveBuilder::<UInt8Type>::new());
488 builder.values().append_value(0xFF);
489 builder.append(true);
490 let list = builder.finish();
491 let _ = StringArray::from(list);
492 }
493
494 #[test]
495 fn test_string_array_invalid_bytes_outside_offsets() {
496 let values = Buffer::from_slice_ref(b"\xFFa\xC3\xA9\xFF");
499 let offsets = OffsetBuffer::new(vec![1, 2, 4].into());
500 let string = StringArray::try_new(offsets, values.clone(), None).unwrap();
501 assert_eq!(string, StringArray::from(vec!["a", "é"]));
502
503 let offsets = OffsetBuffer::new(vec![1, 3, 4].into());
504 let err = StringArray::try_new(offsets, values, None).unwrap_err();
505 assert_eq!(
506 err.to_string(),
507 "Invalid argument error: Split UTF-8 codepoint at offset 3"
508 );
509
510 let offsets = OffsetBuffer::new(vec![1, 1].into());
513 let string = StringArray::try_new(offsets, Buffer::from_slice_ref("é"), None).unwrap();
514 string.to_data().validate_full().unwrap();
515 }
516
517 #[test]
518 fn test_empty_offsets() {
519 let string = StringArray::from(
520 ArrayData::builder(DataType::Utf8)
521 .buffers(vec![Buffer::from(&[]), Buffer::from(&[])])
522 .build()
523 .unwrap(),
524 );
525 assert_eq!(string.len(), 0);
526 assert_eq!(string.value_offsets(), &[0]);
527
528 let string = LargeStringArray::from(
529 ArrayData::builder(DataType::LargeUtf8)
530 .buffers(vec![Buffer::from(&[]), Buffer::from(&[])])
531 .build()
532 .unwrap(),
533 );
534 assert_eq!(string.len(), 0);
535 assert_eq!(string.value_offsets(), &[0]);
536 }
537
538 #[test]
539 fn test_into_builder() {
540 let array: StringArray = vec!["hello", "arrow"].into();
541
542 let mut builder = array.into_builder().unwrap();
544
545 builder.append_value("rust");
546
547 let expected: StringArray = vec!["hello", "arrow", "rust"].into();
548 let array = builder.finish();
549 assert_eq!(expected, array);
550 }
551
552 #[test]
553 fn test_into_builder_err() {
554 let array: StringArray = vec!["hello", "arrow"].into();
555
556 let shared_array = array.clone();
558
559 let err_return = array.into_builder().unwrap_err();
560 assert_eq!(&err_return, &shared_array);
561 }
562
563 #[test]
564 fn test_into_builder_non_zero_first_offset() {
565 let array: StringArray = vec!["abcde", "fgh", "ij"].into();
567 let sliced = array.slice(1, 2);
568 drop(array);
569 let err_return = sliced.into_builder().unwrap_err();
570 assert_eq!(err_return, StringArray::from(vec!["fgh", "ij"]));
571
572 let array = StringArray::new(
575 OffsetBuffer::new(vec![3, 5, 8].into()),
576 Buffer::from(b"xxxabcdeyy"),
577 None,
578 );
579 let mut builder = array.into_builder().unwrap();
580 builder.append_value("fg");
581 assert_eq!(builder.finish(), StringArray::from(vec!["ab", "cde", "fg"]));
582 }
583
584 #[test]
585 fn test_into_builder_err_returns_original_buffers() {
586 let values = Buffer::from(b"abxyz");
588 let array = StringArray::new(
589 OffsetBuffer::new(vec![0, 2, 2].into()),
590 values.clone(),
591 None,
592 );
593 let offsets_ptr = array.offsets().as_ptr();
594
595 let err_return = array.into_builder().unwrap_err();
596 assert_eq!(err_return.offsets().as_ptr(), offsets_ptr);
597 assert_eq!(err_return.values(), &values);
598 }
599
600 #[test]
601 fn test_into_builder_nulls() {
602 let array = StringArray::from(vec![Some("ab"), None]);
604 let nulls_ptr = array.nulls().unwrap().buffer().as_ptr();
605 let builder = array.into_builder().unwrap();
606 assert_eq!(builder.validity_slice().unwrap().as_ptr(), nulls_ptr);
607
608 let nulls = NullBuffer::from(vec![true, false]);
610 let array = StringArray::new(
611 OffsetBuffer::new(vec![0, 2, 2].into()),
612 Buffer::from(b"ab"),
613 Some(nulls.clone()),
614 );
615 let mut builder = array.into_builder().unwrap();
616 builder.append_value("c");
617 assert_eq!(
618 builder.finish(),
619 StringArray::from(vec![Some("ab"), None, Some("c")])
620 );
621 }
622
623 #[test]
624 fn test_non_null_string_array_equal() {
625 let a = StringArray::from(vec![Some("ab"), Some("c")]);
626 let b = StringArray::from(vec![Some("a"), Some("bc")]);
627
628 assert_ne!(a, b);
629 }
630}