1use crate::array::{get_offsets, print_long_array};
19use crate::builder::GenericByteBuilder;
20use crate::iterator::ArrayIter;
21use crate::types::bytes::ByteArrayNativeType;
22use crate::types::ByteArrayType;
23use crate::{Array, ArrayAccessor, ArrayRef, OffsetSizeTrait, Scalar};
24use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer};
25use arrow_buffer::{NullBuffer, OffsetBuffer};
26use arrow_data::{ArrayData, ArrayDataBuilder};
27use arrow_schema::{ArrowError, DataType};
28use std::any::Any;
29use std::sync::Arc;
30
31pub struct GenericByteArray<T: ByteArrayType> {
88 data_type: DataType,
89 value_offsets: OffsetBuffer<T::Offset>,
90 value_data: Buffer,
91 nulls: Option<NullBuffer>,
92}
93
94impl<T: ByteArrayType> Clone for GenericByteArray<T> {
95 fn clone(&self) -> Self {
96 Self {
97 data_type: T::DATA_TYPE,
98 value_offsets: self.value_offsets.clone(),
99 value_data: self.value_data.clone(),
100 nulls: self.nulls.clone(),
101 }
102 }
103}
104
105impl<T: ByteArrayType> GenericByteArray<T> {
106 pub const DATA_TYPE: DataType = T::DATA_TYPE;
108
109 pub fn new(
115 offsets: OffsetBuffer<T::Offset>,
116 values: Buffer,
117 nulls: Option<NullBuffer>,
118 ) -> Self {
119 Self::try_new(offsets, values, nulls).unwrap()
120 }
121
122 pub fn try_new(
129 offsets: OffsetBuffer<T::Offset>,
130 values: Buffer,
131 nulls: Option<NullBuffer>,
132 ) -> Result<Self, ArrowError> {
133 let len = offsets.len() - 1;
134
135 T::validate(&offsets, &values)?;
137
138 if let Some(n) = nulls.as_ref() {
139 if n.len() != len {
140 return Err(ArrowError::InvalidArgumentError(format!(
141 "Incorrect length of null buffer for {}{}Array, expected {len} got {}",
142 T::Offset::PREFIX,
143 T::PREFIX,
144 n.len(),
145 )));
146 }
147 }
148
149 Ok(Self {
150 data_type: T::DATA_TYPE,
151 value_offsets: offsets,
152 value_data: values,
153 nulls,
154 })
155 }
156
157 pub unsafe fn new_unchecked(
163 offsets: OffsetBuffer<T::Offset>,
164 values: Buffer,
165 nulls: Option<NullBuffer>,
166 ) -> Self {
167 if cfg!(feature = "force_validate") {
168 return Self::new(offsets, values, nulls);
169 }
170 Self {
171 data_type: T::DATA_TYPE,
172 value_offsets: offsets,
173 value_data: values,
174 nulls,
175 }
176 }
177
178 pub fn new_null(len: usize) -> Self {
180 Self {
181 data_type: T::DATA_TYPE,
182 value_offsets: OffsetBuffer::new_zeroed(len),
183 value_data: MutableBuffer::new(0).into(),
184 nulls: Some(NullBuffer::new_null(len)),
185 }
186 }
187
188 pub fn new_scalar(value: impl AsRef<T::Native>) -> Scalar<Self> {
190 Scalar::new(Self::from_iter_values(std::iter::once(value)))
191 }
192
193 pub fn from_iter_values<Ptr, I>(iter: I) -> Self
195 where
196 Ptr: AsRef<T::Native>,
197 I: IntoIterator<Item = Ptr>,
198 {
199 let iter = iter.into_iter();
200 let (_, data_len) = iter.size_hint();
201 let data_len = data_len.expect("Iterator must be sized"); let mut offsets = MutableBuffer::new((data_len + 1) * std::mem::size_of::<T::Offset>());
204 offsets.push(T::Offset::usize_as(0));
205
206 let mut values = MutableBuffer::new(0);
207 for s in iter {
208 let s: &[u8] = s.as_ref().as_ref();
209 values.extend_from_slice(s);
210 offsets.push(T::Offset::usize_as(values.len()));
211 }
212
213 T::Offset::from_usize(values.len()).expect("offset overflow");
214 let offsets = Buffer::from(offsets);
215
216 let value_offsets = unsafe { OffsetBuffer::new_unchecked(offsets.into()) };
218
219 Self {
220 data_type: T::DATA_TYPE,
221 value_data: values.into(),
222 value_offsets,
223 nulls: None,
224 }
225 }
226
227 pub fn into_parts(self) -> (OffsetBuffer<T::Offset>, Buffer, Option<NullBuffer>) {
229 (self.value_offsets, self.value_data, self.nulls)
230 }
231
232 #[inline]
236 pub fn value_length(&self, i: usize) -> T::Offset {
237 let offsets = self.value_offsets();
238 offsets[i + 1] - offsets[i]
239 }
240
241 #[inline]
246 pub fn offsets(&self) -> &OffsetBuffer<T::Offset> {
247 &self.value_offsets
248 }
249
250 #[inline]
255 pub fn values(&self) -> &Buffer {
256 &self.value_data
257 }
258
259 pub fn value_data(&self) -> &[u8] {
261 self.value_data.as_slice()
262 }
263
264 pub fn is_ascii(&self) -> bool {
266 let offsets = self.value_offsets();
267 let start = offsets.first().unwrap();
268 let end = offsets.last().unwrap();
269 self.value_data()[start.as_usize()..end.as_usize()].is_ascii()
270 }
271
272 #[inline]
274 pub fn value_offsets(&self) -> &[T::Offset] {
275 &self.value_offsets
276 }
277
278 pub unsafe fn value_unchecked(&self, i: usize) -> &T::Native {
282 let end = *self.value_offsets().get_unchecked(i + 1);
283 let start = *self.value_offsets().get_unchecked(i);
284
285 let b = std::slice::from_raw_parts(
295 self.value_data
296 .as_ptr()
297 .offset(start.to_isize().unwrap_unchecked()),
298 (end - start).to_usize().unwrap_unchecked(),
299 );
300
301 T::Native::from_bytes_unchecked(b)
304 }
305
306 pub fn value(&self, i: usize) -> &T::Native {
310 assert!(
311 i < self.len(),
312 "Trying to access an element at index {} from a {}{}Array of length {}",
313 i,
314 T::Offset::PREFIX,
315 T::PREFIX,
316 self.len()
317 );
318 unsafe { self.value_unchecked(i) }
321 }
322
323 pub fn iter(&self) -> ArrayIter<&Self> {
325 ArrayIter::new(self)
326 }
327
328 pub fn slice(&self, offset: usize, length: usize) -> Self {
330 Self {
331 data_type: T::DATA_TYPE,
332 value_offsets: self.value_offsets.slice(offset, length),
333 value_data: self.value_data.clone(),
334 nulls: self.nulls.as_ref().map(|n| n.slice(offset, length)),
335 }
336 }
337
338 pub fn into_builder(self) -> Result<GenericByteBuilder<T>, Self> {
341 let len = self.len();
342 let value_len = T::Offset::as_usize(self.value_offsets()[len] - self.value_offsets()[0]);
343
344 let data = self.into_data();
345 let null_bit_buffer = data.nulls().map(|b| b.inner().sliced());
346
347 let element_len = std::mem::size_of::<T::Offset>();
348 let offset_buffer = data.buffers()[0]
349 .slice_with_length(data.offset() * element_len, (len + 1) * element_len);
350
351 let element_len = std::mem::size_of::<u8>();
352 let value_buffer = data.buffers()[1]
353 .slice_with_length(data.offset() * element_len, value_len * element_len);
354
355 drop(data);
356
357 let try_mutable_null_buffer = match null_bit_buffer {
358 None => Ok(None),
359 Some(null_buffer) => {
360 null_buffer.into_mutable().map(Some)
362 }
363 };
364
365 let try_mutable_buffers = match try_mutable_null_buffer {
366 Ok(mutable_null_buffer) => {
367 let try_mutable_offset_buffer = offset_buffer.into_mutable();
369 let try_mutable_value_buffer = value_buffer.into_mutable();
370
371 match (try_mutable_offset_buffer, try_mutable_value_buffer) {
374 (Ok(mutable_offset_buffer), Ok(mutable_value_buffer)) => unsafe {
375 Ok(GenericByteBuilder::<T>::new_from_buffer(
376 mutable_offset_buffer,
377 mutable_value_buffer,
378 mutable_null_buffer,
379 ))
380 },
381 (Ok(mutable_offset_buffer), Err(value_buffer)) => Err((
382 mutable_offset_buffer.into(),
383 value_buffer,
384 mutable_null_buffer.map(|b| b.into()),
385 )),
386 (Err(offset_buffer), Ok(mutable_value_buffer)) => Err((
387 offset_buffer,
388 mutable_value_buffer.into(),
389 mutable_null_buffer.map(|b| b.into()),
390 )),
391 (Err(offset_buffer), Err(value_buffer)) => Err((
392 offset_buffer,
393 value_buffer,
394 mutable_null_buffer.map(|b| b.into()),
395 )),
396 }
397 }
398 Err(mutable_null_buffer) => {
399 Err((offset_buffer, value_buffer, Some(mutable_null_buffer)))
401 }
402 };
403
404 match try_mutable_buffers {
405 Ok(builder) => Ok(builder),
406 Err((offset_buffer, value_buffer, null_bit_buffer)) => {
407 let builder = ArrayData::builder(T::DATA_TYPE)
408 .len(len)
409 .add_buffer(offset_buffer)
410 .add_buffer(value_buffer)
411 .null_bit_buffer(null_bit_buffer);
412
413 let array_data = unsafe { builder.build_unchecked() };
414 let array = GenericByteArray::<T>::from(array_data);
415
416 Err(array)
417 }
418 }
419 }
420}
421
422impl<T: ByteArrayType> std::fmt::Debug for GenericByteArray<T> {
423 fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
424 write!(f, "{}{}Array\n[\n", T::Offset::PREFIX, T::PREFIX)?;
425 print_long_array(self, f, |array, index, f| {
426 std::fmt::Debug::fmt(&array.value(index), f)
427 })?;
428 write!(f, "]")
429 }
430}
431
432impl<T: ByteArrayType> Array for GenericByteArray<T> {
433 fn as_any(&self) -> &dyn Any {
434 self
435 }
436
437 fn to_data(&self) -> ArrayData {
438 self.clone().into()
439 }
440
441 fn into_data(self) -> ArrayData {
442 self.into()
443 }
444
445 fn data_type(&self) -> &DataType {
446 &self.data_type
447 }
448
449 fn slice(&self, offset: usize, length: usize) -> ArrayRef {
450 Arc::new(self.slice(offset, length))
451 }
452
453 fn len(&self) -> usize {
454 self.value_offsets.len() - 1
455 }
456
457 fn is_empty(&self) -> bool {
458 self.value_offsets.len() <= 1
459 }
460
461 fn shrink_to_fit(&mut self) {
462 self.value_offsets.shrink_to_fit();
463 self.value_data.shrink_to_fit();
464 if let Some(nulls) = &mut self.nulls {
465 nulls.shrink_to_fit();
466 }
467 }
468
469 fn offset(&self) -> usize {
470 0
471 }
472
473 fn nulls(&self) -> Option<&NullBuffer> {
474 self.nulls.as_ref()
475 }
476
477 fn logical_null_count(&self) -> usize {
478 self.null_count()
480 }
481
482 fn get_buffer_memory_size(&self) -> usize {
483 let mut sum = self.value_offsets.inner().inner().capacity();
484 sum += self.value_data.capacity();
485 if let Some(x) = &self.nulls {
486 sum += x.buffer().capacity()
487 }
488 sum
489 }
490
491 fn get_array_memory_size(&self) -> usize {
492 std::mem::size_of::<Self>() + self.get_buffer_memory_size()
493 }
494}
495
496impl<'a, T: ByteArrayType> ArrayAccessor for &'a GenericByteArray<T> {
497 type Item = &'a T::Native;
498
499 fn value(&self, index: usize) -> Self::Item {
500 GenericByteArray::value(self, index)
501 }
502
503 unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
504 GenericByteArray::value_unchecked(self, index)
505 }
506}
507
508impl<T: ByteArrayType> From<ArrayData> for GenericByteArray<T> {
509 fn from(data: ArrayData) -> Self {
510 assert_eq!(
511 data.data_type(),
512 &Self::DATA_TYPE,
513 "{}{}Array expects DataType::{}",
514 T::Offset::PREFIX,
515 T::PREFIX,
516 Self::DATA_TYPE
517 );
518 assert_eq!(
519 data.buffers().len(),
520 2,
521 "{}{}Array data should contain 2 buffers only (offsets and values)",
522 T::Offset::PREFIX,
523 T::PREFIX,
524 );
525 let value_offsets = unsafe { get_offsets(&data) };
528 let value_data = data.buffers()[1].clone();
529 Self {
530 value_offsets,
531 value_data,
532 data_type: T::DATA_TYPE,
533 nulls: data.nulls().cloned(),
534 }
535 }
536}
537
538impl<T: ByteArrayType> From<GenericByteArray<T>> for ArrayData {
539 fn from(array: GenericByteArray<T>) -> Self {
540 let len = array.len();
541
542 let offsets = array.value_offsets.into_inner().into_inner();
543 let builder = ArrayDataBuilder::new(array.data_type)
544 .len(len)
545 .buffers(vec![offsets, array.value_data])
546 .nulls(array.nulls);
547
548 unsafe { builder.build_unchecked() }
549 }
550}
551
552impl<'a, T: ByteArrayType> IntoIterator for &'a GenericByteArray<T> {
553 type Item = Option<&'a T::Native>;
554 type IntoIter = ArrayIter<Self>;
555
556 fn into_iter(self) -> Self::IntoIter {
557 ArrayIter::new(self)
558 }
559}
560
561impl<'a, Ptr, T: ByteArrayType> FromIterator<&'a Option<Ptr>> for GenericByteArray<T>
562where
563 Ptr: AsRef<T::Native> + 'a,
564{
565 fn from_iter<I: IntoIterator<Item = &'a Option<Ptr>>>(iter: I) -> Self {
566 iter.into_iter()
567 .map(|o| o.as_ref().map(|p| p.as_ref()))
568 .collect()
569 }
570}
571
572impl<Ptr, T: ByteArrayType> FromIterator<Option<Ptr>> for GenericByteArray<T>
573where
574 Ptr: AsRef<T::Native>,
575{
576 fn from_iter<I: IntoIterator<Item = Option<Ptr>>>(iter: I) -> Self {
577 let iter = iter.into_iter();
578 let mut builder = GenericByteBuilder::with_capacity(iter.size_hint().0, 1024);
579 builder.extend(iter);
580 builder.finish()
581 }
582}
583
584#[cfg(test)]
585mod tests {
586 use crate::{BinaryArray, StringArray};
587 use arrow_buffer::{Buffer, NullBuffer, OffsetBuffer};
588
589 #[test]
590 fn try_new() {
591 let data = Buffer::from_slice_ref("helloworld");
592 let offsets = OffsetBuffer::new(vec![0, 5, 10].into());
593 StringArray::new(offsets.clone(), data.clone(), None);
594
595 let nulls = NullBuffer::new_null(3);
596 let err =
597 StringArray::try_new(offsets.clone(), data.clone(), Some(nulls.clone())).unwrap_err();
598 assert_eq!(err.to_string(), "Invalid argument error: Incorrect length of null buffer for StringArray, expected 2 got 3");
599
600 let err = BinaryArray::try_new(offsets.clone(), data.clone(), Some(nulls)).unwrap_err();
601 assert_eq!(err.to_string(), "Invalid argument error: Incorrect length of null buffer for BinaryArray, expected 2 got 3");
602
603 let non_utf8_data = Buffer::from_slice_ref(b"he\xFFloworld");
604 let err = StringArray::try_new(offsets.clone(), non_utf8_data.clone(), None).unwrap_err();
605 assert_eq!(err.to_string(), "Invalid argument error: Encountered non UTF-8 data: invalid utf-8 sequence of 1 bytes from index 2");
606
607 BinaryArray::new(offsets, non_utf8_data, None);
608
609 let offsets = OffsetBuffer::new(vec![0, 5, 11].into());
610 let err = StringArray::try_new(offsets.clone(), data.clone(), None).unwrap_err();
611 assert_eq!(
612 err.to_string(),
613 "Invalid argument error: Offset of 11 exceeds length of values 10"
614 );
615
616 let err = BinaryArray::try_new(offsets.clone(), data, None).unwrap_err();
617 assert_eq!(
618 err.to_string(),
619 "Invalid argument error: Maximum offset of 11 is larger than values of length 10"
620 );
621
622 let non_ascii_data = Buffer::from_slice_ref("heìloworld");
623 StringArray::new(offsets.clone(), non_ascii_data.clone(), None);
624 BinaryArray::new(offsets, non_ascii_data.clone(), None);
625
626 let offsets = OffsetBuffer::new(vec![0, 3, 10].into());
627 let err = StringArray::try_new(offsets.clone(), non_ascii_data.clone(), None).unwrap_err();
628 assert_eq!(
629 err.to_string(),
630 "Invalid argument error: Split UTF-8 codepoint at offset 3"
631 );
632
633 BinaryArray::new(offsets, non_ascii_data, None);
634 }
635}