1use crate::VariantArray;
21use arrow::array::builder::make_view;
22use arrow::array::{ArrayRef, BinaryViewArray, NullBufferBuilder, StructArray};
23use arrow::buffer::Buffer;
24use arrow_schema::{ArrowError, DataType, Field, Fields};
25use parquet_variant::{
26 BuilderSpecificState, ListBuilder, MetadataBuilder, ObjectBuilder, Variant, VariantBuilderExt,
27 VariantMetadata,
28};
29use parquet_variant::{
30 ParentState, ReadOnlyMetadataBuilder, ValueBuilder, WritableMetadataBuilder,
31};
32use std::sync::Arc;
33
34#[derive(Debug)]
96pub struct VariantArrayBuilder {
97 nulls: NullBufferBuilder,
99 metadata_builder: WritableMetadataBuilder,
101 metadata_offsets: Vec<usize>,
103 value_builder: ValueBuilder,
105 value_offsets: Vec<usize>,
107 fields: Fields,
112}
113
114impl VariantArrayBuilder {
115 pub fn new(row_capacity: usize) -> Self {
116 let metadata_field = Field::new("metadata", DataType::BinaryView, false);
118 let value_field = Field::new("value", DataType::BinaryView, false);
119
120 Self {
121 nulls: NullBufferBuilder::new(row_capacity),
122 metadata_builder: WritableMetadataBuilder::default(),
123 metadata_offsets: Vec::with_capacity(row_capacity),
124 value_builder: ValueBuilder::new(),
125 value_offsets: Vec::with_capacity(row_capacity),
126 fields: Fields::from(vec![metadata_field, value_field]),
127 }
128 }
129
130 pub fn build(self) -> VariantArray {
132 let Self {
133 mut nulls,
134 metadata_builder,
135 metadata_offsets,
136 value_builder,
137 value_offsets,
138 fields,
139 } = self;
140
141 let metadata_buffer = metadata_builder.into_inner();
142 let metadata_array = binary_view_array_from_buffers(metadata_buffer, metadata_offsets);
143
144 let value_buffer = value_builder.into_inner();
145 let value_array = binary_view_array_from_buffers(value_buffer, value_offsets);
146
147 let inner = StructArray::new(
149 fields,
150 vec![
151 Arc::new(metadata_array) as ArrayRef,
152 Arc::new(value_array) as ArrayRef,
153 ],
154 nulls.finish(),
155 );
156 VariantArray::try_new(&inner).expect("valid VariantArray by construction")
159 }
160
161 pub fn append_null(&mut self) {
163 self.nulls.append_null();
164 self.metadata_offsets.push(self.metadata_builder.offset());
166 self.value_offsets.push(self.value_builder.offset());
167 }
168
169 pub fn append_nulls(&mut self, n: usize) {
171 self.nulls.append_n_nulls(n);
172 let metadata_offset = self.metadata_builder.offset();
174 let value_offset = self.value_builder.offset();
175 self.metadata_offsets
176 .extend(std::iter::repeat_n(metadata_offset, n));
177 self.value_offsets
178 .extend(std::iter::repeat_n(value_offset, n));
179 }
180
181 pub fn append_variant(&mut self, variant: Variant) {
183 ValueBuilder::append_variant(self.parent_state(), variant);
184 }
185
186 fn parent_state(&mut self) -> ParentState<'_, ArrayBuilderState<'_>> {
188 let state = ArrayBuilderState {
189 metadata_offsets: &mut self.metadata_offsets,
190 value_offsets: &mut self.value_offsets,
191 nulls: &mut self.nulls,
192 };
193
194 ParentState::new(&mut self.value_builder, &mut self.metadata_builder, state)
195 }
196}
197
198impl<'m, 'v> Extend<Option<Variant<'m, 'v>>> for VariantArrayBuilder {
199 fn extend<T: IntoIterator<Item = Option<Variant<'m, 'v>>>>(&mut self, iter: T) {
200 for v in iter {
201 match v {
202 Some(v) => self.append_variant(v),
203 None => self.append_null(),
204 }
205 }
206 }
207}
208
209#[derive(Debug)]
212pub struct ArrayBuilderState<'a> {
213 metadata_offsets: &'a mut Vec<usize>,
214 value_offsets: &'a mut Vec<usize>,
215 nulls: &'a mut NullBufferBuilder,
216}
217
218impl BuilderSpecificState for ArrayBuilderState<'_> {
220 fn finish(
221 &mut self,
222 metadata_builder: &mut dyn MetadataBuilder,
223 value_builder: &mut ValueBuilder,
224 ) {
225 self.metadata_offsets.push(metadata_builder.finish());
226 self.value_offsets.push(value_builder.offset());
227 self.nulls.append_non_null();
228 }
229}
230
231impl VariantBuilderExt for VariantArrayBuilder {
232 type State<'a>
233 = ArrayBuilderState<'a>
234 where
235 Self: 'a;
236
237 fn append_null(&mut self) {
239 self.append_null();
240 }
241
242 fn append_value<'m, 'v>(&mut self, value: impl Into<Variant<'m, 'v>>) {
243 self.append_variant(value.into());
244 }
245
246 fn try_new_list(&mut self) -> Result<ListBuilder<'_, Self::State<'_>>, ArrowError> {
247 Ok(ListBuilder::new(self.parent_state(), false))
248 }
249
250 fn try_new_object(&mut self) -> Result<ObjectBuilder<'_, Self::State<'_>>, ArrowError> {
251 Ok(ObjectBuilder::new(self.parent_state(), false))
252 }
253}
254
255#[derive(Debug)]
283pub struct VariantValueArrayBuilder {
284 value_builder: ValueBuilder,
285 value_offsets: Vec<usize>,
286 nulls: NullBufferBuilder,
287}
288
289impl VariantValueArrayBuilder {
290 pub fn new(row_capacity: usize) -> Self {
292 Self {
293 value_builder: ValueBuilder::new(),
294 value_offsets: Vec::with_capacity(row_capacity),
295 nulls: NullBufferBuilder::new(row_capacity),
296 }
297 }
298
299 pub fn build(mut self) -> Result<BinaryViewArray, ArrowError> {
304 let value_buffer = self.value_builder.into_inner();
305 let mut array = binary_view_array_from_buffers(value_buffer, self.value_offsets);
306 if let Some(nulls) = self.nulls.finish() {
307 let (views, buffers, _) = array.into_parts();
308 array = BinaryViewArray::try_new(views, buffers, Some(nulls))?;
309 }
310 Ok(array)
311 }
312
313 pub fn append_null(&mut self) {
320 self.value_offsets.push(self.value_builder.offset());
321 self.nulls.append_null();
322 }
323
324 pub fn append_value(&mut self, value: Variant<'_, '_>) {
342 self.builder_ext(&value.metadata().clone())
344 .append_value(value);
345 }
346
347 pub fn parent_state<'a>(
372 &'a mut self,
373 metadata_builder: &'a mut dyn MetadataBuilder,
374 ) -> ParentState<'a, ValueArrayBuilderState<'a>> {
375 let state = ValueArrayBuilderState {
376 value_offsets: &mut self.value_offsets,
377 nulls: &mut self.nulls,
378 };
379
380 ParentState::new(&mut self.value_builder, metadata_builder, state)
381 }
382
383 pub fn builder_ext<'a>(
386 &'a mut self,
387 metadata: &'a VariantMetadata<'a>,
388 ) -> VariantValueArrayBuilderExt<'a> {
389 VariantValueArrayBuilderExt {
390 metadata_builder: ReadOnlyMetadataBuilder::new(metadata),
391 value_builder: self,
392 }
393 }
394}
395
396#[derive(Debug)]
399pub struct ValueArrayBuilderState<'a> {
400 value_offsets: &'a mut Vec<usize>,
401 nulls: &'a mut NullBufferBuilder,
402}
403
404impl BuilderSpecificState for ValueArrayBuilderState<'_> {
406 fn finish(
407 &mut self,
408 _metadata_builder: &mut dyn MetadataBuilder,
409 value_builder: &mut ValueBuilder,
410 ) {
411 self.value_offsets.push(value_builder.offset());
412 self.nulls.append_non_null();
413 }
414}
415
416pub struct VariantValueArrayBuilderExt<'a> {
419 metadata_builder: ReadOnlyMetadataBuilder<'a>,
420 value_builder: &'a mut VariantValueArrayBuilder,
421}
422
423impl<'a> VariantValueArrayBuilderExt<'a> {
424 pub fn new(
426 metadata_builder: ReadOnlyMetadataBuilder<'a>,
427 value_builder: &'a mut VariantValueArrayBuilder,
428 ) -> Self {
429 Self {
430 metadata_builder,
431 value_builder,
432 }
433 }
434}
435
436impl VariantBuilderExt for VariantValueArrayBuilderExt<'_> {
437 type State<'b>
438 = ValueArrayBuilderState<'b>
439 where
440 Self: 'b;
441
442 fn append_null(&mut self) {
443 self.value_builder.append_null()
444 }
445
446 fn append_value<'m, 'v>(&mut self, value: impl Into<Variant<'m, 'v>>) {
447 let state = self.value_builder.parent_state(&mut self.metadata_builder);
448 ValueBuilder::append_variant_bytes(state, value.into());
449 }
450
451 fn try_new_list(&mut self) -> Result<ListBuilder<'_, Self::State<'_>>, ArrowError> {
452 let state = self.value_builder.parent_state(&mut self.metadata_builder);
453 Ok(ListBuilder::new(state, false))
454 }
455
456 fn try_new_object(&mut self) -> Result<ObjectBuilder<'_, Self::State<'_>>, ArrowError> {
457 let state = self.value_builder.parent_state(&mut self.metadata_builder);
458 Ok(ObjectBuilder::new(state, false))
459 }
460}
461
462fn binary_view_array_from_buffers(buffer: Vec<u8>, offsets: Vec<usize>) -> BinaryViewArray {
463 assert!(buffer.len() < u32::MAX as usize);
466
467 let buffer = Buffer::from(buffer);
468 let mut views = Vec::with_capacity(offsets.len());
469 let mut start = 0_usize;
470 for end in offsets {
471 views.push(make_view(&buffer[start..end], 0, start as u32));
472 start = end;
473 }
474
475 unsafe { BinaryViewArray::new_unchecked(views.into(), vec![buffer].into(), None) }
478}
479
480#[cfg(test)]
481mod test {
482 use super::*;
483 use arrow::array::Array;
484 use parquet_variant::{ShortString, Variant};
485
486 #[test]
488 fn test_variant_array_builder_non_nullable() {
489 let mut builder = VariantArrayBuilder::new(10);
490
491 builder.extend([
492 None, Some(Variant::from(42_i32)),
494 ]);
495
496 let variant_array = builder.build();
497
498 assert_eq!(variant_array.len(), 2);
499 assert!(variant_array.is_null(0));
500 assert!(!variant_array.is_null(1));
501 assert_eq!(variant_array.value(1), Variant::from(42i32));
502
503 assert!(variant_array.metadata_column().nulls().is_none());
505 assert!(variant_array.value_column().nulls().is_none());
506 let DataType::Struct(fields) = variant_array.data_type() else {
507 panic!("Expected VariantArray to have Struct data type");
508 };
509 for field in fields {
510 assert!(
511 !field.is_nullable(),
512 "Field {} should be non-nullable",
513 field.name()
514 );
515 }
516 }
517
518 #[test]
520 fn test_variant_array_builder() {
521 let mut builder = VariantArrayBuilder::new(10);
522 builder.append_null(); builder.append_variant(Variant::from(42i32));
524
525 builder.new_object().with_field("foo", "bar").finish();
527
528 builder
530 .new_list()
531 .with_value(Variant::from(1i32))
532 .with_value(Variant::from(2i32))
533 .finish();
534 let variant_array = builder.build();
535
536 assert_eq!(variant_array.len(), 4);
537 assert!(variant_array.is_null(0));
538 assert!(!variant_array.is_null(1));
539 assert_eq!(variant_array.value(1), Variant::from(42i32));
540 assert!(!variant_array.is_null(2));
541 let variant = variant_array.value(2);
542 let variant = variant.as_object().expect("variant to be an object");
543 assert_eq!(variant.get("foo").unwrap(), Variant::from("bar"));
544 assert!(!variant_array.is_null(3));
545 let variant = variant_array.value(3);
546 let list = variant.as_list().expect("variant to be a list");
547 assert_eq!(list.len(), 2);
548 }
549
550 #[test]
551 fn test_variant_array_builder_append_nulls() {
552 let mut builder = VariantArrayBuilder::new(6);
553 builder.append_variant(Variant::from(1i32));
554 builder.append_nulls(0); builder.append_nulls(3);
556 builder.append_variant(Variant::from(2i32));
557
558 let variant_array = builder.build();
559
560 assert_eq!(variant_array.len(), 5);
561 assert_eq!(variant_array.value(0), Variant::from(1i32));
562 assert!(variant_array.is_null(1));
563 assert!(variant_array.is_null(2));
564 assert!(variant_array.is_null(3));
565 assert_eq!(variant_array.value(4), Variant::from(2i32));
566 }
567
568 #[test]
569 fn test_extend_variant_array_builder() {
570 let mut b = VariantArrayBuilder::new(3);
571 b.extend([None, Some(Variant::Null), Some(Variant::from("norm"))]);
572
573 let variant_array = b.build();
574
575 assert_eq!(variant_array.len(), 3);
576 assert!(variant_array.is_null(0));
577 assert_eq!(variant_array.value(1), Variant::Null);
578 assert_eq!(
579 variant_array.value(2),
580 Variant::ShortString(ShortString::try_new("norm").unwrap())
581 );
582 }
583
584 #[test]
585 fn test_variant_value_array_builder_basic() {
586 let mut builder = VariantValueArrayBuilder::new(10);
587
588 builder.append_value(Variant::from(42i32));
590 builder.append_null();
591 builder.append_value(Variant::from("hello"));
592
593 let value_array = builder.build().unwrap();
594 assert_eq!(value_array.len(), 3);
595 }
596
597 #[test]
598 fn test_variant_value_array_builder_with_objects() {
599 let mut builder = VariantArrayBuilder::new(3);
601 builder
602 .new_object()
603 .with_field("name", "Alice")
604 .with_field("age", 30i32)
605 .finish();
606
607 builder
608 .new_object()
609 .with_field("name", "Bob")
610 .with_field("age", 42i32)
611 .with_field("city", "Wonderland")
612 .finish();
613
614 builder
615 .new_object()
616 .with_field("name", "Charlie")
617 .with_field("age", 1i32)
618 .finish();
619
620 let array = builder.build();
621
622 let mut value_builder = VariantValueArrayBuilder::new(3);
627
628 value_builder.append_value(array.value(0));
630
631 let value = array.value(1);
633 let mut builder = value_builder.builder_ext(value.metadata());
634 builder
635 .new_object()
636 .with_field("name", value.get_object_field("name").unwrap())
637 .with_field("age", value.get_object_field("age").unwrap())
638 .finish();
639
640 let value = array.value(2);
642 let mut builder = value_builder.builder_ext(value.metadata());
643 builder
644 .new_list()
645 .with_value(value.clone())
646 .with_value(value.clone())
647 .finish();
648
649 let array2 = VariantArray::from_parts(
650 array.metadata_column().clone(),
651 Arc::new(value_builder.build().unwrap()),
652 None,
653 None,
654 );
655
656 assert_eq!(array2.len(), 3);
657 assert_eq!(array.value(0), array2.value(0));
658
659 assert_eq!(
660 array.value(1).get_object_field("name"),
661 array2.value(1).get_object_field("name")
662 );
663 assert_eq!(
664 array.value(1).get_object_field("age"),
665 array2.value(1).get_object_field("age")
666 );
667
668 assert_eq!(array.value(2), array2.value(2).get_list_element(0).unwrap());
669 assert_eq!(array.value(2), array2.value(2).get_list_element(1).unwrap());
670 }
671}