1use crate::VariantArray;
8use arrow::array::builder::make_view;
9use arrow::array::{ArrayRef, BinaryViewArray, NullBufferBuilder, StructArray};
10use arrow::buffer::Buffer;
11use arrow_schema::{ArrowError, DataType, Field, Fields};
12use parquet_variant::{
13 BuilderSpecificState, ListBuilder, MetadataBuilder, ObjectBuilder, Variant, VariantBuilderExt,
14 VariantMetadata,
15};
16use parquet_variant::{
17 ParentState, ReadOnlyMetadataBuilder, ValueBuilder, WritableMetadataBuilder,
18};
19use std::sync::Arc;
20
21#[derive(Debug)]
83pub struct VariantArrayBuilder {
84 nulls: NullBufferBuilder,
86 metadata_builder: WritableMetadataBuilder,
88 metadata_offsets: Vec<usize>,
90 value_builder: ValueBuilder,
92 value_offsets: Vec<usize>,
94 fields: Fields,
99}
100
101impl VariantArrayBuilder {
102 pub fn new(row_capacity: usize) -> Self {
103 let metadata_field = Field::new("metadata", DataType::BinaryView, false);
105 let value_field = Field::new("value", DataType::BinaryView, false);
106
107 Self {
108 nulls: NullBufferBuilder::new(row_capacity),
109 metadata_builder: WritableMetadataBuilder::default(),
110 metadata_offsets: Vec::with_capacity(row_capacity),
111 value_builder: ValueBuilder::new(),
112 value_offsets: Vec::with_capacity(row_capacity),
113 fields: Fields::from(vec![metadata_field, value_field]),
114 }
115 }
116
117 pub fn build(self) -> VariantArray {
119 let Self {
120 mut nulls,
121 metadata_builder,
122 metadata_offsets,
123 value_builder,
124 value_offsets,
125 fields,
126 } = self;
127
128 let metadata_buffer = metadata_builder.into_inner();
129 let metadata_array = binary_view_array_from_buffers(metadata_buffer, metadata_offsets);
130
131 let value_buffer = value_builder.into_inner();
132 let value_array = binary_view_array_from_buffers(value_buffer, value_offsets);
133
134 let inner = StructArray::new(
136 fields,
137 vec![
138 Arc::new(metadata_array) as ArrayRef,
139 Arc::new(value_array) as ArrayRef,
140 ],
141 nulls.finish(),
142 );
143 VariantArray::try_new(&inner).expect("valid VariantArray by construction")
146 }
147
148 pub fn append_null(&mut self) {
150 self.nulls.append_null();
151 self.metadata_offsets.push(self.metadata_builder.offset());
153 self.value_offsets.push(self.value_builder.offset());
154 }
155
156 pub fn append_nulls(&mut self, n: usize) {
158 self.nulls.append_n_nulls(n);
159 let metadata_offset = self.metadata_builder.offset();
161 let value_offset = self.value_builder.offset();
162 self.metadata_offsets
163 .extend(std::iter::repeat_n(metadata_offset, n));
164 self.value_offsets
165 .extend(std::iter::repeat_n(value_offset, n));
166 }
167
168 pub fn append_variant(&mut self, variant: Variant) {
170 ValueBuilder::append_variant(self.parent_state(), variant);
171 }
172
173 fn parent_state(&mut self) -> ParentState<'_, ArrayBuilderState<'_>> {
175 let state = ArrayBuilderState {
176 metadata_offsets: &mut self.metadata_offsets,
177 value_offsets: &mut self.value_offsets,
178 nulls: &mut self.nulls,
179 };
180
181 ParentState::new(&mut self.value_builder, &mut self.metadata_builder, state)
182 }
183}
184
185impl<'m, 'v> Extend<Option<Variant<'m, 'v>>> for VariantArrayBuilder {
186 fn extend<T: IntoIterator<Item = Option<Variant<'m, 'v>>>>(&mut self, iter: T) {
187 for v in iter {
188 match v {
189 Some(v) => self.append_variant(v),
190 None => self.append_null(),
191 }
192 }
193 }
194}
195
196#[derive(Debug)]
199pub struct ArrayBuilderState<'a> {
200 metadata_offsets: &'a mut Vec<usize>,
201 value_offsets: &'a mut Vec<usize>,
202 nulls: &'a mut NullBufferBuilder,
203}
204
205impl BuilderSpecificState for ArrayBuilderState<'_> {
207 fn finish(
208 &mut self,
209 metadata_builder: &mut dyn MetadataBuilder,
210 value_builder: &mut ValueBuilder,
211 ) {
212 self.metadata_offsets.push(metadata_builder.finish());
213 self.value_offsets.push(value_builder.offset());
214 self.nulls.append_non_null();
215 }
216}
217
218impl VariantBuilderExt for VariantArrayBuilder {
219 type State<'a>
220 = ArrayBuilderState<'a>
221 where
222 Self: 'a;
223
224 fn append_null(&mut self) {
226 self.append_null();
227 }
228
229 fn append_value<'m, 'v>(&mut self, value: impl Into<Variant<'m, 'v>>) {
230 self.append_variant(value.into());
231 }
232
233 fn try_new_list(&mut self) -> Result<ListBuilder<'_, Self::State<'_>>, ArrowError> {
234 Ok(ListBuilder::new(self.parent_state(), false))
235 }
236
237 fn try_new_object(&mut self) -> Result<ObjectBuilder<'_, Self::State<'_>>, ArrowError> {
238 Ok(ObjectBuilder::new(self.parent_state(), false))
239 }
240}
241
242#[derive(Debug)]
270pub struct VariantValueArrayBuilder {
271 value_builder: ValueBuilder,
272 value_offsets: Vec<usize>,
273 nulls: NullBufferBuilder,
274}
275
276impl VariantValueArrayBuilder {
277 pub fn new(row_capacity: usize) -> Self {
279 Self {
280 value_builder: ValueBuilder::new(),
281 value_offsets: Vec::with_capacity(row_capacity),
282 nulls: NullBufferBuilder::new(row_capacity),
283 }
284 }
285
286 pub fn build(mut self) -> Result<BinaryViewArray, ArrowError> {
291 let value_buffer = self.value_builder.into_inner();
292 let mut array = binary_view_array_from_buffers(value_buffer, self.value_offsets);
293 if let Some(nulls) = self.nulls.finish() {
294 let (views, buffers, _) = array.into_parts();
295 array = BinaryViewArray::try_new(views, buffers, Some(nulls))?;
296 }
297 Ok(array)
298 }
299
300 pub fn append_null(&mut self) {
307 self.value_offsets.push(self.value_builder.offset());
308 self.nulls.append_null();
309 }
310
311 pub fn append_value(&mut self, value: Variant<'_, '_>) {
329 self.builder_ext(&value.metadata().clone())
331 .append_value(value);
332 }
333
334 pub fn parent_state<'a>(
359 &'a mut self,
360 metadata_builder: &'a mut dyn MetadataBuilder,
361 ) -> ParentState<'a, ValueArrayBuilderState<'a>> {
362 let state = ValueArrayBuilderState {
363 value_offsets: &mut self.value_offsets,
364 nulls: &mut self.nulls,
365 };
366
367 ParentState::new(&mut self.value_builder, metadata_builder, state)
368 }
369
370 pub fn builder_ext<'a>(
373 &'a mut self,
374 metadata: &'a VariantMetadata<'a>,
375 ) -> VariantValueArrayBuilderExt<'a> {
376 VariantValueArrayBuilderExt {
377 metadata_builder: ReadOnlyMetadataBuilder::new(metadata),
378 value_builder: self,
379 }
380 }
381}
382
383#[derive(Debug)]
386pub struct ValueArrayBuilderState<'a> {
387 value_offsets: &'a mut Vec<usize>,
388 nulls: &'a mut NullBufferBuilder,
389}
390
391impl BuilderSpecificState for ValueArrayBuilderState<'_> {
393 fn finish(
394 &mut self,
395 _metadata_builder: &mut dyn MetadataBuilder,
396 value_builder: &mut ValueBuilder,
397 ) {
398 self.value_offsets.push(value_builder.offset());
399 self.nulls.append_non_null();
400 }
401}
402
403pub struct VariantValueArrayBuilderExt<'a> {
406 metadata_builder: ReadOnlyMetadataBuilder<'a>,
407 value_builder: &'a mut VariantValueArrayBuilder,
408}
409
410impl<'a> VariantValueArrayBuilderExt<'a> {
411 pub fn new(
413 metadata_builder: ReadOnlyMetadataBuilder<'a>,
414 value_builder: &'a mut VariantValueArrayBuilder,
415 ) -> Self {
416 Self {
417 metadata_builder,
418 value_builder,
419 }
420 }
421}
422
423impl VariantBuilderExt for VariantValueArrayBuilderExt<'_> {
424 type State<'b>
425 = ValueArrayBuilderState<'b>
426 where
427 Self: 'b;
428
429 fn append_null(&mut self) {
432 self.append_value(Variant::Null);
433 }
434
435 fn append_value<'m, 'v>(&mut self, value: impl Into<Variant<'m, 'v>>) {
436 let state = self.value_builder.parent_state(&mut self.metadata_builder);
437 ValueBuilder::append_variant_bytes(state, value.into());
438 }
439
440 fn try_new_list(&mut self) -> Result<ListBuilder<'_, Self::State<'_>>, ArrowError> {
441 let state = self.value_builder.parent_state(&mut self.metadata_builder);
442 Ok(ListBuilder::new(state, false))
443 }
444
445 fn try_new_object(&mut self) -> Result<ObjectBuilder<'_, Self::State<'_>>, ArrowError> {
446 let state = self.value_builder.parent_state(&mut self.metadata_builder);
447 Ok(ObjectBuilder::new(state, false))
448 }
449}
450
451fn binary_view_array_from_buffers(buffer: Vec<u8>, offsets: Vec<usize>) -> BinaryViewArray {
452 assert!(buffer.len() < u32::MAX as usize);
455
456 let buffer = Buffer::from(buffer);
457 let mut views = Vec::with_capacity(offsets.len());
458 let mut start = 0_usize;
459 for end in offsets {
460 views.push(make_view(&buffer[start..end], 0, start as u32));
461 start = end;
462 }
463
464 unsafe { BinaryViewArray::new_unchecked(views.into(), vec![buffer].into(), None) }
467}
468
469#[cfg(test)]
470mod test {
471 use super::*;
472 use arrow::array::Array;
473 use parquet_variant::{ShortString, Variant};
474
475 #[test]
477 fn test_variant_array_builder_non_nullable() {
478 let mut builder = VariantArrayBuilder::new(10);
479
480 builder.extend([
481 None, Some(Variant::from(42_i32)),
483 ]);
484
485 let variant_array = builder.build();
486
487 assert_eq!(variant_array.len(), 2);
488 assert!(variant_array.is_null(0));
489 assert!(!variant_array.is_null(1));
490 assert_eq!(variant_array.value(1), Variant::from(42i32));
491
492 assert!(variant_array.metadata_column().nulls().is_none());
494 assert!(variant_array.value_column().nulls().is_none());
495 let DataType::Struct(fields) = variant_array.data_type() else {
496 panic!("Expected VariantArray to have Struct data type");
497 };
498 for field in fields {
499 assert!(
500 !field.is_nullable(),
501 "Field {} should be non-nullable",
502 field.name()
503 );
504 }
505 }
506
507 #[test]
509 fn test_variant_array_builder() {
510 let mut builder = VariantArrayBuilder::new(10);
511 builder.append_null(); builder.append_variant(Variant::from(42i32));
513
514 builder.new_object().with_field("foo", "bar").finish();
516
517 builder
519 .new_list()
520 .with_value(Variant::from(1i32))
521 .with_value(Variant::from(2i32))
522 .finish();
523 let variant_array = builder.build();
524
525 assert_eq!(variant_array.len(), 4);
526 assert!(variant_array.is_null(0));
527 assert!(!variant_array.is_null(1));
528 assert_eq!(variant_array.value(1), Variant::from(42i32));
529 assert!(!variant_array.is_null(2));
530 let variant = variant_array.value(2);
531 let variant = variant.as_object().expect("variant to be an object");
532 assert_eq!(variant.get("foo").unwrap(), Variant::from("bar"));
533 assert!(!variant_array.is_null(3));
534 let variant = variant_array.value(3);
535 let list = variant.as_list().expect("variant to be a list");
536 assert_eq!(list.len(), 2);
537 }
538
539 #[test]
540 fn test_variant_array_builder_append_nulls() {
541 let mut builder = VariantArrayBuilder::new(6);
542 builder.append_variant(Variant::from(1i32));
543 builder.append_nulls(0); builder.append_nulls(3);
545 builder.append_variant(Variant::from(2i32));
546
547 let variant_array = builder.build();
548
549 assert_eq!(variant_array.len(), 5);
550 assert_eq!(variant_array.value(0), Variant::from(1i32));
551 assert!(variant_array.is_null(1));
552 assert!(variant_array.is_null(2));
553 assert!(variant_array.is_null(3));
554 assert_eq!(variant_array.value(4), Variant::from(2i32));
555 }
556
557 #[test]
558 fn test_extend_variant_array_builder() {
559 let mut b = VariantArrayBuilder::new(3);
560 b.extend([None, Some(Variant::Null), Some(Variant::from("norm"))]);
561
562 let variant_array = b.build();
563
564 assert_eq!(variant_array.len(), 3);
565 assert!(variant_array.is_null(0));
566 assert_eq!(variant_array.value(1), Variant::Null);
567 assert_eq!(
568 variant_array.value(2),
569 Variant::ShortString(ShortString::try_new("norm").unwrap())
570 );
571 }
572
573 #[test]
574 fn test_variant_value_array_builder_basic() {
575 let mut builder = VariantValueArrayBuilder::new(10);
576
577 builder.append_value(Variant::from(42i32));
579 builder.append_null();
580 builder.append_value(Variant::from("hello"));
581
582 let value_array = builder.build().unwrap();
583 assert_eq!(value_array.len(), 3);
584 }
585
586 #[test]
587 fn test_variant_value_array_builder_with_objects() {
588 let mut builder = VariantArrayBuilder::new(3);
590 builder
591 .new_object()
592 .with_field("name", "Alice")
593 .with_field("age", 30i32)
594 .finish();
595
596 builder
597 .new_object()
598 .with_field("name", "Bob")
599 .with_field("age", 42i32)
600 .with_field("city", "Wonderland")
601 .finish();
602
603 builder
604 .new_object()
605 .with_field("name", "Charlie")
606 .with_field("age", 1i32)
607 .finish();
608
609 let array = builder.build();
610
611 let mut value_builder = VariantValueArrayBuilder::new(3);
616
617 value_builder.append_value(array.value(0));
619
620 let value = array.value(1);
622 let mut builder = value_builder.builder_ext(value.metadata());
623 builder
624 .new_object()
625 .with_field("name", value.get_object_field("name").unwrap())
626 .with_field("age", value.get_object_field("age").unwrap())
627 .finish();
628
629 let value = array.value(2);
631 let mut builder = value_builder.builder_ext(value.metadata());
632 builder
633 .new_list()
634 .with_value(value.clone())
635 .with_value(value.clone())
636 .finish();
637
638 let array2 = VariantArray::from_parts(
639 array.metadata_column().clone(),
640 Arc::new(value_builder.build().unwrap()),
641 None,
642 None,
643 );
644
645 assert_eq!(array2.len(), 3);
646 assert_eq!(array.value(0), array2.value(0));
647
648 assert_eq!(
649 array.value(1).get_object_field("name"),
650 array2.value(1).get_object_field("name")
651 );
652 assert_eq!(
653 array.value(1).get_object_field("age"),
654 array2.value(1).get_object_field("age")
655 );
656
657 assert_eq!(array.value(2), array2.value(2).get_list_element(0).unwrap());
658 assert_eq!(array.value(2), array2.value(2).get_list_element(1).unwrap());
659 }
660}