parquet_variant/
builder.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17use crate::decoder::{VariantBasicType, VariantPrimitiveType};
18use crate::{
19    ShortString, Variant, VariantDecimal16, VariantDecimal4, VariantDecimal8, VariantList,
20    VariantMetadata, VariantObject,
21};
22use arrow_schema::ArrowError;
23use chrono::Timelike;
24use indexmap::{IndexMap, IndexSet};
25use uuid::Uuid;
26
27use std::collections::HashMap;
28
29const BASIC_TYPE_BITS: u8 = 2;
30const UNIX_EPOCH_DATE: chrono::NaiveDate = chrono::NaiveDate::from_ymd_opt(1970, 1, 1).unwrap();
31
32fn primitive_header(primitive_type: VariantPrimitiveType) -> u8 {
33    (primitive_type as u8) << 2 | VariantBasicType::Primitive as u8
34}
35
36fn short_string_header(len: usize) -> u8 {
37    (len as u8) << 2 | VariantBasicType::ShortString as u8
38}
39
40fn array_header(large: bool, offset_size: u8) -> u8 {
41    let large_bit = if large { 1 } else { 0 };
42    (large_bit << (BASIC_TYPE_BITS + 2))
43        | ((offset_size - 1) << BASIC_TYPE_BITS)
44        | VariantBasicType::Array as u8
45}
46
47fn object_header(large: bool, id_size: u8, offset_size: u8) -> u8 {
48    let large_bit = if large { 1 } else { 0 };
49    (large_bit << (BASIC_TYPE_BITS + 4))
50        | ((id_size - 1) << (BASIC_TYPE_BITS + 2))
51        | ((offset_size - 1) << BASIC_TYPE_BITS)
52        | VariantBasicType::Object as u8
53}
54
55fn int_size(v: usize) -> u8 {
56    match v {
57        0..=0xFF => 1,
58        0x100..=0xFFFF => 2,
59        0x10000..=0xFFFFFF => 3,
60        _ => 4,
61    }
62}
63
64/// Write little-endian integer to buffer
65fn write_offset(buf: &mut Vec<u8>, value: usize, nbytes: u8) {
66    let bytes = value.to_le_bytes();
67    buf.extend_from_slice(&bytes[..nbytes as usize]);
68}
69
70/// Write little-endian integer to buffer at a specific position
71fn write_offset_at_pos(buf: &mut [u8], start_pos: usize, value: usize, nbytes: u8) {
72    let bytes = value.to_le_bytes();
73    buf[start_pos..start_pos + nbytes as usize].copy_from_slice(&bytes[..nbytes as usize]);
74}
75
76/// Append `value_size` bytes of given `value` into `dest`.
77fn append_packed_u32(dest: &mut Vec<u8>, value: u32, value_size: usize) {
78    let n = dest.len() + value_size;
79    dest.extend(value.to_le_bytes());
80    dest.truncate(n);
81}
82
83/// Wrapper around a `Vec<u8>` that provides methods for appending
84/// primitive values, variant types, and metadata.
85///
86/// This is used internally by the builders to construct the
87/// the `value` field for [`Variant`] values.
88///
89/// You can reuse an existing `Vec<u8>` by using the `from` impl
90#[derive(Debug, Default)]
91pub struct ValueBuilder(Vec<u8>);
92
93impl ValueBuilder {
94    /// Construct a ValueBuffer that will write to a new underlying `Vec`
95    pub fn new() -> Self {
96        Default::default()
97    }
98}
99
100/// Macro to generate the match statement for each append_variant, try_append_variant, and
101/// append_variant_bytes -- they each have slightly different handling for object and list handling.
102macro_rules! variant_append_value {
103    ($builder:expr, $value:expr, $object_pat:pat => $object_arm:expr, $list_pat:pat => $list_arm:expr) => {
104        match $value {
105            Variant::Null => $builder.append_null(),
106            Variant::BooleanTrue => $builder.append_bool(true),
107            Variant::BooleanFalse => $builder.append_bool(false),
108            Variant::Int8(v) => $builder.append_int8(v),
109            Variant::Int16(v) => $builder.append_int16(v),
110            Variant::Int32(v) => $builder.append_int32(v),
111            Variant::Int64(v) => $builder.append_int64(v),
112            Variant::Date(v) => $builder.append_date(v),
113            Variant::Time(v) => $builder.append_time_micros(v),
114            Variant::TimestampMicros(v) => $builder.append_timestamp_micros(v),
115            Variant::TimestampNtzMicros(v) => $builder.append_timestamp_ntz_micros(v),
116            Variant::TimestampNanos(v) => $builder.append_timestamp_nanos(v),
117            Variant::TimestampNtzNanos(v) => $builder.append_timestamp_ntz_nanos(v),
118            Variant::Decimal4(decimal4) => $builder.append_decimal4(decimal4),
119            Variant::Decimal8(decimal8) => $builder.append_decimal8(decimal8),
120            Variant::Decimal16(decimal16) => $builder.append_decimal16(decimal16),
121            Variant::Float(v) => $builder.append_float(v),
122            Variant::Double(v) => $builder.append_double(v),
123            Variant::Binary(v) => $builder.append_binary(v),
124            Variant::String(s) => $builder.append_string(s),
125            Variant::ShortString(s) => $builder.append_short_string(s),
126            Variant::Uuid(v) => $builder.append_uuid(v),
127            $object_pat => $object_arm,
128            $list_pat => $list_arm,
129        }
130    };
131}
132
133impl ValueBuilder {
134    fn append_u8(&mut self, term: u8) {
135        self.0.push(term);
136    }
137
138    fn append_slice(&mut self, other: &[u8]) {
139        self.0.extend_from_slice(other);
140    }
141
142    fn append_primitive_header(&mut self, primitive_type: VariantPrimitiveType) {
143        self.0.push(primitive_header(primitive_type));
144    }
145
146    /// Returns the underlying buffer, consuming self
147    pub fn into_inner(self) -> Vec<u8> {
148        self.0
149    }
150
151    fn inner_mut(&mut self) -> &mut Vec<u8> {
152        &mut self.0
153    }
154
155    // Variant types below
156
157    fn append_null(&mut self) {
158        self.append_primitive_header(VariantPrimitiveType::Null);
159    }
160
161    fn append_bool(&mut self, value: bool) {
162        let primitive_type = if value {
163            VariantPrimitiveType::BooleanTrue
164        } else {
165            VariantPrimitiveType::BooleanFalse
166        };
167        self.append_primitive_header(primitive_type);
168    }
169
170    fn append_int8(&mut self, value: i8) {
171        self.append_primitive_header(VariantPrimitiveType::Int8);
172        self.append_u8(value as u8);
173    }
174
175    fn append_int16(&mut self, value: i16) {
176        self.append_primitive_header(VariantPrimitiveType::Int16);
177        self.append_slice(&value.to_le_bytes());
178    }
179
180    fn append_int32(&mut self, value: i32) {
181        self.append_primitive_header(VariantPrimitiveType::Int32);
182        self.append_slice(&value.to_le_bytes());
183    }
184
185    fn append_int64(&mut self, value: i64) {
186        self.append_primitive_header(VariantPrimitiveType::Int64);
187        self.append_slice(&value.to_le_bytes());
188    }
189
190    fn append_float(&mut self, value: f32) {
191        self.append_primitive_header(VariantPrimitiveType::Float);
192        self.append_slice(&value.to_le_bytes());
193    }
194
195    fn append_double(&mut self, value: f64) {
196        self.append_primitive_header(VariantPrimitiveType::Double);
197        self.append_slice(&value.to_le_bytes());
198    }
199
200    fn append_date(&mut self, value: chrono::NaiveDate) {
201        self.append_primitive_header(VariantPrimitiveType::Date);
202        let days_since_epoch = value.signed_duration_since(UNIX_EPOCH_DATE).num_days() as i32;
203        self.append_slice(&days_since_epoch.to_le_bytes());
204    }
205
206    fn append_timestamp_micros(&mut self, value: chrono::DateTime<chrono::Utc>) {
207        self.append_primitive_header(VariantPrimitiveType::TimestampMicros);
208        let micros = value.timestamp_micros();
209        self.append_slice(&micros.to_le_bytes());
210    }
211
212    fn append_timestamp_ntz_micros(&mut self, value: chrono::NaiveDateTime) {
213        self.append_primitive_header(VariantPrimitiveType::TimestampNtzMicros);
214        let micros = value.and_utc().timestamp_micros();
215        self.append_slice(&micros.to_le_bytes());
216    }
217
218    fn append_time_micros(&mut self, value: chrono::NaiveTime) {
219        self.append_primitive_header(VariantPrimitiveType::Time);
220        let micros_from_midnight = value.num_seconds_from_midnight() as u64 * 1_000_000
221            + value.nanosecond() as u64 / 1_000;
222        self.append_slice(&micros_from_midnight.to_le_bytes());
223    }
224
225    fn append_timestamp_nanos(&mut self, value: chrono::DateTime<chrono::Utc>) {
226        self.append_primitive_header(VariantPrimitiveType::TimestampNanos);
227        let nanos = value.timestamp_nanos_opt().unwrap();
228        self.append_slice(&nanos.to_le_bytes());
229    }
230
231    fn append_timestamp_ntz_nanos(&mut self, value: chrono::NaiveDateTime) {
232        self.append_primitive_header(VariantPrimitiveType::TimestampNtzNanos);
233        let nanos = value.and_utc().timestamp_nanos_opt().unwrap();
234        self.append_slice(&nanos.to_le_bytes());
235    }
236
237    fn append_uuid(&mut self, value: Uuid) {
238        self.append_primitive_header(VariantPrimitiveType::Uuid);
239        self.append_slice(&value.into_bytes());
240    }
241
242    fn append_decimal4(&mut self, decimal4: VariantDecimal4) {
243        self.append_primitive_header(VariantPrimitiveType::Decimal4);
244        self.append_u8(decimal4.scale());
245        self.append_slice(&decimal4.integer().to_le_bytes());
246    }
247
248    fn append_decimal8(&mut self, decimal8: VariantDecimal8) {
249        self.append_primitive_header(VariantPrimitiveType::Decimal8);
250        self.append_u8(decimal8.scale());
251        self.append_slice(&decimal8.integer().to_le_bytes());
252    }
253
254    fn append_decimal16(&mut self, decimal16: VariantDecimal16) {
255        self.append_primitive_header(VariantPrimitiveType::Decimal16);
256        self.append_u8(decimal16.scale());
257        self.append_slice(&decimal16.integer().to_le_bytes());
258    }
259
260    fn append_binary(&mut self, value: &[u8]) {
261        self.append_primitive_header(VariantPrimitiveType::Binary);
262        self.append_slice(&(value.len() as u32).to_le_bytes());
263        self.append_slice(value);
264    }
265
266    fn append_short_string(&mut self, value: ShortString) {
267        let inner = value.0;
268        self.append_u8(short_string_header(inner.len()));
269        self.append_slice(inner.as_bytes());
270    }
271
272    fn append_string(&mut self, value: &str) {
273        self.append_primitive_header(VariantPrimitiveType::String);
274        self.append_slice(&(value.len() as u32).to_le_bytes());
275        self.append_slice(value.as_bytes());
276    }
277
278    fn append_object<S: BuilderSpecificState>(state: ParentState<'_, S>, obj: VariantObject) {
279        let mut object_builder = ObjectBuilder::new(state, false);
280
281        for (field_name, value) in obj.iter() {
282            object_builder.insert(field_name, value);
283        }
284
285        object_builder.finish();
286    }
287
288    fn try_append_object<S: BuilderSpecificState>(
289        state: ParentState<'_, S>,
290        obj: VariantObject,
291    ) -> Result<(), ArrowError> {
292        let mut object_builder = ObjectBuilder::new(state, false);
293
294        for res in obj.iter_try() {
295            let (field_name, value) = res?;
296            object_builder.try_insert(field_name, value)?;
297        }
298
299        object_builder.finish();
300        Ok(())
301    }
302
303    fn append_list<S: BuilderSpecificState>(state: ParentState<'_, S>, list: VariantList) {
304        let mut list_builder = ListBuilder::new(state, false);
305        for value in list.iter() {
306            list_builder.append_value(value);
307        }
308        list_builder.finish();
309    }
310
311    fn try_append_list<S: BuilderSpecificState>(
312        state: ParentState<'_, S>,
313        list: VariantList,
314    ) -> Result<(), ArrowError> {
315        let mut list_builder = ListBuilder::new(state, false);
316        for res in list.iter_try() {
317            let value = res?;
318            list_builder.try_append_value(value)?;
319        }
320
321        list_builder.finish();
322
323        Ok(())
324    }
325
326    /// Returns the current size of the underlying buffer
327    pub fn offset(&self) -> usize {
328        self.0.len()
329    }
330
331    /// Appends a variant to the builder.
332    ///
333    /// # Panics
334    ///
335    /// This method will panic if the variant contains duplicate field names in objects
336    /// when validation is enabled. For a fallible version, use [`ValueBuilder::try_append_variant`]
337    pub fn append_variant<S: BuilderSpecificState>(
338        mut state: ParentState<'_, S>,
339        variant: Variant<'_, '_>,
340    ) {
341        variant_append_value!(
342            state.value_builder(),
343            variant,
344            Variant::Object(obj) => return Self::append_object(state, obj),
345            Variant::List(list) => return Self::append_list(state, list)
346        );
347        state.finish();
348    }
349
350    /// Tries to append a variant to the provided [`ParentState`] instance.
351    ///
352    /// The attempt fails if the variant contains duplicate field names in objects when validation
353    /// is enabled.
354    pub fn try_append_variant<S: BuilderSpecificState>(
355        mut state: ParentState<'_, S>,
356        variant: Variant<'_, '_>,
357    ) -> Result<(), ArrowError> {
358        variant_append_value!(
359            state.value_builder(),
360            variant,
361            Variant::Object(obj) => return Self::try_append_object(state, obj),
362            Variant::List(list) => return Self::try_append_list(state, list)
363        );
364        state.finish();
365        Ok(())
366    }
367
368    /// Appends a variant to the buffer by copying raw bytes when possible.
369    ///
370    /// For objects and lists, this directly copies their underlying byte representation instead of
371    /// performing a logical copy and without touching the metadata builder. For other variant
372    /// types, this falls back to the standard append behavior.
373    ///
374    /// The caller must ensure that the metadata dictionary is already built and correct for
375    /// any objects or lists being appended.
376    pub fn append_variant_bytes<S: BuilderSpecificState>(
377        mut state: ParentState<'_, S>,
378        variant: Variant<'_, '_>,
379    ) {
380        let builder = state.value_builder();
381        variant_append_value!(
382            builder,
383            variant,
384            Variant::Object(obj) => builder.append_slice(obj.value),
385            Variant::List(list) => builder.append_slice(list.value)
386        );
387        state.finish();
388    }
389
390    /// Writes out the header byte for a variant object or list, from the starting position
391    /// of the builder, will return the position after this write
392    fn append_header_start_from_buf_pos(
393        &mut self,
394        start_pos: usize, // the start position where the header will be inserted
395        header_byte: u8,
396        is_large: bool,
397        num_fields: usize,
398    ) -> usize {
399        let buffer = self.inner_mut();
400
401        // Write header at the original start position
402        let mut header_pos = start_pos;
403
404        // Write header byte
405        buffer[header_pos] = header_byte;
406        header_pos += 1;
407
408        // Write number of fields
409        if is_large {
410            buffer[header_pos..header_pos + 4].copy_from_slice(&(num_fields as u32).to_le_bytes());
411            header_pos += 4;
412        } else {
413            buffer[header_pos] = num_fields as u8;
414            header_pos += 1;
415        }
416
417        header_pos
418    }
419
420    /// Writes out the offsets for an array of offsets, including the final offset (data size).
421    /// from the starting position of the buffer, will return the position after this write
422    fn append_offset_array_start_from_buf_pos(
423        &mut self,
424        start_pos: usize,
425        offsets: impl IntoIterator<Item = usize>,
426        data_size: Option<usize>,
427        nbytes: u8,
428    ) -> usize {
429        let buf = self.inner_mut();
430
431        let mut current_pos = start_pos;
432        for relative_offset in offsets {
433            write_offset_at_pos(buf, current_pos, relative_offset, nbytes);
434            current_pos += nbytes as usize;
435        }
436
437        // Write data_size
438        if let Some(data_size) = data_size {
439            // Write data_size at the end of the offsets
440            write_offset_at_pos(buf, current_pos, data_size, nbytes);
441            current_pos += nbytes as usize;
442        }
443
444        current_pos
445    }
446}
447
448/// A trait for building variant metadata dictionaries, to be used in conjunction with a
449/// [`ValueBuilder`]. The trait provides methods for managing field names and their IDs, as well as
450/// rolling back a failed builder operation that might have created new field ids.
451pub trait MetadataBuilder: std::fmt::Debug {
452    /// Attempts to register a field name, returning the corresponding (possibly newly-created)
453    /// field id on success. Attempting to register the same field name twice will _generally_
454    /// produce the same field id both times, but the variant spec does not actually require it.
455    fn try_upsert_field_name(&mut self, field_name: &str) -> Result<u32, ArrowError>;
456
457    /// Retrieves the field name for a given field id, which must be less than
458    /// [`Self::num_field_names`]. Panics if the field id is out of bounds.
459    fn field_name(&self, field_id: usize) -> &str;
460
461    /// Returns the number of field names stored in this metadata builder. Any number less than this
462    /// is a valid field id. The builder can be reverted back to this size later on (discarding any
463    /// newer/higher field ids) by calling [`Self::truncate_field_names`].
464    fn num_field_names(&self) -> usize;
465
466    /// Reverts the field names to a previous size, discarding any newly out of bounds field ids.
467    fn truncate_field_names(&mut self, new_size: usize);
468
469    /// Finishes the current metadata dictionary, returning the new size of the underlying buffer.
470    fn finish(&mut self) -> usize;
471}
472
473impl MetadataBuilder for WritableMetadataBuilder {
474    fn try_upsert_field_name(&mut self, field_name: &str) -> Result<u32, ArrowError> {
475        Ok(self.upsert_field_name(field_name))
476    }
477    fn field_name(&self, field_id: usize) -> &str {
478        self.field_name(field_id)
479    }
480    fn num_field_names(&self) -> usize {
481        self.num_field_names()
482    }
483    fn truncate_field_names(&mut self, new_size: usize) {
484        self.field_names.truncate(new_size)
485    }
486    fn finish(&mut self) -> usize {
487        self.finish()
488    }
489}
490
491/// A metadata builder that cannot register new field names, and merely returns the field id
492/// associated with a known field name. This is useful for variant unshredding operations, where the
493/// metadata column is fixed and -- per variant shredding spec -- already contains all field names
494/// from the typed_value column. It is also useful when projecting a subset of fields from a variant
495/// object value, since the bytes can be copied across directly without re-encoding their field ids.
496///
497/// NOTE: [`Self::finish`] is a no-op. If the intent is to make a copy of the underlying bytes each
498/// time `finish` is called, a different trait impl will be needed.
499#[derive(Debug)]
500pub struct ReadOnlyMetadataBuilder<'m> {
501    metadata: &'m VariantMetadata<'m>,
502    // A cache that tracks field names this builder has already seen, because finding the field id
503    // for a given field name is expensive -- O(n) for a large and unsorted metadata dictionary.
504    known_field_names: HashMap<&'m str, u32>,
505}
506
507impl<'m> ReadOnlyMetadataBuilder<'m> {
508    /// Creates a new read-only metadata builder from the given metadata dictionary.
509    pub fn new(metadata: &'m VariantMetadata<'m>) -> Self {
510        Self {
511            metadata,
512            known_field_names: HashMap::new(),
513        }
514    }
515}
516
517impl MetadataBuilder for ReadOnlyMetadataBuilder<'_> {
518    fn try_upsert_field_name(&mut self, field_name: &str) -> Result<u32, ArrowError> {
519        if let Some(field_id) = self.known_field_names.get(field_name) {
520            return Ok(*field_id);
521        }
522
523        let Some((field_id, field_name)) = self.metadata.get_entry(field_name) else {
524            return Err(ArrowError::InvalidArgumentError(format!(
525                "Field name '{field_name}' not found in metadata dictionary"
526            )));
527        };
528
529        self.known_field_names.insert(field_name, field_id);
530        Ok(field_id)
531    }
532    fn field_name(&self, field_id: usize) -> &str {
533        &self.metadata[field_id]
534    }
535    fn num_field_names(&self) -> usize {
536        self.metadata.len()
537    }
538    fn truncate_field_names(&mut self, new_size: usize) {
539        debug_assert_eq!(self.metadata.len(), new_size);
540    }
541    fn finish(&mut self) -> usize {
542        self.metadata.bytes.len()
543    }
544}
545
546/// Builder for constructing metadata for [`Variant`] values.
547///
548/// This is used internally by the [`VariantBuilder`] to construct the metadata
549///
550/// You can use an existing `Vec<u8>` as the metadata buffer by using the `from` impl.
551#[derive(Default, Debug)]
552pub struct WritableMetadataBuilder {
553    // Field names -- field_ids are assigned in insert order
554    field_names: IndexSet<String>,
555
556    // flag that checks if field names by insertion order are also lexicographically sorted
557    is_sorted: bool,
558
559    /// Output buffer. Metadata is written to the end of this buffer
560    metadata_buffer: Vec<u8>,
561}
562
563impl WritableMetadataBuilder {
564    /// Upsert field name to dictionary, return its ID
565    pub fn upsert_field_name(&mut self, field_name: &str) -> u32 {
566        let (id, new_entry) = self.field_names.insert_full(field_name.to_string());
567
568        if new_entry {
569            let n = self.num_field_names();
570
571            // Dictionary sort order tracking:
572            // - An empty dictionary is unsorted (ambiguous in spec but required by interop tests)
573            // - A single-entry dictionary is trivially sorted
574            // - Otherwise, an already-sorted dictionary becomes unsorted if the new entry breaks order
575            self.is_sorted =
576                n == 1 || self.is_sorted && (self.field_names[n - 2] < self.field_names[n - 1]);
577        }
578
579        id as u32
580    }
581
582    /// The current length of the underlying metadata buffer
583    pub fn offset(&self) -> usize {
584        self.metadata_buffer.len()
585    }
586
587    /// Returns the number of field names stored in the metadata builder.
588    /// Note: this method should be the only place to call `self.field_names.len()`
589    ///
590    /// # Panics
591    ///
592    /// If the number of field names exceeds the maximum allowed value for `u32`.
593    fn num_field_names(&self) -> usize {
594        let n = self.field_names.len();
595        assert!(n <= u32::MAX as usize);
596
597        n
598    }
599
600    fn field_name(&self, i: usize) -> &str {
601        &self.field_names[i]
602    }
603
604    fn metadata_size(&self) -> usize {
605        self.field_names.iter().map(|k| k.len()).sum()
606    }
607
608    /// Finalizes the metadata dictionary and appends its serialized bytes to the underlying buffer,
609    /// returning the resulting [`Self::offset`]. The builder state is reset and ready to start
610    /// building a new metadata dictionary.
611    pub fn finish(&mut self) -> usize {
612        let nkeys = self.num_field_names();
613
614        // Calculate metadata size
615        let total_dict_size: usize = self.metadata_size();
616
617        let metadata_buffer = &mut self.metadata_buffer;
618        let is_sorted = std::mem::take(&mut self.is_sorted);
619        let field_names = std::mem::take(&mut self.field_names);
620
621        // Determine appropriate offset size based on the larger of dict size or total string size
622        let max_offset = std::cmp::max(total_dict_size, nkeys);
623        let offset_size = int_size(max_offset);
624
625        let offset_start = 1 + offset_size as usize;
626        let string_start = offset_start + (nkeys + 1) * offset_size as usize;
627        let metadata_size = string_start + total_dict_size;
628
629        metadata_buffer.reserve(metadata_size);
630
631        // Write header: version=1, field names are sorted, with calculated offset_size
632        metadata_buffer.push(0x01 | (is_sorted as u8) << 4 | ((offset_size - 1) << 6));
633
634        // Write dictionary size
635        write_offset(metadata_buffer, nkeys, offset_size);
636
637        // Write offsets
638        let mut cur_offset = 0;
639        for key in field_names.iter() {
640            write_offset(metadata_buffer, cur_offset, offset_size);
641            cur_offset += key.len();
642        }
643        // Write final offset
644        write_offset(metadata_buffer, cur_offset, offset_size);
645
646        // Write string data
647        for key in field_names {
648            metadata_buffer.extend_from_slice(key.as_bytes());
649        }
650
651        metadata_buffer.len()
652    }
653
654    /// Returns the inner buffer, consuming self without finalizing any in progress metadata.
655    pub fn into_inner(self) -> Vec<u8> {
656        self.metadata_buffer
657    }
658}
659
660impl<S: AsRef<str>> FromIterator<S> for WritableMetadataBuilder {
661    fn from_iter<T: IntoIterator<Item = S>>(iter: T) -> Self {
662        let mut this = Self::default();
663        this.extend(iter);
664
665        this
666    }
667}
668
669impl<S: AsRef<str>> Extend<S> for WritableMetadataBuilder {
670    fn extend<T: IntoIterator<Item = S>>(&mut self, iter: T) {
671        let iter = iter.into_iter();
672        let (min, _) = iter.size_hint();
673
674        self.field_names.reserve(min);
675
676        for field_name in iter {
677            self.upsert_field_name(field_name.as_ref());
678        }
679    }
680}
681
682/// A trait for managing state specific to different builder types.
683pub trait BuilderSpecificState: std::fmt::Debug {
684    /// Called by [`ParentState::finish`] to apply any pending builder-specific changes.
685    ///
686    /// The provided implementation does nothing by default.
687    ///
688    /// Parameters:
689    /// - `metadata_builder`: The metadata builder that was used
690    /// - `value_builder`: The value builder that was used
691    fn finish(
692        &mut self,
693        _metadata_builder: &mut dyn MetadataBuilder,
694        _value_builder: &mut ValueBuilder,
695    ) {
696    }
697
698    /// Called by [`ParentState::drop`] to revert any changes that were eagerly applied, if
699    /// [`ParentState::finish`] was never invoked.
700    ///
701    /// The provided implementation does nothing by default.
702    ///
703    /// The base [`ParentState`] will handle rolling back the value and metadata builders,
704    /// but builder-specific state may need to revert its own changes.
705    fn rollback(&mut self) {}
706}
707
708/// Empty no-op implementation for top-level variant building
709impl BuilderSpecificState for () {}
710
711/// Internal state for list building
712#[derive(Debug)]
713pub struct ListState<'a> {
714    offsets: &'a mut Vec<usize>,
715    saved_offsets_size: usize,
716}
717
718// `ListBuilder::finish()` eagerly updates the list offsets, which we should rollback on failure.
719impl BuilderSpecificState for ListState<'_> {
720    fn rollback(&mut self) {
721        self.offsets.truncate(self.saved_offsets_size);
722    }
723}
724
725/// Internal state for object building
726#[derive(Debug)]
727pub struct ObjectState<'a> {
728    fields: &'a mut IndexMap<u32, usize>,
729    saved_fields_size: usize,
730}
731
732// `ObjectBuilder::finish()` eagerly updates the field offsets, which we should rollback on failure.
733impl BuilderSpecificState for ObjectState<'_> {
734    fn rollback(&mut self) {
735        self.fields.truncate(self.saved_fields_size);
736    }
737}
738
739/// Tracks information needed to correctly finalize a nested builder.
740///
741/// A child builder has no effect on its parent unless/until its `finalize` method is called, at
742/// which point the child appends the new value to the parent. As a (desirable) side effect,
743/// creating a parent state instance captures mutable references to a subset of the parent's fields,
744/// rendering the parent object completely unusable until the parent state goes out of scope. This
745/// ensures that at most one child builder can exist at a time.
746///
747/// The redundancy in `value_builder` and `metadata_builder` is because all the references come from
748/// the parent, and we cannot "split" a mutable reference across two objects (parent state and the
749/// child builder that uses it). So everything has to be here.
750#[derive(Debug)]
751pub struct ParentState<'a, S: BuilderSpecificState> {
752    value_builder: &'a mut ValueBuilder,
753    saved_value_builder_offset: usize,
754    metadata_builder: &'a mut dyn MetadataBuilder,
755    saved_metadata_builder_dict_size: usize,
756    builder_state: S,
757    finished: bool,
758}
759
760impl<'a, S: BuilderSpecificState> ParentState<'a, S> {
761    /// Creates a new ParentState instance. The value and metadata builder
762    /// state is checkpointed and will roll back on drop, unless [`Self::finish`] is called. The
763    /// builder-specific state is governed by its own `finish` and `rollback` calls.
764    pub fn new(
765        value_builder: &'a mut ValueBuilder,
766        metadata_builder: &'a mut dyn MetadataBuilder,
767        builder_state: S,
768    ) -> Self {
769        Self {
770            saved_value_builder_offset: value_builder.offset(),
771            value_builder,
772            saved_metadata_builder_dict_size: metadata_builder.num_field_names(),
773            metadata_builder,
774            builder_state,
775            finished: false,
776        }
777    }
778
779    /// Marks the insertion as having succeeded and invokes
780    /// [`BuilderSpecificState::finish`]. Internal state will no longer roll back on drop.
781    pub fn finish(&mut self) {
782        self.builder_state
783            .finish(self.metadata_builder, self.value_builder);
784        self.finished = true
785    }
786
787    // Rolls back value and metadata builder changes and invokes [`BuilderSpecificState::rollback`].
788    fn rollback(&mut self) {
789        if self.finished {
790            return;
791        }
792
793        self.value_builder
794            .inner_mut()
795            .truncate(self.saved_value_builder_offset);
796        self.metadata_builder
797            .truncate_field_names(self.saved_metadata_builder_dict_size);
798        self.builder_state.rollback();
799    }
800
801    // Useful because e.g. `let b = self.value_builder;` fails compilation.
802    fn value_builder(&mut self) -> &mut ValueBuilder {
803        self.value_builder
804    }
805
806    // Useful because e.g. `let b = self.metadata_builder;` fails compilation.
807    fn metadata_builder(&mut self) -> &mut dyn MetadataBuilder {
808        self.metadata_builder
809    }
810}
811
812impl<'a> ParentState<'a, ()> {
813    /// Creates a new instance suitable for a top-level variant builder
814    /// (e.g. [`VariantBuilder`]). The value and metadata builder state is checkpointed and will
815    /// roll back on drop, unless [`Self::finish`] is called.
816    pub fn variant(
817        value_builder: &'a mut ValueBuilder,
818        metadata_builder: &'a mut dyn MetadataBuilder,
819    ) -> Self {
820        Self::new(value_builder, metadata_builder, ())
821    }
822}
823
824impl<'a> ParentState<'a, ListState<'a>> {
825    /// Creates a new instance suitable for a [`ListBuilder`]. The value and metadata builder state
826    /// is checkpointed and will roll back on drop, unless [`Self::finish`] is called. The new
827    /// element's offset is also captured eagerly and will also roll back if not finished.
828    pub fn list(
829        value_builder: &'a mut ValueBuilder,
830        metadata_builder: &'a mut dyn MetadataBuilder,
831        offsets: &'a mut Vec<usize>,
832        saved_parent_value_builder_offset: usize,
833    ) -> Self {
834        // The saved_parent_buffer_offset is the buffer size as of when the parent builder was
835        // constructed. The saved_buffer_offset is the buffer size as of now (when a child builder
836        // is created). The variant field_offset entry for this list element is their difference.
837        let saved_value_builder_offset = value_builder.offset();
838        let saved_offsets_size = offsets.len();
839        offsets.push(saved_value_builder_offset - saved_parent_value_builder_offset);
840
841        let builder_state = ListState {
842            offsets,
843            saved_offsets_size,
844        };
845        Self {
846            saved_metadata_builder_dict_size: metadata_builder.num_field_names(),
847            saved_value_builder_offset,
848            metadata_builder,
849            value_builder,
850            builder_state,
851            finished: false,
852        }
853    }
854}
855
856impl<'a> ParentState<'a, ObjectState<'a>> {
857    /// Creates a new instance suitable for an [`ObjectBuilder`]. The value and metadata builder state
858    /// is checkpointed and will roll back on drop, unless [`Self::finish`] is called. The new
859    /// field's name and offset are also captured eagerly and will also roll back if not finished.
860    ///
861    /// The call fails if the field name is invalid (e.g. because it duplicates an existing field).
862    pub fn try_object(
863        value_builder: &'a mut ValueBuilder,
864        metadata_builder: &'a mut dyn MetadataBuilder,
865        fields: &'a mut IndexMap<u32, usize>,
866        saved_parent_value_builder_offset: usize,
867        field_name: &str,
868        validate_unique_fields: bool,
869    ) -> Result<Self, ArrowError> {
870        // The saved_parent_buffer_offset is the buffer size as of when the parent builder was
871        // constructed. The saved_buffer_offset is the buffer size as of now (when a child builder
872        // is created). The variant field_offset entry for this field is their difference.
873        let saved_value_builder_offset = value_builder.offset();
874        let saved_fields_size = fields.len();
875        let saved_metadata_builder_dict_size = metadata_builder.num_field_names();
876        let field_id = metadata_builder.try_upsert_field_name(field_name)?;
877        let field_start = saved_value_builder_offset - saved_parent_value_builder_offset;
878        if fields.insert(field_id, field_start).is_some() && validate_unique_fields {
879            return Err(ArrowError::InvalidArgumentError(format!(
880                "Duplicate field name: {field_name}"
881            )));
882        }
883
884        let builder_state = ObjectState {
885            fields,
886            saved_fields_size,
887        };
888        Ok(Self {
889            saved_metadata_builder_dict_size,
890            saved_value_builder_offset,
891            value_builder,
892            metadata_builder,
893            builder_state,
894            finished: false,
895        })
896    }
897}
898
899/// Automatically rolls back any unfinished `ParentState`.
900impl<S: BuilderSpecificState> Drop for ParentState<'_, S> {
901    fn drop(&mut self) {
902        self.rollback()
903    }
904}
905
906/// Top level builder for [`Variant`] values
907///
908/// # Example: create a Primitive Int8
909/// ```
910/// # use parquet_variant::{Variant, VariantBuilder};
911/// let mut builder = VariantBuilder::new();
912/// builder.append_value(Variant::Int8(42));
913/// // Finish the builder to get the metadata and value
914/// let (metadata, value) = builder.finish();
915/// // use the Variant API to verify the result
916/// let variant = Variant::try_new(&metadata, &value).unwrap();
917/// assert_eq!(variant, Variant::Int8(42));
918/// ```
919///
920/// # Example: Create a [`Variant::Object`]
921///
922/// This example shows how to create an object with two fields:
923/// ```json
924/// {
925///  "first_name": "Jiaying",
926///  "last_name": "Li"
927/// }
928/// ```
929///
930/// ```
931/// # use parquet_variant::{Variant, VariantBuilder};
932/// let mut builder = VariantBuilder::new();
933/// // Create an object builder that will write fields to the object
934/// let mut object_builder = builder.new_object();
935/// object_builder.insert("first_name", "Jiaying");
936/// object_builder.insert("last_name", "Li");
937/// object_builder.finish(); // call finish to finalize the object
938/// // Finish the builder to get the metadata and value
939/// let (metadata, value) = builder.finish();
940/// // use the Variant API to verify the result
941/// let variant = Variant::try_new(&metadata, &value).unwrap();
942/// let variant_object = variant.as_object().unwrap();
943/// assert_eq!(
944///   variant_object.get("first_name"),
945///   Some(Variant::from("Jiaying"))
946/// );
947/// assert_eq!(
948///   variant_object.get("last_name"),
949///   Some(Variant::from("Li"))
950/// );
951/// ```
952///
953///
954/// You can also use the [`ObjectBuilder::with_field`] to add fields to the
955/// object
956/// ```
957/// # use parquet_variant::{Variant, VariantBuilder};
958/// // build the same object as above
959/// let mut builder = VariantBuilder::new();
960/// builder.new_object()
961///   .with_field("first_name", "Jiaying")
962///   .with_field("last_name", "Li")
963///   .finish();
964/// let (metadata, value) = builder.finish();
965/// let variant = Variant::try_new(&metadata, &value).unwrap();
966/// let variant_object = variant.as_object().unwrap();
967/// assert_eq!(
968///   variant_object.get("first_name"),
969///   Some(Variant::from("Jiaying"))
970/// );
971/// assert_eq!(
972///   variant_object.get("last_name"),
973///   Some(Variant::from("Li"))
974/// );
975/// ```
976/// # Example: Create a [`Variant::List`] (an Array)
977///
978/// This example shows how to create an array of integers: `[1, 2, 3]`.
979/// ```
980///  # use parquet_variant::{Variant, VariantBuilder};
981///  let mut builder = VariantBuilder::new();
982///  // Create a builder that will write elements to the list
983///  let mut list_builder = builder.new_list();
984///  list_builder.append_value(1i8);
985///  list_builder.append_value(2i8);
986///  list_builder.append_value(3i8);
987/// // call finish to finalize the list
988///  list_builder.finish();
989/// // Finish the builder to get the metadata and value
990/// let (metadata, value) = builder.finish();
991/// // use the Variant API to verify the result
992/// let variant = Variant::try_new(&metadata, &value).unwrap();
993/// let variant_list = variant.as_list().unwrap();
994/// // Verify the list contents
995/// assert_eq!(variant_list.get(0).unwrap(), Variant::Int8(1));
996/// assert_eq!(variant_list.get(1).unwrap(), Variant::Int8(2));
997/// assert_eq!(variant_list.get(2).unwrap(), Variant::Int8(3));
998/// ```
999///
1000/// You can also use the [`ListBuilder::with_value`] to append values to the
1001/// list.
1002/// ```
1003///  # use parquet_variant::{Variant, VariantBuilder};
1004///  let mut builder = VariantBuilder::new();
1005///  builder.new_list()
1006///      .with_value(1i8)
1007///      .with_value(2i8)
1008///      .with_value(3i8)
1009///      .finish();
1010/// let (metadata, value) = builder.finish();
1011/// let variant = Variant::try_new(&metadata, &value).unwrap();
1012/// let variant_list = variant.as_list().unwrap();
1013/// assert_eq!(variant_list.get(0).unwrap(), Variant::Int8(1));
1014/// assert_eq!(variant_list.get(1).unwrap(), Variant::Int8(2));
1015/// assert_eq!(variant_list.get(2).unwrap(), Variant::Int8(3));
1016/// ```
1017///
1018/// # Example: [`Variant::List`] of  [`Variant::Object`]s
1019///
1020/// This example shows how to create an list of objects:
1021/// ```json
1022/// [
1023///   {
1024///      "id": 1,
1025///      "type": "Cauliflower"
1026///   },
1027///   {
1028///      "id": 2,
1029///      "type": "Beets"
1030///   }
1031/// ]
1032/// ```
1033/// ```
1034/// use parquet_variant::{Variant, VariantBuilder};
1035/// let mut builder = VariantBuilder::new();
1036///
1037/// // Create a builder that will write elements to the list
1038/// let mut list_builder = builder.new_list();
1039///
1040/// {
1041///     let mut object_builder = list_builder.new_object();
1042///     object_builder.insert("id", 1);
1043///     object_builder.insert("type", "Cauliflower");
1044///     object_builder.finish();
1045/// }
1046///
1047/// {
1048///     let mut object_builder = list_builder.new_object();
1049///     object_builder.insert("id", 2);
1050///     object_builder.insert("type", "Beets");
1051///     object_builder.finish();
1052/// }
1053///
1054/// list_builder.finish();
1055/// // Finish the builder to get the metadata and value
1056/// let (metadata, value) = builder.finish();
1057/// // use the Variant API to verify the result
1058/// let variant = Variant::try_new(&metadata, &value).unwrap();
1059/// let variant_list = variant.as_list().unwrap();
1060///
1061///
1062/// let obj1_variant = variant_list.get(0).unwrap();
1063/// let obj1 = obj1_variant.as_object().unwrap();
1064/// assert_eq!(
1065///     obj1.get("id"),
1066///     Some(Variant::from(1))
1067/// );
1068/// assert_eq!(
1069///     obj1.get("type"),
1070///     Some(Variant::from("Cauliflower"))
1071/// );
1072///
1073/// let obj2_variant = variant_list.get(1).unwrap();
1074/// let obj2 = obj2_variant.as_object().unwrap();
1075///
1076/// assert_eq!(
1077///     obj2.get("id"),
1078///     Some(Variant::from(2))
1079/// );
1080/// assert_eq!(
1081///     obj2.get("type"),
1082///     Some(Variant::from("Beets"))
1083/// );
1084///
1085/// ```
1086/// # Example: Unique Field Validation
1087///
1088/// This example shows how enabling unique field validation will cause an error
1089/// if the same field is inserted more than once.
1090/// ```
1091/// # use parquet_variant::VariantBuilder;
1092/// #
1093/// let mut builder = VariantBuilder::new().with_validate_unique_fields(true);
1094///
1095/// // When validation is enabled, try_with_field will return an error
1096/// let result = builder
1097///     .new_object()
1098///     .with_field("a", 1)
1099///     .try_with_field("a", 2);
1100/// assert!(result.is_err());
1101/// ```
1102///
1103/// # Example: Sorted dictionaries
1104///
1105/// This example shows how to create a [`VariantBuilder`] with a pre-sorted field dictionary
1106/// to improve field access performance when reading [`Variant`] objects.
1107///
1108/// You can use [`VariantBuilder::with_field_names`] to add multiple field names at once:
1109/// ```
1110/// use parquet_variant::{Variant, VariantBuilder};
1111/// let mut builder = VariantBuilder::new()
1112///     .with_field_names(["age", "name", "score"].into_iter());
1113///
1114/// let mut obj = builder.new_object();
1115/// obj.insert("name", "Alice");
1116/// obj.insert("age", 30);
1117/// obj.insert("score", 95.5);
1118/// obj.finish();
1119///
1120/// let (metadata, value) = builder.finish();
1121/// let variant = Variant::try_new(&metadata, &value).unwrap();
1122/// ```
1123///
1124/// Alternatively, you can use [`VariantBuilder::add_field_name`] to add field names one by one:
1125/// ```
1126/// use parquet_variant::{Variant, VariantBuilder};
1127/// let mut builder = VariantBuilder::new();
1128/// builder.add_field_name("age"); // field id = 0
1129/// builder.add_field_name("name"); // field id = 1
1130/// builder.add_field_name("score"); // field id = 2
1131///
1132/// let mut obj = builder.new_object();
1133/// obj.insert("name", "Bob"); // field id = 3
1134/// obj.insert("age", 25);
1135/// obj.insert("score", 88.0);
1136/// obj.finish();
1137///
1138/// let (metadata, value) = builder.finish();
1139/// let variant = Variant::try_new(&metadata, &value).unwrap();
1140/// ```
1141#[derive(Default, Debug)]
1142pub struct VariantBuilder {
1143    value_builder: ValueBuilder,
1144    metadata_builder: WritableMetadataBuilder,
1145    validate_unique_fields: bool,
1146}
1147
1148impl VariantBuilder {
1149    /// Create a new VariantBuilder with new underlying buffers
1150    pub fn new() -> Self {
1151        Self {
1152            value_builder: ValueBuilder::new(),
1153            metadata_builder: WritableMetadataBuilder::default(),
1154            validate_unique_fields: false,
1155        }
1156    }
1157
1158    /// Create a new VariantBuilder with pre-existing [`VariantMetadata`].
1159    pub fn with_metadata(mut self, metadata: VariantMetadata) -> Self {
1160        self.metadata_builder.extend(metadata.iter());
1161
1162        self
1163    }
1164
1165    /// Enables validation of unique field keys in nested objects.
1166    ///
1167    /// This setting is propagated to all [`ObjectBuilder`]s created through this [`VariantBuilder`]
1168    /// (including via any [`ListBuilder`]), and causes [`ObjectBuilder::finish()`] to return
1169    /// an error if duplicate keys were inserted.
1170    pub fn with_validate_unique_fields(mut self, validate_unique_fields: bool) -> Self {
1171        self.validate_unique_fields = validate_unique_fields;
1172        self
1173    }
1174
1175    /// This method pre-populates the field name directory in the Variant metadata with
1176    /// the specific field names, in order.
1177    ///
1178    /// You can use this to pre-populate a [`VariantBuilder`] with a sorted dictionary if you
1179    /// know the field names beforehand. Sorted dictionaries can accelerate field access when
1180    /// reading [`Variant`]s.
1181    pub fn with_field_names<'a>(mut self, field_names: impl IntoIterator<Item = &'a str>) -> Self {
1182        self.metadata_builder.extend(field_names);
1183
1184        self
1185    }
1186
1187    /// Builder-style API for appending a value to the list and returning self to enable method chaining.
1188    ///
1189    /// # Panics
1190    ///
1191    /// This method will panic if the variant contains duplicate field names in objects
1192    /// when validation is enabled. For a fallible version, use [`ListBuilder::try_with_value`].
1193    pub fn with_value<'m, 'd, T: Into<Variant<'m, 'd>>>(mut self, value: T) -> Self {
1194        self.append_value(value);
1195        self
1196    }
1197
1198    /// Builder-style API for appending a value to the list and returns self for method chaining.
1199    ///
1200    /// This is the fallible version of [`ListBuilder::with_value`].
1201    pub fn try_with_value<'m, 'd, T: Into<Variant<'m, 'd>>>(
1202        mut self,
1203        value: T,
1204    ) -> Result<Self, ArrowError> {
1205        self.try_append_value(value)?;
1206        Ok(self)
1207    }
1208
1209    /// This method reserves capacity for field names in the Variant metadata,
1210    /// which can improve performance when you know the approximate number of unique field
1211    /// names that will be used across all objects in the [`Variant`].
1212    pub fn reserve(&mut self, capacity: usize) {
1213        self.metadata_builder.field_names.reserve(capacity);
1214    }
1215
1216    /// Adds a single field name to the field name directory in the Variant metadata.
1217    ///
1218    /// This method does the same thing as [`VariantBuilder::with_field_names`] but adds one field name at a time.
1219    pub fn add_field_name(&mut self, field_name: &str) {
1220        self.metadata_builder.upsert_field_name(field_name);
1221    }
1222
1223    /// Create an [`ListBuilder`] for creating [`Variant::List`] values.
1224    ///
1225    /// See the examples on [`VariantBuilder`] for usage.
1226    pub fn new_list(&mut self) -> ListBuilder<'_, ()> {
1227        let parent_state =
1228            ParentState::variant(&mut self.value_builder, &mut self.metadata_builder);
1229        ListBuilder::new(parent_state, self.validate_unique_fields)
1230    }
1231
1232    /// Create an [`ObjectBuilder`] for creating [`Variant::Object`] values.
1233    ///
1234    /// See the examples on [`VariantBuilder`] for usage.
1235    pub fn new_object(&mut self) -> ObjectBuilder<'_, ()> {
1236        let parent_state =
1237            ParentState::variant(&mut self.value_builder, &mut self.metadata_builder);
1238        ObjectBuilder::new(parent_state, self.validate_unique_fields)
1239    }
1240
1241    /// Append a value to the builder.
1242    ///
1243    /// # Panics
1244    ///
1245    /// This method will panic if the variant contains duplicate field names in objects
1246    /// when validation is enabled. For a fallible version, use [`VariantBuilder::try_append_value`]
1247    ///
1248    /// # Example
1249    /// ```
1250    /// # use parquet_variant::{Variant, VariantBuilder};
1251    /// let mut builder = VariantBuilder::new();
1252    /// // most primitive types can be appended directly as they implement `Into<Variant>`
1253    /// builder.append_value(42i8);
1254    /// ```
1255    pub fn append_value<'m, 'd, T: Into<Variant<'m, 'd>>>(&mut self, value: T) {
1256        let state = ParentState::variant(&mut self.value_builder, &mut self.metadata_builder);
1257        ValueBuilder::append_variant(state, value.into())
1258    }
1259
1260    /// Append a value to the builder.
1261    pub fn try_append_value<'m, 'd, T: Into<Variant<'m, 'd>>>(
1262        &mut self,
1263        value: T,
1264    ) -> Result<(), ArrowError> {
1265        let state = ParentState::variant(&mut self.value_builder, &mut self.metadata_builder);
1266        ValueBuilder::try_append_variant(state, value.into())
1267    }
1268
1269    /// Appends a variant value to the builder by copying raw bytes when possible.
1270    ///
1271    /// For objects and lists, this directly copies their underlying byte representation instead of
1272    /// performing a logical copy and without touching the metadata builder. For other variant
1273    /// types, this falls back to the standard append behavior.
1274    ///
1275    /// The caller must ensure that the metadata dictionary entries are already built and correct for
1276    /// any objects or lists being appended.
1277    pub fn append_value_bytes<'m, 'd>(&mut self, value: impl Into<Variant<'m, 'd>>) {
1278        let state = ParentState::variant(&mut self.value_builder, &mut self.metadata_builder);
1279        ValueBuilder::append_variant_bytes(state, value.into());
1280    }
1281
1282    /// Finish the builder and return the metadata and value buffers.
1283    pub fn finish(mut self) -> (Vec<u8>, Vec<u8>) {
1284        self.metadata_builder.finish();
1285        (
1286            self.metadata_builder.into_inner(),
1287            self.value_builder.into_inner(),
1288        )
1289    }
1290}
1291
1292/// A builder for creating [`Variant::List`] values.
1293///
1294/// See the examples on [`VariantBuilder`] for usage.
1295#[derive(Debug)]
1296pub struct ListBuilder<'a, S: BuilderSpecificState> {
1297    parent_state: ParentState<'a, S>,
1298    offsets: Vec<usize>,
1299    validate_unique_fields: bool,
1300}
1301
1302impl<'a, S: BuilderSpecificState> ListBuilder<'a, S> {
1303    /// Creates a new list builder, nested on top of the given parent state.
1304    pub fn new(parent_state: ParentState<'a, S>, validate_unique_fields: bool) -> Self {
1305        Self {
1306            parent_state,
1307            offsets: vec![],
1308            validate_unique_fields,
1309        }
1310    }
1311
1312    /// Enables unique field key validation for objects created within this list.
1313    ///
1314    /// Propagates the validation flag to any [`ObjectBuilder`]s created using
1315    /// [`ListBuilder::new_object`].
1316    pub fn with_validate_unique_fields(mut self, validate_unique_fields: bool) -> Self {
1317        self.validate_unique_fields = validate_unique_fields;
1318        self
1319    }
1320
1321    // Returns validate_unique_fields because we can no longer reference self once this method returns.
1322    fn parent_state(&mut self) -> (ParentState<'_, ListState<'_>>, bool) {
1323        let state = ParentState::list(
1324            self.parent_state.value_builder,
1325            self.parent_state.metadata_builder,
1326            &mut self.offsets,
1327            self.parent_state.saved_value_builder_offset,
1328        );
1329        (state, self.validate_unique_fields)
1330    }
1331
1332    /// Returns an object builder that can be used to append a new (nested) object to this list.
1333    ///
1334    /// WARNING: The builder will have no effect unless/until [`ObjectBuilder::finish`] is called.
1335    pub fn new_object(&mut self) -> ObjectBuilder<'_, ListState<'_>> {
1336        let (parent_state, validate_unique_fields) = self.parent_state();
1337        ObjectBuilder::new(parent_state, validate_unique_fields)
1338    }
1339
1340    /// Returns a list builder that can be used to append a new (nested) list to this list.
1341    ///
1342    /// WARNING: The builder will have no effect unless/until [`ListBuilder::finish`] is called.
1343    pub fn new_list(&mut self) -> ListBuilder<'_, ListState<'_>> {
1344        let (parent_state, validate_unique_fields) = self.parent_state();
1345        ListBuilder::new(parent_state, validate_unique_fields)
1346    }
1347
1348    /// Appends a variant to the list.
1349    ///
1350    /// # Panics
1351    ///
1352    /// This method will panic if the variant contains duplicate field names in objects
1353    /// when validation is enabled. For a fallible version, use [`ListBuilder::try_append_value`].
1354    pub fn append_value<'m, 'd, T: Into<Variant<'m, 'd>>>(&mut self, value: T) {
1355        let (state, _) = self.parent_state();
1356        ValueBuilder::append_variant(state, value.into())
1357    }
1358
1359    /// Appends a new primitive value to this list
1360    pub fn try_append_value<'m, 'd, T: Into<Variant<'m, 'd>>>(
1361        &mut self,
1362        value: T,
1363    ) -> Result<(), ArrowError> {
1364        let (state, _) = self.parent_state();
1365        ValueBuilder::try_append_variant(state, value.into())
1366    }
1367
1368    /// Appends a variant value to this list by copying raw bytes when possible.
1369    ///
1370    /// For objects and lists, this directly copies their underlying byte representation instead of
1371    /// performing a logical copy. For other variant types, this falls back to the standard append
1372    /// behavior.
1373    ///
1374    /// The caller must ensure that the metadata dictionary is already built and correct for
1375    /// any objects or lists being appended.
1376    pub fn append_value_bytes<'m, 'd>(&mut self, value: impl Into<Variant<'m, 'd>>) {
1377        let (state, _) = self.parent_state();
1378        ValueBuilder::append_variant_bytes(state, value.into())
1379    }
1380
1381    /// Builder-style API for appending a value to the list and returning self to enable method chaining.
1382    ///
1383    /// # Panics
1384    ///
1385    /// This method will panic if the variant contains duplicate field names in objects
1386    /// when validation is enabled. For a fallible version, use [`ListBuilder::try_with_value`].
1387    pub fn with_value<'m, 'd, T: Into<Variant<'m, 'd>>>(mut self, value: T) -> Self {
1388        self.append_value(value);
1389        self
1390    }
1391
1392    /// Builder-style API for appending a value to the list and returns self for method chaining.
1393    ///
1394    /// This is the fallible version of [`ListBuilder::with_value`].
1395    pub fn try_with_value<'m, 'd, T: Into<Variant<'m, 'd>>>(
1396        mut self,
1397        value: T,
1398    ) -> Result<Self, ArrowError> {
1399        self.try_append_value(value)?;
1400        Ok(self)
1401    }
1402
1403    /// Finalizes this list and appends it to its parent, which otherwise remains unmodified.
1404    pub fn finish(mut self) {
1405        let starting_offset = self.parent_state.saved_value_builder_offset;
1406        let value_builder = self.parent_state.value_builder();
1407
1408        let data_size = value_builder
1409            .offset()
1410            .checked_sub(starting_offset)
1411            .expect("Data size overflowed usize");
1412
1413        let num_elements = self.offsets.len();
1414        let is_large = num_elements > u8::MAX as usize;
1415        let offset_size = int_size(data_size);
1416
1417        let num_elements_size = if is_large { 4 } else { 1 }; // is_large: 4 bytes, else 1 byte.
1418        let num_elements = self.offsets.len();
1419        let header_size = 1 +      // header (i.e., `array_header`)
1420            num_elements_size +  // num_element_size
1421            (num_elements + 1) * offset_size as usize; // offsets and data size
1422
1423        // Calculated header size becomes a hint; being wrong only risks extra allocations.
1424        // Make sure to reserve enough capacity to handle the extra bytes we'll truncate.
1425        let mut bytes_to_splice = Vec::with_capacity(header_size + 3);
1426        // Write header
1427        let header = array_header(is_large, offset_size);
1428        bytes_to_splice.push(header);
1429
1430        append_packed_u32(&mut bytes_to_splice, num_elements as u32, num_elements_size);
1431
1432        for offset in &self.offsets {
1433            append_packed_u32(&mut bytes_to_splice, *offset as u32, offset_size as usize);
1434        }
1435
1436        append_packed_u32(&mut bytes_to_splice, data_size as u32, offset_size as usize);
1437
1438        value_builder
1439            .inner_mut()
1440            .splice(starting_offset..starting_offset, bytes_to_splice);
1441
1442        self.parent_state.finish();
1443    }
1444}
1445
1446/// A builder for creating [`Variant::Object`] values.
1447///
1448/// See the examples on [`VariantBuilder`] for usage.
1449#[derive(Debug)]
1450pub struct ObjectBuilder<'a, S: BuilderSpecificState> {
1451    parent_state: ParentState<'a, S>,
1452    fields: IndexMap<u32, usize>, // (field_id, offset)
1453    validate_unique_fields: bool,
1454}
1455
1456impl<'a, S: BuilderSpecificState> ObjectBuilder<'a, S> {
1457    /// Creates a new object builder, nested on top of the given parent state.
1458    pub fn new(parent_state: ParentState<'a, S>, validate_unique_fields: bool) -> Self {
1459        Self {
1460            parent_state,
1461            fields: IndexMap::new(),
1462            validate_unique_fields,
1463        }
1464    }
1465
1466    /// Add a field with key and value to the object
1467    ///
1468    /// # See Also
1469    /// - [`ObjectBuilder::try_insert`] for a fallible version.
1470    /// - [`ObjectBuilder::with_field`] for a builder-style API.
1471    ///
1472    /// # Panics
1473    ///
1474    /// This method will panic if the variant contains duplicate field names in objects
1475    /// when validation is enabled. For a fallible version, use [`ObjectBuilder::try_insert`]
1476    pub fn insert<'m, 'd, T: Into<Variant<'m, 'd>>>(&mut self, key: &str, value: T) {
1477        let (state, _) = self.parent_state(key).unwrap();
1478        ValueBuilder::append_variant(state, value.into())
1479    }
1480
1481    /// Add a field with key and value to the object
1482    ///
1483    /// # See Also
1484    /// - [`ObjectBuilder::insert`] for an infallible version that panics
1485    /// - [`ObjectBuilder::try_with_field`] for a builder-style API.
1486    ///
1487    /// # Note
1488    /// Attempting to insert a duplicate field name produces an error if unique field
1489    /// validation is enabled. Otherwise, the new value overwrites the previous field mapping
1490    /// without erasing the old value, resulting in a larger variant
1491    pub fn try_insert<'m, 'd, T: Into<Variant<'m, 'd>>>(
1492        &mut self,
1493        key: &str,
1494        value: T,
1495    ) -> Result<(), ArrowError> {
1496        let (state, _) = self.parent_state(key)?;
1497        ValueBuilder::try_append_variant(state, value.into())
1498    }
1499
1500    /// Add a field with key and value to the object by copying raw bytes when possible.
1501    ///
1502    /// For objects and lists, this directly copies their underlying byte representation instead of
1503    /// performing a logical copy, and without touching the metadata builder. For other variant
1504    /// types, this falls back to the standard append behavior.
1505    ///
1506    /// The caller must ensure that the metadata dictionary is already built and correct for
1507    /// any objects or lists being appended, but the value's new field name is handled normally.
1508    ///
1509    /// # Panics
1510    ///
1511    /// This method will panic if the variant contains duplicate field names in objects
1512    /// when validation is enabled. For a fallible version, use [`ObjectBuilder::try_insert_bytes`]
1513    pub fn insert_bytes<'m, 'd>(&mut self, key: &str, value: impl Into<Variant<'m, 'd>>) {
1514        self.try_insert_bytes(key, value).unwrap()
1515    }
1516
1517    /// Add a field with key and value to the object by copying raw bytes when possible.
1518    ///
1519    /// For objects and lists, this directly copies their underlying byte representation instead of
1520    /// performing a logical copy, and without touching the metadata builder. For other variant
1521    /// types, this falls back to the standard append behavior.
1522    ///
1523    /// The caller must ensure that the metadata dictionary is already built and correct for
1524    /// any objects or lists being appended, but the value's new field name is handled normally.
1525    ///
1526    /// # Note
1527    /// When inserting duplicate keys, the new value overwrites the previous mapping,
1528    /// but the old value remains in the buffer, resulting in a larger variant
1529    pub fn try_insert_bytes<'m, 'd>(
1530        &mut self,
1531        key: &str,
1532        value: impl Into<Variant<'m, 'd>>,
1533    ) -> Result<(), ArrowError> {
1534        let (state, _) = self.parent_state(key)?;
1535        ValueBuilder::append_variant_bytes(state, value.into());
1536        Ok(())
1537    }
1538
1539    /// Builder style API for adding a field with key and value to the object
1540    ///
1541    /// Same as [`ObjectBuilder::insert`], but returns `self` for chaining.
1542    pub fn with_field<'m, 'd, T: Into<Variant<'m, 'd>>>(mut self, key: &str, value: T) -> Self {
1543        self.insert(key, value);
1544        self
1545    }
1546
1547    /// Builder style API for adding a field with key and value to the object
1548    ///
1549    /// Same as [`ObjectBuilder::try_insert`], but returns `self` for chaining.
1550    pub fn try_with_field<'m, 'd, T: Into<Variant<'m, 'd>>>(
1551        mut self,
1552        key: &str,
1553        value: T,
1554    ) -> Result<Self, ArrowError> {
1555        self.try_insert(key, value)?;
1556        Ok(self)
1557    }
1558
1559    /// Enables validation for unique field keys when inserting into this object.
1560    ///
1561    /// When this is enabled, calling [`ObjectBuilder::finish`] will return an error
1562    /// if any duplicate field keys were added using [`ObjectBuilder::insert`].
1563    pub fn with_validate_unique_fields(mut self, validate_unique_fields: bool) -> Self {
1564        self.validate_unique_fields = validate_unique_fields;
1565        self
1566    }
1567
1568    // Returns validate_unique_fields because we can no longer reference self once this method returns.
1569    fn parent_state<'b>(
1570        &'b mut self,
1571        field_name: &str,
1572    ) -> Result<(ParentState<'b, ObjectState<'b>>, bool), ArrowError> {
1573        let validate_unique_fields = self.validate_unique_fields;
1574        let state = ParentState::try_object(
1575            self.parent_state.value_builder,
1576            self.parent_state.metadata_builder,
1577            &mut self.fields,
1578            self.parent_state.saved_value_builder_offset,
1579            field_name,
1580            validate_unique_fields,
1581        )?;
1582        Ok((state, validate_unique_fields))
1583    }
1584
1585    /// Returns an object builder that can be used to append a new (nested) object to this object.
1586    ///
1587    /// Panics if the proposed key was a duplicate
1588    ///
1589    /// WARNING: The builder will have no effect unless/until [`ObjectBuilder::finish`] is called.
1590    pub fn new_object<'b>(&'b mut self, key: &'b str) -> ObjectBuilder<'b, ObjectState<'b>> {
1591        self.try_new_object(key).unwrap()
1592    }
1593
1594    /// Returns an object builder that can be used to append a new (nested) object to this object.
1595    ///
1596    /// Fails if the proposed key was a duplicate
1597    ///
1598    /// WARNING: The builder will have no effect unless/until [`ObjectBuilder::finish`] is called.
1599    pub fn try_new_object<'b>(
1600        &'b mut self,
1601        key: &str,
1602    ) -> Result<ObjectBuilder<'b, ObjectState<'b>>, ArrowError> {
1603        let (parent_state, validate_unique_fields) = self.parent_state(key)?;
1604        Ok(ObjectBuilder::new(parent_state, validate_unique_fields))
1605    }
1606
1607    /// Returns a list builder that can be used to append a new (nested) list to this object.
1608    ///
1609    /// Panics if the proposed key was a duplicate
1610    ///
1611    /// WARNING: The builder will have no effect unless/until [`ListBuilder::finish`] is called.
1612    pub fn new_list<'b>(&'b mut self, key: &str) -> ListBuilder<'b, ObjectState<'b>> {
1613        self.try_new_list(key).unwrap()
1614    }
1615
1616    /// Returns a list builder that can be used to append a new (nested) list to this object.
1617    ///
1618    /// Fails if the proposed key was a duplicate
1619    ///
1620    /// WARNING: The builder will have no effect unless/until [`ListBuilder::finish`] is called.
1621    pub fn try_new_list<'b>(
1622        &'b mut self,
1623        key: &str,
1624    ) -> Result<ListBuilder<'b, ObjectState<'b>>, ArrowError> {
1625        let (parent_state, validate_unique_fields) = self.parent_state(key)?;
1626        Ok(ListBuilder::new(parent_state, validate_unique_fields))
1627    }
1628
1629    /// Finalizes this object and appends it to its parent, which otherwise remains unmodified.
1630    pub fn finish(mut self) {
1631        let metadata_builder = self.parent_state.metadata_builder();
1632
1633        self.fields.sort_by(|&field_a_id, _, &field_b_id, _| {
1634            let field_a_name = metadata_builder.field_name(field_a_id as usize);
1635            let field_b_name = metadata_builder.field_name(field_b_id as usize);
1636            field_a_name.cmp(field_b_name)
1637        });
1638
1639        let max_id = self.fields.iter().map(|(i, _)| *i).max().unwrap_or(0);
1640        let id_size = int_size(max_id as usize);
1641
1642        let starting_offset = self.parent_state.saved_value_builder_offset;
1643        let value_builder = self.parent_state.value_builder();
1644        let current_offset = value_builder.offset();
1645        // Current object starts from `object_start_offset`
1646        let data_size = current_offset - starting_offset;
1647        let offset_size = int_size(data_size);
1648
1649        let num_fields = self.fields.len();
1650        let is_large = num_fields > u8::MAX as usize;
1651
1652        let header_size = 1 + // header byte
1653            (if is_large { 4 } else { 1 }) + // num_fields
1654            (num_fields * id_size as usize) + // field IDs
1655            ((num_fields + 1) * offset_size as usize); // field offsets + data_size
1656
1657        // Shift existing data to make room for the header
1658        value_builder.inner_mut().splice(
1659            starting_offset..starting_offset,
1660            std::iter::repeat_n(0u8, header_size),
1661        );
1662
1663        // Write header at the original start position
1664        let mut header_pos = starting_offset;
1665
1666        // Write header byte
1667        let header = object_header(is_large, id_size, offset_size);
1668
1669        header_pos = self
1670            .parent_state
1671            .value_builder()
1672            .append_header_start_from_buf_pos(header_pos, header, is_large, num_fields);
1673
1674        header_pos = self
1675            .parent_state
1676            .value_builder()
1677            .append_offset_array_start_from_buf_pos(
1678                header_pos,
1679                self.fields.keys().copied().map(|id| id as usize),
1680                None,
1681                id_size,
1682            );
1683
1684        self.parent_state
1685            .value_builder()
1686            .append_offset_array_start_from_buf_pos(
1687                header_pos,
1688                self.fields.values().copied(),
1689                Some(data_size),
1690                offset_size,
1691            );
1692        self.parent_state.finish();
1693    }
1694}
1695
1696/// Extends [`VariantBuilder`] to help building nested [`Variant`]s
1697///
1698/// Allows users to append values to a [`VariantBuilder`], [`ListBuilder`] or
1699/// [`ObjectBuilder`]. using the same interface.
1700pub trait VariantBuilderExt {
1701    /// The builder specific state used by nested builders
1702    type State<'a>: BuilderSpecificState + 'a
1703    where
1704        Self: 'a;
1705
1706    /// Appends a NULL value to this builder. The semantics depend on the implementation, but will
1707    /// often translate to appending a [`Variant::Null`] value.
1708    fn append_null(&mut self);
1709
1710    /// Appends a new variant value to this builder. See e.g. [`VariantBuilder::append_value`].
1711    fn append_value<'m, 'v>(&mut self, value: impl Into<Variant<'m, 'v>>);
1712
1713    /// Creates a nested list builder. See e.g. [`VariantBuilder::new_list`]. Panics if the nested
1714    /// builder cannot be created, see e.g. [`ObjectBuilder::new_list`].
1715    fn new_list(&mut self) -> ListBuilder<'_, Self::State<'_>> {
1716        self.try_new_list().unwrap()
1717    }
1718
1719    /// Creates a nested object builder. See e.g. [`VariantBuilder::new_object`]. Panics if the
1720    /// nested builder cannot be created, see e.g. [`ObjectBuilder::new_object`].
1721    fn new_object(&mut self) -> ObjectBuilder<'_, Self::State<'_>> {
1722        self.try_new_object().unwrap()
1723    }
1724
1725    /// Creates a nested list builder. See e.g. [`VariantBuilder::new_list`]. Returns an error if
1726    /// the nested builder cannot be created, see e.g. [`ObjectBuilder::try_new_list`].
1727    fn try_new_list(&mut self) -> Result<ListBuilder<'_, Self::State<'_>>, ArrowError>;
1728
1729    /// Creates a nested object builder. See e.g. [`VariantBuilder::new_object`]. Returns an error
1730    /// if the nested builder cannot be created, see e.g. [`ObjectBuilder::try_new_object`].
1731    fn try_new_object(&mut self) -> Result<ObjectBuilder<'_, Self::State<'_>>, ArrowError>;
1732}
1733
1734impl<'a, S: BuilderSpecificState> VariantBuilderExt for ListBuilder<'a, S> {
1735    type State<'s>
1736        = ListState<'s>
1737    where
1738        Self: 's;
1739
1740    /// Variant arrays cannot encode NULL values, only `Variant::Null`.
1741    fn append_null(&mut self) {
1742        self.append_value(Variant::Null);
1743    }
1744    fn append_value<'m, 'v>(&mut self, value: impl Into<Variant<'m, 'v>>) {
1745        self.append_value(value);
1746    }
1747
1748    fn try_new_list(&mut self) -> Result<ListBuilder<'_, Self::State<'_>>, ArrowError> {
1749        Ok(self.new_list())
1750    }
1751
1752    fn try_new_object(&mut self) -> Result<ObjectBuilder<'_, Self::State<'_>>, ArrowError> {
1753        Ok(self.new_object())
1754    }
1755}
1756
1757impl VariantBuilderExt for VariantBuilder {
1758    type State<'a>
1759        = ()
1760    where
1761        Self: 'a;
1762
1763    /// Variant values cannot encode NULL, only [`Variant::Null`]. This is different from the column
1764    /// that holds variant values being NULL at some positions.
1765    fn append_null(&mut self) {
1766        self.append_value(Variant::Null);
1767    }
1768    fn append_value<'m, 'v>(&mut self, value: impl Into<Variant<'m, 'v>>) {
1769        self.append_value(value);
1770    }
1771
1772    fn try_new_list(&mut self) -> Result<ListBuilder<'_, Self::State<'_>>, ArrowError> {
1773        Ok(self.new_list())
1774    }
1775
1776    fn try_new_object(&mut self) -> Result<ObjectBuilder<'_, Self::State<'_>>, ArrowError> {
1777        Ok(self.new_object())
1778    }
1779}
1780
1781/// A [`VariantBuilderExt`] that inserts a new field into a variant object.
1782pub struct ObjectFieldBuilder<'o, 'v, 's, S: BuilderSpecificState> {
1783    key: &'s str,
1784    builder: &'o mut ObjectBuilder<'v, S>,
1785}
1786
1787impl<'o, 'v, 's, S: BuilderSpecificState> ObjectFieldBuilder<'o, 'v, 's, S> {
1788    pub fn new(key: &'s str, builder: &'o mut ObjectBuilder<'v, S>) -> Self {
1789        Self { key, builder }
1790    }
1791}
1792
1793impl<S: BuilderSpecificState> VariantBuilderExt for ObjectFieldBuilder<'_, '_, '_, S> {
1794    type State<'a>
1795        = ObjectState<'a>
1796    where
1797        Self: 'a;
1798
1799    /// A NULL object field is interpreted as missing, so nothing gets inserted at all.
1800    fn append_null(&mut self) {}
1801    fn append_value<'m, 'v>(&mut self, value: impl Into<Variant<'m, 'v>>) {
1802        self.builder.insert(self.key, value);
1803    }
1804
1805    fn try_new_list(&mut self) -> Result<ListBuilder<'_, Self::State<'_>>, ArrowError> {
1806        self.builder.try_new_list(self.key)
1807    }
1808
1809    fn try_new_object(&mut self) -> Result<ObjectBuilder<'_, Self::State<'_>>, ArrowError> {
1810        self.builder.try_new_object(self.key)
1811    }
1812}
1813
1814#[cfg(test)]
1815mod tests {
1816    use crate::VariantMetadata;
1817
1818    use super::*;
1819
1820    #[test]
1821    fn test_simple_usage() {
1822        {
1823            let mut builder = VariantBuilder::new();
1824            builder.append_value(());
1825            let (metadata, value) = builder.finish();
1826            let variant = Variant::try_new(&metadata, &value).unwrap();
1827            assert_eq!(variant, Variant::Null);
1828        }
1829
1830        {
1831            let mut builder = VariantBuilder::new();
1832            builder.append_value(true);
1833            let (metadata, value) = builder.finish();
1834            let variant = Variant::try_new(&metadata, &value).unwrap();
1835            assert_eq!(variant, Variant::BooleanTrue);
1836        }
1837
1838        {
1839            let mut builder = VariantBuilder::new();
1840            builder.append_value(false);
1841            let (metadata, value) = builder.finish();
1842            let variant = Variant::try_new(&metadata, &value).unwrap();
1843            assert_eq!(variant, Variant::BooleanFalse);
1844        }
1845
1846        {
1847            let mut builder = VariantBuilder::new();
1848            builder.append_value(42i8);
1849            let (metadata, value) = builder.finish();
1850            let variant = Variant::try_new(&metadata, &value).unwrap();
1851            assert_eq!(variant, Variant::Int8(42));
1852        }
1853
1854        {
1855            let mut builder = VariantBuilder::new();
1856            builder.append_value(1234i16);
1857            let (metadata, value) = builder.finish();
1858            let variant = Variant::try_new(&metadata, &value).unwrap();
1859            assert_eq!(variant, Variant::Int16(1234));
1860        }
1861
1862        {
1863            let mut builder = VariantBuilder::new();
1864            builder.append_value(123456i32);
1865            let (metadata, value) = builder.finish();
1866            let variant = Variant::try_new(&metadata, &value).unwrap();
1867            assert_eq!(variant, Variant::Int32(123456));
1868        }
1869
1870        {
1871            let mut builder = VariantBuilder::new();
1872            builder.append_value(123456789i64);
1873            let (metadata, value) = builder.finish();
1874            let variant = Variant::try_new(&metadata, &value).unwrap();
1875            assert_eq!(variant, Variant::Int64(123456789));
1876        }
1877
1878        {
1879            let mut builder = VariantBuilder::new();
1880            builder.append_value(1.5f32);
1881            let (metadata, value) = builder.finish();
1882            let variant = Variant::try_new(&metadata, &value).unwrap();
1883            assert_eq!(variant, Variant::Float(1.5));
1884        }
1885
1886        {
1887            let mut builder = VariantBuilder::new();
1888            builder.append_value(2.5f64);
1889            let (metadata, value) = builder.finish();
1890            let variant = Variant::try_new(&metadata, &value).unwrap();
1891            assert_eq!(variant, Variant::Double(2.5));
1892        }
1893
1894        {
1895            let mut builder = VariantBuilder::new();
1896            builder.append_value("hello");
1897            let (metadata, value) = builder.finish();
1898            let variant = Variant::try_new(&metadata, &value).unwrap();
1899            assert_eq!(variant, Variant::ShortString(ShortString("hello")));
1900        }
1901
1902        {
1903            let mut builder = VariantBuilder::new();
1904            let long_string = "This is a very long string that exceeds the short string limit of 63 bytes and should be encoded as a regular string type instead of a short string";
1905            builder.append_value(long_string);
1906            let (metadata, value) = builder.finish();
1907            let variant = Variant::try_new(&metadata, &value).unwrap();
1908            assert_eq!(variant, Variant::String(long_string));
1909        }
1910
1911        {
1912            let mut builder = VariantBuilder::new();
1913            let binary_data = b"binary data";
1914            builder.append_value(binary_data.as_slice());
1915            let (metadata, value) = builder.finish();
1916            let variant = Variant::try_new(&metadata, &value).unwrap();
1917            assert_eq!(variant, Variant::Binary(binary_data.as_slice()));
1918        }
1919    }
1920
1921    #[test]
1922    fn test_list() {
1923        let mut builder = VariantBuilder::new();
1924
1925        builder
1926            .new_list()
1927            .with_value(1i8)
1928            .with_value(2i8)
1929            .with_value("test")
1930            .finish();
1931
1932        let (metadata, value) = builder.finish();
1933        assert!(!metadata.is_empty());
1934        assert!(!value.is_empty());
1935
1936        let variant = Variant::try_new(&metadata, &value).unwrap();
1937
1938        match variant {
1939            Variant::List(list) => {
1940                let val0 = list.get(0).unwrap();
1941                assert_eq!(val0, Variant::Int8(1));
1942
1943                let val1 = list.get(1).unwrap();
1944                assert_eq!(val1, Variant::Int8(2));
1945
1946                let val2 = list.get(2).unwrap();
1947                assert_eq!(val2, Variant::ShortString(ShortString("test")));
1948            }
1949            _ => panic!("Expected an array variant, got: {variant:?}"),
1950        }
1951    }
1952
1953    #[test]
1954    fn test_object() {
1955        let mut builder = VariantBuilder::new();
1956
1957        builder
1958            .new_object()
1959            .with_field("name", "John")
1960            .with_field("age", 42i8)
1961            .finish();
1962
1963        let (metadata, value) = builder.finish();
1964        assert!(!metadata.is_empty());
1965        assert!(!value.is_empty());
1966    }
1967
1968    #[test]
1969    fn test_object_field_ordering() {
1970        let mut builder = VariantBuilder::new();
1971
1972        builder
1973            .new_object()
1974            .with_field("zebra", "stripes")
1975            .with_field("apple", "red")
1976            .with_field("banana", "yellow")
1977            .finish();
1978
1979        let (_, value) = builder.finish();
1980
1981        let header = value[0];
1982        assert_eq!(header & 0x03, VariantBasicType::Object as u8);
1983
1984        let field_count = value[1] as usize;
1985        assert_eq!(field_count, 3);
1986
1987        // Get field IDs from the object header
1988        let field_ids: Vec<u8> = value[2..5].to_vec();
1989
1990        // apple(1), banana(2), zebra(0)
1991        assert_eq!(field_ids, vec![1, 2, 0]);
1992    }
1993
1994    #[test]
1995    fn test_duplicate_fields_in_object() {
1996        let mut builder = VariantBuilder::new();
1997        builder
1998            .new_object()
1999            .with_field("name", "Ron Artest")
2000            .with_field("name", "Metta World Peace") // Duplicate field
2001            .finish();
2002
2003        let (metadata, value) = builder.finish();
2004        let variant = Variant::try_new(&metadata, &value).unwrap();
2005
2006        let obj = variant.as_object().unwrap();
2007        assert_eq!(obj.len(), 1);
2008        assert_eq!(obj.field(0).unwrap(), Variant::from("Metta World Peace"));
2009
2010        assert_eq!(
2011            vec![("name", Variant::from("Metta World Peace"))],
2012            obj.iter().collect::<Vec<_>>()
2013        );
2014    }
2015
2016    #[test]
2017    fn test_nested_list() {
2018        let mut builder = VariantBuilder::new();
2019
2020        let mut outer_list_builder = builder.new_list();
2021
2022        // create inner list
2023        outer_list_builder
2024            .new_list()
2025            .with_value("a")
2026            .with_value("b")
2027            .with_value("c")
2028            .with_value("d")
2029            .finish();
2030
2031        outer_list_builder.finish();
2032
2033        let (metadata, value) = builder.finish();
2034
2035        let variant = Variant::try_new(&metadata, &value).unwrap();
2036        let outer_list = variant.as_list().unwrap();
2037
2038        assert_eq!(outer_list.len(), 1);
2039
2040        let inner_variant = outer_list.get(0).unwrap();
2041        let inner_list = inner_variant.as_list().unwrap();
2042
2043        assert_eq!(
2044            vec![
2045                Variant::from("a"),
2046                Variant::from("b"),
2047                Variant::from("c"),
2048                Variant::from("d"),
2049            ],
2050            inner_list.iter().collect::<Vec<_>>()
2051        );
2052    }
2053
2054    #[test]
2055    fn test_super_nested_list() {
2056        /*
2057        [[[[[1]]]]]
2058        */
2059
2060        let mut builder = VariantBuilder::new();
2061        {
2062            let mut list_builder1 = builder.new_list();
2063            {
2064                let mut list_builder2 = list_builder1.new_list();
2065                {
2066                    let mut list_builder3 = list_builder2.new_list();
2067                    {
2068                        let mut list_builder4 = list_builder3.new_list();
2069                        {
2070                            let mut list_builder5 = list_builder4.new_list();
2071                            list_builder5.append_value(1);
2072                            list_builder5.finish();
2073                        }
2074                        list_builder4.finish();
2075                    }
2076                    list_builder3.finish();
2077                }
2078                list_builder2.finish();
2079            }
2080            list_builder1.finish();
2081        }
2082
2083        let (metadata, value) = builder.finish();
2084
2085        let variant = Variant::try_new(&metadata, &value).unwrap();
2086        let list1 = variant.as_list().unwrap();
2087        assert_eq!(list1.len(), 1);
2088
2089        let list2_variant = list1.get(0).unwrap();
2090        let list2 = list2_variant.as_list().unwrap();
2091        assert_eq!(list2.len(), 1);
2092
2093        let list3_variant = list2.get(0).unwrap();
2094        let list3 = list3_variant.as_list().unwrap();
2095        assert_eq!(list3.len(), 1);
2096
2097        let list4_variant = list3.get(0).unwrap();
2098        let list4 = list4_variant.as_list().unwrap();
2099        assert_eq!(list4.len(), 1);
2100
2101        let list5_variant = list4.get(0).unwrap();
2102        let list5 = list5_variant.as_list().unwrap();
2103        assert_eq!(list5.len(), 1);
2104
2105        assert_eq!(list5.len(), 1);
2106
2107        assert_eq!(list5.get(0).unwrap(), Variant::from(1));
2108    }
2109
2110    #[test]
2111    fn test_object_list() {
2112        let mut builder = VariantBuilder::new();
2113
2114        let mut list_builder = builder.new_list();
2115
2116        list_builder
2117            .new_object()
2118            .with_field("id", 1)
2119            .with_field("type", "Cauliflower")
2120            .finish();
2121
2122        list_builder
2123            .new_object()
2124            .with_field("id", 2)
2125            .with_field("type", "Beets")
2126            .finish();
2127
2128        list_builder.finish();
2129
2130        let (metadata, value) = builder.finish();
2131
2132        let variant = Variant::try_new(&metadata, &value).unwrap();
2133        let list = variant.as_list().unwrap();
2134
2135        assert_eq!(list.len(), 2);
2136
2137        let obj1_variant = list.get(0).unwrap();
2138        let obj1 = obj1_variant.as_object().unwrap();
2139
2140        assert_eq!(
2141            vec![
2142                ("id", Variant::from(1)),
2143                ("type", Variant::from("Cauliflower")),
2144            ],
2145            obj1.iter().collect::<Vec<_>>()
2146        );
2147
2148        let obj2_variant = list.get(1).unwrap();
2149        let obj2 = obj2_variant.as_object().unwrap();
2150
2151        assert_eq!(
2152            vec![("id", Variant::from(2)), ("type", Variant::from("Beets")),],
2153            obj2.iter().collect::<Vec<_>>()
2154        );
2155    }
2156
2157    #[test]
2158    fn test_object_list2() {
2159        let mut builder = VariantBuilder::new();
2160
2161        let mut list_builder = builder.new_list();
2162
2163        list_builder.new_object().with_field("a", 1).finish();
2164
2165        list_builder.new_object().with_field("b", 2).finish();
2166
2167        list_builder.finish();
2168
2169        let (metadata, value) = builder.finish();
2170
2171        let variant = Variant::try_new(&metadata, &value).unwrap();
2172        let list = variant.as_list().unwrap();
2173        assert_eq!(list.len(), 2);
2174
2175        let obj1_variant = list.get(0).unwrap();
2176        let obj1 = obj1_variant.as_object().unwrap();
2177        assert_eq!(
2178            vec![("a", Variant::from(1)),],
2179            obj1.iter().collect::<Vec<_>>()
2180        );
2181
2182        let obj2_variant = list.get(1).unwrap();
2183        let obj2 = obj2_variant.as_object().unwrap();
2184        assert_eq!(
2185            vec![("b", Variant::from(2)),],
2186            obj2.iter().collect::<Vec<_>>()
2187        );
2188    }
2189
2190    #[test]
2191    fn test_hetergenous_list() {
2192        /*
2193        [
2194            1,
2195            { "a": 1 },
2196            2,
2197            { "b": 2},
2198            3
2199        ]
2200        */
2201
2202        let mut builder = VariantBuilder::new();
2203
2204        let mut list_builder = builder.new_list();
2205
2206        list_builder.append_value(1);
2207
2208        {
2209            let mut object_builder = list_builder.new_object();
2210            object_builder.insert("a", 1);
2211            object_builder.finish();
2212        }
2213
2214        list_builder.append_value(2);
2215
2216        {
2217            let mut object_builder = list_builder.new_object();
2218            object_builder.insert("b", 2);
2219            object_builder.finish();
2220        }
2221
2222        list_builder.append_value(3);
2223
2224        list_builder.finish();
2225
2226        let (metadata, value) = builder.finish();
2227
2228        let variant = Variant::try_new(&metadata, &value).unwrap();
2229        let list = variant.as_list().unwrap();
2230        assert_eq!(list.len(), 5);
2231        assert_eq!(list.get(0).unwrap(), Variant::from(1));
2232
2233        let obj1_variant = list.get(1).unwrap();
2234        let obj1 = obj1_variant.as_object().unwrap();
2235        assert_eq!(
2236            vec![("a", Variant::from(1)),],
2237            obj1.iter().collect::<Vec<_>>()
2238        );
2239
2240        assert_eq!(list.get(2).unwrap(), Variant::from(2));
2241
2242        let obj2_variant = list.get(3).unwrap();
2243        let obj2 = obj2_variant.as_object().unwrap();
2244        assert_eq!(
2245            vec![("b", Variant::from(2)),],
2246            obj2.iter().collect::<Vec<_>>()
2247        );
2248
2249        assert_eq!(list.get(4).unwrap(), Variant::from(3));
2250    }
2251
2252    #[test]
2253    fn test_nested_object() {
2254        /*
2255        {
2256            "c": {
2257                "b": "a"
2258            }
2259        }
2260
2261        */
2262
2263        let mut builder = VariantBuilder::new();
2264        {
2265            let mut outer_object_builder = builder.new_object();
2266            {
2267                let mut inner_object_builder = outer_object_builder.new_object("c");
2268                inner_object_builder.insert("b", "a");
2269                inner_object_builder.finish();
2270            }
2271
2272            outer_object_builder.finish();
2273        }
2274
2275        let (metadata, value) = builder.finish();
2276        let variant = Variant::try_new(&metadata, &value).unwrap();
2277        let outer_object = variant.as_object().unwrap();
2278
2279        assert_eq!(outer_object.len(), 1);
2280        assert_eq!(outer_object.field_name(0).unwrap(), "c");
2281
2282        let inner_object_variant = outer_object.field(0).unwrap();
2283        let inner_object = inner_object_variant.as_object().unwrap();
2284
2285        assert_eq!(inner_object.len(), 1);
2286        assert_eq!(inner_object.field_name(0).unwrap(), "b");
2287        assert_eq!(inner_object.field(0).unwrap(), Variant::from("a"));
2288    }
2289
2290    #[test]
2291    fn test_nested_object_with_duplicate_field_names_per_object() {
2292        /*
2293        {
2294            "c": {
2295                "b": false,
2296                "c": "a"
2297            },
2298            "b": false,
2299        }
2300
2301        */
2302
2303        let mut builder = VariantBuilder::new();
2304        {
2305            let mut outer_object_builder = builder.new_object();
2306            {
2307                let mut inner_object_builder = outer_object_builder.new_object("c");
2308                inner_object_builder.insert("b", false);
2309                inner_object_builder.insert("c", "a");
2310
2311                inner_object_builder.finish();
2312            }
2313
2314            outer_object_builder.insert("b", false);
2315            outer_object_builder.finish();
2316        }
2317
2318        let (metadata, value) = builder.finish();
2319        let variant = Variant::try_new(&metadata, &value).unwrap();
2320        let outer_object = variant.as_object().unwrap();
2321
2322        assert_eq!(outer_object.len(), 2);
2323        assert_eq!(outer_object.field_name(0).unwrap(), "b");
2324
2325        let inner_object_variant = outer_object.field(1).unwrap();
2326        let inner_object = inner_object_variant.as_object().unwrap();
2327
2328        assert_eq!(inner_object.len(), 2);
2329        assert_eq!(inner_object.field_name(0).unwrap(), "b");
2330        assert_eq!(inner_object.field(0).unwrap(), Variant::from(false));
2331        assert_eq!(inner_object.field_name(1).unwrap(), "c");
2332        assert_eq!(inner_object.field(1).unwrap(), Variant::from("a"));
2333    }
2334
2335    #[test]
2336    fn test_nested_object_with_lists() {
2337        /*
2338        {
2339            "door 1": {
2340                "items": ["apple", false ]
2341            }
2342        }
2343
2344        */
2345
2346        let mut builder = VariantBuilder::new();
2347        {
2348            let mut outer_object_builder = builder.new_object();
2349            {
2350                let mut inner_object_builder = outer_object_builder.new_object("door 1");
2351
2352                // create inner_object_list
2353                inner_object_builder
2354                    .new_list("items")
2355                    .with_value("apple")
2356                    .with_value(false)
2357                    .finish();
2358
2359                inner_object_builder.finish();
2360            }
2361
2362            outer_object_builder.finish();
2363        }
2364
2365        let (metadata, value) = builder.finish();
2366        let variant = Variant::try_new(&metadata, &value).unwrap();
2367        let outer_object = variant.as_object().unwrap();
2368
2369        assert_eq!(outer_object.len(), 1);
2370        assert_eq!(outer_object.field_name(0).unwrap(), "door 1");
2371
2372        let inner_object_variant = outer_object.field(0).unwrap();
2373        let inner_object = inner_object_variant.as_object().unwrap();
2374
2375        assert_eq!(inner_object.len(), 1);
2376        assert_eq!(inner_object.field_name(0).unwrap(), "items");
2377
2378        let items_variant = inner_object.field(0).unwrap();
2379        let items_list = items_variant.as_list().unwrap();
2380
2381        assert_eq!(items_list.len(), 2);
2382        assert_eq!(items_list.get(0).unwrap(), Variant::from("apple"));
2383        assert_eq!(items_list.get(1).unwrap(), Variant::from(false));
2384    }
2385
2386    #[test]
2387    fn test_nested_object_with_heterogeneous_fields() {
2388        /*
2389        {
2390            "a": false,
2391            "c": {
2392                "b": "a",
2393                "c": {
2394                   "aa": "bb",
2395                },
2396                "d": {
2397                    "cc": "dd"
2398                }
2399            },
2400            "b": true,
2401            "d": {
2402               "e": 1,
2403               "f": [1, true],
2404               "g": ["tree", false],
2405            }
2406        }
2407        */
2408
2409        let mut builder = VariantBuilder::new();
2410        {
2411            let mut outer_object_builder = builder.new_object();
2412
2413            outer_object_builder.insert("a", false);
2414
2415            {
2416                let mut inner_object_builder = outer_object_builder.new_object("c");
2417                inner_object_builder.insert("b", "a");
2418
2419                {
2420                    let mut inner_inner_object_builder = inner_object_builder.new_object("c");
2421                    inner_inner_object_builder.insert("aa", "bb");
2422                    inner_inner_object_builder.finish();
2423                }
2424
2425                {
2426                    let mut inner_inner_object_builder = inner_object_builder.new_object("d");
2427                    inner_inner_object_builder.insert("cc", "dd");
2428                    inner_inner_object_builder.finish();
2429                }
2430                inner_object_builder.finish();
2431            }
2432
2433            outer_object_builder.insert("b", true);
2434
2435            {
2436                let mut inner_object_builder = outer_object_builder.new_object("d");
2437                inner_object_builder.insert("e", 1);
2438                {
2439                    let mut inner_list_builder = inner_object_builder.new_list("f");
2440                    inner_list_builder.append_value(1);
2441                    inner_list_builder.append_value(true);
2442
2443                    inner_list_builder.finish();
2444                }
2445
2446                {
2447                    let mut inner_list_builder = inner_object_builder.new_list("g");
2448                    inner_list_builder.append_value("tree");
2449                    inner_list_builder.append_value(false);
2450
2451                    inner_list_builder.finish();
2452                }
2453
2454                inner_object_builder.finish();
2455            }
2456
2457            outer_object_builder.finish();
2458        }
2459
2460        let (metadata, value) = builder.finish();
2461
2462        // note, object fields are now sorted lexigraphically by field name
2463        /*
2464         {
2465            "a": false,
2466            "b": true,
2467            "c": {
2468                "b": "a",
2469                "c": {
2470                   "aa": "bb",
2471                },
2472                "d": {
2473                    "cc": "dd"
2474                }
2475            },
2476            "d": {
2477               "e": 1,
2478               "f": [1, true],
2479               "g": ["tree", false],
2480            }
2481        }
2482        */
2483
2484        let variant = Variant::try_new(&metadata, &value).unwrap();
2485        let outer_object = variant.as_object().unwrap();
2486
2487        assert_eq!(outer_object.len(), 4);
2488
2489        assert_eq!(outer_object.field_name(0).unwrap(), "a");
2490        assert_eq!(outer_object.field(0).unwrap(), Variant::from(false));
2491
2492        assert_eq!(outer_object.field_name(2).unwrap(), "c");
2493
2494        let inner_object_variant = outer_object.field(2).unwrap();
2495        let inner_object = inner_object_variant.as_object().unwrap();
2496
2497        assert_eq!(inner_object.len(), 3);
2498        assert_eq!(inner_object.field_name(0).unwrap(), "b");
2499        assert_eq!(inner_object.field(0).unwrap(), Variant::from("a"));
2500
2501        let inner_iner_object_variant_c = inner_object.field(1).unwrap();
2502        let inner_inner_object_c = inner_iner_object_variant_c.as_object().unwrap();
2503        assert_eq!(inner_inner_object_c.len(), 1);
2504        assert_eq!(inner_inner_object_c.field_name(0).unwrap(), "aa");
2505        assert_eq!(inner_inner_object_c.field(0).unwrap(), Variant::from("bb"));
2506
2507        let inner_iner_object_variant_d = inner_object.field(2).unwrap();
2508        let inner_inner_object_d = inner_iner_object_variant_d.as_object().unwrap();
2509        assert_eq!(inner_inner_object_d.len(), 1);
2510        assert_eq!(inner_inner_object_d.field_name(0).unwrap(), "cc");
2511        assert_eq!(inner_inner_object_d.field(0).unwrap(), Variant::from("dd"));
2512
2513        assert_eq!(outer_object.field_name(1).unwrap(), "b");
2514        assert_eq!(outer_object.field(1).unwrap(), Variant::from(true));
2515
2516        let out_object_variant_d = outer_object.field(3).unwrap();
2517        let out_object_d = out_object_variant_d.as_object().unwrap();
2518        assert_eq!(out_object_d.len(), 3);
2519        assert_eq!("e", out_object_d.field_name(0).unwrap());
2520        assert_eq!(Variant::from(1), out_object_d.field(0).unwrap());
2521        assert_eq!("f", out_object_d.field_name(1).unwrap());
2522
2523        let first_inner_list_variant_f = out_object_d.field(1).unwrap();
2524        let first_inner_list_f = first_inner_list_variant_f.as_list().unwrap();
2525        assert_eq!(2, first_inner_list_f.len());
2526        assert_eq!(Variant::from(1), first_inner_list_f.get(0).unwrap());
2527        assert_eq!(Variant::from(true), first_inner_list_f.get(1).unwrap());
2528
2529        let second_inner_list_variant_g = out_object_d.field(2).unwrap();
2530        let second_inner_list_g = second_inner_list_variant_g.as_list().unwrap();
2531        assert_eq!(2, second_inner_list_g.len());
2532        assert_eq!(Variant::from("tree"), second_inner_list_g.get(0).unwrap());
2533        assert_eq!(Variant::from(false), second_inner_list_g.get(1).unwrap());
2534    }
2535
2536    // This test wants to cover the logic for reuse parent buffer for list builder
2537    // the builder looks like
2538    // [ "apple", "false", [{"a": "b", "b": "c"}, {"c":"d", "d":"e"}], [[1, true], ["tree", false]], 1]
2539    #[test]
2540    fn test_nested_list_with_heterogeneous_fields_for_buffer_reuse() {
2541        let mut builder = VariantBuilder::new();
2542
2543        {
2544            let mut outer_list_builder = builder.new_list();
2545
2546            outer_list_builder.append_value("apple");
2547            outer_list_builder.append_value(false);
2548
2549            {
2550                // the list here wants to cover the logic object builder inside list builder
2551                let mut inner_list_builder = outer_list_builder.new_list();
2552
2553                {
2554                    let mut inner_object_builder = inner_list_builder.new_object();
2555                    inner_object_builder.insert("a", "b");
2556                    inner_object_builder.insert("b", "c");
2557                    inner_object_builder.finish();
2558                }
2559
2560                {
2561                    // the seconde object builder here wants to cover the logic for
2562                    // list builder resue the parent buffer.
2563                    let mut inner_object_builder = inner_list_builder.new_object();
2564                    inner_object_builder.insert("c", "d");
2565                    inner_object_builder.insert("d", "e");
2566                    inner_object_builder.finish();
2567                }
2568
2569                inner_list_builder.finish();
2570            }
2571
2572            {
2573                // the list here wants to cover the logic list builder inside list builder
2574                let mut inner_list_builder = outer_list_builder.new_list();
2575
2576                {
2577                    let mut double_inner_list_builder = inner_list_builder.new_list();
2578                    double_inner_list_builder.append_value(1);
2579                    double_inner_list_builder.append_value(true);
2580
2581                    double_inner_list_builder.finish();
2582                }
2583
2584                {
2585                    let mut double_inner_list_builder = inner_list_builder.new_list();
2586                    double_inner_list_builder.append_value("tree");
2587                    double_inner_list_builder.append_value(false);
2588
2589                    double_inner_list_builder.finish();
2590                }
2591                inner_list_builder.finish();
2592            }
2593
2594            outer_list_builder.append_value(1);
2595
2596            outer_list_builder.finish();
2597        }
2598
2599        let (metadata, value) = builder.finish();
2600
2601        let variant = Variant::try_new(&metadata, &value).unwrap();
2602        let outer_list = variant.as_list().unwrap();
2603
2604        assert_eq!(5, outer_list.len());
2605
2606        // Primitive value
2607        assert_eq!(Variant::from("apple"), outer_list.get(0).unwrap());
2608        assert_eq!(Variant::from(false), outer_list.get(1).unwrap());
2609        assert_eq!(Variant::from(1), outer_list.get(4).unwrap());
2610
2611        // The first inner list [{"a": "b", "b": "c"}, {"c":"d", "d":"e"}]
2612        let list1_variant = outer_list.get(2).unwrap();
2613        let list1 = list1_variant.as_list().unwrap();
2614        assert_eq!(2, list1.len());
2615
2616        let list1_obj1_variant = list1.get(0).unwrap();
2617        let list1_obj1 = list1_obj1_variant.as_object().unwrap();
2618        assert_eq!("a", list1_obj1.field_name(0).unwrap());
2619        assert_eq!(Variant::from("b"), list1_obj1.field(0).unwrap());
2620
2621        assert_eq!("b", list1_obj1.field_name(1).unwrap());
2622        assert_eq!(Variant::from("c"), list1_obj1.field(1).unwrap());
2623
2624        // The second inner list [[1, true], ["tree", false]]
2625        let list2_variant = outer_list.get(3).unwrap();
2626        let list2 = list2_variant.as_list().unwrap();
2627        assert_eq!(2, list2.len());
2628
2629        // The list [1, true]
2630        let list2_list1_variant = list2.get(0).unwrap();
2631        let list2_list1 = list2_list1_variant.as_list().unwrap();
2632        assert_eq!(2, list2_list1.len());
2633        assert_eq!(Variant::from(1), list2_list1.get(0).unwrap());
2634        assert_eq!(Variant::from(true), list2_list1.get(1).unwrap());
2635
2636        // The list ["true", false]
2637        let list2_list2_variant = list2.get(1).unwrap();
2638        let list2_list2 = list2_list2_variant.as_list().unwrap();
2639        assert_eq!(2, list2_list2.len());
2640        assert_eq!(Variant::from("tree"), list2_list2.get(0).unwrap());
2641        assert_eq!(Variant::from(false), list2_list2.get(1).unwrap());
2642    }
2643
2644    #[test]
2645    fn test_object_without_unique_field_validation() {
2646        let mut builder = VariantBuilder::new();
2647
2648        // Root object with duplicates
2649        let mut obj = builder.new_object();
2650        obj.insert("a", 1);
2651        obj.insert("a", 2);
2652        obj.finish();
2653
2654        // Deeply nested list structure with duplicates
2655        let mut builder = VariantBuilder::new();
2656        let mut outer_list = builder.new_list();
2657        let mut inner_list = outer_list.new_list();
2658        let mut nested_obj = inner_list.new_object();
2659        nested_obj.insert("x", 1);
2660        nested_obj.insert("x", 2);
2661        nested_obj.new_list("x").with_value(3).finish();
2662        nested_obj.new_object("x").with_field("y", 4).finish();
2663        nested_obj.finish();
2664        inner_list.finish();
2665        outer_list.finish();
2666
2667        // Verify the nested object is built correctly -- the nested object "x" should have "won"
2668        let (metadata, value) = builder.finish();
2669        let variant = Variant::try_new(&metadata, &value).unwrap();
2670        let outer_element = variant.get_list_element(0).unwrap();
2671        let inner_element = outer_element.get_list_element(0).unwrap();
2672        let outer_field = inner_element.get_object_field("x").unwrap();
2673        let inner_field = outer_field.get_object_field("y").unwrap();
2674        assert_eq!(inner_field, Variant::from(4));
2675    }
2676
2677    #[test]
2678    fn test_object_with_unique_field_validation() {
2679        let mut builder = VariantBuilder::new().with_validate_unique_fields(true);
2680
2681        // Root-level object with duplicates
2682        let result = builder
2683            .new_object()
2684            .with_field("a", 1)
2685            .with_field("b", 2)
2686            .try_with_field("a", 3);
2687        assert_eq!(
2688            result.unwrap_err().to_string(),
2689            "Invalid argument error: Duplicate field name: a"
2690        );
2691
2692        // Deeply nested list -> list -> object with duplicate
2693        let mut outer_list = builder.new_list();
2694        let mut inner_list = outer_list.new_list();
2695        let mut object = inner_list.new_object().with_field("x", 1);
2696        let nested_result = object.try_insert("x", 2);
2697        assert_eq!(
2698            nested_result.unwrap_err().to_string(),
2699            "Invalid argument error: Duplicate field name: x"
2700        );
2701        let nested_result = object.try_new_list("x");
2702        assert_eq!(
2703            nested_result.unwrap_err().to_string(),
2704            "Invalid argument error: Duplicate field name: x"
2705        );
2706
2707        let nested_result = object.try_new_object("x");
2708        assert_eq!(
2709            nested_result.unwrap_err().to_string(),
2710            "Invalid argument error: Duplicate field name: x"
2711        );
2712
2713        drop(object);
2714        inner_list.finish();
2715        outer_list.finish();
2716
2717        // Valid object should succeed
2718        let mut list = builder.new_list();
2719        let mut valid_obj = list.new_object();
2720        valid_obj.insert("m", 1);
2721        valid_obj.insert("n", 2);
2722
2723        valid_obj.finish();
2724        list.finish();
2725    }
2726
2727    #[test]
2728    fn test_sorted_dictionary() {
2729        // check if variant metadatabuilders are equivalent from different ways of constructing them
2730        let mut variant1 = VariantBuilder::new().with_field_names(["b", "c", "d"]);
2731
2732        let mut variant2 = {
2733            let mut builder = VariantBuilder::new();
2734
2735            builder.add_field_name("b");
2736            builder.add_field_name("c");
2737            builder.add_field_name("d");
2738
2739            builder
2740        };
2741
2742        assert_eq!(
2743            variant1.metadata_builder.field_names,
2744            variant2.metadata_builder.field_names
2745        );
2746
2747        // check metadata builders say it's sorted
2748        assert!(variant1.metadata_builder.is_sorted);
2749        assert!(variant2.metadata_builder.is_sorted);
2750
2751        {
2752            // test the bad case and break the sort order
2753            variant2.add_field_name("a");
2754            assert!(!variant2.metadata_builder.is_sorted);
2755
2756            // per the spec, make sure the variant will fail to build if only metadata is provided
2757            let (m, v) = variant2.finish();
2758            let res = Variant::try_new(&m, &v);
2759            assert!(res.is_err());
2760
2761            // since it is not sorted, make sure the metadata says so
2762            let header = VariantMetadata::try_new(&m).unwrap();
2763            assert!(!header.is_sorted());
2764        }
2765
2766        // write out variant1 and make sure the sorted flag is properly encoded
2767        variant1.append_value(false);
2768
2769        let (m, v) = variant1.finish();
2770        let res = Variant::try_new(&m, &v);
2771        assert!(res.is_ok());
2772
2773        let header = VariantMetadata::try_new(&m).unwrap();
2774        assert!(header.is_sorted());
2775    }
2776
2777    #[test]
2778    fn test_object_sorted_dictionary() {
2779        // predefine the list of field names
2780        let mut variant1 = VariantBuilder::new().with_field_names(["a", "b", "c"]);
2781        let mut obj = variant1.new_object();
2782
2783        obj.insert("c", true);
2784        obj.insert("a", false);
2785        obj.insert("b", ());
2786
2787        // verify the field ids are correctly
2788        let field_ids_by_insert_order = obj.fields.iter().map(|(&id, _)| id).collect::<Vec<_>>();
2789        assert_eq!(field_ids_by_insert_order, vec![2, 0, 1]);
2790
2791        // add a field name that wasn't pre-defined but doesn't break the sort order
2792        obj.insert("d", 2);
2793        obj.finish();
2794
2795        let (metadata, value) = variant1.finish();
2796        let variant = Variant::try_new(&metadata, &value).unwrap();
2797
2798        let metadata = VariantMetadata::try_new(&metadata).unwrap();
2799        assert!(metadata.is_sorted());
2800
2801        // verify object is sorted by field name order
2802        let object = variant.as_object().unwrap();
2803        let field_names = object
2804            .iter()
2805            .map(|(field_name, _)| field_name)
2806            .collect::<Vec<_>>();
2807
2808        assert_eq!(field_names, vec!["a", "b", "c", "d"]);
2809    }
2810
2811    #[test]
2812    fn test_object_not_sorted_dictionary() {
2813        // predefine the list of field names
2814        let mut variant1 = VariantBuilder::new().with_field_names(["b", "c", "d"]);
2815        let mut obj = variant1.new_object();
2816
2817        obj.insert("c", true);
2818        obj.insert("d", false);
2819        obj.insert("b", ());
2820
2821        // verify the field ids are correctly
2822        let field_ids_by_insert_order = obj.fields.iter().map(|(&id, _)| id).collect::<Vec<_>>();
2823        assert_eq!(field_ids_by_insert_order, vec![1, 2, 0]);
2824
2825        // add a field name that wasn't pre-defined but breaks the sort order
2826        obj.insert("a", 2);
2827        obj.finish();
2828
2829        let (metadata, value) = variant1.finish();
2830        let variant = Variant::try_new(&metadata, &value).unwrap();
2831
2832        let metadata = VariantMetadata::try_new(&metadata).unwrap();
2833        assert!(!metadata.is_sorted());
2834
2835        // verify object field names are sorted by field name order
2836        let object = variant.as_object().unwrap();
2837        let field_names = object
2838            .iter()
2839            .map(|(field_name, _)| field_name)
2840            .collect::<Vec<_>>();
2841
2842        assert_eq!(field_names, vec!["a", "b", "c", "d"]);
2843    }
2844
2845    #[test]
2846    fn test_building_sorted_dictionary() {
2847        let mut builder = VariantBuilder::new();
2848        assert!(!builder.metadata_builder.is_sorted);
2849        assert_eq!(builder.metadata_builder.num_field_names(), 0);
2850
2851        builder.add_field_name("a");
2852
2853        assert!(builder.metadata_builder.is_sorted);
2854        assert_eq!(builder.metadata_builder.num_field_names(), 1);
2855
2856        let builder = builder.with_field_names(["b", "c", "d"]);
2857
2858        assert!(builder.metadata_builder.is_sorted);
2859        assert_eq!(builder.metadata_builder.num_field_names(), 4);
2860
2861        let builder = builder.with_field_names(["z", "y"]);
2862        assert!(!builder.metadata_builder.is_sorted);
2863        assert_eq!(builder.metadata_builder.num_field_names(), 6);
2864    }
2865
2866    #[test]
2867    fn test_metadata_builder_from_iter() {
2868        let metadata = WritableMetadataBuilder::from_iter(vec!["apple", "banana", "cherry"]);
2869        assert_eq!(metadata.num_field_names(), 3);
2870        assert_eq!(metadata.field_name(0), "apple");
2871        assert_eq!(metadata.field_name(1), "banana");
2872        assert_eq!(metadata.field_name(2), "cherry");
2873        assert!(metadata.is_sorted);
2874
2875        let metadata = WritableMetadataBuilder::from_iter(["zebra", "apple", "banana"]);
2876        assert_eq!(metadata.num_field_names(), 3);
2877        assert_eq!(metadata.field_name(0), "zebra");
2878        assert_eq!(metadata.field_name(1), "apple");
2879        assert_eq!(metadata.field_name(2), "banana");
2880        assert!(!metadata.is_sorted);
2881
2882        let metadata = WritableMetadataBuilder::from_iter(Vec::<&str>::new());
2883        assert_eq!(metadata.num_field_names(), 0);
2884        assert!(!metadata.is_sorted);
2885    }
2886
2887    #[test]
2888    fn test_metadata_builder_extend() {
2889        let mut metadata = WritableMetadataBuilder::default();
2890        assert_eq!(metadata.num_field_names(), 0);
2891        assert!(!metadata.is_sorted);
2892
2893        metadata.extend(["apple", "cherry"]);
2894        assert_eq!(metadata.num_field_names(), 2);
2895        assert_eq!(metadata.field_name(0), "apple");
2896        assert_eq!(metadata.field_name(1), "cherry");
2897        assert!(metadata.is_sorted);
2898
2899        // extend with more field names that maintain sort order
2900        metadata.extend(vec!["dinosaur", "monkey"]);
2901        assert_eq!(metadata.num_field_names(), 4);
2902        assert_eq!(metadata.field_name(2), "dinosaur");
2903        assert_eq!(metadata.field_name(3), "monkey");
2904        assert!(metadata.is_sorted);
2905
2906        // test extending with duplicate field names
2907        let initial_count = metadata.num_field_names();
2908        metadata.extend(["apple", "monkey"]);
2909        assert_eq!(metadata.num_field_names(), initial_count); // No new fields added
2910    }
2911
2912    #[test]
2913    fn test_metadata_builder_extend_sort_order() {
2914        let mut metadata = WritableMetadataBuilder::default();
2915
2916        metadata.extend(["middle"]);
2917        assert!(metadata.is_sorted);
2918
2919        metadata.extend(["zebra"]);
2920        assert!(metadata.is_sorted);
2921
2922        // add field that breaks sort order
2923        metadata.extend(["apple"]);
2924        assert!(!metadata.is_sorted);
2925    }
2926
2927    #[test]
2928    fn test_metadata_builder_from_iter_with_string_types() {
2929        // &str
2930        let metadata = WritableMetadataBuilder::from_iter(["a", "b", "c"]);
2931        assert_eq!(metadata.num_field_names(), 3);
2932
2933        // string
2934        let metadata = WritableMetadataBuilder::from_iter(vec![
2935            "a".to_string(),
2936            "b".to_string(),
2937            "c".to_string(),
2938        ]);
2939        assert_eq!(metadata.num_field_names(), 3);
2940
2941        // mixed types (anything that implements AsRef<str>)
2942        let field_names: Vec<Box<str>> = vec!["a".into(), "b".into(), "c".into()];
2943        let metadata = WritableMetadataBuilder::from_iter(field_names);
2944        assert_eq!(metadata.num_field_names(), 3);
2945    }
2946
2947    #[test]
2948    fn test_variant_builder_to_list_builder_no_finish() {
2949        // Create a list builder but never finish it
2950        let mut builder = VariantBuilder::new();
2951        let mut list_builder = builder.new_list();
2952        list_builder.append_value("hi");
2953        drop(list_builder);
2954
2955        builder.append_value(42i8);
2956
2957        // The original builder should be unchanged
2958        let (metadata, value) = builder.finish();
2959        let metadata = VariantMetadata::try_new(&metadata).unwrap();
2960        assert!(metadata.is_empty());
2961
2962        let variant = Variant::try_new_with_metadata(metadata, &value).unwrap();
2963        assert_eq!(variant, Variant::Int8(42));
2964    }
2965
2966    #[test]
2967    fn test_variant_builder_to_object_builder_no_finish() {
2968        // Create an object builder but never finish it
2969        let mut builder = VariantBuilder::new();
2970        let mut object_builder = builder.new_object();
2971        object_builder.insert("name", "unknown");
2972        drop(object_builder);
2973
2974        builder.append_value(42i8);
2975
2976        // The original builder should be unchanged
2977        let (metadata, value) = builder.finish();
2978        let metadata = VariantMetadata::try_new(&metadata).unwrap();
2979        assert!(metadata.is_empty()); // rolled back
2980
2981        let variant = Variant::try_new_with_metadata(metadata, &value).unwrap();
2982        assert_eq!(variant, Variant::Int8(42));
2983    }
2984
2985    #[test]
2986    fn test_list_builder_to_list_builder_inner_no_finish() {
2987        let mut builder = VariantBuilder::new();
2988        let mut list_builder = builder.new_list();
2989        list_builder.append_value(1i8);
2990
2991        // Create a nested list builder but never finish it
2992        let mut nested_list_builder = list_builder.new_list();
2993        nested_list_builder.append_value("hi");
2994        drop(nested_list_builder);
2995
2996        list_builder.append_value(2i8);
2997
2998        // The parent list should only contain the original values
2999        list_builder.finish();
3000        let (metadata, value) = builder.finish();
3001        let metadata = VariantMetadata::try_new(&metadata).unwrap();
3002        assert!(metadata.is_empty());
3003
3004        let variant = Variant::try_new_with_metadata(metadata, &value).unwrap();
3005        let list = variant.as_list().unwrap();
3006        assert_eq!(list.len(), 2);
3007        assert_eq!(list.get(0).unwrap(), Variant::Int8(1));
3008        assert_eq!(list.get(1).unwrap(), Variant::Int8(2));
3009    }
3010
3011    #[test]
3012    fn test_list_builder_to_list_builder_outer_no_finish() {
3013        let mut builder = VariantBuilder::new();
3014        let mut list_builder = builder.new_list();
3015        list_builder.append_value(1i8);
3016
3017        // Create a nested list builder and finish it
3018        let mut nested_list_builder = list_builder.new_list();
3019        nested_list_builder.append_value("hi");
3020        nested_list_builder.finish();
3021
3022        // Drop the outer list builder without finishing it
3023        drop(list_builder);
3024
3025        builder.append_value(2i8);
3026
3027        // Only the second attempt should appear in the final variant
3028        let (metadata, value) = builder.finish();
3029        let metadata = VariantMetadata::try_new(&metadata).unwrap();
3030        assert!(metadata.is_empty());
3031
3032        let variant = Variant::try_new_with_metadata(metadata, &value).unwrap();
3033        assert_eq!(variant, Variant::Int8(2));
3034    }
3035
3036    #[test]
3037    fn test_list_builder_to_object_builder_inner_no_finish() {
3038        let mut builder = VariantBuilder::new();
3039        let mut list_builder = builder.new_list();
3040        list_builder.append_value(1i8);
3041
3042        // Create a nested object builder but never finish it
3043        let mut nested_object_builder = list_builder.new_object();
3044        nested_object_builder.insert("name", "unknown");
3045        drop(nested_object_builder);
3046
3047        list_builder.append_value(2i8);
3048
3049        // The parent list should only contain the original values
3050        list_builder.finish();
3051        let (metadata, value) = builder.finish();
3052        let metadata = VariantMetadata::try_new(&metadata).unwrap();
3053        assert!(metadata.is_empty());
3054
3055        let variant = Variant::try_new_with_metadata(metadata, &value).unwrap();
3056        let list = variant.as_list().unwrap();
3057        assert_eq!(list.len(), 2);
3058        assert_eq!(list.get(0).unwrap(), Variant::Int8(1));
3059        assert_eq!(list.get(1).unwrap(), Variant::Int8(2));
3060    }
3061
3062    #[test]
3063    fn test_list_builder_to_object_builder_outer_no_finish() {
3064        let mut builder = VariantBuilder::new();
3065        let mut list_builder = builder.new_list();
3066        list_builder.append_value(1i8);
3067
3068        // Create a nested object builder and finish it
3069        let mut nested_object_builder = list_builder.new_object();
3070        nested_object_builder.insert("name", "unknown");
3071        nested_object_builder.finish();
3072
3073        // Drop the outer list builder without finishing it
3074        drop(list_builder);
3075
3076        builder.append_value(2i8);
3077
3078        // Only the second attempt should appear in the final variant
3079        let (metadata, value) = builder.finish();
3080        let metadata = VariantMetadata::try_new(&metadata).unwrap();
3081        assert!(metadata.is_empty()); // rolled back
3082
3083        let variant = Variant::try_new_with_metadata(metadata, &value).unwrap();
3084        assert_eq!(variant, Variant::Int8(2));
3085    }
3086
3087    #[test]
3088    fn test_object_builder_to_list_builder_inner_no_finish() {
3089        let mut builder = VariantBuilder::new();
3090        let mut object_builder = builder.new_object();
3091        object_builder.insert("first", 1i8);
3092
3093        // Create a nested list builder but never finish it
3094        let mut nested_list_builder = object_builder.new_list("nested");
3095        nested_list_builder.append_value("hi");
3096        drop(nested_list_builder);
3097
3098        object_builder.insert("second", 2i8);
3099
3100        // The parent object should only contain the original fields
3101        object_builder.finish();
3102        let (metadata, value) = builder.finish();
3103
3104        let metadata = VariantMetadata::try_new(&metadata).unwrap();
3105        assert_eq!(metadata.len(), 2);
3106        assert_eq!(&metadata[0], "first");
3107        assert_eq!(&metadata[1], "second");
3108
3109        let variant = Variant::try_new_with_metadata(metadata, &value).unwrap();
3110        let obj = variant.as_object().unwrap();
3111        assert_eq!(obj.len(), 2);
3112        assert_eq!(obj.get("first"), Some(Variant::Int8(1)));
3113        assert_eq!(obj.get("second"), Some(Variant::Int8(2)));
3114    }
3115
3116    #[test]
3117    fn test_object_builder_to_list_builder_outer_no_finish() {
3118        let mut builder = VariantBuilder::new();
3119        let mut object_builder = builder.new_object();
3120        object_builder.insert("first", 1i8);
3121
3122        // Create a nested list builder and finish it
3123        let mut nested_list_builder = object_builder.new_list("nested");
3124        nested_list_builder.append_value("hi");
3125        nested_list_builder.finish();
3126
3127        // Drop the outer object builder without finishing it
3128        drop(object_builder);
3129
3130        builder.append_value(2i8);
3131
3132        // Only the second attempt should appear in the final variant
3133        let (metadata, value) = builder.finish();
3134        let metadata = VariantMetadata::try_new(&metadata).unwrap();
3135        assert!(metadata.is_empty()); // rolled back
3136
3137        let variant = Variant::try_new_with_metadata(metadata, &value).unwrap();
3138        assert_eq!(variant, Variant::Int8(2));
3139    }
3140
3141    #[test]
3142    fn test_object_builder_to_object_builder_inner_no_finish() {
3143        let mut builder = VariantBuilder::new();
3144        let mut object_builder = builder.new_object();
3145        object_builder.insert("first", 1i8);
3146
3147        // Create a nested object builder but never finish it
3148        let mut nested_object_builder = object_builder.new_object("nested");
3149        nested_object_builder.insert("name", "unknown");
3150        drop(nested_object_builder);
3151
3152        object_builder.insert("second", 2i8);
3153
3154        // The parent object should only contain the original fields
3155        object_builder.finish();
3156        let (metadata, value) = builder.finish();
3157
3158        let metadata = VariantMetadata::try_new(&metadata).unwrap();
3159        assert_eq!(metadata.len(), 2); // the fields of nested_object_builder has been rolled back
3160        assert_eq!(&metadata[0], "first");
3161        assert_eq!(&metadata[1], "second");
3162
3163        let variant = Variant::try_new_with_metadata(metadata, &value).unwrap();
3164        let obj = variant.as_object().unwrap();
3165        assert_eq!(obj.len(), 2);
3166        assert_eq!(obj.get("first"), Some(Variant::Int8(1)));
3167        assert_eq!(obj.get("second"), Some(Variant::Int8(2)));
3168    }
3169
3170    #[test]
3171    fn test_object_builder_to_object_builder_outer_no_finish() {
3172        let mut builder = VariantBuilder::new();
3173        let mut object_builder = builder.new_object();
3174        object_builder.insert("first", 1i8);
3175
3176        // Create a nested object builder and finish it
3177        let mut nested_object_builder = object_builder.new_object("nested");
3178        nested_object_builder.insert("name", "unknown");
3179        nested_object_builder.finish();
3180
3181        // Drop the outer object builder without finishing it
3182        drop(object_builder);
3183
3184        builder.append_value(2i8);
3185
3186        // Only the second attempt should appear in the final variant
3187        let (metadata, value) = builder.finish();
3188        let metadata = VariantMetadata::try_new(&metadata).unwrap();
3189        assert_eq!(metadata.len(), 0); // rolled back
3190
3191        let variant = Variant::try_new_with_metadata(metadata, &value).unwrap();
3192        assert_eq!(variant, Variant::Int8(2));
3193    }
3194
3195    // matthew
3196    #[test]
3197    fn test_append_object() {
3198        let (m1, v1) = make_object();
3199        let variant = Variant::new(&m1, &v1);
3200
3201        let mut builder = VariantBuilder::new().with_metadata(VariantMetadata::new(&m1));
3202
3203        builder.append_value(variant.clone());
3204
3205        let (metadata, value) = builder.finish();
3206        assert_eq!(variant, Variant::new(&metadata, &value));
3207    }
3208
3209    /// make an object variant with field names in reverse lexicographical order
3210    fn make_object() -> (Vec<u8>, Vec<u8>) {
3211        let mut builder = VariantBuilder::new();
3212
3213        let mut obj = builder.new_object();
3214
3215        obj.insert("b", true);
3216        obj.insert("a", false);
3217        obj.finish();
3218        builder.finish()
3219    }
3220
3221    #[test]
3222    fn test_append_nested_object() {
3223        let (m1, v1) = make_nested_object();
3224        let variant = Variant::new(&m1, &v1);
3225
3226        // because we can guarantee metadata is validated through the builder
3227        let mut builder = VariantBuilder::new().with_metadata(VariantMetadata::new(&m1));
3228        builder.append_value(variant.clone());
3229
3230        let (metadata, value) = builder.finish();
3231        let result_variant = Variant::new(&metadata, &value);
3232
3233        assert_eq!(variant, result_variant);
3234    }
3235
3236    /// make a nested object variant
3237    fn make_nested_object() -> (Vec<u8>, Vec<u8>) {
3238        let mut builder = VariantBuilder::new();
3239
3240        {
3241            let mut outer_obj = builder.new_object();
3242
3243            {
3244                let mut inner_obj = outer_obj.new_object("b");
3245                inner_obj.insert("a", "inner_value");
3246                inner_obj.finish();
3247            }
3248
3249            outer_obj.finish();
3250        }
3251
3252        builder.finish()
3253    }
3254
3255    #[test]
3256    fn test_append_list() {
3257        let (m1, v1) = make_list();
3258        let variant = Variant::new(&m1, &v1);
3259        let mut builder = VariantBuilder::new();
3260        builder.append_value(variant.clone());
3261        let (metadata, value) = builder.finish();
3262        assert_eq!(variant, Variant::new(&metadata, &value));
3263    }
3264
3265    /// make a simple List variant
3266    fn make_list() -> (Vec<u8>, Vec<u8>) {
3267        let mut builder = VariantBuilder::new();
3268
3269        builder
3270            .new_list()
3271            .with_value(1234)
3272            .with_value("a string value")
3273            .finish();
3274
3275        builder.finish()
3276    }
3277
3278    #[test]
3279    fn test_append_nested_list() {
3280        let (m1, v1) = make_nested_list();
3281        let variant = Variant::new(&m1, &v1);
3282        let mut builder = VariantBuilder::new();
3283        builder.append_value(variant.clone());
3284        let (metadata, value) = builder.finish();
3285        assert_eq!(variant, Variant::new(&metadata, &value));
3286    }
3287
3288    fn make_nested_list() -> (Vec<u8>, Vec<u8>) {
3289        let mut builder = VariantBuilder::new();
3290        let mut list = builder.new_list();
3291
3292        //create inner list
3293        list.new_list()
3294            .with_value("the dog licked the oil")
3295            .with_value(4.3)
3296            .finish();
3297
3298        list.finish();
3299
3300        builder.finish()
3301    }
3302
3303    // Make sure that we can correctly build deeply nested objects even when some of the nested
3304    // builders don't finish.
3305    #[test]
3306    fn test_append_list_object_list_object() {
3307        // An infinite counter
3308        let mut counter = 0..;
3309        let mut take = move |i| (&mut counter).take(i).collect::<Vec<_>>();
3310        let mut builder = VariantBuilder::new();
3311        let skip = 5;
3312        {
3313            let mut list = builder.new_list();
3314            for i in take(4) {
3315                let mut object = list.new_object();
3316                for i in take(4) {
3317                    let field_name = format!("field{i}");
3318                    let mut list = object.new_list(&field_name);
3319                    for i in take(3) {
3320                        let mut object = list.new_object();
3321                        for i in take(3) {
3322                            if i % skip != 0 {
3323                                object.insert(&format!("field{i}"), i);
3324                            }
3325                        }
3326                        if i % skip != 0 {
3327                            object.finish();
3328                        }
3329                    }
3330                    if i % skip != 0 {
3331                        list.finish();
3332                    }
3333                }
3334                if i % skip != 0 {
3335                    object.finish();
3336                }
3337            }
3338            list.finish();
3339        }
3340        let (metadata, value) = builder.finish();
3341        let v1 = Variant::try_new(&metadata, &value).unwrap();
3342
3343        let (metadata, value) = VariantBuilder::new().with_value(v1.clone()).finish();
3344        let v2 = Variant::try_new(&metadata, &value).unwrap();
3345
3346        assert_eq!(format!("{v1:?}"), format!("{v2:?}"));
3347    }
3348
3349    #[test]
3350    fn test_read_only_metadata_builder() {
3351        // First create some metadata with a few field names
3352        let mut default_builder = VariantBuilder::new();
3353        default_builder.add_field_name("name");
3354        default_builder.add_field_name("age");
3355        default_builder.add_field_name("active");
3356        let (metadata_bytes, _) = default_builder.finish();
3357
3358        // Use the metadata to build new variant values
3359        let metadata = VariantMetadata::try_new(&metadata_bytes).unwrap();
3360        let mut metadata_builder = ReadOnlyMetadataBuilder::new(&metadata);
3361        let mut value_builder = ValueBuilder::new();
3362
3363        {
3364            let state = ParentState::variant(&mut value_builder, &mut metadata_builder);
3365            let mut obj = ObjectBuilder::new(state, false);
3366
3367            // These should succeed because the fields exist in the metadata
3368            obj.insert("name", "Alice");
3369            obj.insert("age", 30i8);
3370            obj.insert("active", true);
3371            obj.finish();
3372        }
3373
3374        let value = value_builder.into_inner();
3375
3376        // Verify the variant was built correctly
3377        let variant = Variant::try_new(&metadata_bytes, &value).unwrap();
3378        let obj = variant.as_object().unwrap();
3379        assert_eq!(obj.get("name"), Some(Variant::from("Alice")));
3380        assert_eq!(obj.get("age"), Some(Variant::Int8(30)));
3381        assert_eq!(obj.get("active"), Some(Variant::from(true)));
3382    }
3383
3384    #[test]
3385    fn test_read_only_metadata_builder_fails_on_unknown_field() {
3386        // Create metadata with only one field
3387        let mut default_builder = VariantBuilder::new();
3388        default_builder.add_field_name("known_field");
3389        let (metadata_bytes, _) = default_builder.finish();
3390
3391        // Use the metadata to build new variant values
3392        let metadata = VariantMetadata::try_new(&metadata_bytes).unwrap();
3393        let mut metadata_builder = ReadOnlyMetadataBuilder::new(&metadata);
3394        let mut value_builder = ValueBuilder::new();
3395
3396        {
3397            let state = ParentState::variant(&mut value_builder, &mut metadata_builder);
3398            let mut obj = ObjectBuilder::new(state, false);
3399
3400            // This should succeed
3401            obj.insert("known_field", "value");
3402
3403            // This should fail because "unknown_field" is not in the metadata
3404            let result = obj.try_insert("unknown_field", "value");
3405            assert!(result.is_err());
3406            assert!(result
3407                .unwrap_err()
3408                .to_string()
3409                .contains("Field name 'unknown_field' not found"));
3410        }
3411    }
3412
3413    #[test]
3414    fn test_append_variant_bytes_round_trip() {
3415        // Create a complex variant with the normal builder
3416        let mut builder = VariantBuilder::new();
3417        {
3418            let mut obj = builder.new_object();
3419            obj.insert("name", "Alice");
3420            obj.insert("age", 30i32);
3421            {
3422                let mut scores_list = obj.new_list("scores");
3423                scores_list.append_value(95i32);
3424                scores_list.append_value(87i32);
3425                scores_list.append_value(92i32);
3426                scores_list.finish();
3427            }
3428            {
3429                let mut address = obj.new_object("address");
3430                address.insert("street", "123 Main St");
3431                address.insert("city", "Anytown");
3432                address.finish();
3433            }
3434            obj.finish();
3435        }
3436        let (metadata, value1) = builder.finish();
3437        let variant1 = Variant::try_new(&metadata, &value1).unwrap();
3438
3439        // Copy using the new bytes API
3440        let metadata = VariantMetadata::new(&metadata);
3441        let mut metadata = ReadOnlyMetadataBuilder::new(&metadata);
3442        let mut builder2 = ValueBuilder::new();
3443        let state = ParentState::variant(&mut builder2, &mut metadata);
3444        ValueBuilder::append_variant_bytes(state, variant1);
3445        let value2 = builder2.into_inner();
3446
3447        // The bytes should be identical, we merely copied them across.
3448        assert_eq!(value1, value2);
3449    }
3450
3451    #[test]
3452    fn test_object_insert_bytes_subset() {
3453        // Create an original object, making sure to inject the field names we'll add later.
3454        let mut builder = VariantBuilder::new().with_field_names(["new_field", "another_field"]);
3455        {
3456            let mut obj = builder.new_object();
3457            obj.insert("field1", "value1");
3458            obj.insert("field2", 42i32);
3459            obj.insert("field3", true);
3460            obj.insert("field4", "value4");
3461            obj.finish();
3462        }
3463        let (metadata1, value1) = builder.finish();
3464        let original_variant = Variant::try_new(&metadata1, &value1).unwrap();
3465        let original_obj = original_variant.as_object().unwrap();
3466
3467        // Create a new object copying subset of fields interleaved with new ones
3468        let metadata2 = VariantMetadata::new(&metadata1);
3469        let mut metadata2 = ReadOnlyMetadataBuilder::new(&metadata2);
3470        let mut builder2 = ValueBuilder::new();
3471        let state = ParentState::variant(&mut builder2, &mut metadata2);
3472        {
3473            let mut obj = ObjectBuilder::new(state, true);
3474
3475            // Copy field1 using bytes API
3476            obj.insert_bytes("field1", original_obj.get("field1").unwrap());
3477
3478            // Add new field
3479            obj.insert("new_field", "new_value");
3480
3481            // Copy field3 using bytes API
3482            obj.insert_bytes("field3", original_obj.get("field3").unwrap());
3483
3484            // Add another new field
3485            obj.insert("another_field", 99i32);
3486
3487            // Copy field2 using bytes API
3488            obj.insert_bytes("field2", original_obj.get("field2").unwrap());
3489
3490            obj.finish();
3491        }
3492        let value2 = builder2.into_inner();
3493        let result_variant = Variant::try_new(&metadata1, &value2).unwrap();
3494        let result_obj = result_variant.as_object().unwrap();
3495
3496        // Verify the object contains expected fields
3497        assert_eq!(result_obj.len(), 5);
3498        assert_eq!(
3499            result_obj.get("field1").unwrap().as_string().unwrap(),
3500            "value1"
3501        );
3502        assert_eq!(result_obj.get("field2").unwrap().as_int32().unwrap(), 42);
3503        assert!(result_obj.get("field3").unwrap().as_boolean().unwrap());
3504        assert_eq!(
3505            result_obj.get("new_field").unwrap().as_string().unwrap(),
3506            "new_value"
3507        );
3508        assert_eq!(
3509            result_obj.get("another_field").unwrap().as_int32().unwrap(),
3510            99
3511        );
3512    }
3513
3514    #[test]
3515    fn test_list_append_bytes_subset() {
3516        // Create an original list
3517        let mut builder = VariantBuilder::new();
3518        {
3519            let mut list = builder.new_list();
3520            list.append_value("item1");
3521            list.append_value(42i32);
3522            list.append_value(true);
3523            list.append_value("item4");
3524            list.append_value(1.234f64);
3525            list.finish();
3526        }
3527        let (metadata1, value1) = builder.finish();
3528        let original_variant = Variant::try_new(&metadata1, &value1).unwrap();
3529        let original_list = original_variant.as_list().unwrap();
3530
3531        // Create a new list copying subset of elements interleaved with new ones
3532        let metadata2 = VariantMetadata::new(&metadata1);
3533        let mut metadata2 = ReadOnlyMetadataBuilder::new(&metadata2);
3534        let mut builder2 = ValueBuilder::new();
3535        let state = ParentState::variant(&mut builder2, &mut metadata2);
3536        {
3537            let mut list = ListBuilder::new(state, true);
3538
3539            // Copy first element using bytes API
3540            list.append_value_bytes(original_list.get(0).unwrap());
3541
3542            // Add new element
3543            list.append_value("new_item");
3544
3545            // Copy third element using bytes API
3546            list.append_value_bytes(original_list.get(2).unwrap());
3547
3548            // Add another new element
3549            list.append_value(99i32);
3550
3551            // Copy last element using bytes API
3552            list.append_value_bytes(original_list.get(4).unwrap());
3553
3554            list.finish();
3555        }
3556        let value2 = builder2.into_inner();
3557        let result_variant = Variant::try_new(&metadata1, &value2).unwrap();
3558        let result_list = result_variant.as_list().unwrap();
3559
3560        // Verify the list contains expected elements
3561        assert_eq!(result_list.len(), 5);
3562        assert_eq!(result_list.get(0).unwrap().as_string().unwrap(), "item1");
3563        assert_eq!(result_list.get(1).unwrap().as_string().unwrap(), "new_item");
3564        assert!(result_list.get(2).unwrap().as_boolean().unwrap());
3565        assert_eq!(result_list.get(3).unwrap().as_int32().unwrap(), 99);
3566        assert_eq!(result_list.get(4).unwrap().as_f64().unwrap(), 1.234);
3567    }
3568
3569    #[test]
3570    fn test_complex_nested_filtering_injection() {
3571        // Create a complex nested structure: object -> list -> objects. Make sure to pre-register
3572        // the extra field names we'll need later while manipulating variant bytes.
3573        let mut builder = VariantBuilder::new().with_field_names([
3574            "active_count",
3575            "active_users",
3576            "computed_score",
3577            "processed_at",
3578            "status",
3579        ]);
3580
3581        {
3582            let mut root_obj = builder.new_object();
3583            root_obj.insert("metadata", "original");
3584
3585            {
3586                let mut users_list = root_obj.new_list("users");
3587
3588                // User 1
3589                {
3590                    let mut user1 = users_list.new_object();
3591                    user1.insert("id", 1i32);
3592                    user1.insert("name", "Alice");
3593                    user1.insert("active", true);
3594                    user1.finish();
3595                }
3596
3597                // User 2
3598                {
3599                    let mut user2 = users_list.new_object();
3600                    user2.insert("id", 2i32);
3601                    user2.insert("name", "Bob");
3602                    user2.insert("active", false);
3603                    user2.finish();
3604                }
3605
3606                // User 3
3607                {
3608                    let mut user3 = users_list.new_object();
3609                    user3.insert("id", 3i32);
3610                    user3.insert("name", "Charlie");
3611                    user3.insert("active", true);
3612                    user3.finish();
3613                }
3614
3615                users_list.finish();
3616            }
3617
3618            root_obj.insert("total_count", 3i32);
3619            root_obj.finish();
3620        }
3621        let (metadata1, value1) = builder.finish();
3622        let original_variant = Variant::try_new(&metadata1, &value1).unwrap();
3623        let original_obj = original_variant.as_object().unwrap();
3624        let original_users = original_obj.get("users").unwrap();
3625        let original_users = original_users.as_list().unwrap();
3626
3627        // Create filtered/modified version: only copy active users and inject new data
3628        let metadata2 = VariantMetadata::new(&metadata1);
3629        let mut metadata2 = ReadOnlyMetadataBuilder::new(&metadata2);
3630        let mut builder2 = ValueBuilder::new();
3631        let state = ParentState::variant(&mut builder2, &mut metadata2);
3632        {
3633            let mut root_obj = ObjectBuilder::new(state, true);
3634
3635            // Copy metadata using bytes API
3636            root_obj.insert_bytes("metadata", original_obj.get("metadata").unwrap());
3637
3638            // Add processing timestamp
3639            root_obj.insert("processed_at", "2024-01-01T00:00:00Z");
3640
3641            {
3642                let mut filtered_users = root_obj.new_list("active_users");
3643
3644                // Copy only active users and inject additional data
3645                for i in 0..original_users.len() {
3646                    let user = original_users.get(i).unwrap();
3647                    let user = user.as_object().unwrap();
3648                    if user.get("active").unwrap().as_boolean().unwrap() {
3649                        {
3650                            let mut new_user = filtered_users.new_object();
3651
3652                            // Copy existing fields using bytes API
3653                            new_user.insert_bytes("id", user.get("id").unwrap());
3654                            new_user.insert_bytes("name", user.get("name").unwrap());
3655
3656                            // Inject new computed field
3657                            let user_id = user.get("id").unwrap().as_int32().unwrap();
3658                            new_user.insert("computed_score", user_id * 10);
3659
3660                            // Add status transformation (don't copy the 'active' field)
3661                            new_user.insert("status", "verified");
3662
3663                            new_user.finish();
3664                        }
3665                    }
3666                }
3667
3668                // Inject a completely new user
3669                {
3670                    let mut new_user = filtered_users.new_object();
3671                    new_user.insert("id", 999i32);
3672                    new_user.insert("name", "System User");
3673                    new_user.insert("computed_score", 0i32);
3674                    new_user.insert("status", "system");
3675                    new_user.finish();
3676                }
3677
3678                filtered_users.finish();
3679            }
3680
3681            // Update count
3682            root_obj.insert("active_count", 3i32); // 2 active + 1 new
3683
3684            root_obj.finish();
3685        }
3686        let value2 = builder2.into_inner();
3687        let result_variant = Variant::try_new(&metadata1, &value2).unwrap();
3688        let result_obj = result_variant.as_object().unwrap();
3689
3690        // Verify the filtered/modified structure
3691        assert_eq!(
3692            result_obj.get("metadata").unwrap().as_string().unwrap(),
3693            "original"
3694        );
3695        assert_eq!(
3696            result_obj.get("processed_at").unwrap().as_string().unwrap(),
3697            "2024-01-01T00:00:00Z"
3698        );
3699        assert_eq!(
3700            result_obj.get("active_count").unwrap().as_int32().unwrap(),
3701            3
3702        );
3703
3704        let active_users = result_obj.get("active_users").unwrap();
3705        let active_users = active_users.as_list().unwrap();
3706        assert_eq!(active_users.len(), 3);
3707
3708        // Verify Alice (id=1, was active)
3709        let alice = active_users.get(0).unwrap();
3710        let alice = alice.as_object().unwrap();
3711        assert_eq!(alice.get("id").unwrap().as_int32().unwrap(), 1);
3712        assert_eq!(alice.get("name").unwrap().as_string().unwrap(), "Alice");
3713        assert_eq!(alice.get("computed_score").unwrap().as_int32().unwrap(), 10);
3714        assert_eq!(
3715            alice.get("status").unwrap().as_string().unwrap(),
3716            "verified"
3717        );
3718        assert!(alice.get("active").is_none()); // This field was not copied
3719
3720        // Verify Charlie (id=3, was active) - Bob (id=2) was not active so not included
3721        let charlie = active_users.get(1).unwrap();
3722        let charlie = charlie.as_object().unwrap();
3723        assert_eq!(charlie.get("id").unwrap().as_int32().unwrap(), 3);
3724        assert_eq!(charlie.get("name").unwrap().as_string().unwrap(), "Charlie");
3725        assert_eq!(
3726            charlie.get("computed_score").unwrap().as_int32().unwrap(),
3727            30
3728        );
3729        assert_eq!(
3730            charlie.get("status").unwrap().as_string().unwrap(),
3731            "verified"
3732        );
3733
3734        // Verify injected system user
3735        let system_user = active_users.get(2).unwrap();
3736        let system_user = system_user.as_object().unwrap();
3737        assert_eq!(system_user.get("id").unwrap().as_int32().unwrap(), 999);
3738        assert_eq!(
3739            system_user.get("name").unwrap().as_string().unwrap(),
3740            "System User"
3741        );
3742        assert_eq!(
3743            system_user
3744                .get("computed_score")
3745                .unwrap()
3746                .as_int32()
3747                .unwrap(),
3748            0
3749        );
3750        assert_eq!(
3751            system_user.get("status").unwrap().as_string().unwrap(),
3752            "system"
3753        );
3754    }
3755}