Skip to main content

arrow_avro/
schema.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Avro Schema representations for Arrow.
6
7#[cfg(feature = "canonical_extension_types")]
8use arrow_schema::extension::ExtensionType;
9use arrow_schema::{
10    ArrowError, DataType, Field as ArrowField, IntervalUnit, Metadata, Schema as ArrowSchema,
11    TimeUnit, UnionMode,
12};
13use serde::{Deserialize, Serialize};
14use serde_json::{Map as JsonMap, Value, json};
15#[cfg(feature = "sha256")]
16use sha2::{Digest, Sha256};
17use std::borrow::Cow;
18use std::cmp::PartialEq;
19use std::collections::hash_map::Entry;
20use std::collections::{HashMap, HashSet};
21use strum_macros::AsRefStr;
22
23/// The Avro single‑object encoding “magic” bytes (`0xC3 0x01`)
24pub const SINGLE_OBJECT_MAGIC: [u8; 2] = [0xC3, 0x01];
25
26/// The Confluent "magic" byte (`0x00`)
27pub const CONFLUENT_MAGIC: [u8; 1] = [0x00];
28
29/// The maximum possible length of a prefix.
30/// SHA256 (32) + single-object magic (2)
31pub const MAX_PREFIX_LEN: usize = 34;
32
33/// The metadata key used for storing the JSON encoded `Schema`
34pub const SCHEMA_METADATA_KEY: &str = "avro.schema";
35
36/// Metadata key used to represent Avro enum symbols in an Arrow schema.
37pub const AVRO_ENUM_SYMBOLS_METADATA_KEY: &str = "avro.enum.symbols";
38
39/// Metadata key used to store the default value of a field in an Avro schema.
40pub const AVRO_FIELD_DEFAULT_METADATA_KEY: &str = "avro.field.default";
41
42/// Metadata key used to store the name of a type in an Avro schema.
43pub const AVRO_NAME_METADATA_KEY: &str = "avro.name";
44
45/// Metadata key used to store the name of a type in an Avro schema.
46pub const AVRO_NAMESPACE_METADATA_KEY: &str = "avro.namespace";
47
48/// Metadata key used to store the documentation for a type in an Avro schema.
49pub const AVRO_DOC_METADATA_KEY: &str = "avro.doc";
50
51/// Default name for the root record in an Avro schema.
52pub const AVRO_ROOT_RECORD_DEFAULT_NAME: &str = "topLevelRecord";
53
54/// Avro types are not nullable, with nullability instead encoded as a union
55/// where one of the variants is the null type.
56///
57/// To accommodate this, we specially case two-variant unions where one of the
58/// variants is the null type, and use this to derive arrow's notion of nullability
59#[derive(Debug, Copy, Clone, PartialEq, Default)]
60pub(crate) enum Nullability {
61    /// The nulls are encoded as the first union variant
62    #[default]
63    NullFirst,
64    /// The nulls are encoded as the second union variant
65    NullSecond,
66}
67
68impl Nullability {
69    /// Returns the index of the non-null variant in the union.
70    pub(crate) fn non_null_index(&self) -> usize {
71        match self {
72            Nullability::NullFirst => 1,
73            Nullability::NullSecond => 0,
74        }
75    }
76}
77
78/// Either a [`PrimitiveType`] or a reference to a previously defined named type
79///
80/// <https://avro.apache.org/docs/1.11.1/specification/#names>
81#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
82#[serde(untagged)]
83/// A type name in an Avro schema
84///
85/// This represents the different ways a type can be referenced in an Avro schema.
86pub(crate) enum TypeName<'a> {
87    /// A primitive type like null, boolean, int, etc.
88    Primitive(PrimitiveType),
89    /// A reference to another named type
90    Ref(&'a str),
91}
92
93/// A primitive type
94///
95/// <https://avro.apache.org/docs/1.11.1/specification/#primitive-types>
96#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, AsRefStr)]
97#[serde(rename_all = "camelCase")]
98#[strum(serialize_all = "lowercase")]
99pub(crate) enum PrimitiveType {
100    /// null: no value
101    Null,
102    /// boolean: a binary value
103    Boolean,
104    /// int: 32-bit signed integer
105    Int,
106    /// long: 64-bit signed integer
107    Long,
108    /// float: single precision (32-bit) IEEE 754 floating-point number
109    Float,
110    /// double: double precision (64-bit) IEEE 754 floating-point number
111    Double,
112    /// bytes: sequence of 8-bit unsigned bytes
113    Bytes,
114    /// string: Unicode character sequence
115    String,
116}
117
118/// Additional attributes within a `Schema`
119///
120/// <https://avro.apache.org/docs/1.11.1/specification/#schema-declaration>
121#[derive(Debug, Clone, PartialEq, Eq, Default, Deserialize, Serialize)]
122#[serde(rename_all = "camelCase")]
123pub(crate) struct Attributes<'a> {
124    /// A logical type name
125    ///
126    /// <https://avro.apache.org/docs/1.11.1/specification/#logical-types>
127    #[serde(default)]
128    pub(crate) logical_type: Option<&'a str>,
129
130    /// Additional JSON attributes
131    #[serde(flatten)]
132    pub(crate) additional: HashMap<&'a str, Value>,
133}
134
135impl Attributes<'_> {
136    /// Returns the field metadata for this [`Attributes`]
137    pub(crate) fn field_metadata(&self) -> HashMap<String, String> {
138        self.additional
139            .iter()
140            .map(|(k, v)| (k.to_string(), v.to_string()))
141            .collect()
142    }
143}
144
145/// A type definition that is not a variant of [`ComplexType`]
146#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
147#[serde(rename_all = "camelCase")]
148pub(crate) struct Type<'a> {
149    /// The type of this Avro data structure
150    #[serde(borrow)]
151    pub(crate) r#type: TypeName<'a>,
152    /// Additional attributes associated with this type
153    #[serde(flatten)]
154    pub(crate) attributes: Attributes<'a>,
155}
156
157/// An Avro schema
158///
159/// This represents the different shapes of Avro schemas as defined in the specification.
160/// See <https://avro.apache.org/docs/1.11.1/specification/#schemas> for more details.
161#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
162#[serde(untagged)]
163pub(crate) enum Schema<'a> {
164    /// A direct type name (primitive or reference)
165    #[serde(borrow)]
166    TypeName(TypeName<'a>),
167    /// A union of multiple schemas (e.g., `["null", "string"]`)
168    #[serde(borrow)]
169    Union(Vec<Schema<'a>>),
170    /// A complex type such as record, array, map, etc.
171    #[serde(borrow)]
172    Complex(ComplexType<'a>),
173    /// A type with attributes
174    #[serde(borrow)]
175    Type(Type<'a>),
176}
177
178/// A complex type
179///
180/// <https://avro.apache.org/docs/1.11.1/specification/#complex-types>
181#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
182#[serde(tag = "type", rename_all = "camelCase")]
183pub(crate) enum ComplexType<'a> {
184    /// Record type: a sequence of fields with names and types
185    #[serde(borrow)]
186    Record(Record<'a>),
187    /// Enum type: a set of named values
188    #[serde(borrow)]
189    Enum(Enum<'a>),
190    /// Array type: a sequence of values of the same type
191    #[serde(borrow)]
192    Array(Array<'a>),
193    /// Map type: a mapping from strings to values of the same type
194    #[serde(borrow)]
195    Map(Map<'a>),
196    /// Fixed type: a fixed-size byte array
197    #[serde(borrow)]
198    Fixed(Fixed<'a>),
199}
200
201/// A record
202///
203/// <https://avro.apache.org/docs/1.11.1/specification/#schema-record>
204#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
205pub(crate) struct Record<'a> {
206    /// Name of the record
207    #[serde(borrow)]
208    pub(crate) name: &'a str,
209    /// Optional namespace for the record, provides a way to organize names
210    #[serde(borrow, default)]
211    pub(crate) namespace: Option<&'a str>,
212    /// Optional documentation string for the record
213    #[serde(borrow, default)]
214    pub(crate) doc: Option<Cow<'a, str>>,
215    /// Alternative names for this record
216    #[serde(borrow, default)]
217    pub(crate) aliases: Vec<&'a str>,
218    /// The fields contained in this record
219    #[serde(borrow)]
220    pub(crate) fields: Vec<Field<'a>>,
221    /// Additional attributes for this record
222    #[serde(flatten)]
223    pub(crate) attributes: Attributes<'a>,
224}
225
226fn deserialize_default<'de, D>(deserializer: D) -> Result<Option<Value>, D::Error>
227where
228    D: serde::Deserializer<'de>,
229{
230    Value::deserialize(deserializer).map(Some)
231}
232
233/// A field within a [`Record`]
234#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
235pub(crate) struct Field<'a> {
236    /// Name of the field within the record
237    #[serde(borrow)]
238    pub(crate) name: &'a str,
239    /// Optional documentation for this field
240    #[serde(borrow, default)]
241    pub(crate) doc: Option<Cow<'a, str>>,
242    /// The field's type definition
243    #[serde(borrow)]
244    pub(crate) r#type: Schema<'a>,
245    /// Optional default value for this field
246    #[serde(deserialize_with = "deserialize_default", default)]
247    pub(crate) default: Option<Value>,
248    /// Alternative names (aliases) for this field (Avro spec: field-level aliases).
249    /// Borrowed from input JSON where possible.
250    #[serde(borrow, default)]
251    pub(crate) aliases: Vec<&'a str>,
252}
253
254/// An enumeration
255///
256/// <https://avro.apache.org/docs/1.11.1/specification/#enums>
257#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
258pub(crate) struct Enum<'a> {
259    /// Name of the enum
260    #[serde(borrow)]
261    pub(crate) name: &'a str,
262    /// Optional namespace for the enum, provides organizational structure
263    #[serde(borrow, default)]
264    pub(crate) namespace: Option<&'a str>,
265    /// Optional documentation string describing the enum
266    #[serde(borrow, default)]
267    pub(crate) doc: Option<Cow<'a, str>>,
268    /// Alternative names for this enum
269    #[serde(borrow, default)]
270    pub(crate) aliases: Vec<&'a str>,
271    /// The symbols (values) that this enum can have
272    #[serde(borrow)]
273    pub(crate) symbols: Vec<&'a str>,
274    /// Optional default value for this enum
275    #[serde(borrow, default)]
276    pub(crate) default: Option<&'a str>,
277    /// Additional attributes for this enum
278    #[serde(flatten)]
279    pub(crate) attributes: Attributes<'a>,
280}
281
282/// An array
283///
284/// <https://avro.apache.org/docs/1.11.1/specification/#arrays>
285#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
286pub(crate) struct Array<'a> {
287    /// The schema for items in this array
288    #[serde(borrow)]
289    pub(crate) items: Box<Schema<'a>>,
290    /// Additional attributes for this array
291    #[serde(flatten)]
292    pub(crate) attributes: Attributes<'a>,
293}
294
295/// A map
296///
297/// <https://avro.apache.org/docs/1.11.1/specification/#maps>
298#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
299pub(crate) struct Map<'a> {
300    /// The schema for values in this map
301    #[serde(borrow)]
302    pub(crate) values: Box<Schema<'a>>,
303    /// Additional attributes for this map
304    #[serde(flatten)]
305    pub(crate) attributes: Attributes<'a>,
306}
307
308/// A fixed length binary array
309///
310/// <https://avro.apache.org/docs/1.11.1/specification/#fixed>
311#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
312pub(crate) struct Fixed<'a> {
313    /// Name of the fixed type
314    #[serde(borrow)]
315    pub(crate) name: &'a str,
316    /// Optional namespace for the fixed type
317    #[serde(borrow, default)]
318    pub(crate) namespace: Option<&'a str>,
319    /// Alternative names for this fixed type
320    #[serde(borrow, default)]
321    pub(crate) aliases: Vec<&'a str>,
322    /// The number of bytes in this fixed type
323    pub(crate) size: usize,
324    /// Additional attributes for this fixed type
325    #[serde(flatten)]
326    pub(crate) attributes: Attributes<'a>,
327}
328
329#[derive(Debug, Copy, Clone, PartialEq, Default)]
330pub(crate) struct AvroSchemaOptions {
331    pub(crate) null_order: Option<Nullability>,
332    pub(crate) strip_metadata: bool,
333}
334
335/// A wrapper for an Avro schema in its JSON string representation.
336#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
337pub struct AvroSchema {
338    /// The Avro schema as a JSON string.
339    pub json_string: String,
340}
341
342impl TryFrom<&ArrowSchema> for AvroSchema {
343    type Error = ArrowError;
344
345    /// Converts an `ArrowSchema` to `AvroSchema`, delegating to
346    /// `AvroSchema::from_arrow_with_options` with `None` so that the
347    /// union null ordering is decided by `Nullability::default()`.
348    fn try_from(schema: &ArrowSchema) -> Result<Self, Self::Error> {
349        AvroSchema::from_arrow_with_options(schema, None)
350    }
351}
352
353impl AvroSchema {
354    /// Creates a new `AvroSchema` from a JSON string.
355    pub fn new(json_string: String) -> Self {
356        Self { json_string }
357    }
358
359    pub(crate) fn schema(&self) -> Result<Schema<'_>, ArrowError> {
360        serde_json::from_str(self.json_string.as_str())
361            .map_err(|e| ArrowError::ParseError(format!("Invalid Avro schema JSON: {e}")))
362    }
363
364    /// Returns the fingerprint of the schema, computed using the specified [`FingerprintAlgorithm`].
365    ///
366    /// The fingerprint is computed over the schema's Parsed Canonical Form
367    /// as defined by the Avro specification. Depending on `hash_type`, this
368    /// will return one of the supported [`Fingerprint`] variants:
369    /// - [`Fingerprint::Rabin`] for [`FingerprintAlgorithm::Rabin`]
370    /// - `Fingerprint::MD5` for `FingerprintAlgorithm::MD5`
371    /// - `Fingerprint::SHA256` for `FingerprintAlgorithm::SHA256`
372    ///
373    /// Note: [`FingerprintAlgorithm::Id`] or [`FingerprintAlgorithm::Id64`] cannot be used to generate a fingerprint
374    /// and will result in an error. If you intend to use a Schema Registry ID-based
375    /// wire format, either use [`SchemaStore::set`] or load the [`Fingerprint::Id`] directly via [`Fingerprint::load_fingerprint_id`] or for
376    /// [`Fingerprint::Id64`] via [`Fingerprint::load_fingerprint_id64`].
377    ///
378    /// See also: <https://avro.apache.org/docs/1.11.1/specification/#schema-fingerprints>
379    ///
380    /// # Errors
381    /// Returns an error if deserializing the schema fails, if generating the
382    /// canonical form of the schema fails, or if `hash_type` is [`FingerprintAlgorithm::Id`].
383    ///
384    /// # Examples
385    /// ```
386    /// use arrow_avro::schema::{AvroSchema, FingerprintAlgorithm};
387    ///
388    /// let avro = AvroSchema::new("\"string\"".to_string());
389    /// let fp = avro.fingerprint(FingerprintAlgorithm::Rabin).unwrap();
390    /// ```
391    pub fn fingerprint(&self, hash_type: FingerprintAlgorithm) -> Result<Fingerprint, ArrowError> {
392        Self::generate_fingerprint(&self.schema()?, hash_type)
393    }
394
395    pub(crate) fn project(&self, projection: &[usize]) -> Result<Self, ArrowError> {
396        let mut value: Value = serde_json::from_str(&self.json_string)
397            .map_err(|e| ArrowError::AvroError(format!("Invalid Avro schema JSON: {e}")))?;
398        let obj = value.as_object_mut().ok_or_else(|| {
399            ArrowError::AvroError(
400                "Projected schema must be a JSON object Avro record schema".to_string(),
401            )
402        })?;
403        match obj.get("type").and_then(|v| v.as_str()) {
404            Some("record") => {}
405            Some(other) => {
406                return Err(ArrowError::AvroError(format!(
407                    "Projected schema must be an Avro record, found type '{other}'"
408                )));
409            }
410            None => {
411                return Err(ArrowError::AvroError(
412                    "Projected schema missing required 'type' field".to_string(),
413                ));
414            }
415        }
416        let fields_val = obj.get_mut("fields").ok_or_else(|| {
417            ArrowError::AvroError("Avro record schema missing required 'fields'".to_string())
418        })?;
419        let projected_fields = {
420            let mut original_fields = match fields_val {
421                Value::Array(arr) => std::mem::take(arr),
422                _ => {
423                    return Err(ArrowError::AvroError(
424                        "Avro record schema 'fields' must be an array".to_string(),
425                    ));
426                }
427            };
428            let len = original_fields.len();
429            let mut seen: HashSet<usize> = HashSet::with_capacity(projection.len());
430            let mut out: Vec<Value> = Vec::with_capacity(projection.len());
431            for &i in projection {
432                if i >= len {
433                    return Err(ArrowError::AvroError(format!(
434                        "Projection index {i} out of bounds for record with {len} fields"
435                    )));
436                }
437                if !seen.insert(i) {
438                    return Err(ArrowError::AvroError(format!(
439                        "Duplicate projection index {i}"
440                    )));
441                }
442                out.push(std::mem::replace(&mut original_fields[i], Value::Null));
443            }
444            out
445        };
446        *fields_val = Value::Array(projected_fields);
447        let json_string = serde_json::to_string(&value).map_err(|e| {
448            ArrowError::AvroError(format!(
449                "Failed to serialize projected Avro schema JSON: {e}"
450            ))
451        })?;
452        Ok(Self::new(json_string))
453    }
454
455    pub(crate) fn generate_fingerprint(
456        schema: &Schema,
457        hash_type: FingerprintAlgorithm,
458    ) -> Result<Fingerprint, ArrowError> {
459        let canonical = Self::generate_canonical_form(schema).map_err(|e| {
460            ArrowError::ComputeError(format!("Failed to generate canonical form for schema: {e}"))
461        })?;
462        match hash_type {
463            FingerprintAlgorithm::Rabin => {
464                Ok(Fingerprint::Rabin(compute_fingerprint_rabin(&canonical)))
465            }
466            FingerprintAlgorithm::Id | FingerprintAlgorithm::Id64 => Err(ArrowError::SchemaError(
467                "FingerprintAlgorithm of Id or Id64 cannot be used to generate a fingerprint; \
468                if using Fingerprint::Id, pass the registry ID in instead using the set method."
469                    .to_string(),
470            )),
471            #[cfg(feature = "md5")]
472            FingerprintAlgorithm::MD5 => Ok(Fingerprint::MD5(compute_fingerprint_md5(&canonical))),
473            #[cfg(feature = "sha256")]
474            FingerprintAlgorithm::SHA256 => {
475                Ok(Fingerprint::SHA256(compute_fingerprint_sha256(&canonical)))
476            }
477        }
478    }
479
480    /// Generates the Parsed Canonical Form for the given `Schema`.
481    ///
482    /// The canonical form is a standardized JSON representation of the schema,
483    /// primarily used for generating a schema fingerprint for equality checking.
484    ///
485    /// This form strips attributes that do not affect the schema's identity,
486    /// such as `doc` fields, `aliases`, and any properties not defined in the
487    /// Avro specification.
488    ///
489    /// <https://avro.apache.org/docs/1.11.1/specification/#parsing-canonical-form-for-schemas>
490    pub(crate) fn generate_canonical_form(schema: &Schema) -> Result<String, ArrowError> {
491        build_canonical(schema, None)
492    }
493
494    /// Build Avro JSON from an Arrow [`ArrowSchema`], applying the given null‑union order and optionally stripping internal Arrow metadata.
495    ///
496    /// If the input Arrow schema already contains Avro JSON in
497    /// [`SCHEMA_METADATA_KEY`], that JSON is returned verbatim to preserve
498    /// the exact header encoding alignment; otherwise, a new JSON is generated
499    /// honoring `null_union_order` at **all nullable sites**.
500    pub(crate) fn from_arrow_with_options(
501        schema: &ArrowSchema,
502        options: Option<AvroSchemaOptions>,
503    ) -> Result<AvroSchema, ArrowError> {
504        let opts = options.unwrap_or_default();
505        let order = opts.null_order.unwrap_or_default();
506        let strip = opts.strip_metadata;
507        if !strip && let Some(json) = schema.metadata.get(SCHEMA_METADATA_KEY) {
508            return Ok(AvroSchema::new(json.clone()));
509        }
510        let mut name_gen = NameGenerator::default();
511        let fields_json = schema
512            .fields()
513            .iter()
514            .map(|f| arrow_field_to_avro(f, &mut name_gen, order, strip))
515            .collect::<Result<Vec<_>, _>>()?;
516        let record_name = schema
517            .metadata
518            .get(AVRO_NAME_METADATA_KEY)
519            .map_or(AVRO_ROOT_RECORD_DEFAULT_NAME, |s| s.as_str());
520        let mut record = JsonMap::with_capacity(schema.metadata.len() + 4);
521        record.insert("type".into(), Value::String("record".into()));
522        record.insert(
523            "name".into(),
524            Value::String(sanitise_avro_name(record_name)),
525        );
526        if let Some(ns) = schema.metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
527            record.insert("namespace".into(), Value::String(ns.clone()));
528        }
529        if let Some(doc) = schema.metadata.get(AVRO_DOC_METADATA_KEY) {
530            record.insert("doc".into(), Value::String(doc.clone()));
531        }
532        record.insert("fields".into(), Value::Array(fields_json));
533        extend_with_passthrough_metadata(&mut record, &schema.metadata);
534        let json_string = serde_json::to_string(&Value::Object(record))
535            .map_err(|e| ArrowError::SchemaError(format!("Serializing Avro JSON failed: {e}")))?;
536        Ok(AvroSchema::new(json_string))
537    }
538}
539
540/// A stack-allocated, fixed-size buffer for the prefix.
541#[derive(Debug, Copy, Clone)]
542pub(crate) struct Prefix {
543    buf: [u8; MAX_PREFIX_LEN],
544    len: u8,
545}
546
547impl Prefix {
548    #[inline]
549    pub(crate) fn as_slice(&self) -> &[u8] {
550        &self.buf[..self.len as usize]
551    }
552}
553
554/// Defines the strategy for generating the per-record prefix for an Avro binary stream.
555#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
556pub enum FingerprintStrategy {
557    /// Use the 64-bit Rabin fingerprint (default for single-object encoding).
558    #[default]
559    Rabin,
560    /// Use a Confluent Schema Registry 32-bit ID.
561    Id(u32),
562    /// Use an Apicurio Schema Registry 64-bit ID.
563    Id64(u64),
564    #[cfg(feature = "md5")]
565    /// Use the 128-bit MD5 fingerprint.
566    MD5,
567    #[cfg(feature = "sha256")]
568    /// Use the 256-bit SHA-256 fingerprint.
569    SHA256,
570}
571
572impl From<Fingerprint> for FingerprintStrategy {
573    fn from(f: Fingerprint) -> Self {
574        Self::from(&f)
575    }
576}
577
578impl From<FingerprintAlgorithm> for FingerprintStrategy {
579    fn from(f: FingerprintAlgorithm) -> Self {
580        match f {
581            FingerprintAlgorithm::Rabin => FingerprintStrategy::Rabin,
582            FingerprintAlgorithm::Id => FingerprintStrategy::Id(0),
583            FingerprintAlgorithm::Id64 => FingerprintStrategy::Id64(0),
584            #[cfg(feature = "md5")]
585            FingerprintAlgorithm::MD5 => FingerprintStrategy::MD5,
586            #[cfg(feature = "sha256")]
587            FingerprintAlgorithm::SHA256 => FingerprintStrategy::SHA256,
588        }
589    }
590}
591
592impl From<&Fingerprint> for FingerprintStrategy {
593    fn from(f: &Fingerprint) -> Self {
594        match f {
595            Fingerprint::Rabin(_) => FingerprintStrategy::Rabin,
596            Fingerprint::Id(_) => FingerprintStrategy::Id(0),
597            Fingerprint::Id64(_) => FingerprintStrategy::Id64(0),
598            #[cfg(feature = "md5")]
599            Fingerprint::MD5(_) => FingerprintStrategy::MD5,
600            #[cfg(feature = "sha256")]
601            Fingerprint::SHA256(_) => FingerprintStrategy::SHA256,
602        }
603    }
604}
605
606/// Supported fingerprint algorithms for Avro schema identification.
607/// For use with Confluent Schema Registry IDs, set to None.
608#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)]
609pub enum FingerprintAlgorithm {
610    /// 64‑bit CRC‑64‑AVRO Rabin fingerprint.
611    #[default]
612    Rabin,
613    /// Represents a 32 bit fingerprint not based on a hash algorithm, (e.g., a 32-bit Schema Registry ID.)
614    Id,
615    /// Represents a 64 bit fingerprint not based on a hash algorithm, (e.g., a 64-bit Schema Registry ID.)
616    Id64,
617    #[cfg(feature = "md5")]
618    /// 128-bit MD5 message digest.
619    MD5,
620    #[cfg(feature = "sha256")]
621    /// 256-bit SHA-256 digest.
622    SHA256,
623}
624
625/// Allow easy extraction of the algorithm used to create a fingerprint.
626impl From<&Fingerprint> for FingerprintAlgorithm {
627    fn from(fp: &Fingerprint) -> Self {
628        match fp {
629            Fingerprint::Rabin(_) => FingerprintAlgorithm::Rabin,
630            Fingerprint::Id(_) => FingerprintAlgorithm::Id,
631            Fingerprint::Id64(_) => FingerprintAlgorithm::Id64,
632            #[cfg(feature = "md5")]
633            Fingerprint::MD5(_) => FingerprintAlgorithm::MD5,
634            #[cfg(feature = "sha256")]
635            Fingerprint::SHA256(_) => FingerprintAlgorithm::SHA256,
636        }
637    }
638}
639
640impl From<FingerprintStrategy> for FingerprintAlgorithm {
641    fn from(s: FingerprintStrategy) -> Self {
642        Self::from(&s)
643    }
644}
645
646impl From<&FingerprintStrategy> for FingerprintAlgorithm {
647    fn from(s: &FingerprintStrategy) -> Self {
648        match s {
649            FingerprintStrategy::Rabin => FingerprintAlgorithm::Rabin,
650            FingerprintStrategy::Id(_) => FingerprintAlgorithm::Id,
651            FingerprintStrategy::Id64(_) => FingerprintAlgorithm::Id64,
652            #[cfg(feature = "md5")]
653            FingerprintStrategy::MD5 => FingerprintAlgorithm::MD5,
654            #[cfg(feature = "sha256")]
655            FingerprintStrategy::SHA256 => FingerprintAlgorithm::SHA256,
656        }
657    }
658}
659
660/// A schema fingerprint in one of the supported formats.
661///
662/// This is used as the key inside `SchemaStore` `HashMap`. Each `SchemaStore`
663/// instance always stores only one variant, matching its configured
664/// `FingerprintAlgorithm`, but the enum makes the API uniform.
665///
666/// <https://avro.apache.org/docs/1.11.1/specification/#schema-fingerprints>
667/// <https://docs.confluent.io/platform/current/schema-registry/fundamentals/serdes-develop/index.html#wire-format>
668#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
669pub enum Fingerprint {
670    /// A 64-bit Rabin fingerprint.
671    Rabin(u64),
672    /// A 32-bit Schema Registry ID.
673    Id(u32),
674    /// A 64-bit Schema Registry ID.
675    Id64(u64),
676    #[cfg(feature = "md5")]
677    /// A 128-bit MD5 fingerprint.
678    MD5([u8; 16]),
679    #[cfg(feature = "sha256")]
680    /// A 256-bit SHA-256 fingerprint.
681    SHA256([u8; 32]),
682}
683
684impl From<FingerprintStrategy> for Fingerprint {
685    fn from(s: FingerprintStrategy) -> Self {
686        Self::from(&s)
687    }
688}
689
690impl From<&FingerprintStrategy> for Fingerprint {
691    fn from(s: &FingerprintStrategy) -> Self {
692        match s {
693            FingerprintStrategy::Rabin => Fingerprint::Rabin(0),
694            FingerprintStrategy::Id(id) => Fingerprint::Id(*id),
695            FingerprintStrategy::Id64(id) => Fingerprint::Id64(*id),
696            #[cfg(feature = "md5")]
697            FingerprintStrategy::MD5 => Fingerprint::MD5([0; 16]),
698            #[cfg(feature = "sha256")]
699            FingerprintStrategy::SHA256 => Fingerprint::SHA256([0; 32]),
700        }
701    }
702}
703
704impl From<FingerprintAlgorithm> for Fingerprint {
705    fn from(s: FingerprintAlgorithm) -> Self {
706        match s {
707            FingerprintAlgorithm::Rabin => Fingerprint::Rabin(0),
708            FingerprintAlgorithm::Id => Fingerprint::Id(0),
709            FingerprintAlgorithm::Id64 => Fingerprint::Id64(0),
710            #[cfg(feature = "md5")]
711            FingerprintAlgorithm::MD5 => Fingerprint::MD5([0; 16]),
712            #[cfg(feature = "sha256")]
713            FingerprintAlgorithm::SHA256 => Fingerprint::SHA256([0; 32]),
714        }
715    }
716}
717
718impl Fingerprint {
719    /// Loads the 32-bit Schema Registry fingerprint (Confluent Schema Registry ID).
720    ///
721    /// The provided `id` is in big-endian wire order; this converts it to host order
722    /// and returns `Fingerprint::Id`.
723    ///
724    /// # Returns
725    /// A `Fingerprint::Id` variant containing the 32-bit fingerprint.
726    pub fn load_fingerprint_id(id: u32) -> Self {
727        Fingerprint::Id(u32::from_be(id))
728    }
729
730    /// Loads the 64-bit Schema Registry fingerprint (Apicurio Schema Registry ID).
731    ///
732    /// The provided `id` is in big-endian wire order; this converts it to host order
733    /// and returns `Fingerprint::Id64`.
734    ///
735    /// # Returns
736    /// A `Fingerprint::Id64` variant containing the 64-bit fingerprint.
737    pub fn load_fingerprint_id64(id: u64) -> Self {
738        Fingerprint::Id64(u64::from_be(id))
739    }
740
741    /// Constructs a serialized prefix represented as a `Vec<u8>` based on the variant of the enum.
742    ///
743    /// This method serializes data in different formats depending on the variant of `self`:
744    /// - **`Id(id)`**: Uses the Confluent wire format, which includes a predefined magic header (`CONFLUENT_MAGIC`)
745    ///   followed by the big-endian byte representation of the `id`.
746    /// - **`Id64(id)`**: Uses the Apicurio wire format, which includes a predefined magic header (`CONFLUENT_MAGIC`)
747    ///   followed by the big-endian 8-byte representation of the `id`.
748    /// - **`Rabin(val)`**: Uses the Avro single-object specification format. This includes a different magic header
749    ///   (`SINGLE_OBJECT_MAGIC`) followed by the little-endian byte representation of the `val`.
750    /// - **`MD5(bytes)`** (optional, `md5` feature enabled): A non-standard extension that adds the
751    ///   `SINGLE_OBJECT_MAGIC` header followed by the provided `bytes`.
752    /// - **`SHA256(bytes)`** (optional, `sha256` feature enabled): Similar to the `MD5` variant, this is
753    ///   a non-standard extension that attaches the `SINGLE_OBJECT_MAGIC` header followed by the given `bytes`.
754    ///
755    /// # Returns
756    ///
757    /// A `Prefix` containing the serialized prefix data.
758    ///
759    /// # Features
760    ///
761    /// - You can optionally enable the `md5` feature to include the `MD5` variant.
762    /// - You can optionally enable the `sha256` feature to include the `SHA256` variant.
763    ///
764    pub(crate) fn make_prefix(&self) -> Prefix {
765        let mut buf = [0u8; MAX_PREFIX_LEN];
766        let len = match self {
767            Self::Id(val) => write_prefix(&mut buf, &CONFLUENT_MAGIC, &val.to_be_bytes()),
768            Self::Id64(val) => write_prefix(&mut buf, &CONFLUENT_MAGIC, &val.to_be_bytes()),
769            Self::Rabin(val) => write_prefix(&mut buf, &SINGLE_OBJECT_MAGIC, &val.to_le_bytes()),
770            #[cfg(feature = "md5")]
771            Self::MD5(val) => write_prefix(&mut buf, &SINGLE_OBJECT_MAGIC, val),
772            #[cfg(feature = "sha256")]
773            Self::SHA256(val) => write_prefix(&mut buf, &SINGLE_OBJECT_MAGIC, val),
774        };
775        Prefix { buf, len }
776    }
777}
778
779fn write_prefix<const MAGIC_LEN: usize, const PAYLOAD_LEN: usize>(
780    buf: &mut [u8; MAX_PREFIX_LEN],
781    magic: &[u8; MAGIC_LEN],
782    payload: &[u8; PAYLOAD_LEN],
783) -> u8 {
784    debug_assert!(MAGIC_LEN + PAYLOAD_LEN <= MAX_PREFIX_LEN);
785    let total = MAGIC_LEN + PAYLOAD_LEN;
786    let prefix_slice = &mut buf[..total];
787    prefix_slice[..MAGIC_LEN].copy_from_slice(magic);
788    prefix_slice[MAGIC_LEN..total].copy_from_slice(payload);
789    total as u8
790}
791
792/// An in-memory cache of Avro schemas, indexed by their fingerprint.
793///
794/// `SchemaStore` provides a mechanism to store and retrieve Avro schemas efficiently.
795/// Each schema is associated with a unique [`Fingerprint`], which is generated based
796/// on the schema's canonical form and a specific hashing algorithm.
797///
798/// A `SchemaStore` instance is configured to use a single [`FingerprintAlgorithm`] such as Rabin,
799/// MD5 (not yet supported), or SHA256 (not yet supported) for all its operations.
800/// This ensures consistency when generating fingerprints and looking up schemas.
801/// All schemas registered will have their fingerprint computed with this algorithm, and
802/// lookups must use a matching fingerprint.
803///
804/// # Examples
805///
806/// ```no_run
807/// // Create a new store with the default Rabin fingerprinting.
808/// use arrow_avro::schema::{AvroSchema, SchemaStore};
809///
810/// let mut store = SchemaStore::new();
811/// let schema = AvroSchema::new("\"string\"".to_string());
812/// // Register the schema to get its fingerprint.
813/// let fingerprint = store.register(schema.clone()).unwrap();
814/// // Use the fingerprint to look up the schema.
815/// let retrieved_schema = store.lookup(&fingerprint).cloned();
816/// assert_eq!(retrieved_schema, Some(schema));
817/// ```
818#[derive(Debug, Clone, Default)]
819pub struct SchemaStore {
820    /// The hashing algorithm used for generating fingerprints.
821    fingerprint_algorithm: FingerprintAlgorithm,
822    /// A map from a schema's fingerprint to the schema itself.
823    schemas: HashMap<Fingerprint, AvroSchema>,
824}
825
826impl TryFrom<HashMap<Fingerprint, AvroSchema>> for SchemaStore {
827    type Error = ArrowError;
828
829    /// Creates a `SchemaStore` from a HashMap of schemas.
830    /// Each schema in the HashMap is registered with the new store.
831    fn try_from(schemas: HashMap<Fingerprint, AvroSchema>) -> Result<Self, Self::Error> {
832        Ok(Self {
833            schemas,
834            ..Self::default()
835        })
836    }
837}
838
839impl SchemaStore {
840    /// Creates an empty `SchemaStore` using the default fingerprinting algorithm (64-bit Rabin).
841    pub fn new() -> Self {
842        Self::default()
843    }
844
845    /// Creates an empty `SchemaStore` using the default fingerprinting algorithm (64-bit Rabin).
846    pub fn new_with_type(fingerprint_algorithm: FingerprintAlgorithm) -> Self {
847        Self {
848            fingerprint_algorithm,
849            ..Self::default()
850        }
851    }
852
853    /// Registers a schema with the store and the provided fingerprint.
854    /// Note: Confluent wire format implementations should leverage this method.
855    ///
856    /// A schema is set in the store, using the provided fingerprint. If a schema
857    /// with the same fingerprint does not already exist in the store, the new schema
858    /// is inserted. If the fingerprint already exists, the existing schema is not overwritten.
859    ///
860    /// # Arguments
861    ///
862    /// * `fingerprint` - A reference to the `Fingerprint` of the schema to register.
863    /// * `schema` - The `AvroSchema` to register.
864    ///
865    /// # Returns
866    ///
867    /// A `Result` returning the provided `Fingerprint` of the schema if successful,
868    /// or an `ArrowError` on failure.
869    pub fn set(
870        &mut self,
871        fingerprint: Fingerprint,
872        schema: AvroSchema,
873    ) -> Result<Fingerprint, ArrowError> {
874        match self.schemas.entry(fingerprint) {
875            Entry::Occupied(entry) => {
876                if entry.get() != &schema {
877                    return Err(ArrowError::ComputeError(format!(
878                        "Schema fingerprint collision detected for fingerprint {fingerprint:?}"
879                    )));
880                }
881            }
882            Entry::Vacant(entry) => {
883                entry.insert(schema);
884            }
885        }
886        Ok(fingerprint)
887    }
888
889    /// Registers a schema with the store and returns its fingerprint.
890    ///
891    /// A fingerprint is calculated for the given schema using the store's configured
892    /// hash type. If a schema with the same fingerprint does not already exist in the
893    /// store, the new schema is inserted. If the fingerprint already exists, the
894    /// existing schema is not overwritten. If FingerprintAlgorithm is set to Id or Id64, this
895    /// method will return an error. Confluent wire format implementations should leverage the
896    /// set method instead.
897    ///
898    /// # Arguments
899    ///
900    /// * `schema` - The `AvroSchema` to register.
901    ///
902    /// # Returns
903    ///
904    /// A `Result` containing the `Fingerprint` of the schema if successful,
905    /// or an `ArrowError` on failure.
906    pub fn register(&mut self, schema: AvroSchema) -> Result<Fingerprint, ArrowError> {
907        if self.fingerprint_algorithm == FingerprintAlgorithm::Id
908            || self.fingerprint_algorithm == FingerprintAlgorithm::Id64
909        {
910            return Err(ArrowError::SchemaError(
911                "Invalid FingerprintAlgorithm; unable to generate fingerprint. \
912            Use the set method directly instead, providing a valid fingerprint"
913                    .to_string(),
914            ));
915        }
916        let fingerprint =
917            AvroSchema::generate_fingerprint(&schema.schema()?, self.fingerprint_algorithm)?;
918        self.set(fingerprint, schema)?;
919        Ok(fingerprint)
920    }
921
922    /// Looks up a schema by its `Fingerprint`.
923    ///
924    /// # Arguments
925    ///
926    /// * `fingerprint` - A reference to the `Fingerprint` of the schema to look up.
927    ///
928    /// # Returns
929    ///
930    /// An `Option` containing a clone of the `AvroSchema` if found, otherwise `None`.
931    pub fn lookup(&self, fingerprint: &Fingerprint) -> Option<&AvroSchema> {
932        self.schemas.get(fingerprint)
933    }
934
935    /// Returns a `Vec` containing **all unique [`Fingerprint`]s** currently
936    /// held by this [`SchemaStore`].
937    ///
938    /// The order of the returned fingerprints is unspecified and should not be
939    /// relied upon.
940    pub fn fingerprints(&self) -> Vec<Fingerprint> {
941        self.schemas.keys().copied().collect()
942    }
943
944    /// Returns the `FingerprintAlgorithm` used by the `SchemaStore` for fingerprinting.
945    pub(crate) fn fingerprint_algorithm(&self) -> FingerprintAlgorithm {
946        self.fingerprint_algorithm
947    }
948}
949
950fn quote(s: &str) -> Result<String, ArrowError> {
951    serde_json::to_string(s)
952        .map_err(|e| ArrowError::ComputeError(format!("Failed to quote string: {e}")))
953}
954
955// Avro names are defined by a `name` and an optional `namespace`.
956// The full name is composed of the namespace and the name, separated by a dot.
957//
958// Avro specification defines two ways to specify a full name:
959// 1. The `name` attribute contains the full name (e.g., "a.b.c.d").
960//    In this case, the `namespace` attribute is ignored.
961// 2. The `name` attribute contains the simple name (e.g., "d") and the
962//    `namespace` attribute contains the namespace (e.g., "a.b.c").
963//
964// Each part of the name must match the regex `^[A-Za-z_][A-Za-z0-9_]*$`.
965// Complex paths with quotes or backticks like `a."hi".b` are not supported.
966//
967// This function constructs the full name and extracts the namespace,
968// handling both ways of specifying the name. It prioritizes a namespace
969// defined within the `name` attribute itself, then the explicit `namespace_attr`,
970// and finally the `enclosing_ns`.
971pub(crate) fn make_full_name(
972    name: &str,
973    namespace_attr: Option<&str>,
974    enclosing_ns: Option<&str>,
975) -> (String, Option<String>) {
976    // `name` already contains a dot then treat as full-name, ignore namespace.
977    if let Some((ns, _)) = name.rsplit_once('.') {
978        return (name.to_string(), Some(ns.to_string()));
979    }
980    match namespace_attr.or(enclosing_ns) {
981        Some(ns) => (format!("{ns}.{name}"), Some(ns.to_string())),
982        None => (name.to_string(), None),
983    }
984}
985
986fn build_canonical(schema: &Schema, enclosing_ns: Option<&str>) -> Result<String, ArrowError> {
987    Ok(match schema {
988        Schema::TypeName(tn) | Schema::Type(Type { r#type: tn, .. }) => match tn {
989            TypeName::Primitive(pt) => quote(pt.as_ref())?,
990            TypeName::Ref(name) => {
991                let (full_name, _) = make_full_name(name, None, enclosing_ns);
992                quote(&full_name)?
993            }
994        },
995        Schema::Union(branches) => format!(
996            "[{}]",
997            branches
998                .iter()
999                .map(|b| build_canonical(b, enclosing_ns))
1000                .collect::<Result<Vec<_>, _>>()?
1001                .join(",")
1002        ),
1003        Schema::Complex(ct) => match ct {
1004            ComplexType::Record(r) => {
1005                let (full_name, child_ns) = make_full_name(r.name, r.namespace, enclosing_ns);
1006                let fields = r
1007                    .fields
1008                    .iter()
1009                    .map(|f| {
1010                        // PCF [STRIP] per Avro spec: keep only attributes relevant to parsing
1011                        // ("name" and "type" for fields) and **strip others** such as doc,
1012                        // default, order, and **aliases**. This preserves canonicalization. See:
1013                        // https://avro.apache.org/docs/1.11.1/specification/#parsing-canonical-form-for-schemas
1014                        let field_type =
1015                            build_canonical(&f.r#type, child_ns.as_deref().or(enclosing_ns))?;
1016                        Ok(format!(
1017                            r#"{{"name":{},"type":{}}}"#,
1018                            quote(f.name)?,
1019                            field_type
1020                        ))
1021                    })
1022                    .collect::<Result<Vec<_>, ArrowError>>()?
1023                    .join(",");
1024                format!(
1025                    r#"{{"name":{},"type":"record","fields":[{fields}]}}"#,
1026                    quote(&full_name)?,
1027                )
1028            }
1029            ComplexType::Enum(e) => {
1030                let (full_name, _) = make_full_name(e.name, e.namespace, enclosing_ns);
1031                let symbols = e
1032                    .symbols
1033                    .iter()
1034                    .map(|s| quote(s))
1035                    .collect::<Result<Vec<_>, _>>()?
1036                    .join(",");
1037                format!(
1038                    r#"{{"name":{},"type":"enum","symbols":[{symbols}]}}"#,
1039                    quote(&full_name)?
1040                )
1041            }
1042            ComplexType::Array(arr) => format!(
1043                r#"{{"type":"array","items":{}}}"#,
1044                build_canonical(&arr.items, enclosing_ns)?
1045            ),
1046            ComplexType::Map(map) => format!(
1047                r#"{{"type":"map","values":{}}}"#,
1048                build_canonical(&map.values, enclosing_ns)?
1049            ),
1050            ComplexType::Fixed(f) => {
1051                let (full_name, _) = make_full_name(f.name, f.namespace, enclosing_ns);
1052                format!(
1053                    r#"{{"name":{},"type":"fixed","size":{}}}"#,
1054                    quote(&full_name)?,
1055                    f.size
1056                )
1057            }
1058        },
1059    })
1060}
1061
1062/// 64‑bit Rabin fingerprint as described in the Avro spec.
1063const EMPTY: u64 = 0xc15d_213a_a4d7_a795;
1064
1065/// Build one entry of the polynomial‑division table.
1066///
1067/// We cannot yet write `for _ in 0..8` here: `for` loops rely on
1068/// `Iterator::next`, which is not `const` on stable Rust.  Until the
1069/// `const_for` feature (tracking issue #87575) is stabilized, a `while`
1070/// loop is the only option in a `const fn`
1071const fn one_entry(i: usize) -> u64 {
1072    let mut fp = i as u64;
1073    let mut j = 0;
1074    while j < 8 {
1075        fp = (fp >> 1) ^ (EMPTY & (0u64.wrapping_sub(fp & 1)));
1076        j += 1;
1077    }
1078    fp
1079}
1080
1081/// Build the full 256‑entry table at compile time.
1082///
1083/// We cannot yet write `for _ in 0..256` here: `for` loops rely on
1084/// `Iterator::next`, which is not `const` on stable Rust.  Until the
1085/// `const_for` feature (tracking issue #87575) is stabilized, a `while`
1086/// loop is the only option in a `const fn`
1087const fn build_table() -> [u64; 256] {
1088    let mut table = [0u64; 256];
1089    let mut i = 0;
1090    while i < 256 {
1091        table[i] = one_entry(i);
1092        i += 1;
1093    }
1094    table
1095}
1096
1097/// The pre‑computed table.
1098static FINGERPRINT_TABLE: [u64; 256] = build_table();
1099
1100/// Computes the 64-bit Rabin fingerprint for a given canonical schema string.
1101/// This implementation is based on the Avro specification for schema fingerprinting.
1102pub(crate) fn compute_fingerprint_rabin(canonical_form: &str) -> u64 {
1103    let mut fp = EMPTY;
1104    for &byte in canonical_form.as_bytes() {
1105        let idx = ((fp as u8) ^ byte) as usize;
1106        fp = (fp >> 8) ^ FINGERPRINT_TABLE[idx];
1107    }
1108    fp
1109}
1110
1111#[cfg(feature = "md5")]
1112/// Compute the **128‑bit MD5** fingerprint of the canonical form.
1113///
1114/// Returns a 16‑byte array (`[u8; 16]`) containing the full MD5 digest,
1115/// exactly as required by the Avro specification.
1116#[inline]
1117pub(crate) fn compute_fingerprint_md5(canonical_form: &str) -> [u8; 16] {
1118    let digest = md5::compute(canonical_form.as_bytes());
1119    digest.0
1120}
1121
1122#[cfg(feature = "sha256")]
1123/// Compute the **256‑bit SHA‑256** fingerprint of the canonical form.
1124///
1125/// Returns a 32‑byte array (`[u8; 32]`) containing the full SHA‑256 digest.
1126#[inline]
1127pub(crate) fn compute_fingerprint_sha256(canonical_form: &str) -> [u8; 32] {
1128    let mut hasher = Sha256::new();
1129    hasher.update(canonical_form.as_bytes());
1130    let digest = hasher.finalize();
1131    digest.into()
1132}
1133
1134#[inline]
1135fn is_internal_arrow_key(key: &str) -> bool {
1136    key.starts_with("ARROW:") || key == SCHEMA_METADATA_KEY
1137}
1138
1139/// Copies Arrow schema metadata entries to the provided JSON map,
1140/// skipping keys that are Avro-reserved, internal Arrow keys, or
1141/// nested under the `avro.schema.` namespace. Values that parse as
1142/// JSON are inserted as JSON; otherwise the raw string is preserved.
1143fn extend_with_passthrough_metadata(target: &mut JsonMap<String, Value>, metadata: &Metadata) {
1144    for (meta_key, meta_val) in metadata {
1145        if meta_key.starts_with("avro.") || is_internal_arrow_key(meta_key) {
1146            continue;
1147        }
1148        let json_val =
1149            serde_json::from_str(meta_val).unwrap_or_else(|_| Value::String(meta_val.clone()));
1150        target.insert(meta_key.clone(), json_val);
1151    }
1152}
1153
1154// Sanitize an arbitrary string so it is a valid Avro field or type name
1155fn sanitise_avro_name(base_name: &str) -> String {
1156    if base_name.is_empty() {
1157        return "_".to_owned();
1158    }
1159    let mut out: String = base_name
1160        .chars()
1161        .map(|char| {
1162            if char.is_ascii_alphanumeric() || char == '_' {
1163                char
1164            } else {
1165                '_'
1166            }
1167        })
1168        .collect();
1169    if out.as_bytes()[0].is_ascii_digit() {
1170        out.insert(0, '_');
1171    }
1172    out
1173}
1174
1175#[derive(Default)]
1176struct NameGenerator {
1177    used: HashSet<String>,
1178    counters: HashMap<String, usize>,
1179}
1180
1181impl NameGenerator {
1182    fn make_unique(&mut self, field_name: &str) -> String {
1183        let field_name = sanitise_avro_name(field_name);
1184        if self.used.insert(field_name.clone()) {
1185            self.counters.insert(field_name.clone(), 1);
1186            return field_name;
1187        }
1188        let counter = self.counters.entry(field_name.clone()).or_insert(1);
1189        loop {
1190            let candidate = format!("{field_name}_{}", *counter);
1191            if self.used.insert(candidate.clone()) {
1192                return candidate;
1193            }
1194            *counter += 1;
1195        }
1196    }
1197}
1198
1199fn merge_extras(schema: Value, extras: JsonMap<String, Value>) -> Value {
1200    if extras.is_empty() {
1201        return schema;
1202    }
1203    match schema {
1204        Value::Object(mut map) => {
1205            map.extend(extras);
1206            Value::Object(map)
1207        }
1208        Value::Array(mut union) => {
1209            // For unions, we cannot attach attributes to the array itself (per Avro spec).
1210            // As a fallback for extension metadata, attach extras to the first non-null branch object.
1211            if let Some(non_null) = union.iter_mut().find(|val| val.as_str() != Some("null")) {
1212                let original = std::mem::take(non_null);
1213                *non_null = merge_extras(original, extras);
1214            }
1215            Value::Array(union)
1216        }
1217        primitive => {
1218            let mut map = JsonMap::with_capacity(extras.len() + 1);
1219            map.insert("type".into(), primitive);
1220            map.extend(extras);
1221            Value::Object(map)
1222        }
1223    }
1224}
1225
1226#[inline]
1227fn is_avro_json_null(v: &Value) -> bool {
1228    matches!(v, Value::String(s) if s == "null")
1229}
1230
1231fn wrap_nullable(inner: Value, null_order: Nullability) -> Value {
1232    let null = Value::String("null".into());
1233    match inner {
1234        Value::Array(mut union) => {
1235            // If this site is already a union and already contains "null",
1236            // preserve the branch order exactly. Reordering "null" breaks
1237            // the correspondence between Arrow union child order (type_ids)
1238            // and the Avro branch index written on the wire.
1239            if union.iter().any(is_avro_json_null) {
1240                return Value::Array(union);
1241            }
1242            // Otherwise, inject "null" without reordering existing branches.
1243            match null_order {
1244                Nullability::NullFirst => union.insert(0, null),
1245                Nullability::NullSecond => union.push(null),
1246            }
1247            Value::Array(union)
1248        }
1249        other => match null_order {
1250            Nullability::NullFirst => Value::Array(vec![null, other]),
1251            Nullability::NullSecond => Value::Array(vec![other, null]),
1252        },
1253    }
1254}
1255
1256fn min_fixed_bytes_for_precision(p: usize) -> usize {
1257    // From the spec: max precision for n=1..=32 bytes:
1258    // [2,4,6,9,11,14,16,18,21,23,26,28,31,33,35,38,40,43,45,47,50,52,55,57,59,62,64,67,69,71,74,76]
1259    const MAX_P: [usize; 32] = [
1260        2, 4, 6, 9, 11, 14, 16, 18, 21, 23, 26, 28, 31, 33, 35, 38, 40, 43, 45, 47, 50, 52, 55, 57,
1261        59, 62, 64, 67, 69, 71, 74, 76,
1262    ];
1263    for (i, &max_p) in MAX_P.iter().enumerate() {
1264        if p <= max_p {
1265            return i + 1;
1266        }
1267    }
1268    32 // saturate at Decimal256
1269}
1270
1271fn union_branch_signature(branch: &Value) -> Result<String, ArrowError> {
1272    match branch {
1273        Value::String(t) => Ok(format!("P:{t}")),
1274        Value::Object(map) => {
1275            let t = map.get("type").and_then(|v| v.as_str()).ok_or_else(|| {
1276                ArrowError::SchemaError("Union branch object missing string 'type'".into())
1277            })?;
1278            match t {
1279                "record" | "enum" | "fixed" => {
1280                    let name = map.get("name").and_then(|v| v.as_str()).ok_or_else(|| {
1281                        ArrowError::SchemaError(format!(
1282                            "Union branch '{t}' missing required 'name'"
1283                        ))
1284                    })?;
1285                    Ok(format!("N:{t}:{name}"))
1286                }
1287                "array" | "map" => Ok(format!("C:{t}")),
1288                other => Ok(format!("P:{other}")),
1289            }
1290        }
1291        Value::Array(_) => Err(ArrowError::SchemaError(
1292            "Avro union may not immediately contain another union".into(),
1293        )),
1294        _ => Err(ArrowError::SchemaError(
1295            "Invalid JSON for Avro union branch".into(),
1296        )),
1297    }
1298}
1299
1300fn datatype_to_avro(
1301    dt: &DataType,
1302    field_name: &str,
1303    metadata: &Metadata,
1304    name_gen: &mut NameGenerator,
1305    null_order: Nullability,
1306    strip: bool,
1307) -> Result<(Value, JsonMap<String, Value>), ArrowError> {
1308    let mut extras = JsonMap::new();
1309    let mut handle_decimal = |precision: &u8, scale: &i8| -> Result<Value, ArrowError> {
1310        if *scale < 0 {
1311            return Err(ArrowError::SchemaError(format!(
1312                "Invalid Avro decimal for field '{field_name}': scale ({scale}) must be >= 0"
1313            )));
1314        }
1315        if (*scale as usize) > (*precision as usize) {
1316            return Err(ArrowError::SchemaError(format!(
1317                "Invalid Avro decimal for field '{field_name}': scale ({scale}) \
1318                 must be <= precision ({precision})"
1319            )));
1320        }
1321        let mut meta = JsonMap::from_iter([
1322            ("logicalType".into(), json!("decimal")),
1323            ("precision".into(), json!(*precision)),
1324            ("scale".into(), json!(*scale)),
1325        ]);
1326        let mut fixed_size = metadata.get("size").and_then(|v| v.parse::<usize>().ok());
1327        let carries_name = metadata.contains_key(AVRO_NAME_METADATA_KEY)
1328            || metadata.contains_key(AVRO_NAMESPACE_METADATA_KEY);
1329        if fixed_size.is_none() && carries_name {
1330            fixed_size = Some(min_fixed_bytes_for_precision(*precision as usize));
1331        }
1332        if let Some(size) = fixed_size {
1333            meta.insert("type".into(), json!("fixed"));
1334            meta.insert("size".into(), json!(size));
1335            let chosen_name = metadata
1336                .get(AVRO_NAME_METADATA_KEY)
1337                .map(|s| sanitise_avro_name(s))
1338                .unwrap_or_else(|| name_gen.make_unique(field_name));
1339            meta.insert("name".into(), json!(chosen_name));
1340            if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1341                meta.insert("namespace".into(), json!(ns));
1342            }
1343        } else {
1344            // default to bytes-backed decimal
1345            meta.insert("type".into(), json!("bytes"));
1346        }
1347        Ok(Value::Object(meta))
1348    };
1349    let val = match dt {
1350        DataType::Null => Value::String("null".into()),
1351        DataType::Boolean => Value::String("boolean".into()),
1352        #[cfg(not(feature = "avro_custom_types"))]
1353        DataType::Int8 | DataType::Int16 | DataType::UInt8 | DataType::UInt16 => {
1354            Value::String("int".into())
1355        }
1356        DataType::Int32 => Value::String("int".into()),
1357        #[cfg(feature = "avro_custom_types")]
1358        DataType::Int8 => json!({ "type": "int", "logicalType": "arrow.int8" }),
1359        #[cfg(feature = "avro_custom_types")]
1360        DataType::Int16 => json!({ "type": "int", "logicalType": "arrow.int16" }),
1361        #[cfg(feature = "avro_custom_types")]
1362        DataType::UInt8 => json!({ "type": "int", "logicalType": "arrow.uint8" }),
1363        #[cfg(feature = "avro_custom_types")]
1364        DataType::UInt16 => json!({ "type": "int", "logicalType": "arrow.uint16" }),
1365        #[cfg(not(feature = "avro_custom_types"))]
1366        DataType::UInt32 => Value::String("long".into()),
1367        #[cfg(feature = "avro_custom_types")]
1368        DataType::UInt32 => json!({ "type": "long", "logicalType": "arrow.uint32" }),
1369        DataType::Int64 => Value::String("long".into()),
1370        #[cfg(not(feature = "avro_custom_types"))]
1371        DataType::UInt64 => Value::String("long".into()),
1372        #[cfg(feature = "avro_custom_types")]
1373        DataType::UInt64 => {
1374            // UInt64 must use fixed(8) to avoid overflow
1375            let chosen_name = metadata
1376                .get(AVRO_NAME_METADATA_KEY)
1377                .map(|s| sanitise_avro_name(s))
1378                .unwrap_or_else(|| name_gen.make_unique(field_name));
1379            let mut obj = JsonMap::from_iter([
1380                ("type".into(), json!("fixed")),
1381                ("name".into(), json!(chosen_name)),
1382                ("size".into(), json!(8)),
1383                ("logicalType".into(), json!("arrow.uint64")),
1384            ]);
1385            if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1386                obj.insert("namespace".into(), json!(ns));
1387            }
1388            json!(obj)
1389        }
1390        #[cfg(not(feature = "avro_custom_types"))]
1391        DataType::Float16 => Value::String("float".into()),
1392        #[cfg(feature = "avro_custom_types")]
1393        DataType::Float16 => {
1394            // Float16 uses fixed(2) for IEEE-754 bits
1395            let chosen_name = metadata
1396                .get(AVRO_NAME_METADATA_KEY)
1397                .map(|s| sanitise_avro_name(s))
1398                .unwrap_or_else(|| name_gen.make_unique(field_name));
1399            let mut obj = JsonMap::from_iter([
1400                ("type".into(), json!("fixed")),
1401                ("name".into(), json!(chosen_name)),
1402                ("size".into(), json!(2)),
1403                ("logicalType".into(), json!("arrow.float16")),
1404            ]);
1405            if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1406                obj.insert("namespace".into(), json!(ns));
1407            }
1408            json!(obj)
1409        }
1410        DataType::Float32 => Value::String("float".into()),
1411        DataType::Float64 => Value::String("double".into()),
1412        DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Value::String("string".into()),
1413        DataType::Binary | DataType::LargeBinary => Value::String("bytes".into()),
1414        DataType::BinaryView => {
1415            if !strip {
1416                extras.insert("arrowBinaryView".into(), Value::Bool(true));
1417            }
1418            Value::String("bytes".into())
1419        }
1420        DataType::FixedSizeBinary(len) => {
1421            let md_is_uuid = metadata
1422                .get("logicalType")
1423                .map(|s| s.trim_matches('"') == "uuid")
1424                .unwrap_or(false);
1425            #[cfg(feature = "canonical_extension_types")]
1426            let ext_is_uuid = metadata
1427                .get(arrow_schema::extension::EXTENSION_TYPE_NAME_KEY)
1428                .map(|v| v == arrow_schema::extension::Uuid::NAME || v == "uuid")
1429                .unwrap_or(false);
1430            #[cfg(not(feature = "canonical_extension_types"))]
1431            let ext_is_uuid = false;
1432            let is_uuid = (*len == 16) && (md_is_uuid || ext_is_uuid);
1433            if is_uuid {
1434                json!({ "type": "string", "logicalType": "uuid" })
1435            } else {
1436                let chosen_name = metadata
1437                    .get(AVRO_NAME_METADATA_KEY)
1438                    .map(|s| sanitise_avro_name(s))
1439                    .unwrap_or_else(|| name_gen.make_unique(field_name));
1440                let mut obj = JsonMap::from_iter([
1441                    ("type".into(), json!("fixed")),
1442                    ("name".into(), json!(chosen_name)),
1443                    ("size".into(), json!(len)),
1444                ]);
1445                if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1446                    obj.insert("namespace".into(), json!(ns));
1447                }
1448                Value::Object(obj)
1449            }
1450        }
1451        #[cfg(feature = "small_decimals")]
1452        DataType::Decimal32(precision, scale) | DataType::Decimal64(precision, scale) => {
1453            handle_decimal(precision, scale)?
1454        }
1455        DataType::Decimal128(precision, scale) | DataType::Decimal256(precision, scale) => {
1456            handle_decimal(precision, scale)?
1457        }
1458        DataType::Date32 => json!({ "type": "int", "logicalType": "date" }),
1459        #[cfg(not(feature = "avro_custom_types"))]
1460        DataType::Date64 => json!({ "type": "long", "logicalType": "local-timestamp-millis" }),
1461        #[cfg(feature = "avro_custom_types")]
1462        DataType::Date64 => json!({ "type": "long", "logicalType": "arrow.date64" }),
1463        DataType::Time32(unit) => match unit {
1464            TimeUnit::Millisecond => json!({ "type": "int", "logicalType": "time-millis" }),
1465            #[cfg(not(feature = "avro_custom_types"))]
1466            TimeUnit::Second => {
1467                // Encoder converts seconds to milliseconds, so use time-millis
1468                if !strip {
1469                    extras.insert("arrowTimeUnit".into(), Value::String("second".into()));
1470                }
1471                json!({ "type": "int", "logicalType": "time-millis" })
1472            }
1473            #[cfg(feature = "avro_custom_types")]
1474            TimeUnit::Second => {
1475                json!({ "type": "int", "logicalType": "arrow.time32-second" })
1476            }
1477            _ => Value::String("int".into()),
1478        },
1479        DataType::Time64(unit) => match unit {
1480            TimeUnit::Microsecond => json!({ "type": "long", "logicalType": "time-micros" }),
1481            #[cfg(not(feature = "avro_custom_types"))]
1482            TimeUnit::Nanosecond => {
1483                // Encoder truncates nanoseconds to microseconds, so use time-micros
1484                if !strip {
1485                    extras.insert("arrowTimeUnit".into(), Value::String("nanosecond".into()));
1486                }
1487                json!({ "type": "long", "logicalType": "time-micros" })
1488            }
1489            #[cfg(feature = "avro_custom_types")]
1490            TimeUnit::Nanosecond => {
1491                json!({ "type": "long", "logicalType": "arrow.time64-nanosecond" })
1492            }
1493            _ => Value::String("long".into()),
1494        },
1495        DataType::Timestamp(unit, tz) => {
1496            #[cfg(feature = "avro_custom_types")]
1497            if matches!(unit, TimeUnit::Second) {
1498                let logical_type = if tz.is_some() {
1499                    "arrow.timestamp-second"
1500                } else {
1501                    "arrow.local-timestamp-second"
1502                };
1503                return Ok((
1504                    json!({ "type": "long", "logicalType": logical_type }),
1505                    extras,
1506                ));
1507            }
1508            let logical_type = match (unit, tz.is_some()) {
1509                (TimeUnit::Millisecond, true) => "timestamp-millis",
1510                (TimeUnit::Millisecond, false) => "local-timestamp-millis",
1511                (TimeUnit::Microsecond, true) => "timestamp-micros",
1512                (TimeUnit::Microsecond, false) => "local-timestamp-micros",
1513                (TimeUnit::Nanosecond, true) => "timestamp-nanos",
1514                (TimeUnit::Nanosecond, false) => "local-timestamp-nanos",
1515                (TimeUnit::Second, has_tz) => {
1516                    // Encoder converts seconds to milliseconds, so use timestamp-millis
1517                    if !strip {
1518                        extras.insert("arrowTimeUnit".into(), Value::String("second".into()));
1519                    }
1520                    let ts_logical_type = if has_tz {
1521                        "timestamp-millis"
1522                    } else {
1523                        "local-timestamp-millis"
1524                    };
1525                    return Ok((
1526                        json!({ "type": "long", "logicalType": ts_logical_type }),
1527                        extras,
1528                    ));
1529                }
1530            };
1531            if !strip && matches!(unit, TimeUnit::Nanosecond) {
1532                extras.insert("arrowTimeUnit".into(), Value::String("nanosecond".into()));
1533            }
1534            json!({ "type": "long", "logicalType": logical_type })
1535        }
1536        #[cfg(not(feature = "avro_custom_types"))]
1537        DataType::Duration(_unit) => Value::String("long".into()),
1538        #[cfg(feature = "avro_custom_types")]
1539        DataType::Duration(unit) => {
1540            // When the feature is enabled, create an Avro schema object
1541            // with the correct `logicalType` annotation.
1542            let logical_type = match unit {
1543                TimeUnit::Second => "arrow.duration-seconds",
1544                TimeUnit::Millisecond => "arrow.duration-millis",
1545                TimeUnit::Microsecond => "arrow.duration-micros",
1546                TimeUnit::Nanosecond => "arrow.duration-nanos",
1547            };
1548            json!({ "type": "long", "logicalType": logical_type })
1549        }
1550        #[cfg(not(feature = "avro_custom_types"))]
1551        DataType::Interval(IntervalUnit::MonthDayNano) => {
1552            // Avro duration logical type: fixed(12) with months/days/millis per spec.
1553            let chosen_name = metadata
1554                .get(AVRO_NAME_METADATA_KEY)
1555                .map(|s| sanitise_avro_name(s))
1556                .unwrap_or_else(|| name_gen.make_unique(field_name));
1557            let mut obj = JsonMap::from_iter([
1558                ("type".into(), json!("fixed")),
1559                ("name".into(), json!(chosen_name)),
1560                ("size".into(), json!(12)),
1561                ("logicalType".into(), json!("duration")),
1562            ]);
1563            if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1564                obj.insert("namespace".into(), json!(ns));
1565            }
1566            json!(obj)
1567        }
1568        #[cfg(feature = "avro_custom_types")]
1569        DataType::Interval(IntervalUnit::MonthDayNano) => {
1570            // Arrow MonthDayNano interval: i32 months + i32 days + i64 nanos (16 bytes).
1571            // We preserve the Arrow native representation via a custom logical type.
1572            let chosen_name = metadata
1573                .get(AVRO_NAME_METADATA_KEY)
1574                .map(|s| sanitise_avro_name(s))
1575                .unwrap_or_else(|| name_gen.make_unique(field_name));
1576            let mut obj = JsonMap::from_iter([
1577                ("type".into(), json!("fixed")),
1578                ("name".into(), json!(chosen_name)),
1579                ("size".into(), json!(16)),
1580                ("logicalType".into(), json!("arrow.interval-month-day-nano")),
1581            ]);
1582            if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1583                obj.insert("namespace".into(), json!(ns));
1584            }
1585            json!(obj)
1586        }
1587        #[cfg(not(feature = "avro_custom_types"))]
1588        DataType::Interval(IntervalUnit::YearMonth) => {
1589            // Encode as Avro `duration` (fixed(12)) like MonthDayNano
1590            let chosen_name = metadata
1591                .get(AVRO_NAME_METADATA_KEY)
1592                .map(|s| sanitise_avro_name(s))
1593                .unwrap_or_else(|| name_gen.make_unique(field_name));
1594            let mut extras = JsonMap::from_iter([
1595                ("type".into(), json!("fixed")),
1596                ("name".into(), json!(chosen_name)),
1597                ("size".into(), json!(12)),
1598                ("logicalType".into(), json!("duration")),
1599            ]);
1600            if !strip {
1601                extras.insert(
1602                    "arrowIntervalUnit".into(),
1603                    Value::String("yearmonth".into()),
1604                );
1605            }
1606            if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1607                extras.insert("namespace".into(), json!(ns));
1608            }
1609            json!(extras)
1610        }
1611        #[cfg(feature = "avro_custom_types")]
1612        DataType::Interval(IntervalUnit::YearMonth) => {
1613            let chosen_name = metadata
1614                .get(AVRO_NAME_METADATA_KEY)
1615                .map(|s| sanitise_avro_name(s))
1616                .unwrap_or_else(|| name_gen.make_unique(field_name));
1617            let mut obj = JsonMap::from_iter([
1618                ("type".into(), json!("fixed")),
1619                ("name".into(), json!(chosen_name)),
1620                ("size".into(), json!(4)),
1621                ("logicalType".into(), json!("arrow.interval-year-month")),
1622            ]);
1623            if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1624                obj.insert("namespace".into(), json!(ns));
1625            }
1626            json!(obj)
1627        }
1628        #[cfg(not(feature = "avro_custom_types"))]
1629        DataType::Interval(IntervalUnit::DayTime) => {
1630            // Encode as Avro `duration` (fixed(12)) like MonthDayNano
1631            let chosen_name = metadata
1632                .get(AVRO_NAME_METADATA_KEY)
1633                .map(|s| sanitise_avro_name(s))
1634                .unwrap_or_else(|| name_gen.make_unique(field_name));
1635            let mut obj = JsonMap::from_iter([
1636                ("type".into(), json!("fixed")),
1637                ("name".into(), json!(chosen_name)),
1638                ("size".into(), json!(12)),
1639                ("logicalType".into(), json!("duration")),
1640            ]);
1641            if !strip {
1642                obj.insert("arrowIntervalUnit".into(), Value::String("daytime".into()));
1643            }
1644            if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1645                obj.insert("namespace".into(), json!(ns));
1646            }
1647            json!(obj)
1648        }
1649        #[cfg(feature = "avro_custom_types")]
1650        DataType::Interval(IntervalUnit::DayTime) => {
1651            let chosen_name = metadata
1652                .get(AVRO_NAME_METADATA_KEY)
1653                .map(|s| sanitise_avro_name(s))
1654                .unwrap_or_else(|| name_gen.make_unique(field_name));
1655            let mut obj = JsonMap::from_iter([
1656                ("type".into(), json!("fixed")),
1657                ("name".into(), json!(chosen_name)),
1658                ("size".into(), json!(8)),
1659                ("logicalType".into(), json!("arrow.interval-day-time")),
1660            ]);
1661            if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1662                obj.insert("namespace".into(), json!(ns));
1663            }
1664            json!(obj)
1665        }
1666        DataType::List(child) | DataType::LargeList(child) => {
1667            if matches!(dt, DataType::LargeList(_)) && !strip {
1668                extras.insert("arrowLargeList".into(), Value::Bool(true));
1669            }
1670            let items_schema = process_datatype(
1671                child.data_type(),
1672                child.name(),
1673                child.metadata(),
1674                name_gen,
1675                null_order,
1676                child.is_nullable(),
1677                strip,
1678            )?;
1679            json!({
1680                "type": "array",
1681                "items": items_schema
1682            })
1683        }
1684        DataType::ListView(child) | DataType::LargeListView(child) => {
1685            if matches!(dt, DataType::LargeListView(_)) && !strip {
1686                extras.insert("arrowLargeList".into(), Value::Bool(true));
1687            }
1688            if !strip {
1689                extras.insert("arrowListView".into(), Value::Bool(true));
1690            }
1691            let items_schema = process_datatype(
1692                child.data_type(),
1693                child.name(),
1694                child.metadata(),
1695                name_gen,
1696                null_order,
1697                child.is_nullable(),
1698                strip,
1699            )?;
1700            json!({
1701                "type": "array",
1702                "items": items_schema
1703            })
1704        }
1705        DataType::FixedSizeList(child, len) => {
1706            if !strip {
1707                extras.insert("arrowFixedSize".into(), json!(len));
1708            }
1709            let items_schema = process_datatype(
1710                child.data_type(),
1711                child.name(),
1712                child.metadata(),
1713                name_gen,
1714                null_order,
1715                child.is_nullable(),
1716                strip,
1717            )?;
1718            json!({
1719                "type": "array",
1720                "items": items_schema
1721            })
1722        }
1723        DataType::Map(entries, _) => {
1724            let value_field = match entries.data_type() {
1725                DataType::Struct(fs) => &fs[1],
1726                _ => {
1727                    return Err(ArrowError::SchemaError(
1728                        "Map 'entries' field must be Struct(key,value)".into(),
1729                    ));
1730                }
1731            };
1732            let values_schema = process_datatype(
1733                value_field.data_type(),
1734                value_field.name(),
1735                value_field.metadata(),
1736                name_gen,
1737                null_order,
1738                value_field.is_nullable(),
1739                strip,
1740            )?;
1741            json!({
1742                "type": "map",
1743                "values": values_schema
1744            })
1745        }
1746        DataType::Struct(fields) => {
1747            let avro_fields = fields
1748                .iter()
1749                .map(|field| arrow_field_to_avro(field, name_gen, null_order, strip))
1750                .collect::<Result<Vec<_>, _>>()?;
1751            // Prefer avro.name/avro.namespace when provided on the struct field metadata
1752            let chosen_name = metadata
1753                .get(AVRO_NAME_METADATA_KEY)
1754                .map(|s| sanitise_avro_name(s))
1755                .unwrap_or_else(|| name_gen.make_unique(field_name));
1756            let mut obj = JsonMap::from_iter([
1757                ("type".into(), json!("record")),
1758                ("name".into(), json!(chosen_name)),
1759                ("fields".into(), Value::Array(avro_fields)),
1760            ]);
1761            if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1762                obj.insert("namespace".into(), json!(ns));
1763            }
1764            Value::Object(obj)
1765        }
1766        DataType::Dictionary(_, value) => {
1767            if let Some(j) = metadata.get(AVRO_ENUM_SYMBOLS_METADATA_KEY) {
1768                let symbols: Vec<&str> =
1769                    serde_json::from_str(j).map_err(|e| ArrowError::ParseError(e.to_string()))?;
1770                // Prefer avro.name/namespace when provided for enums
1771                let chosen_name = metadata
1772                    .get(AVRO_NAME_METADATA_KEY)
1773                    .map(|s| sanitise_avro_name(s))
1774                    .unwrap_or_else(|| name_gen.make_unique(field_name));
1775                let mut obj = JsonMap::from_iter([
1776                    ("type".into(), json!("enum")),
1777                    ("name".into(), json!(chosen_name)),
1778                    ("symbols".into(), json!(symbols)),
1779                ]);
1780                if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1781                    obj.insert("namespace".into(), json!(ns));
1782                }
1783                Value::Object(obj)
1784            } else {
1785                process_datatype(
1786                    value.as_ref(),
1787                    field_name,
1788                    metadata,
1789                    name_gen,
1790                    null_order,
1791                    false,
1792                    strip,
1793                )?
1794            }
1795        }
1796        #[cfg(feature = "avro_custom_types")]
1797        DataType::RunEndEncoded(run_ends, values) => {
1798            let bits = match run_ends.data_type() {
1799                DataType::Int16 => 16,
1800                DataType::Int32 => 32,
1801                DataType::Int64 => 64,
1802                other => {
1803                    return Err(ArrowError::SchemaError(format!(
1804                        "RunEndEncoded requires Int16/Int32/Int64 for run_ends, found: {other:?}"
1805                    )));
1806                }
1807            };
1808            // Build the value site schema, preserving its own nullability
1809            let (value_schema, value_extras) = datatype_to_avro(
1810                values.data_type(),
1811                values.name(),
1812                values.metadata(),
1813                name_gen,
1814                null_order,
1815                strip,
1816            )?;
1817            let mut merged = merge_extras(value_schema, value_extras);
1818            if values.is_nullable() {
1819                merged = wrap_nullable(merged, null_order);
1820            }
1821            let mut extras = JsonMap::new();
1822            extras.insert("logicalType".into(), json!("arrow.run-end-encoded"));
1823            extras.insert("arrow.runEndIndexBits".into(), json!(bits));
1824            return Ok((merged, extras));
1825        }
1826        #[cfg(not(feature = "avro_custom_types"))]
1827        DataType::RunEndEncoded(_run_ends, values) => {
1828            let (value_schema, _extras) = datatype_to_avro(
1829                values.data_type(),
1830                values.name(),
1831                values.metadata(),
1832                name_gen,
1833                null_order,
1834                strip,
1835            )?;
1836            return Ok((value_schema, JsonMap::new()));
1837        }
1838        DataType::Union(fields, mode) => {
1839            let mut branches: Vec<Value> = Vec::with_capacity(fields.len());
1840            let mut type_ids: Vec<i32> = Vec::with_capacity(fields.len());
1841            for (type_id, field_ref) in fields.iter() {
1842                // NOTE: `process_datatype` would wrap nullability; force is_nullable=false here.
1843                let (branch_schema, _branch_extras) = datatype_to_avro(
1844                    field_ref.data_type(),
1845                    field_ref.name(),
1846                    field_ref.metadata(),
1847                    name_gen,
1848                    null_order,
1849                    strip,
1850                )?;
1851                // Avro unions cannot immediately contain another union
1852                if matches!(branch_schema, Value::Array(_)) {
1853                    return Err(ArrowError::SchemaError(
1854                        "Avro union may not immediately contain another union".into(),
1855                    ));
1856                }
1857                branches.push(branch_schema);
1858                type_ids.push(type_id as i32);
1859            }
1860            let mut seen: HashSet<String> = HashSet::with_capacity(branches.len());
1861            for b in &branches {
1862                let sig = union_branch_signature(b)?;
1863                if !seen.insert(sig) {
1864                    return Err(ArrowError::SchemaError(
1865                        "Avro union contains duplicate branch types (disallowed by spec)".into(),
1866                    ));
1867                }
1868            }
1869            if !strip {
1870                extras.insert(
1871                    "arrowUnionMode".into(),
1872                    Value::String(
1873                        match mode {
1874                            UnionMode::Sparse => "sparse",
1875                            UnionMode::Dense => "dense",
1876                        }
1877                        .to_string(),
1878                    ),
1879                );
1880                extras.insert(
1881                    "arrowUnionTypeIds".into(),
1882                    Value::Array(type_ids.into_iter().map(|id| json!(id)).collect()),
1883                );
1884            }
1885            Value::Array(branches)
1886        }
1887        #[cfg(not(feature = "small_decimals"))]
1888        other => {
1889            return Err(ArrowError::NotYetImplemented(format!(
1890                "Arrow type {other:?} has no Avro representation"
1891            )));
1892        }
1893    };
1894    Ok((val, extras))
1895}
1896
1897fn process_datatype(
1898    dt: &DataType,
1899    field_name: &str,
1900    metadata: &Metadata,
1901    name_gen: &mut NameGenerator,
1902    null_order: Nullability,
1903    is_nullable: bool,
1904    strip: bool,
1905) -> Result<Value, ArrowError> {
1906    let (schema, extras) = datatype_to_avro(dt, field_name, metadata, name_gen, null_order, strip)?;
1907    let mut merged = merge_extras(schema, extras);
1908    if is_nullable {
1909        merged = wrap_nullable(merged, null_order)
1910    }
1911    Ok(merged)
1912}
1913
1914fn arrow_field_to_avro(
1915    field: &ArrowField,
1916    name_gen: &mut NameGenerator,
1917    null_order: Nullability,
1918    strip: bool,
1919) -> Result<Value, ArrowError> {
1920    let avro_name = sanitise_avro_name(field.name());
1921    let schema_value = process_datatype(
1922        field.data_type(),
1923        &avro_name,
1924        field.metadata(),
1925        name_gen,
1926        null_order,
1927        field.is_nullable(),
1928        strip,
1929    )?;
1930    // Build the field map
1931    let mut map = JsonMap::with_capacity(field.metadata().len() + 3);
1932    map.insert("name".into(), Value::String(avro_name));
1933    map.insert("type".into(), schema_value);
1934    // Transfer selected metadata
1935    for (meta_key, meta_val) in field.metadata() {
1936        if is_internal_arrow_key(meta_key) {
1937            continue;
1938        }
1939        match meta_key.as_str() {
1940            AVRO_DOC_METADATA_KEY => {
1941                map.insert("doc".into(), Value::String(meta_val.clone()));
1942            }
1943            AVRO_FIELD_DEFAULT_METADATA_KEY => {
1944                let default_value = serde_json::from_str(meta_val)
1945                    .unwrap_or_else(|_| Value::String(meta_val.clone()));
1946                map.insert("default".into(), default_value);
1947            }
1948            _ => {
1949                let json_val = serde_json::from_str(meta_val)
1950                    .unwrap_or_else(|_| Value::String(meta_val.clone()));
1951                map.insert(meta_key.clone(), json_val);
1952            }
1953        }
1954    }
1955    Ok(Value::Object(map))
1956}
1957
1958#[cfg(test)]
1959mod tests {
1960    use super::*;
1961    use crate::codec::{AvroField, AvroFieldBuilder};
1962    use arrow_schema::{DataType, Fields, SchemaBuilder, TimeUnit, UnionFields};
1963    use serde_json::json;
1964    use std::sync::Arc;
1965
1966    fn int_schema() -> Schema<'static> {
1967        Schema::TypeName(TypeName::Primitive(PrimitiveType::Int))
1968    }
1969
1970    fn record_schema() -> Schema<'static> {
1971        Schema::Complex(ComplexType::Record(Record {
1972            name: "record1",
1973            namespace: Some("test.namespace"),
1974            doc: Some(Cow::from("A test record")),
1975            aliases: vec![],
1976            fields: vec![
1977                Field {
1978                    name: "field1",
1979                    doc: Some(Cow::from("An integer field")),
1980                    r#type: int_schema(),
1981                    default: None,
1982                    aliases: vec![],
1983                },
1984                Field {
1985                    name: "field2",
1986                    doc: None,
1987                    r#type: Schema::TypeName(TypeName::Primitive(PrimitiveType::String)),
1988                    default: None,
1989                    aliases: vec![],
1990                },
1991            ],
1992            attributes: Attributes::default(),
1993        }))
1994    }
1995
1996    fn single_field_schema(field: ArrowField) -> arrow_schema::Schema {
1997        let mut sb = SchemaBuilder::new();
1998        sb.push(field);
1999        sb.finish()
2000    }
2001
2002    fn assert_json_contains(avro_json: &str, needle: &str) {
2003        assert!(
2004            avro_json.contains(needle),
2005            "JSON did not contain `{needle}` : {avro_json}"
2006        )
2007    }
2008
2009    #[test]
2010    fn test_deserialize() {
2011        let t: Schema = serde_json::from_str("\"string\"").unwrap();
2012        assert_eq!(
2013            t,
2014            Schema::TypeName(TypeName::Primitive(PrimitiveType::String))
2015        );
2016
2017        let t: Schema = serde_json::from_str("[\"int\", \"null\"]").unwrap();
2018        assert_eq!(
2019            t,
2020            Schema::Union(vec![
2021                Schema::TypeName(TypeName::Primitive(PrimitiveType::Int)),
2022                Schema::TypeName(TypeName::Primitive(PrimitiveType::Null)),
2023            ])
2024        );
2025
2026        let t: Type = serde_json::from_str(
2027            r#"{
2028                   "type":"long",
2029                   "logicalType":"timestamp-micros"
2030                }"#,
2031        )
2032        .unwrap();
2033
2034        let timestamp = Type {
2035            r#type: TypeName::Primitive(PrimitiveType::Long),
2036            attributes: Attributes {
2037                logical_type: Some("timestamp-micros"),
2038                additional: Default::default(),
2039            },
2040        };
2041
2042        assert_eq!(t, timestamp);
2043
2044        let t: ComplexType = serde_json::from_str(
2045            r#"{
2046                   "type":"fixed",
2047                   "name":"fixed",
2048                   "namespace":"topLevelRecord.value",
2049                   "size":11,
2050                   "logicalType":"decimal",
2051                   "precision":25,
2052                   "scale":2
2053                }"#,
2054        )
2055        .unwrap();
2056
2057        let decimal = ComplexType::Fixed(Fixed {
2058            name: "fixed",
2059            namespace: Some("topLevelRecord.value"),
2060            aliases: vec![],
2061            size: 11,
2062            attributes: Attributes {
2063                logical_type: Some("decimal"),
2064                additional: vec![("precision", json!(25)), ("scale", json!(2))]
2065                    .into_iter()
2066                    .collect(),
2067            },
2068        });
2069
2070        assert_eq!(t, decimal);
2071
2072        let schema: Schema = serde_json::from_str(
2073            r#"{
2074               "type":"record",
2075               "name":"topLevelRecord",
2076               "fields":[
2077                  {
2078                     "name":"value",
2079                     "type":[
2080                        {
2081                           "type":"fixed",
2082                           "name":"fixed",
2083                           "namespace":"topLevelRecord.value",
2084                           "size":11,
2085                           "logicalType":"decimal",
2086                           "precision":25,
2087                           "scale":2
2088                        },
2089                        "null"
2090                     ]
2091                  }
2092               ]
2093            }"#,
2094        )
2095        .unwrap();
2096
2097        assert_eq!(
2098            schema,
2099            Schema::Complex(ComplexType::Record(Record {
2100                name: "topLevelRecord",
2101                namespace: None,
2102                doc: None,
2103                aliases: vec![],
2104                fields: vec![Field {
2105                    name: "value",
2106                    doc: None,
2107                    r#type: Schema::Union(vec![
2108                        Schema::Complex(decimal),
2109                        Schema::TypeName(TypeName::Primitive(PrimitiveType::Null)),
2110                    ]),
2111                    default: None,
2112                    aliases: vec![],
2113                },],
2114                attributes: Default::default(),
2115            }))
2116        );
2117
2118        let schema: Schema = serde_json::from_str(
2119            r#"{
2120                  "type": "record",
2121                  "name": "LongList",
2122                  "aliases": ["LinkedLongs"],
2123                  "fields" : [
2124                    {"name": "value", "type": "long"},
2125                    {"name": "next", "type": ["null", "LongList"]}
2126                  ]
2127                }"#,
2128        )
2129        .unwrap();
2130
2131        assert_eq!(
2132            schema,
2133            Schema::Complex(ComplexType::Record(Record {
2134                name: "LongList",
2135                namespace: None,
2136                doc: None,
2137                aliases: vec!["LinkedLongs"],
2138                fields: vec![
2139                    Field {
2140                        name: "value",
2141                        doc: None,
2142                        r#type: Schema::TypeName(TypeName::Primitive(PrimitiveType::Long)),
2143                        default: None,
2144                        aliases: vec![],
2145                    },
2146                    Field {
2147                        name: "next",
2148                        doc: None,
2149                        r#type: Schema::Union(vec![
2150                            Schema::TypeName(TypeName::Primitive(PrimitiveType::Null)),
2151                            Schema::TypeName(TypeName::Ref("LongList")),
2152                        ]),
2153                        default: None,
2154                        aliases: vec![],
2155                    }
2156                ],
2157                attributes: Attributes::default(),
2158            }))
2159        );
2160
2161        // Recursive schema are not supported
2162        let err = AvroField::try_from(&schema).unwrap_err().to_string();
2163        assert_eq!(err, "Parser error: Failed to resolve .LongList");
2164
2165        let schema: Schema = serde_json::from_str(
2166            r#"{
2167               "type":"record",
2168               "name":"topLevelRecord",
2169               "fields":[
2170                  {
2171                     "name":"id",
2172                     "type":[
2173                        "int",
2174                        "null"
2175                     ]
2176                  },
2177                  {
2178                     "name":"timestamp_col",
2179                     "type":[
2180                        {
2181                           "type":"long",
2182                           "logicalType":"timestamp-micros"
2183                        },
2184                        "null"
2185                     ]
2186                  }
2187               ]
2188            }"#,
2189        )
2190        .unwrap();
2191
2192        assert_eq!(
2193            schema,
2194            Schema::Complex(ComplexType::Record(Record {
2195                name: "topLevelRecord",
2196                namespace: None,
2197                doc: None,
2198                aliases: vec![],
2199                fields: vec![
2200                    Field {
2201                        name: "id",
2202                        doc: None,
2203                        r#type: Schema::Union(vec![
2204                            Schema::TypeName(TypeName::Primitive(PrimitiveType::Int)),
2205                            Schema::TypeName(TypeName::Primitive(PrimitiveType::Null)),
2206                        ]),
2207                        default: None,
2208                        aliases: vec![],
2209                    },
2210                    Field {
2211                        name: "timestamp_col",
2212                        doc: None,
2213                        r#type: Schema::Union(vec![
2214                            Schema::Type(timestamp),
2215                            Schema::TypeName(TypeName::Primitive(PrimitiveType::Null)),
2216                        ]),
2217                        default: None,
2218                        aliases: vec![],
2219                    }
2220                ],
2221                attributes: Default::default(),
2222            }))
2223        );
2224        let codec = AvroField::try_from(&schema).unwrap();
2225        let expected_arrow_field = arrow_schema::Field::new(
2226            "topLevelRecord",
2227            DataType::Struct(Fields::from(vec![
2228                arrow_schema::Field::new("id", DataType::Int32, true),
2229                arrow_schema::Field::new(
2230                    "timestamp_col",
2231                    DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
2232                    true,
2233                ),
2234            ])),
2235            false,
2236        )
2237        .with_metadata(std::collections::HashMap::from([(
2238            AVRO_NAME_METADATA_KEY.to_string(),
2239            "topLevelRecord".to_string(),
2240        )]));
2241
2242        assert_eq!(codec.field(), expected_arrow_field);
2243
2244        let schema: Schema = serde_json::from_str(
2245            r#"{
2246                  "type": "record",
2247                  "name": "HandshakeRequest", "namespace":"org.apache.avro.ipc",
2248                  "fields": [
2249                    {"name": "clientHash", "type": {"type": "fixed", "name": "MD5", "size": 16}},
2250                    {"name": "clientProtocol", "type": ["null", "string"]},
2251                    {"name": "serverHash", "type": "MD5"},
2252                    {"name": "meta", "type": ["null", {"type": "map", "values": "bytes"}]}
2253                  ]
2254            }"#,
2255        )
2256        .unwrap();
2257
2258        assert_eq!(
2259            schema,
2260            Schema::Complex(ComplexType::Record(Record {
2261                name: "HandshakeRequest",
2262                namespace: Some("org.apache.avro.ipc"),
2263                doc: None,
2264                aliases: vec![],
2265                fields: vec![
2266                    Field {
2267                        name: "clientHash",
2268                        doc: None,
2269                        r#type: Schema::Complex(ComplexType::Fixed(Fixed {
2270                            name: "MD5",
2271                            namespace: None,
2272                            aliases: vec![],
2273                            size: 16,
2274                            attributes: Default::default(),
2275                        })),
2276                        default: None,
2277                        aliases: vec![],
2278                    },
2279                    Field {
2280                        name: "clientProtocol",
2281                        doc: None,
2282                        r#type: Schema::Union(vec![
2283                            Schema::TypeName(TypeName::Primitive(PrimitiveType::Null)),
2284                            Schema::TypeName(TypeName::Primitive(PrimitiveType::String)),
2285                        ]),
2286                        default: None,
2287                        aliases: vec![],
2288                    },
2289                    Field {
2290                        name: "serverHash",
2291                        doc: None,
2292                        r#type: Schema::TypeName(TypeName::Ref("MD5")),
2293                        default: None,
2294                        aliases: vec![],
2295                    },
2296                    Field {
2297                        name: "meta",
2298                        doc: None,
2299                        r#type: Schema::Union(vec![
2300                            Schema::TypeName(TypeName::Primitive(PrimitiveType::Null)),
2301                            Schema::Complex(ComplexType::Map(Map {
2302                                values: Box::new(Schema::TypeName(TypeName::Primitive(
2303                                    PrimitiveType::Bytes
2304                                ))),
2305                                attributes: Default::default(),
2306                            })),
2307                        ]),
2308                        default: None,
2309                        aliases: vec![],
2310                    }
2311                ],
2312                attributes: Default::default(),
2313            }))
2314        );
2315    }
2316
2317    #[test]
2318    fn test_canonical_form_generation_comprehensive_record() {
2319        // NOTE: This schema is identical to the one used in test_deserialize_comprehensive.
2320        let json_str = r#"{
2321          "type": "record",
2322          "name": "E2eComprehensive",
2323          "namespace": "org.apache.arrow.avrotests.v1",
2324          "doc": "Comprehensive Avro writer schema to exercise arrow-avro Reader/Decoder paths.",
2325          "fields": [
2326            {"name": "id", "type": "long", "doc": "Primary row id", "aliases": ["identifier"]},
2327            {"name": "flag", "type": "boolean", "default": true, "doc": "A sample boolean with default true"},
2328            {"name": "ratio_f32", "type": "float", "default": 0.0, "doc": "Float32 example"},
2329            {"name": "ratio_f64", "type": "double", "default": 0.0, "doc": "Float64 example"},
2330            {"name": "count_i32", "type": "int", "default": 0, "doc": "Int32 example"},
2331            {"name": "count_i64", "type": "long", "default": 0, "doc": "Int64 example"},
2332            {"name": "opt_i32_nullfirst", "type": ["null", "int"], "default": null, "doc": "Nullable int (null-first)"},
2333            {"name": "opt_str_nullsecond", "type": ["string", "null"], "default": "", "aliases": ["old_opt_str"], "doc": "Nullable string (null-second). Default is empty string."},
2334            {"name": "tri_union_prim", "type": ["int", "string", "boolean"], "default": 0, "doc": "Union[int, string, boolean] with default on first branch (int=0)."},
2335            {"name": "str_utf8", "type": "string", "default": "default", "doc": "Plain Utf8 string (Reader may use Utf8View)."},
2336            {"name": "raw_bytes", "type": "bytes", "default": "", "doc": "Raw bytes field"},
2337            {"name": "fx16_plain", "type": {"type": "fixed", "name": "Fx16", "namespace": "org.apache.arrow.avrotests.v1.types", "aliases": ["Fixed16Old"], "size": 16}, "doc": "Plain fixed(16)"},
2338            {"name": "dec_bytes_s10_2", "type": {"type": "bytes", "logicalType": "decimal", "precision": 10, "scale": 2}, "doc": "Decimal encoded on bytes, precision 10, scale 2"},
2339            {"name": "dec_fix_s20_4", "type": {"type": "fixed", "name": "DecFix20", "namespace": "org.apache.arrow.avrotests.v1.types", "size": 20, "logicalType": "decimal", "precision": 20, "scale": 4}, "doc": "Decimal encoded on fixed(20), precision 20, scale 4"},
2340            {"name": "uuid_str", "type": {"type": "string", "logicalType": "uuid"}, "doc": "UUID logical type on string"},
2341            {"name": "d_date", "type": {"type": "int", "logicalType": "date"}, "doc": "Date32: days since 1970-01-01"},
2342            {"name": "t_millis", "type": {"type": "int", "logicalType": "time-millis"}, "doc": "Time32-millis"},
2343            {"name": "t_micros", "type": {"type": "long", "logicalType": "time-micros"}, "doc": "Time64-micros"},
2344            {"name": "ts_millis_utc", "type": {"type": "long", "logicalType": "timestamp-millis"}, "doc": "Timestamp ms (UTC)"},
2345            {"name": "ts_micros_utc", "type": {"type": "long", "logicalType": "timestamp-micros"}, "doc": "Timestamp µs (UTC)"},
2346            {"name": "ts_millis_local", "type": {"type": "long", "logicalType": "local-timestamp-millis"}, "doc": "Local timestamp ms"},
2347            {"name": "ts_micros_local", "type": {"type": "long", "logicalType": "local-timestamp-micros"}, "doc": "Local timestamp µs"},
2348            {"name": "interval_mdn", "type": {"type": "fixed", "name": "Dur12", "namespace": "org.apache.arrow.avrotests.v1.types", "size": 12, "logicalType": "duration"}, "doc": "Duration: fixed(12) little-endian (months, days, millis)"},
2349            {"name": "status", "type": {"type": "enum", "name": "Status", "namespace": "org.apache.arrow.avrotests.v1.types", "symbols": ["UNKNOWN", "NEW", "PROCESSING", "DONE"], "aliases": ["State"], "doc": "Processing status enum with default"}, "default": "UNKNOWN", "doc": "Enum field using default when resolving"},
2350            {"name": "arr_union", "type": {"type": "array", "items": ["long", "string", "null"]}, "default": [], "doc": "Array whose items are a union[long,string,null]"},
2351            {"name": "map_union", "type": {"type": "map", "values": ["null", "double", "string"]}, "default": {}, "doc": "Map whose values are a union[null,double,string]"},
2352            {"name": "address", "type": {"type": "record", "name": "Address", "namespace": "org.apache.arrow.avrotests.v1.types", "doc": "Postal address with defaults and field alias", "fields": [
2353                {"name": "street", "type": "string", "default": "", "aliases": ["street_name"], "doc": "Street (field alias = street_name)"},
2354                {"name": "zip", "type": "int", "default": 0, "doc": "ZIP/postal code"},
2355                {"name": "country", "type": "string", "default": "US", "doc": "Country code"}
2356            ]}, "doc": "Embedded Address record"},
2357            {"name": "maybe_auth", "type": {"type": "record", "name": "MaybeAuth", "namespace": "org.apache.arrow.avrotests.v1.types", "doc": "Optional auth token model", "fields": [
2358                {"name": "user", "type": "string", "doc": "Username"},
2359                {"name": "token", "type": ["null", "bytes"], "default": null, "doc": "Nullable auth token"}
2360            ]}},
2361            {"name": "union_enum_record_array_map", "type": [
2362                {"type": "enum", "name": "Color", "namespace": "org.apache.arrow.avrotests.v1.types", "symbols": ["RED", "GREEN", "BLUE"], "doc": "Color enum"},
2363                {"type": "record", "name": "RecA", "namespace": "org.apache.arrow.avrotests.v1.types", "fields": [{"name": "a", "type": "int"}, {"name": "b", "type": "string"}]},
2364                {"type": "record", "name": "RecB", "namespace": "org.apache.arrow.avrotests.v1.types", "fields": [{"name": "x", "type": "long"}, {"name": "y", "type": "bytes"}]},
2365                {"type": "array", "items": "long"},
2366                {"type": "map", "values": "string"}
2367            ], "doc": "Union of enum, two records, array, and map"},
2368            {"name": "union_date_or_fixed4", "type": [
2369                {"type": "int", "logicalType": "date"},
2370                {"type": "fixed", "name": "Fx4", "size": 4}
2371            ], "doc": "Union of date(int) or fixed(4)"},
2372            {"name": "union_interval_or_string", "type": [
2373                {"type": "fixed", "name": "Dur12U", "size": 12, "logicalType": "duration"},
2374                "string"
2375            ], "doc": "Union of duration(fixed12) or string"},
2376            {"name": "union_uuid_or_fixed10", "type": [
2377                {"type": "string", "logicalType": "uuid"},
2378                {"type": "fixed", "name": "Fx10", "size": 10}
2379            ], "doc": "Union of UUID string or fixed(10)"},
2380            {"name": "array_records_with_union", "type": {"type": "array", "items": {
2381                "type": "record", "name": "KV", "namespace": "org.apache.arrow.avrotests.v1.types",
2382                "fields": [
2383                    {"name": "key", "type": "string"},
2384                    {"name": "val", "type": ["null", "int", "long"], "default": null}
2385                ]
2386            }}, "doc": "Array<record{key, val: union[null,int,long]}>", "default": []},
2387            {"name": "union_map_or_array_int", "type": [
2388                {"type": "map", "values": "int"},
2389                {"type": "array", "items": "int"}
2390            ], "doc": "Union[map<string,int>, array<int>]"},
2391            {"name": "renamed_with_default", "type": "int", "default": 42, "aliases": ["old_count"], "doc": "Field with alias and default"},
2392            {"name": "person", "type": {"type": "record", "name": "PersonV2", "namespace": "com.example.v2", "aliases": ["com.example.Person"], "doc": "Person record with alias pointing to previous namespace/name", "fields": [
2393                {"name": "name", "type": "string"},
2394                {"name": "age", "type": "int", "default": 0}
2395            ]}, "doc": "Record using type alias for schema evolution tests"}
2396          ]
2397        }"#;
2398        let avro = AvroSchema::new(json_str.to_string());
2399        let parsed = avro.schema().expect("schema should deserialize");
2400        let expected_canonical_form = r#"{"name":"org.apache.arrow.avrotests.v1.E2eComprehensive","type":"record","fields":[{"name":"id","type":"long"},{"name":"flag","type":"boolean"},{"name":"ratio_f32","type":"float"},{"name":"ratio_f64","type":"double"},{"name":"count_i32","type":"int"},{"name":"count_i64","type":"long"},{"name":"opt_i32_nullfirst","type":["null","int"]},{"name":"opt_str_nullsecond","type":["string","null"]},{"name":"tri_union_prim","type":["int","string","boolean"]},{"name":"str_utf8","type":"string"},{"name":"raw_bytes","type":"bytes"},{"name":"fx16_plain","type":{"name":"org.apache.arrow.avrotests.v1.types.Fx16","type":"fixed","size":16}},{"name":"dec_bytes_s10_2","type":"bytes"},{"name":"dec_fix_s20_4","type":{"name":"org.apache.arrow.avrotests.v1.types.DecFix20","type":"fixed","size":20}},{"name":"uuid_str","type":"string"},{"name":"d_date","type":"int"},{"name":"t_millis","type":"int"},{"name":"t_micros","type":"long"},{"name":"ts_millis_utc","type":"long"},{"name":"ts_micros_utc","type":"long"},{"name":"ts_millis_local","type":"long"},{"name":"ts_micros_local","type":"long"},{"name":"interval_mdn","type":{"name":"org.apache.arrow.avrotests.v1.types.Dur12","type":"fixed","size":12}},{"name":"status","type":{"name":"org.apache.arrow.avrotests.v1.types.Status","type":"enum","symbols":["UNKNOWN","NEW","PROCESSING","DONE"]}},{"name":"arr_union","type":{"type":"array","items":["long","string","null"]}},{"name":"map_union","type":{"type":"map","values":["null","double","string"]}},{"name":"address","type":{"name":"org.apache.arrow.avrotests.v1.types.Address","type":"record","fields":[{"name":"street","type":"string"},{"name":"zip","type":"int"},{"name":"country","type":"string"}]}},{"name":"maybe_auth","type":{"name":"org.apache.arrow.avrotests.v1.types.MaybeAuth","type":"record","fields":[{"name":"user","type":"string"},{"name":"token","type":["null","bytes"]}]}},{"name":"union_enum_record_array_map","type":[{"name":"org.apache.arrow.avrotests.v1.types.Color","type":"enum","symbols":["RED","GREEN","BLUE"]},{"name":"org.apache.arrow.avrotests.v1.types.RecA","type":"record","fields":[{"name":"a","type":"int"},{"name":"b","type":"string"}]},{"name":"org.apache.arrow.avrotests.v1.types.RecB","type":"record","fields":[{"name":"x","type":"long"},{"name":"y","type":"bytes"}]},{"type":"array","items":"long"},{"type":"map","values":"string"}]},{"name":"union_date_or_fixed4","type":["int",{"name":"org.apache.arrow.avrotests.v1.Fx4","type":"fixed","size":4}]},{"name":"union_interval_or_string","type":[{"name":"org.apache.arrow.avrotests.v1.Dur12U","type":"fixed","size":12},"string"]},{"name":"union_uuid_or_fixed10","type":["string",{"name":"org.apache.arrow.avrotests.v1.Fx10","type":"fixed","size":10}]},{"name":"array_records_with_union","type":{"type":"array","items":{"name":"org.apache.arrow.avrotests.v1.types.KV","type":"record","fields":[{"name":"key","type":"string"},{"name":"val","type":["null","int","long"]}]}}},{"name":"union_map_or_array_int","type":[{"type":"map","values":"int"},{"type":"array","items":"int"}]},{"name":"renamed_with_default","type":"int"},{"name":"person","type":{"name":"com.example.v2.PersonV2","type":"record","fields":[{"name":"name","type":"string"},{"name":"age","type":"int"}]}}]}"#;
2401        let canonical_form =
2402            AvroSchema::generate_canonical_form(&parsed).expect("canonical form should be built");
2403        assert_eq!(
2404            canonical_form, expected_canonical_form,
2405            "Canonical form must match Avro spec PCF exactly"
2406        );
2407    }
2408
2409    #[test]
2410    fn test_new_schema_store() {
2411        let store = SchemaStore::new();
2412        assert!(store.schemas.is_empty());
2413    }
2414
2415    #[test]
2416    fn test_try_from_schemas_rabin() {
2417        let int_avro_schema = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2418        let record_avro_schema = AvroSchema::new(serde_json::to_string(&record_schema()).unwrap());
2419        let mut schemas: HashMap<Fingerprint, AvroSchema> = HashMap::new();
2420        schemas.insert(
2421            int_avro_schema
2422                .fingerprint(FingerprintAlgorithm::Rabin)
2423                .unwrap(),
2424            int_avro_schema.clone(),
2425        );
2426        schemas.insert(
2427            record_avro_schema
2428                .fingerprint(FingerprintAlgorithm::Rabin)
2429                .unwrap(),
2430            record_avro_schema.clone(),
2431        );
2432        let store = SchemaStore::try_from(schemas).unwrap();
2433        let int_fp = int_avro_schema
2434            .fingerprint(FingerprintAlgorithm::Rabin)
2435            .unwrap();
2436        assert_eq!(store.lookup(&int_fp).cloned(), Some(int_avro_schema));
2437        let rec_fp = record_avro_schema
2438            .fingerprint(FingerprintAlgorithm::Rabin)
2439            .unwrap();
2440        assert_eq!(store.lookup(&rec_fp).cloned(), Some(record_avro_schema));
2441    }
2442
2443    #[test]
2444    fn test_try_from_with_duplicates() {
2445        let int_avro_schema = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2446        let record_avro_schema = AvroSchema::new(serde_json::to_string(&record_schema()).unwrap());
2447        let mut schemas: HashMap<Fingerprint, AvroSchema> = HashMap::new();
2448        schemas.insert(
2449            int_avro_schema
2450                .fingerprint(FingerprintAlgorithm::Rabin)
2451                .unwrap(),
2452            int_avro_schema.clone(),
2453        );
2454        schemas.insert(
2455            record_avro_schema
2456                .fingerprint(FingerprintAlgorithm::Rabin)
2457                .unwrap(),
2458            record_avro_schema.clone(),
2459        );
2460        // Insert duplicate of int schema
2461        schemas.insert(
2462            int_avro_schema
2463                .fingerprint(FingerprintAlgorithm::Rabin)
2464                .unwrap(),
2465            int_avro_schema.clone(),
2466        );
2467        let store = SchemaStore::try_from(schemas).unwrap();
2468        assert_eq!(store.schemas.len(), 2);
2469        let int_fp = int_avro_schema
2470            .fingerprint(FingerprintAlgorithm::Rabin)
2471            .unwrap();
2472        assert_eq!(store.lookup(&int_fp).cloned(), Some(int_avro_schema));
2473    }
2474
2475    #[test]
2476    fn test_register_and_lookup_rabin() {
2477        let mut store = SchemaStore::new();
2478        let schema = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2479        let fp_enum = store.register(schema.clone()).unwrap();
2480        match fp_enum {
2481            Fingerprint::Rabin(fp_val) => {
2482                assert_eq!(
2483                    store.lookup(&Fingerprint::Rabin(fp_val)).cloned(),
2484                    Some(schema.clone())
2485                );
2486                assert!(
2487                    store
2488                        .lookup(&Fingerprint::Rabin(fp_val.wrapping_add(1)))
2489                        .is_none()
2490                );
2491            }
2492            Fingerprint::Id(_id) => {
2493                unreachable!("This test should only generate Rabin fingerprints")
2494            }
2495            Fingerprint::Id64(_id) => {
2496                unreachable!("This test should only generate Rabin fingerprints")
2497            }
2498            #[cfg(feature = "md5")]
2499            Fingerprint::MD5(_id) => {
2500                unreachable!("This test should only generate Rabin fingerprints")
2501            }
2502            #[cfg(feature = "sha256")]
2503            Fingerprint::SHA256(_id) => {
2504                unreachable!("This test should only generate Rabin fingerprints")
2505            }
2506        }
2507    }
2508
2509    #[test]
2510    fn test_set_and_lookup_id() {
2511        let mut store = SchemaStore::new();
2512        let schema = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2513        let id = 42u32;
2514        let fp = Fingerprint::Id(id);
2515        let out_fp = store.set(fp, schema.clone()).unwrap();
2516        assert_eq!(out_fp, fp);
2517        assert_eq!(store.lookup(&fp).cloned(), Some(schema.clone()));
2518        assert!(store.lookup(&Fingerprint::Id(id.wrapping_add(1))).is_none());
2519    }
2520
2521    #[test]
2522    fn test_set_and_lookup_id64() {
2523        let mut store = SchemaStore::new();
2524        let schema = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2525        let id64: u64 = 0xDEAD_BEEF_DEAD_BEEF;
2526        let fp = Fingerprint::Id64(id64);
2527        let out_fp = store.set(fp, schema.clone()).unwrap();
2528        assert_eq!(out_fp, fp, "set should return the same Id64 fingerprint");
2529        assert_eq!(
2530            store.lookup(&fp).cloned(),
2531            Some(schema.clone()),
2532            "lookup should find the schema by Id64"
2533        );
2534        assert!(
2535            store
2536                .lookup(&Fingerprint::Id64(id64.wrapping_add(1)))
2537                .is_none(),
2538            "lookup with a different Id64 must return None"
2539        );
2540    }
2541
2542    #[test]
2543    fn test_fingerprint_id64_conversions() {
2544        let algo_from_fp = FingerprintAlgorithm::from(&Fingerprint::Id64(123));
2545        assert_eq!(algo_from_fp, FingerprintAlgorithm::Id64);
2546        let fp_from_algo = Fingerprint::from(FingerprintAlgorithm::Id64);
2547        assert!(matches!(fp_from_algo, Fingerprint::Id64(0)));
2548        let strategy_from_fp = FingerprintStrategy::from(Fingerprint::Id64(5));
2549        assert!(matches!(strategy_from_fp, FingerprintStrategy::Id64(0)));
2550        let algo_from_strategy = FingerprintAlgorithm::from(strategy_from_fp);
2551        assert_eq!(algo_from_strategy, FingerprintAlgorithm::Id64);
2552    }
2553
2554    #[test]
2555    fn test_register_duplicate_schema() {
2556        let mut store = SchemaStore::new();
2557        let schema1 = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2558        let schema2 = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2559        let fingerprint1 = store.register(schema1).unwrap();
2560        let fingerprint2 = store.register(schema2).unwrap();
2561        assert_eq!(fingerprint1, fingerprint2);
2562        assert_eq!(store.schemas.len(), 1);
2563    }
2564
2565    #[test]
2566    fn test_set_and_lookup_with_provided_fingerprint() {
2567        let mut store = SchemaStore::new();
2568        let schema = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2569        let fp = schema.fingerprint(FingerprintAlgorithm::Rabin).unwrap();
2570        let out_fp = store.set(fp, schema.clone()).unwrap();
2571        assert_eq!(out_fp, fp);
2572        assert_eq!(store.lookup(&fp).cloned(), Some(schema));
2573    }
2574
2575    #[test]
2576    fn test_set_duplicate_same_schema_ok() {
2577        let mut store = SchemaStore::new();
2578        let schema = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2579        let fp = schema.fingerprint(FingerprintAlgorithm::Rabin).unwrap();
2580        let _ = store.set(fp, schema.clone()).unwrap();
2581        let _ = store.set(fp, schema.clone()).unwrap();
2582        assert_eq!(store.schemas.len(), 1);
2583    }
2584
2585    #[test]
2586    fn test_set_duplicate_different_schema_collision_error() {
2587        let mut store = SchemaStore::new();
2588        let schema1 = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2589        let schema2 = AvroSchema::new(serde_json::to_string(&record_schema()).unwrap());
2590        // Use the same Fingerprint::Id to simulate a collision across different schemas
2591        let fp = Fingerprint::Id(123);
2592        let _ = store.set(fp, schema1).unwrap();
2593        let err = store.set(fp, schema2).unwrap_err();
2594        let msg = format!("{err}");
2595        assert!(msg.contains("Schema fingerprint collision"));
2596    }
2597
2598    #[test]
2599    fn test_canonical_form_generation_primitive() {
2600        let schema = int_schema();
2601        let canonical_form = AvroSchema::generate_canonical_form(&schema).unwrap();
2602        assert_eq!(canonical_form, r#""int""#);
2603    }
2604
2605    #[test]
2606    fn test_canonical_form_generation_record() {
2607        let schema = record_schema();
2608        let expected_canonical_form = r#"{"name":"test.namespace.record1","type":"record","fields":[{"name":"field1","type":"int"},{"name":"field2","type":"string"}]}"#;
2609        let canonical_form = AvroSchema::generate_canonical_form(&schema).unwrap();
2610        assert_eq!(canonical_form, expected_canonical_form);
2611    }
2612
2613    #[test]
2614    fn test_fingerprint_calculation() {
2615        let canonical_form = r#"{"fields":[{"name":"a","type":"long"},{"name":"b","type":"string"}],"name":"test","type":"record"}"#;
2616        let expected_fingerprint = 10505236152925314060;
2617        let fingerprint = compute_fingerprint_rabin(canonical_form);
2618        assert_eq!(fingerprint, expected_fingerprint);
2619    }
2620
2621    #[test]
2622    fn test_register_and_lookup_complex_schema() {
2623        let mut store = SchemaStore::new();
2624        let schema = AvroSchema::new(serde_json::to_string(&record_schema()).unwrap());
2625        let canonical_form = r#"{"name":"test.namespace.record1","type":"record","fields":[{"name":"field1","type":"int"},{"name":"field2","type":"string"}]}"#;
2626        let expected_fingerprint = Fingerprint::Rabin(compute_fingerprint_rabin(canonical_form));
2627        let fingerprint = store.register(schema.clone()).unwrap();
2628        assert_eq!(fingerprint, expected_fingerprint);
2629        let looked_up = store.lookup(&fingerprint).cloned();
2630        assert_eq!(looked_up, Some(schema));
2631    }
2632
2633    #[test]
2634    fn test_fingerprints_returns_all_keys() {
2635        let mut store = SchemaStore::new();
2636        let fp_int = store
2637            .register(AvroSchema::new(
2638                serde_json::to_string(&int_schema()).unwrap(),
2639            ))
2640            .unwrap();
2641        let fp_record = store
2642            .register(AvroSchema::new(
2643                serde_json::to_string(&record_schema()).unwrap(),
2644            ))
2645            .unwrap();
2646        let fps = store.fingerprints();
2647        assert_eq!(fps.len(), 2);
2648        assert!(fps.contains(&fp_int));
2649        assert!(fps.contains(&fp_record));
2650    }
2651
2652    #[test]
2653    fn test_canonical_form_strips_attributes() {
2654        let schema_with_attrs = Schema::Complex(ComplexType::Record(Record {
2655            name: "record_with_attrs",
2656            namespace: None,
2657            doc: Some(Cow::from("This doc should be stripped")),
2658            aliases: vec!["alias1", "alias2"],
2659            fields: vec![Field {
2660                name: "f1",
2661                doc: Some(Cow::from("field doc")),
2662                r#type: Schema::Type(Type {
2663                    r#type: TypeName::Primitive(PrimitiveType::Bytes),
2664                    attributes: Attributes {
2665                        logical_type: None,
2666                        additional: HashMap::from([("precision", json!(4))]),
2667                    },
2668                }),
2669                default: None,
2670                aliases: vec![],
2671            }],
2672            attributes: Attributes {
2673                logical_type: None,
2674                additional: HashMap::from([("custom_attr", json!("value"))]),
2675            },
2676        }));
2677        let expected_canonical_form = r#"{"name":"record_with_attrs","type":"record","fields":[{"name":"f1","type":"bytes"}]}"#;
2678        let canonical_form = AvroSchema::generate_canonical_form(&schema_with_attrs).unwrap();
2679        assert_eq!(canonical_form, expected_canonical_form);
2680    }
2681
2682    #[cfg(not(feature = "avro_custom_types"))]
2683    #[test]
2684    fn test_primitive_mappings() {
2685        let cases = vec![
2686            (DataType::Boolean, "\"boolean\""),
2687            (DataType::Int8, "\"int\""),
2688            (DataType::Int16, "\"int\""),
2689            (DataType::Int32, "\"int\""),
2690            (DataType::Int64, "\"long\""),
2691            (DataType::UInt8, "\"int\""),
2692            (DataType::UInt16, "\"int\""),
2693            (DataType::UInt32, "\"long\""),
2694            (DataType::UInt64, "\"long\""),
2695            (DataType::Float16, "\"float\""),
2696            (DataType::Float32, "\"float\""),
2697            (DataType::Float64, "\"double\""),
2698            (DataType::Utf8, "\"string\""),
2699            (DataType::Binary, "\"bytes\""),
2700        ];
2701        for (dt, avro_token) in cases {
2702            let field = ArrowField::new("col", dt.clone(), false);
2703            let arrow_schema = single_field_schema(field);
2704            let avro = AvroSchema::try_from(&arrow_schema).unwrap();
2705            assert_json_contains(&avro.json_string, avro_token);
2706        }
2707    }
2708
2709    #[cfg(feature = "avro_custom_types")]
2710    #[test]
2711    fn test_primitive_mappings() {
2712        let cases = vec![
2713            (DataType::Boolean, "\"boolean\""),
2714            (DataType::Int8, "\"logicalType\":\"arrow.int8\""),
2715            (DataType::Int16, "\"logicalType\":\"arrow.int16\""),
2716            (DataType::Int32, "\"int\""),
2717            (DataType::Int64, "\"long\""),
2718            (DataType::UInt8, "\"logicalType\":\"arrow.uint8\""),
2719            (DataType::UInt16, "\"logicalType\":\"arrow.uint16\""),
2720            (DataType::UInt32, "\"logicalType\":\"arrow.uint32\""),
2721            (DataType::UInt64, "\"logicalType\":\"arrow.uint64\""),
2722            (DataType::Float16, "\"logicalType\":\"arrow.float16\""),
2723            (DataType::Float32, "\"float\""),
2724            (DataType::Float64, "\"double\""),
2725            (DataType::Utf8, "\"string\""),
2726            (DataType::Binary, "\"bytes\""),
2727        ];
2728        for (dt, avro_token) in cases {
2729            let field = ArrowField::new("col", dt.clone(), false);
2730            let arrow_schema = single_field_schema(field);
2731            let avro = AvroSchema::try_from(&arrow_schema).unwrap();
2732            assert_json_contains(&avro.json_string, avro_token);
2733        }
2734    }
2735
2736    #[cfg(feature = "avro_custom_types")]
2737    #[test]
2738    fn test_custom_fixed_logical_types_preserve_namespace_metadata() {
2739        let namespace = "com.example.types";
2740
2741        let mut md_u64 = HashMap::new();
2742        md_u64.insert(AVRO_NAME_METADATA_KEY.to_string(), "U64Type".to_string());
2743        md_u64.insert(
2744            AVRO_NAMESPACE_METADATA_KEY.to_string(),
2745            namespace.to_string(),
2746        );
2747
2748        let mut md_f16 = HashMap::new();
2749        md_f16.insert(AVRO_NAME_METADATA_KEY.to_string(), "F16Type".to_string());
2750        md_f16.insert(
2751            AVRO_NAMESPACE_METADATA_KEY.to_string(),
2752            namespace.to_string(),
2753        );
2754
2755        let mut md_iv_ym = HashMap::new();
2756        md_iv_ym.insert(AVRO_NAME_METADATA_KEY.to_string(), "IvYmType".to_string());
2757        md_iv_ym.insert(
2758            AVRO_NAMESPACE_METADATA_KEY.to_string(),
2759            namespace.to_string(),
2760        );
2761
2762        let mut md_iv_dt = HashMap::new();
2763        md_iv_dt.insert(AVRO_NAME_METADATA_KEY.to_string(), "IvDtType".to_string());
2764        md_iv_dt.insert(
2765            AVRO_NAMESPACE_METADATA_KEY.to_string(),
2766            namespace.to_string(),
2767        );
2768
2769        let arrow_schema = ArrowSchema::new(vec![
2770            ArrowField::new("u64_col", DataType::UInt64, false).with_metadata(md_u64),
2771            ArrowField::new("f16_col", DataType::Float16, false).with_metadata(md_f16),
2772            ArrowField::new(
2773                "iv_ym_col",
2774                DataType::Interval(IntervalUnit::YearMonth),
2775                false,
2776            )
2777            .with_metadata(md_iv_ym),
2778            ArrowField::new(
2779                "iv_dt_col",
2780                DataType::Interval(IntervalUnit::DayTime),
2781                false,
2782            )
2783            .with_metadata(md_iv_dt),
2784        ]);
2785
2786        let avro = AvroSchema::try_from(&arrow_schema).unwrap();
2787        let root: Value = serde_json::from_str(&avro.json_string).unwrap();
2788        let fields = root
2789            .get("fields")
2790            .and_then(|f| f.as_array())
2791            .expect("record fields array");
2792
2793        let expected = [
2794            ("u64_col", "arrow.uint64"),
2795            ("f16_col", "arrow.float16"),
2796            ("iv_ym_col", "arrow.interval-year-month"),
2797            ("iv_dt_col", "arrow.interval-day-time"),
2798        ];
2799
2800        for (field_name, logical_type) in expected {
2801            let field = fields
2802                .iter()
2803                .find(|f| f.get("name").and_then(Value::as_str) == Some(field_name))
2804                .unwrap_or_else(|| panic!("missing field {field_name}"));
2805            let ty = field
2806                .get("type")
2807                .and_then(Value::as_object)
2808                .unwrap_or_else(|| panic!("field {field_name} type must be object"));
2809
2810            assert_eq!(ty.get("type").and_then(Value::as_str), Some("fixed"));
2811            assert_eq!(
2812                ty.get("logicalType").and_then(Value::as_str),
2813                Some(logical_type)
2814            );
2815            assert_eq!(
2816                ty.get("namespace").and_then(Value::as_str),
2817                Some(namespace),
2818                "field {field_name} must preserve avro.namespace metadata"
2819            );
2820        }
2821    }
2822
2823    #[cfg(feature = "avro_custom_types")]
2824    #[test]
2825    fn test_custom_fixed_logical_types_omit_namespace_without_metadata() {
2826        let mut md_u64 = HashMap::new();
2827        md_u64.insert(AVRO_NAME_METADATA_KEY.to_string(), "U64Type".to_string());
2828
2829        let mut md_f16 = HashMap::new();
2830        md_f16.insert(AVRO_NAME_METADATA_KEY.to_string(), "F16Type".to_string());
2831
2832        let mut md_iv_ym = HashMap::new();
2833        md_iv_ym.insert(AVRO_NAME_METADATA_KEY.to_string(), "IvYmType".to_string());
2834
2835        let mut md_iv_dt = HashMap::new();
2836        md_iv_dt.insert(AVRO_NAME_METADATA_KEY.to_string(), "IvDtType".to_string());
2837
2838        let arrow_schema = ArrowSchema::new(vec![
2839            ArrowField::new("u64_col", DataType::UInt64, false).with_metadata(md_u64),
2840            ArrowField::new("f16_col", DataType::Float16, false).with_metadata(md_f16),
2841            ArrowField::new(
2842                "iv_ym_col",
2843                DataType::Interval(IntervalUnit::YearMonth),
2844                false,
2845            )
2846            .with_metadata(md_iv_ym),
2847            ArrowField::new(
2848                "iv_dt_col",
2849                DataType::Interval(IntervalUnit::DayTime),
2850                false,
2851            )
2852            .with_metadata(md_iv_dt),
2853        ]);
2854
2855        let avro = AvroSchema::try_from(&arrow_schema).unwrap();
2856        let root: Value = serde_json::from_str(&avro.json_string).unwrap();
2857        let fields = root
2858            .get("fields")
2859            .and_then(|f| f.as_array())
2860            .expect("record fields array");
2861
2862        for field_name in ["u64_col", "f16_col", "iv_ym_col", "iv_dt_col"] {
2863            let field = fields
2864                .iter()
2865                .find(|f| f.get("name").and_then(Value::as_str) == Some(field_name))
2866                .unwrap_or_else(|| panic!("missing field {field_name}"));
2867            let ty = field
2868                .get("type")
2869                .and_then(Value::as_object)
2870                .unwrap_or_else(|| panic!("field {field_name} type must be object"));
2871
2872            assert_eq!(ty.get("type").and_then(Value::as_str), Some("fixed"));
2873            assert!(
2874                !ty.contains_key("namespace"),
2875                "field {field_name} should not include namespace when metadata lacks avro.namespace"
2876            );
2877        }
2878    }
2879
2880    #[test]
2881    fn test_temporal_mappings() {
2882        let cases = vec![
2883            (DataType::Date32, "\"logicalType\":\"date\""),
2884            (
2885                DataType::Time32(TimeUnit::Millisecond),
2886                "\"logicalType\":\"time-millis\"",
2887            ),
2888            (
2889                DataType::Time64(TimeUnit::Microsecond),
2890                "\"logicalType\":\"time-micros\"",
2891            ),
2892            (
2893                DataType::Timestamp(TimeUnit::Millisecond, None),
2894                "\"logicalType\":\"local-timestamp-millis\"",
2895            ),
2896            (
2897                DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
2898                "\"logicalType\":\"timestamp-micros\"",
2899            ),
2900        ];
2901        for (dt, needle) in cases {
2902            let field = ArrowField::new("ts", dt.clone(), true);
2903            let arrow_schema = single_field_schema(field);
2904            let avro = AvroSchema::try_from(&arrow_schema).unwrap();
2905            assert_json_contains(&avro.json_string, needle);
2906        }
2907    }
2908
2909    #[test]
2910    fn test_decimal_and_uuid() {
2911        let decimal_field = ArrowField::new("amount", DataType::Decimal128(25, 2), false);
2912        let dec_schema = single_field_schema(decimal_field);
2913        let avro_dec = AvroSchema::try_from(&dec_schema).unwrap();
2914        assert_json_contains(&avro_dec.json_string, "\"logicalType\":\"decimal\"");
2915        assert_json_contains(&avro_dec.json_string, "\"precision\":25");
2916        assert_json_contains(&avro_dec.json_string, "\"scale\":2");
2917        let mut md = HashMap::new();
2918        md.insert("logicalType".into(), "uuid".into());
2919        let uuid_field =
2920            ArrowField::new("id", DataType::FixedSizeBinary(16), false).with_metadata(md);
2921        let uuid_schema = single_field_schema(uuid_field);
2922        let avro_uuid = AvroSchema::try_from(&uuid_schema).unwrap();
2923        assert_json_contains(&avro_uuid.json_string, "\"logicalType\":\"uuid\"");
2924    }
2925
2926    #[cfg(not(feature = "avro_custom_types"))]
2927    #[test]
2928    fn test_interval_month_day_nano_duration_schema() {
2929        let interval_field = ArrowField::new(
2930            "span",
2931            DataType::Interval(IntervalUnit::MonthDayNano),
2932            false,
2933        );
2934        let s = single_field_schema(interval_field);
2935        let avro = AvroSchema::try_from(&s).unwrap();
2936        assert_json_contains(&avro.json_string, "\"logicalType\":\"duration\"");
2937        assert_json_contains(&avro.json_string, "\"size\":12");
2938    }
2939
2940    #[cfg(feature = "avro_custom_types")]
2941    #[test]
2942    fn test_interval_month_day_nano_custom_schema() {
2943        let interval_field = ArrowField::new(
2944            "span",
2945            DataType::Interval(IntervalUnit::MonthDayNano),
2946            false,
2947        );
2948        let s = single_field_schema(interval_field);
2949        let avro = AvroSchema::try_from(&s).unwrap();
2950        assert_json_contains(
2951            &avro.json_string,
2952            "\"logicalType\":\"arrow.interval-month-day-nano\"",
2953        );
2954        assert_json_contains(&avro.json_string, "\"size\":16");
2955    }
2956
2957    #[cfg(feature = "avro_custom_types")]
2958    #[test]
2959    fn test_duration_custom_logical_type() {
2960        let dur_field = ArrowField::new("latency", DataType::Duration(TimeUnit::Nanosecond), false);
2961        let s2 = single_field_schema(dur_field);
2962        let avro2 = AvroSchema::try_from(&s2).unwrap();
2963        assert_json_contains(
2964            &avro2.json_string,
2965            "\"logicalType\":\"arrow.duration-nanos\"",
2966        );
2967    }
2968
2969    #[test]
2970    fn test_complex_types() {
2971        let list_dt = DataType::List(Arc::new(ArrowField::new("item", DataType::Int32, true)));
2972        let list_schema = single_field_schema(ArrowField::new("numbers", list_dt, false));
2973        let avro_list = AvroSchema::try_from(&list_schema).unwrap();
2974        assert_json_contains(&avro_list.json_string, "\"type\":\"array\"");
2975        assert_json_contains(&avro_list.json_string, "\"items\"");
2976        let value_field = ArrowField::new(
2977            arrow_schema::Field::MAP_VALUE_FIELD_DEFAULT_NAME,
2978            DataType::Boolean,
2979            true,
2980        );
2981        let entries_struct = ArrowField::new(
2982            arrow_schema::Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
2983            DataType::Struct(Fields::from(vec![
2984                ArrowField::new(
2985                    arrow_schema::Field::MAP_KEY_FIELD_DEFAULT_NAME,
2986                    DataType::Utf8,
2987                    false,
2988                ),
2989                value_field.clone(),
2990            ])),
2991            false,
2992        );
2993        let map_dt = DataType::Map(Arc::new(entries_struct), false);
2994        let map_schema = single_field_schema(ArrowField::new("props", map_dt, false));
2995        let avro_map = AvroSchema::try_from(&map_schema).unwrap();
2996        assert_json_contains(&avro_map.json_string, "\"type\":\"map\"");
2997        assert_json_contains(&avro_map.json_string, "\"values\"");
2998        let struct_dt = DataType::Struct(Fields::from(vec![
2999            ArrowField::new("f1", DataType::Int64, false),
3000            ArrowField::new("f2", DataType::Utf8, true),
3001        ]));
3002        let struct_schema = single_field_schema(ArrowField::new("person", struct_dt, true));
3003        let avro_struct = AvroSchema::try_from(&struct_schema).unwrap();
3004        assert_json_contains(&avro_struct.json_string, "\"type\":\"record\"");
3005        assert_json_contains(&avro_struct.json_string, "\"null\"");
3006    }
3007
3008    #[test]
3009    fn test_enum_dictionary() {
3010        let mut md = HashMap::new();
3011        md.insert(
3012            AVRO_ENUM_SYMBOLS_METADATA_KEY.into(),
3013            "[\"OPEN\",\"CLOSED\"]".into(),
3014        );
3015        let enum_dt = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
3016        let field = ArrowField::new("status", enum_dt, false).with_metadata(md);
3017        let schema = single_field_schema(field);
3018        let avro = AvroSchema::try_from(&schema).unwrap();
3019        assert_json_contains(&avro.json_string, "\"type\":\"enum\"");
3020        assert_json_contains(&avro.json_string, "\"symbols\":[\"OPEN\",\"CLOSED\"]");
3021    }
3022
3023    #[test]
3024    fn test_run_end_encoded() {
3025        let ree_dt = DataType::RunEndEncoded(
3026            Arc::new(ArrowField::new(
3027                ArrowField::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
3028                DataType::Int32,
3029                false,
3030            )),
3031            Arc::new(ArrowField::new(
3032                ArrowField::REE_VALUES_FIELD_DEFAULT_NAME,
3033                DataType::Utf8,
3034                false,
3035            )),
3036        );
3037        let s = single_field_schema(ArrowField::new("text", ree_dt, false));
3038        let avro = AvroSchema::try_from(&s).unwrap();
3039        assert_json_contains(&avro.json_string, "\"string\"");
3040    }
3041
3042    #[test]
3043    fn test_dense_union() {
3044        let uf: UnionFields = vec![
3045            (2i8, Arc::new(ArrowField::new("a", DataType::Int32, false))),
3046            (7i8, Arc::new(ArrowField::new("b", DataType::Utf8, true))),
3047        ]
3048        .into_iter()
3049        .collect();
3050        let union_dt = DataType::Union(uf, UnionMode::Dense);
3051        let s = single_field_schema(ArrowField::new("u", union_dt, false));
3052        let avro =
3053            AvroSchema::try_from(&s).expect("Arrow Union -> Avro union conversion should succeed");
3054        let v: serde_json::Value = serde_json::from_str(&avro.json_string).unwrap();
3055        let fields = v
3056            .get("fields")
3057            .and_then(|x| x.as_array())
3058            .expect("fields array");
3059        let u_field = fields
3060            .iter()
3061            .find(|f| f.get("name").and_then(|n| n.as_str()) == Some("u"))
3062            .expect("field 'u'");
3063        let union = u_field.get("type").expect("u.type");
3064        let arr = union.as_array().expect("u.type must be Avro union array");
3065        assert_eq!(arr.len(), 2, "expected two union branches");
3066        let first = &arr[0];
3067        let obj = first
3068            .as_object()
3069            .expect("first branch should be an object with metadata");
3070        assert_eq!(obj.get("type").and_then(|t| t.as_str()), Some("int"));
3071        assert_eq!(
3072            obj.get("arrowUnionMode").and_then(|m| m.as_str()),
3073            Some("dense")
3074        );
3075        let type_ids: Vec<i64> = obj
3076            .get("arrowUnionTypeIds")
3077            .and_then(|a| a.as_array())
3078            .expect("arrowUnionTypeIds array")
3079            .iter()
3080            .map(|n| n.as_i64().expect("i64"))
3081            .collect();
3082        assert_eq!(type_ids, vec![2, 7], "type id ordering should be preserved");
3083        assert_eq!(arr[1], Value::String("string".into()));
3084    }
3085
3086    #[test]
3087    fn round_trip_primitive() {
3088        let arrow_schema = ArrowSchema::new(vec![ArrowField::new("f1", DataType::Int32, false)]);
3089        let avro_schema = AvroSchema::try_from(&arrow_schema).unwrap();
3090        let decoded = avro_schema.schema().unwrap();
3091        assert!(matches!(decoded, Schema::Complex(_)));
3092    }
3093
3094    #[test]
3095    fn test_name_generator_sanitization_and_uniqueness() {
3096        let f1 = ArrowField::new("weird-name", DataType::FixedSizeBinary(8), false);
3097        let f2 = ArrowField::new("weird name", DataType::FixedSizeBinary(8), false);
3098        let f3 = ArrowField::new("123bad", DataType::FixedSizeBinary(8), false);
3099        let arrow_schema = ArrowSchema::new(vec![f1, f2, f3]);
3100        let avro = AvroSchema::try_from(&arrow_schema).unwrap();
3101        assert_json_contains(&avro.json_string, "\"name\":\"weird_name\"");
3102        assert_json_contains(&avro.json_string, "\"name\":\"weird_name_1\"");
3103        assert_json_contains(&avro.json_string, "\"name\":\"_123bad\"");
3104    }
3105
3106    #[cfg(not(feature = "avro_custom_types"))]
3107    #[test]
3108    fn test_date64_logical_type_mapping() {
3109        let field = ArrowField::new("d", DataType::Date64, true);
3110        let schema = single_field_schema(field);
3111        let avro = AvroSchema::try_from(&schema).unwrap();
3112        assert_json_contains(
3113            &avro.json_string,
3114            "\"logicalType\":\"local-timestamp-millis\"",
3115        );
3116    }
3117
3118    #[cfg(feature = "avro_custom_types")]
3119    #[test]
3120    fn test_date64_logical_type_mapping_custom() {
3121        let field = ArrowField::new("d", DataType::Date64, true);
3122        let schema = single_field_schema(field);
3123        let avro = AvroSchema::try_from(&schema).unwrap();
3124        assert_json_contains(&avro.json_string, "\"logicalType\":\"arrow.date64\"");
3125    }
3126
3127    #[cfg(feature = "avro_custom_types")]
3128    #[test]
3129    fn test_duration_list_extras_propagated() {
3130        let child = ArrowField::new("lat", DataType::Duration(TimeUnit::Microsecond), false);
3131        let list_dt = DataType::List(Arc::new(child));
3132        let arrow_schema = single_field_schema(ArrowField::new("durations", list_dt, false));
3133        let avro = AvroSchema::try_from(&arrow_schema).unwrap();
3134        assert_json_contains(
3135            &avro.json_string,
3136            "\"logicalType\":\"arrow.duration-micros\"",
3137        );
3138    }
3139
3140    #[cfg(not(feature = "avro_custom_types"))]
3141    #[test]
3142    fn test_interval_yearmonth_extra() {
3143        let field = ArrowField::new("iv", DataType::Interval(IntervalUnit::YearMonth), false);
3144        let schema = single_field_schema(field);
3145        let avro = AvroSchema::try_from(&schema).unwrap();
3146        assert_json_contains(&avro.json_string, "\"arrowIntervalUnit\":\"yearmonth\"");
3147    }
3148
3149    #[cfg(not(feature = "avro_custom_types"))]
3150    #[test]
3151    fn test_interval_daytime_extra() {
3152        let field = ArrowField::new("iv_dt", DataType::Interval(IntervalUnit::DayTime), false);
3153        let schema = single_field_schema(field);
3154        let avro = AvroSchema::try_from(&schema).unwrap();
3155        assert_json_contains(&avro.json_string, "\"arrowIntervalUnit\":\"daytime\"");
3156    }
3157
3158    #[cfg(feature = "avro_custom_types")]
3159    #[test]
3160    fn test_interval_yearmonth_custom() {
3161        let field = ArrowField::new("iv", DataType::Interval(IntervalUnit::YearMonth), false);
3162        let schema = single_field_schema(field);
3163        let avro = AvroSchema::try_from(&schema).unwrap();
3164        assert_json_contains(
3165            &avro.json_string,
3166            "\"logicalType\":\"arrow.interval-year-month\"",
3167        );
3168    }
3169
3170    #[cfg(feature = "avro_custom_types")]
3171    #[test]
3172    fn test_interval_daytime_custom() {
3173        let field = ArrowField::new("iv_dt", DataType::Interval(IntervalUnit::DayTime), false);
3174        let schema = single_field_schema(field);
3175        let avro = AvroSchema::try_from(&schema).unwrap();
3176        assert_json_contains(
3177            &avro.json_string,
3178            "\"logicalType\":\"arrow.interval-day-time\"",
3179        );
3180    }
3181
3182    #[test]
3183    fn test_fixed_size_list_extra() {
3184        let child = ArrowField::new("item", DataType::Int32, false);
3185        let dt = DataType::FixedSizeList(Arc::new(child), 3);
3186        let schema = single_field_schema(ArrowField::new("triples", dt, false));
3187        let avro = AvroSchema::try_from(&schema).unwrap();
3188        assert_json_contains(&avro.json_string, "\"arrowFixedSize\":3");
3189    }
3190
3191    #[cfg(feature = "avro_custom_types")]
3192    #[test]
3193    fn test_map_duration_value_extra() {
3194        let val_field = ArrowField::new(
3195            ArrowField::MAP_VALUE_FIELD_DEFAULT_NAME,
3196            DataType::Duration(TimeUnit::Second),
3197            true,
3198        );
3199        let entries_struct = ArrowField::new(
3200            ArrowField::MAP_ENTRIES_FIELD_DEFAULT_NAME,
3201            DataType::Struct(Fields::from(vec![
3202                ArrowField::new(
3203                    ArrowField::MAP_KEY_FIELD_DEFAULT_NAME,
3204                    DataType::Utf8,
3205                    false,
3206                ),
3207                val_field,
3208            ])),
3209            false,
3210        );
3211        let map_dt = DataType::Map(Arc::new(entries_struct), false);
3212        let schema = single_field_schema(ArrowField::new("metrics", map_dt, false));
3213        let avro = AvroSchema::try_from(&schema).unwrap();
3214        assert_json_contains(
3215            &avro.json_string,
3216            "\"logicalType\":\"arrow.duration-seconds\"",
3217        );
3218    }
3219
3220    #[test]
3221    fn test_schema_with_non_string_defaults_decodes_successfully() {
3222        let schema_json = r#"{
3223            "type": "record",
3224            "name": "R",
3225            "fields": [
3226                {"name": "a", "type": "int", "default": 0},
3227                {"name": "b", "type": {"type": "array", "items": "long"}, "default": [1, 2, 3]},
3228                {"name": "c", "type": {"type": "map", "values": "double"}, "default": {"x": 1.5, "y": 2.5}},
3229                {"name": "inner", "type": {"type": "record", "name": "Inner", "fields": [
3230                    {"name": "flag", "type": "boolean", "default": true},
3231                    {"name": "name", "type": "string", "default": "hi"}
3232                ]}, "default": {"flag": false, "name": "d"}},
3233                {"name": "u", "type": ["int", "null"], "default": 42}
3234            ]
3235        }"#;
3236        let schema: Schema = serde_json::from_str(schema_json).expect("schema should parse");
3237        match &schema {
3238            Schema::Complex(ComplexType::Record(_)) => {}
3239            other => panic!("expected record schema, got: {other:?}"),
3240        }
3241        // Avro to Arrow conversion
3242        let field = crate::codec::AvroField::try_from(&schema)
3243            .expect("Avro->Arrow conversion should succeed");
3244        let arrow_field = field.field();
3245        // Build expected Arrow field
3246        let expected_list_item = ArrowField::new(
3247            arrow_schema::Field::LIST_FIELD_DEFAULT_NAME,
3248            DataType::Int64,
3249            false,
3250        );
3251        let expected_b = ArrowField::new("b", DataType::List(Arc::new(expected_list_item)), false);
3252
3253        let expected_map_value = ArrowField::new(
3254            arrow_schema::Field::MAP_VALUE_FIELD_DEFAULT_NAME,
3255            DataType::Float64,
3256            false,
3257        );
3258        let expected_entries = ArrowField::new(
3259            arrow_schema::Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
3260            DataType::Struct(Fields::from(vec![
3261                ArrowField::new(
3262                    arrow_schema::Field::MAP_KEY_FIELD_DEFAULT_NAME,
3263                    DataType::Utf8,
3264                    false,
3265                ),
3266                expected_map_value,
3267            ])),
3268            false,
3269        );
3270        let expected_c =
3271            ArrowField::new("c", DataType::Map(Arc::new(expected_entries), false), false);
3272        let mut inner_md = std::collections::HashMap::new();
3273        inner_md.insert(AVRO_NAME_METADATA_KEY.to_string(), "Inner".to_string());
3274        let expected_inner = ArrowField::new(
3275            "inner",
3276            DataType::Struct(Fields::from(vec![
3277                ArrowField::new("flag", DataType::Boolean, false),
3278                ArrowField::new("name", DataType::Utf8, false),
3279            ])),
3280            false,
3281        )
3282        .with_metadata(inner_md);
3283        let mut root_md = std::collections::HashMap::new();
3284        root_md.insert(AVRO_NAME_METADATA_KEY.to_string(), "R".to_string());
3285        let expected = ArrowField::new(
3286            "R",
3287            DataType::Struct(Fields::from(vec![
3288                ArrowField::new("a", DataType::Int32, false),
3289                expected_b,
3290                expected_c,
3291                expected_inner,
3292                ArrowField::new("u", DataType::Int32, true),
3293            ])),
3294            false,
3295        )
3296        .with_metadata(root_md);
3297        assert_eq!(arrow_field, expected);
3298    }
3299
3300    #[test]
3301    fn default_order_is_consistent() {
3302        let arrow_schema = ArrowSchema::new(vec![ArrowField::new("s", DataType::Utf8, true)]);
3303        let a = AvroSchema::try_from(&arrow_schema).unwrap().json_string;
3304        let b = AvroSchema::from_arrow_with_options(&arrow_schema, None);
3305        assert_eq!(a, b.unwrap().json_string);
3306    }
3307
3308    #[test]
3309    fn test_union_branch_missing_name_errors() {
3310        for t in ["record", "enum", "fixed"] {
3311            let branch = json!({ "type": t });
3312            let err = union_branch_signature(&branch).unwrap_err().to_string();
3313            assert!(
3314                err.contains(&format!("Union branch '{t}' missing required 'name'")),
3315                "expected missing-name error for {t}, got: {err}"
3316            );
3317        }
3318    }
3319
3320    #[test]
3321    fn test_union_branch_named_type_signature_includes_name() {
3322        let rec = json!({ "type": "record", "name": "Foo" });
3323        assert_eq!(union_branch_signature(&rec).unwrap(), "N:record:Foo");
3324        let en = json!({ "type": "enum", "name": "Color", "symbols": ["R", "G", "B"] });
3325        assert_eq!(union_branch_signature(&en).unwrap(), "N:enum:Color");
3326        let fx = json!({ "type": "fixed", "name": "Bytes16", "size": 16 });
3327        assert_eq!(union_branch_signature(&fx).unwrap(), "N:fixed:Bytes16");
3328    }
3329
3330    #[test]
3331    fn test_record_field_alias_resolution_without_default() {
3332        let writer_json = r#"{
3333          "type":"record",
3334          "name":"R",
3335          "fields":[{"name":"old","type":"int"}]
3336        }"#;
3337        let reader_json = r#"{
3338          "type":"record",
3339          "name":"R",
3340          "fields":[{"name":"new","aliases":["old"],"type":"int"}]
3341        }"#;
3342        let writer: Schema = serde_json::from_str(writer_json).unwrap();
3343        let reader: Schema = serde_json::from_str(reader_json).unwrap();
3344        let resolved = AvroFieldBuilder::new(&writer)
3345            .with_reader_schema(&reader)
3346            .with_utf8view(false)
3347            .with_strict_mode(false)
3348            .build()
3349            .unwrap();
3350        let expected = ArrowField::new(
3351            "R",
3352            DataType::Struct(Fields::from(vec![ArrowField::new(
3353                "new",
3354                DataType::Int32,
3355                false,
3356            )])),
3357            false,
3358        )
3359        .with_metadata(HashMap::from_iter([(
3360            "avro.name".to_owned(),
3361            "R".to_owned(),
3362        )]));
3363        assert_eq!(resolved.field(), expected);
3364    }
3365
3366    #[test]
3367    fn test_record_field_alias_ambiguous_in_strict_mode_errors() {
3368        let writer_json = r#"{
3369          "type":"record",
3370          "name":"R",
3371          "fields":[
3372            {"name":"a","type":"int","aliases":["old"]},
3373            {"name":"b","type":"int","aliases":["old"]}
3374          ]
3375        }"#;
3376        let reader_json = r#"{
3377          "type":"record",
3378          "name":"R",
3379          "fields":[{"name":"target","type":"int","aliases":["old"]}]
3380        }"#;
3381        let writer: Schema = serde_json::from_str(writer_json).unwrap();
3382        let reader: Schema = serde_json::from_str(reader_json).unwrap();
3383        let err = AvroFieldBuilder::new(&writer)
3384            .with_reader_schema(&reader)
3385            .with_utf8view(false)
3386            .with_strict_mode(true)
3387            .build()
3388            .unwrap_err()
3389            .to_string();
3390        assert!(
3391            err.contains("Ambiguous alias 'old'"),
3392            "expected ambiguous-alias error, got: {err}"
3393        );
3394    }
3395
3396    #[test]
3397    fn test_pragmatic_writer_field_alias_mapping_non_strict() {
3398        let writer_json = r#"{
3399          "type":"record",
3400          "name":"R",
3401          "fields":[{"name":"before","type":"int","aliases":["now"]}]
3402        }"#;
3403        let reader_json = r#"{
3404          "type":"record",
3405          "name":"R",
3406          "fields":[{"name":"now","type":"int"}]
3407        }"#;
3408        let writer: Schema = serde_json::from_str(writer_json).unwrap();
3409        let reader: Schema = serde_json::from_str(reader_json).unwrap();
3410        let resolved = AvroFieldBuilder::new(&writer)
3411            .with_reader_schema(&reader)
3412            .with_utf8view(false)
3413            .with_strict_mode(false)
3414            .build()
3415            .unwrap();
3416        let expected = ArrowField::new(
3417            "R",
3418            DataType::Struct(Fields::from(vec![ArrowField::new(
3419                "now",
3420                DataType::Int32,
3421                false,
3422            )])),
3423            false,
3424        )
3425        .with_metadata(HashMap::from_iter([(
3426            "avro.name".to_owned(),
3427            "R".to_owned(),
3428        )]));
3429        assert_eq!(resolved.field(), expected);
3430    }
3431
3432    #[test]
3433    fn test_missing_reader_field_null_first_no_default_is_ok() {
3434        let writer_json = r#"{
3435          "type":"record",
3436          "name":"R",
3437          "fields":[{"name":"a","type":"int"}]
3438        }"#;
3439        let reader_json = r#"{
3440          "type":"record",
3441          "name":"R",
3442          "fields":[
3443            {"name":"a","type":"int"},
3444            {"name":"b","type":["null","int"]}
3445          ]
3446        }"#;
3447        let writer: Schema = serde_json::from_str(writer_json).unwrap();
3448        let reader: Schema = serde_json::from_str(reader_json).unwrap();
3449        let resolved = AvroFieldBuilder::new(&writer)
3450            .with_reader_schema(&reader)
3451            .with_utf8view(false)
3452            .with_strict_mode(false)
3453            .build()
3454            .unwrap();
3455        let expected = ArrowField::new(
3456            "R",
3457            DataType::Struct(Fields::from(vec![
3458                ArrowField::new("a", DataType::Int32, false),
3459                ArrowField::new("b", DataType::Int32, true).with_metadata(HashMap::from([(
3460                    AVRO_FIELD_DEFAULT_METADATA_KEY.to_string(),
3461                    "null".to_string(),
3462                )])),
3463            ])),
3464            false,
3465        )
3466        .with_metadata(HashMap::from_iter([(
3467            "avro.name".to_owned(),
3468            "R".to_owned(),
3469        )]));
3470        assert_eq!(resolved.field(), expected);
3471    }
3472
3473    #[test]
3474    fn test_missing_reader_field_null_second_without_default_errors() {
3475        let writer_json = r#"{
3476          "type":"record",
3477          "name":"R",
3478          "fields":[{"name":"a","type":"int"}]
3479        }"#;
3480        let reader_json = r#"{
3481          "type":"record",
3482          "name":"R",
3483          "fields":[
3484            {"name":"a","type":"int"},
3485            {"name":"b","type":["int","null"]}
3486          ]
3487        }"#;
3488        let writer: Schema = serde_json::from_str(writer_json).unwrap();
3489        let reader: Schema = serde_json::from_str(reader_json).unwrap();
3490        let err = AvroFieldBuilder::new(&writer)
3491            .with_reader_schema(&reader)
3492            .with_utf8view(false)
3493            .with_strict_mode(false)
3494            .build()
3495            .unwrap_err()
3496            .to_string();
3497        assert!(
3498            err.contains("must have a default value"),
3499            "expected missing-default error, got: {err}"
3500        );
3501    }
3502
3503    #[test]
3504    fn test_from_arrow_with_options_respects_schema_metadata_when_not_stripping() {
3505        let field = ArrowField::new("x", DataType::Int32, true);
3506        let injected_json =
3507            r#"{"type":"record","name":"Injected","fields":[{"name":"ignored","type":"int"}]}"#
3508                .to_string();
3509        let mut md = HashMap::new();
3510        md.insert(SCHEMA_METADATA_KEY.to_string(), injected_json.clone());
3511        md.insert("custom".to_string(), "123".to_string());
3512        let arrow_schema = ArrowSchema::new_with_metadata(vec![field], md);
3513        let opts = AvroSchemaOptions {
3514            null_order: Some(Nullability::NullSecond),
3515            strip_metadata: false,
3516        };
3517        let out = AvroSchema::from_arrow_with_options(&arrow_schema, Some(opts)).unwrap();
3518        assert_eq!(
3519            out.json_string, injected_json,
3520            "When strip_metadata=false and avro.schema is present, return the embedded JSON verbatim"
3521        );
3522        let v: Value = serde_json::from_str(&out.json_string).unwrap();
3523        assert_eq!(v.get("type").and_then(|t| t.as_str()), Some("record"));
3524        assert_eq!(v.get("name").and_then(|n| n.as_str()), Some("Injected"));
3525    }
3526
3527    #[test]
3528    fn test_from_arrow_with_options_ignores_schema_metadata_when_stripping_and_keeps_passthrough() {
3529        let field = ArrowField::new("x", DataType::Int32, true);
3530        let injected_json =
3531            r#"{"type":"record","name":"Injected","fields":[{"name":"ignored","type":"int"}]}"#
3532                .to_string();
3533        let mut md = HashMap::new();
3534        md.insert(SCHEMA_METADATA_KEY.to_string(), injected_json);
3535        md.insert("custom_meta".to_string(), "7".to_string());
3536        let arrow_schema = ArrowSchema::new_with_metadata(vec![field], md);
3537        let opts = AvroSchemaOptions {
3538            null_order: Some(Nullability::NullFirst),
3539            strip_metadata: true,
3540        };
3541        let out = AvroSchema::from_arrow_with_options(&arrow_schema, Some(opts)).unwrap();
3542        assert_json_contains(&out.json_string, "\"type\":\"record\"");
3543        assert_json_contains(&out.json_string, "\"name\":\"topLevelRecord\"");
3544        assert_json_contains(&out.json_string, "\"custom_meta\":7");
3545    }
3546
3547    #[test]
3548    fn test_from_arrow_with_options_null_first_for_nullable_primitive() {
3549        let field = ArrowField::new("s", DataType::Utf8, true);
3550        let arrow_schema = single_field_schema(field);
3551        let opts = AvroSchemaOptions {
3552            null_order: Some(Nullability::NullFirst),
3553            strip_metadata: true,
3554        };
3555        let out = AvroSchema::from_arrow_with_options(&arrow_schema, Some(opts)).unwrap();
3556        let v: Value = serde_json::from_str(&out.json_string).unwrap();
3557        let arr = v["fields"][0]["type"]
3558            .as_array()
3559            .expect("nullable primitive should be Avro union array");
3560        assert_eq!(arr[0], Value::String("null".into()));
3561        assert_eq!(arr[1], Value::String("string".into()));
3562    }
3563
3564    #[test]
3565    fn test_from_arrow_with_options_null_second_for_nullable_primitive() {
3566        let field = ArrowField::new("s", DataType::Utf8, true);
3567        let arrow_schema = single_field_schema(field);
3568        let opts = AvroSchemaOptions {
3569            null_order: Some(Nullability::NullSecond),
3570            strip_metadata: true,
3571        };
3572        let out = AvroSchema::from_arrow_with_options(&arrow_schema, Some(opts)).unwrap();
3573        let v: Value = serde_json::from_str(&out.json_string).unwrap();
3574        let arr = v["fields"][0]["type"]
3575            .as_array()
3576            .expect("nullable primitive should be Avro union array");
3577        assert_eq!(arr[0], Value::String("string".into()));
3578        assert_eq!(arr[1], Value::String("null".into()));
3579    }
3580
3581    #[test]
3582    fn test_from_arrow_with_options_union_extras_respected_by_strip_metadata() {
3583        let uf: UnionFields = vec![
3584            (2i8, Arc::new(ArrowField::new("a", DataType::Int32, false))),
3585            (7i8, Arc::new(ArrowField::new("b", DataType::Utf8, false))),
3586        ]
3587        .into_iter()
3588        .collect();
3589        let union_dt = DataType::Union(uf, UnionMode::Dense);
3590        let arrow_schema = single_field_schema(ArrowField::new("u", union_dt, true));
3591        let with_extras = AvroSchema::from_arrow_with_options(
3592            &arrow_schema,
3593            Some(AvroSchemaOptions {
3594                null_order: Some(Nullability::NullFirst),
3595                strip_metadata: false,
3596            }),
3597        )
3598        .unwrap();
3599        let v_with: Value = serde_json::from_str(&with_extras.json_string).unwrap();
3600        let union_arr = v_with["fields"][0]["type"].as_array().expect("union array");
3601        let first_obj = union_arr
3602            .iter()
3603            .find(|b| b.is_object())
3604            .expect("expected an object branch with extras");
3605        let obj = first_obj.as_object().unwrap();
3606        assert_eq!(obj.get("type").and_then(|t| t.as_str()), Some("int"));
3607        assert_eq!(
3608            obj.get("arrowUnionMode").and_then(|m| m.as_str()),
3609            Some("dense")
3610        );
3611        let type_ids: Vec<i64> = obj["arrowUnionTypeIds"]
3612            .as_array()
3613            .expect("arrowUnionTypeIds array")
3614            .iter()
3615            .map(|n| n.as_i64().expect("i64"))
3616            .collect();
3617        assert_eq!(type_ids, vec![2, 7]);
3618        let stripped = AvroSchema::from_arrow_with_options(
3619            &arrow_schema,
3620            Some(AvroSchemaOptions {
3621                null_order: Some(Nullability::NullFirst),
3622                strip_metadata: true,
3623            }),
3624        )
3625        .unwrap();
3626        let v_stripped: Value = serde_json::from_str(&stripped.json_string).unwrap();
3627        let union_arr2 = v_stripped["fields"][0]["type"]
3628            .as_array()
3629            .expect("union array");
3630        assert!(
3631            !union_arr2.iter().any(|b| b
3632                .as_object()
3633                .is_some_and(|m| m.contains_key("arrowUnionMode"))),
3634            "extras must be removed when strip_metadata=true"
3635        );
3636        assert_eq!(union_arr2[0], Value::String("null".into()));
3637        assert_eq!(union_arr2[1], Value::String("int".into()));
3638        assert_eq!(union_arr2[2], Value::String("string".into()));
3639    }
3640
3641    #[test]
3642    fn test_project_empty_projection() {
3643        let schema_json = r#"{
3644            "type": "record",
3645            "name": "Test",
3646            "fields": [
3647                {"name": "a", "type": "int"},
3648                {"name": "b", "type": "string"}
3649            ]
3650        }"#;
3651        let schema = AvroSchema::new(schema_json.to_string());
3652        let projected = schema.project(&[]).unwrap();
3653        let v: Value = serde_json::from_str(&projected.json_string).unwrap();
3654        let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
3655        assert!(
3656            fields.is_empty(),
3657            "Empty projection should yield empty fields"
3658        );
3659    }
3660
3661    #[test]
3662    fn test_project_single_field() {
3663        let schema_json = r#"{
3664            "type": "record",
3665            "name": "Test",
3666            "fields": [
3667                {"name": "a", "type": "int"},
3668                {"name": "b", "type": "string"},
3669                {"name": "c", "type": "long"}
3670            ]
3671        }"#;
3672        let schema = AvroSchema::new(schema_json.to_string());
3673        let projected = schema.project(&[1]).unwrap();
3674        let v: Value = serde_json::from_str(&projected.json_string).unwrap();
3675        let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
3676        assert_eq!(fields.len(), 1);
3677        assert_eq!(fields[0].get("name").and_then(|n| n.as_str()), Some("b"));
3678    }
3679
3680    #[test]
3681    fn test_project_multiple_fields() {
3682        let schema_json = r#"{
3683            "type": "record",
3684            "name": "Test",
3685            "fields": [
3686                {"name": "a", "type": "int"},
3687                {"name": "b", "type": "string"},
3688                {"name": "c", "type": "long"},
3689                {"name": "d", "type": "boolean"}
3690            ]
3691        }"#;
3692        let schema = AvroSchema::new(schema_json.to_string());
3693        let projected = schema.project(&[0, 2, 3]).unwrap();
3694        let v: Value = serde_json::from_str(&projected.json_string).unwrap();
3695        let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
3696        assert_eq!(fields.len(), 3);
3697        assert_eq!(fields[0].get("name").and_then(|n| n.as_str()), Some("a"));
3698        assert_eq!(fields[1].get("name").and_then(|n| n.as_str()), Some("c"));
3699        assert_eq!(fields[2].get("name").and_then(|n| n.as_str()), Some("d"));
3700    }
3701
3702    #[test]
3703    fn test_project_all_fields() {
3704        let schema_json = r#"{
3705            "type": "record",
3706            "name": "Test",
3707            "fields": [
3708                {"name": "a", "type": "int"},
3709                {"name": "b", "type": "string"}
3710            ]
3711        }"#;
3712        let schema = AvroSchema::new(schema_json.to_string());
3713        let projected = schema.project(&[0, 1]).unwrap();
3714        let v: Value = serde_json::from_str(&projected.json_string).unwrap();
3715        let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
3716        assert_eq!(fields.len(), 2);
3717        assert_eq!(fields[0].get("name").and_then(|n| n.as_str()), Some("a"));
3718        assert_eq!(fields[1].get("name").and_then(|n| n.as_str()), Some("b"));
3719    }
3720
3721    #[test]
3722    fn test_project_reorder_fields() {
3723        let schema_json = r#"{
3724            "type": "record",
3725            "name": "Test",
3726            "fields": [
3727                {"name": "a", "type": "int"},
3728                {"name": "b", "type": "string"},
3729                {"name": "c", "type": "long"}
3730            ]
3731        }"#;
3732        let schema = AvroSchema::new(schema_json.to_string());
3733        // Project in reverse order
3734        let projected = schema.project(&[2, 0, 1]).unwrap();
3735        let v: Value = serde_json::from_str(&projected.json_string).unwrap();
3736        let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
3737        assert_eq!(fields.len(), 3);
3738        assert_eq!(fields[0].get("name").and_then(|n| n.as_str()), Some("c"));
3739        assert_eq!(fields[1].get("name").and_then(|n| n.as_str()), Some("a"));
3740        assert_eq!(fields[2].get("name").and_then(|n| n.as_str()), Some("b"));
3741    }
3742
3743    #[test]
3744    fn test_project_preserves_record_metadata() {
3745        let schema_json = r#"{
3746            "type": "record",
3747            "name": "MyRecord",
3748            "namespace": "com.example",
3749            "doc": "A test record",
3750            "aliases": ["OldRecord"],
3751            "fields": [
3752                {"name": "a", "type": "int"},
3753                {"name": "b", "type": "string"}
3754            ]
3755        }"#;
3756        let schema = AvroSchema::new(schema_json.to_string());
3757        let projected = schema.project(&[0]).unwrap();
3758        let v: Value = serde_json::from_str(&projected.json_string).unwrap();
3759        assert_eq!(v.get("name").and_then(|n| n.as_str()), Some("MyRecord"));
3760        assert_eq!(
3761            v.get("namespace").and_then(|n| n.as_str()),
3762            Some("com.example")
3763        );
3764        assert_eq!(v.get("doc").and_then(|n| n.as_str()), Some("A test record"));
3765        assert!(v.get("aliases").is_some());
3766    }
3767
3768    #[test]
3769    fn test_project_preserves_field_metadata() {
3770        let schema_json = r#"{
3771            "type": "record",
3772            "name": "Test",
3773            "fields": [
3774                {"name": "a", "type": "int", "doc": "Field A", "default": 0},
3775                {"name": "b", "type": "string"}
3776            ]
3777        }"#;
3778        let schema = AvroSchema::new(schema_json.to_string());
3779        let projected = schema.project(&[0]).unwrap();
3780        let v: Value = serde_json::from_str(&projected.json_string).unwrap();
3781        let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
3782        assert_eq!(
3783            fields[0].get("doc").and_then(|d| d.as_str()),
3784            Some("Field A")
3785        );
3786        assert_eq!(fields[0].get("default").and_then(|d| d.as_i64()), Some(0));
3787    }
3788
3789    #[test]
3790    fn test_project_with_nested_record() {
3791        let schema_json = r#"{
3792            "type": "record",
3793            "name": "Outer",
3794            "fields": [
3795                {"name": "id", "type": "int"},
3796                {"name": "inner", "type": {
3797                    "type": "record",
3798                    "name": "Inner",
3799                    "fields": [
3800                        {"name": "x", "type": "int"},
3801                        {"name": "y", "type": "string"}
3802                    ]
3803                }},
3804                {"name": "value", "type": "double"}
3805            ]
3806        }"#;
3807        let schema = AvroSchema::new(schema_json.to_string());
3808        let projected = schema.project(&[1]).unwrap();
3809        let v: Value = serde_json::from_str(&projected.json_string).unwrap();
3810        let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
3811        assert_eq!(fields.len(), 1);
3812        assert_eq!(
3813            fields[0].get("name").and_then(|n| n.as_str()),
3814            Some("inner")
3815        );
3816        // Verify nested record structure is preserved
3817        let inner_type = fields[0].get("type").unwrap();
3818        assert_eq!(
3819            inner_type.get("type").and_then(|t| t.as_str()),
3820            Some("record")
3821        );
3822        assert_eq!(
3823            inner_type.get("name").and_then(|n| n.as_str()),
3824            Some("Inner")
3825        );
3826    }
3827
3828    #[test]
3829    fn test_project_with_complex_field_types() {
3830        let schema_json = r#"{
3831            "type": "record",
3832            "name": "Test",
3833            "fields": [
3834                {"name": "arr", "type": {"type": "array", "items": "int"}},
3835                {"name": "map", "type": {"type": "map", "values": "string"}},
3836                {"name": "union", "type": ["null", "int"]}
3837            ]
3838        }"#;
3839        let schema = AvroSchema::new(schema_json.to_string());
3840        let projected = schema.project(&[0, 2]).unwrap();
3841        let v: Value = serde_json::from_str(&projected.json_string).unwrap();
3842        let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
3843        assert_eq!(fields.len(), 2);
3844        // Verify array type is preserved
3845        let arr_type = fields[0].get("type").unwrap();
3846        assert_eq!(arr_type.get("type").and_then(|t| t.as_str()), Some("array"));
3847        // Verify union type is preserved
3848        let union_type = fields[1].get("type").unwrap();
3849        assert!(union_type.is_array());
3850    }
3851
3852    #[test]
3853    fn test_project_error_invalid_json() {
3854        let schema = AvroSchema::new("not valid json".to_string());
3855        let err = schema.project(&[0]).unwrap_err();
3856        let msg = err.to_string();
3857        assert!(
3858            msg.contains("Invalid Avro schema JSON"),
3859            "Expected parse error, got: {msg}"
3860        );
3861    }
3862
3863    #[test]
3864    fn test_project_error_not_object() {
3865        // Primitive type schema (not a JSON object)
3866        let schema = AvroSchema::new(r#""string""#.to_string());
3867        let err = schema.project(&[0]).unwrap_err();
3868        let msg = err.to_string();
3869        assert!(
3870            msg.contains("must be a JSON object"),
3871            "Expected object error, got: {msg}"
3872        );
3873    }
3874
3875    #[test]
3876    fn test_project_error_array_schema() {
3877        // Array (list) is a valid JSON but not a record
3878        let schema = AvroSchema::new(r#"["null", "int"]"#.to_string());
3879        let err = schema.project(&[0]).unwrap_err();
3880        let msg = err.to_string();
3881        assert!(
3882            msg.contains("must be a JSON object"),
3883            "Expected object error for array schema, got: {msg}"
3884        );
3885    }
3886
3887    #[test]
3888    fn test_project_error_type_not_record() {
3889        let schema_json = r#"{
3890            "type": "enum",
3891            "name": "Color",
3892            "symbols": ["RED", "GREEN", "BLUE"]
3893        }"#;
3894        let schema = AvroSchema::new(schema_json.to_string());
3895        let err = schema.project(&[0]).unwrap_err();
3896        let msg = err.to_string();
3897        assert!(
3898            msg.contains("must be an Avro record") && msg.contains("'enum'"),
3899            "Expected type mismatch error, got: {msg}"
3900        );
3901    }
3902
3903    #[test]
3904    fn test_project_error_type_array() {
3905        let schema_json = r#"{
3906            "type": "array",
3907            "items": "int"
3908        }"#;
3909        let schema = AvroSchema::new(schema_json.to_string());
3910        let err = schema.project(&[0]).unwrap_err();
3911        let msg = err.to_string();
3912        assert!(
3913            msg.contains("must be an Avro record") && msg.contains("'array'"),
3914            "Expected type mismatch error for array type, got: {msg}"
3915        );
3916    }
3917
3918    #[test]
3919    fn test_project_error_type_fixed() {
3920        let schema_json = r#"{
3921            "type": "fixed",
3922            "name": "MD5",
3923            "size": 16
3924        }"#;
3925        let schema = AvroSchema::new(schema_json.to_string());
3926        let err = schema.project(&[0]).unwrap_err();
3927        let msg = err.to_string();
3928        assert!(
3929            msg.contains("must be an Avro record") && msg.contains("'fixed'"),
3930            "Expected type mismatch error for fixed type, got: {msg}"
3931        );
3932    }
3933
3934    #[test]
3935    fn test_project_error_type_map() {
3936        let schema_json = r#"{
3937            "type": "map",
3938            "values": "string"
3939        }"#;
3940        let schema = AvroSchema::new(schema_json.to_string());
3941        let err = schema.project(&[0]).unwrap_err();
3942        let msg = err.to_string();
3943        assert!(
3944            msg.contains("must be an Avro record") && msg.contains("'map'"),
3945            "Expected type mismatch error for map type, got: {msg}"
3946        );
3947    }
3948
3949    #[test]
3950    fn test_project_error_missing_type_field() {
3951        let schema_json = r#"{
3952            "name": "Test",
3953            "fields": [{"name": "a", "type": "int"}]
3954        }"#;
3955        let schema = AvroSchema::new(schema_json.to_string());
3956        let err = schema.project(&[0]).unwrap_err();
3957        let msg = err.to_string();
3958        assert!(
3959            msg.contains("missing required 'type' field"),
3960            "Expected missing type error, got: {msg}"
3961        );
3962    }
3963
3964    #[test]
3965    fn test_project_error_missing_fields() {
3966        let schema_json = r#"{
3967            "type": "record",
3968            "name": "Test"
3969        }"#;
3970        let schema = AvroSchema::new(schema_json.to_string());
3971        let err = schema.project(&[0]).unwrap_err();
3972        let msg = err.to_string();
3973        assert!(
3974            msg.contains("missing required 'fields'"),
3975            "Expected missing fields error, got: {msg}"
3976        );
3977    }
3978
3979    #[test]
3980    fn test_project_error_fields_not_array() {
3981        let schema_json = r#"{
3982            "type": "record",
3983            "name": "Test",
3984            "fields": "not an array"
3985        }"#;
3986        let schema = AvroSchema::new(schema_json.to_string());
3987        let err = schema.project(&[0]).unwrap_err();
3988        let msg = err.to_string();
3989        assert!(
3990            msg.contains("'fields' must be an array"),
3991            "Expected fields array error, got: {msg}"
3992        );
3993    }
3994
3995    #[test]
3996    fn test_project_error_index_out_of_bounds() {
3997        let schema_json = r#"{
3998            "type": "record",
3999            "name": "Test",
4000            "fields": [
4001                {"name": "a", "type": "int"},
4002                {"name": "b", "type": "string"}
4003            ]
4004        }"#;
4005        let schema = AvroSchema::new(schema_json.to_string());
4006        let err = schema.project(&[5]).unwrap_err();
4007        let msg = err.to_string();
4008        assert!(
4009            msg.contains("out of bounds") && msg.contains('5') && msg.contains('2'),
4010            "Expected out of bounds error, got: {msg}"
4011        );
4012    }
4013
4014    #[test]
4015    fn test_project_error_index_out_of_bounds_edge() {
4016        let schema_json = r#"{
4017            "type": "record",
4018            "name": "Test",
4019            "fields": [
4020                {"name": "a", "type": "int"}
4021            ]
4022        }"#;
4023        let schema = AvroSchema::new(schema_json.to_string());
4024        // Index 1 is just out of bounds for a 1-element array
4025        let err = schema.project(&[1]).unwrap_err();
4026        let msg = err.to_string();
4027        assert!(
4028            msg.contains("out of bounds") && msg.contains('1'),
4029            "Expected out of bounds error for edge case, got: {msg}"
4030        );
4031    }
4032
4033    #[test]
4034    fn test_project_error_duplicate_index() {
4035        let schema_json = r#"{
4036            "type": "record",
4037            "name": "Test",
4038            "fields": [
4039                {"name": "a", "type": "int"},
4040                {"name": "b", "type": "string"},
4041                {"name": "c", "type": "long"}
4042            ]
4043        }"#;
4044        let schema = AvroSchema::new(schema_json.to_string());
4045        let err = schema.project(&[0, 1, 0]).unwrap_err();
4046        let msg = err.to_string();
4047        assert!(
4048            msg.contains("Duplicate projection index") && msg.contains('0'),
4049            "Expected duplicate index error, got: {msg}"
4050        );
4051    }
4052
4053    #[test]
4054    fn test_project_error_duplicate_index_consecutive() {
4055        let schema_json = r#"{
4056            "type": "record",
4057            "name": "Test",
4058            "fields": [
4059                {"name": "a", "type": "int"},
4060                {"name": "b", "type": "string"}
4061            ]
4062        }"#;
4063        let schema = AvroSchema::new(schema_json.to_string());
4064        let err = schema.project(&[1, 1]).unwrap_err();
4065        let msg = err.to_string();
4066        assert!(
4067            msg.contains("Duplicate projection index") && msg.contains('1'),
4068            "Expected duplicate index error for consecutive duplicates, got: {msg}"
4069        );
4070    }
4071
4072    #[test]
4073    fn test_project_with_empty_fields() {
4074        let schema_json = r#"{
4075            "type": "record",
4076            "name": "EmptyRecord",
4077            "fields": []
4078        }"#;
4079        let schema = AvroSchema::new(schema_json.to_string());
4080        // Projecting empty from empty should succeed
4081        let projected = schema.project(&[]).unwrap();
4082        let v: Value = serde_json::from_str(&projected.json_string).unwrap();
4083        let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
4084        assert!(fields.is_empty());
4085    }
4086
4087    #[test]
4088    fn test_project_empty_fields_index_out_of_bounds() {
4089        let schema_json = r#"{
4090            "type": "record",
4091            "name": "EmptyRecord",
4092            "fields": []
4093        }"#;
4094        let schema = AvroSchema::new(schema_json.to_string());
4095        let err = schema.project(&[0]).unwrap_err();
4096        let msg = err.to_string();
4097        assert!(
4098            msg.contains("out of bounds") && msg.contains("0 fields"),
4099            "Expected out of bounds error for empty record, got: {msg}"
4100        );
4101    }
4102
4103    #[test]
4104    fn test_project_result_is_valid_avro_schema() {
4105        let schema_json = r#"{
4106            "type": "record",
4107            "name": "Test",
4108            "namespace": "com.example",
4109            "fields": [
4110                {"name": "id", "type": "long"},
4111                {"name": "name", "type": "string"},
4112                {"name": "active", "type": "boolean"}
4113            ]
4114        }"#;
4115        let schema = AvroSchema::new(schema_json.to_string());
4116        let projected = schema.project(&[0, 2]).unwrap();
4117        // Verify the projected schema can be parsed as a valid Avro schema
4118        let parsed = projected.schema();
4119        assert!(parsed.is_ok(), "Projected schema should be valid Avro");
4120        match parsed.unwrap() {
4121            Schema::Complex(ComplexType::Record(r)) => {
4122                assert_eq!(r.name, "Test");
4123                assert_eq!(r.namespace, Some("com.example"));
4124                assert_eq!(r.fields.len(), 2);
4125                assert_eq!(r.fields[0].name, "id");
4126                assert_eq!(r.fields[1].name, "active");
4127            }
4128            _ => panic!("Expected Record schema"),
4129        }
4130    }
4131
4132    #[test]
4133    fn test_project_non_contiguous_indices() {
4134        let schema_json = r#"{
4135            "type": "record",
4136            "name": "Test",
4137            "fields": [
4138                {"name": "f0", "type": "int"},
4139                {"name": "f1", "type": "int"},
4140                {"name": "f2", "type": "int"},
4141                {"name": "f3", "type": "int"},
4142                {"name": "f4", "type": "int"}
4143            ]
4144        }"#;
4145        let schema = AvroSchema::new(schema_json.to_string());
4146        // Select every other field
4147        let projected = schema.project(&[0, 2, 4]).unwrap();
4148        let v: Value = serde_json::from_str(&projected.json_string).unwrap();
4149        let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
4150        assert_eq!(fields.len(), 3);
4151        assert_eq!(fields[0].get("name").and_then(|n| n.as_str()), Some("f0"));
4152        assert_eq!(fields[1].get("name").and_then(|n| n.as_str()), Some("f2"));
4153        assert_eq!(fields[2].get("name").and_then(|n| n.as_str()), Some("f4"));
4154    }
4155
4156    #[test]
4157    fn test_project_single_field_from_many() {
4158        let schema_json = r#"{
4159            "type": "record",
4160            "name": "BigRecord",
4161            "fields": [
4162                {"name": "f0", "type": "int"},
4163                {"name": "f1", "type": "int"},
4164                {"name": "f2", "type": "int"},
4165                {"name": "f3", "type": "int"},
4166                {"name": "f4", "type": "int"},
4167                {"name": "f5", "type": "int"},
4168                {"name": "f6", "type": "int"},
4169                {"name": "f7", "type": "int"},
4170                {"name": "f8", "type": "int"},
4171                {"name": "f9", "type": "int"}
4172            ]
4173        }"#;
4174        let schema = AvroSchema::new(schema_json.to_string());
4175        // Select only the last field
4176        let projected = schema.project(&[9]).unwrap();
4177        let v: Value = serde_json::from_str(&projected.json_string).unwrap();
4178        let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
4179        assert_eq!(fields.len(), 1);
4180        assert_eq!(fields[0].get("name").and_then(|n| n.as_str()), Some("f9"));
4181    }
4182}