1#[cfg(feature = "canonical_extension_types")]
8use arrow_schema::extension::ExtensionType;
9use arrow_schema::{
10 ArrowError, DataType, Field as ArrowField, IntervalUnit, Metadata, Schema as ArrowSchema,
11 TimeUnit, UnionMode,
12};
13use serde::{Deserialize, Serialize};
14use serde_json::{Map as JsonMap, Value, json};
15#[cfg(feature = "sha256")]
16use sha2::{Digest, Sha256};
17use std::borrow::Cow;
18use std::cmp::PartialEq;
19use std::collections::hash_map::Entry;
20use std::collections::{HashMap, HashSet};
21use strum_macros::AsRefStr;
22
23pub const SINGLE_OBJECT_MAGIC: [u8; 2] = [0xC3, 0x01];
25
26pub const CONFLUENT_MAGIC: [u8; 1] = [0x00];
28
29pub const MAX_PREFIX_LEN: usize = 34;
32
33pub const SCHEMA_METADATA_KEY: &str = "avro.schema";
35
36pub const AVRO_ENUM_SYMBOLS_METADATA_KEY: &str = "avro.enum.symbols";
38
39pub const AVRO_FIELD_DEFAULT_METADATA_KEY: &str = "avro.field.default";
41
42pub const AVRO_NAME_METADATA_KEY: &str = "avro.name";
44
45pub const AVRO_NAMESPACE_METADATA_KEY: &str = "avro.namespace";
47
48pub const AVRO_DOC_METADATA_KEY: &str = "avro.doc";
50
51pub const AVRO_ROOT_RECORD_DEFAULT_NAME: &str = "topLevelRecord";
53
54#[derive(Debug, Copy, Clone, PartialEq, Default)]
60pub(crate) enum Nullability {
61 #[default]
63 NullFirst,
64 NullSecond,
66}
67
68impl Nullability {
69 pub(crate) fn non_null_index(&self) -> usize {
71 match self {
72 Nullability::NullFirst => 1,
73 Nullability::NullSecond => 0,
74 }
75 }
76}
77
78#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
82#[serde(untagged)]
83pub(crate) enum TypeName<'a> {
87 Primitive(PrimitiveType),
89 Ref(&'a str),
91}
92
93#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, AsRefStr)]
97#[serde(rename_all = "camelCase")]
98#[strum(serialize_all = "lowercase")]
99pub(crate) enum PrimitiveType {
100 Null,
102 Boolean,
104 Int,
106 Long,
108 Float,
110 Double,
112 Bytes,
114 String,
116}
117
118#[derive(Debug, Clone, PartialEq, Eq, Default, Deserialize, Serialize)]
122#[serde(rename_all = "camelCase")]
123pub(crate) struct Attributes<'a> {
124 #[serde(default)]
128 pub(crate) logical_type: Option<&'a str>,
129
130 #[serde(flatten)]
132 pub(crate) additional: HashMap<&'a str, Value>,
133}
134
135impl Attributes<'_> {
136 pub(crate) fn field_metadata(&self) -> HashMap<String, String> {
138 self.additional
139 .iter()
140 .map(|(k, v)| (k.to_string(), v.to_string()))
141 .collect()
142 }
143}
144
145#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
147#[serde(rename_all = "camelCase")]
148pub(crate) struct Type<'a> {
149 #[serde(borrow)]
151 pub(crate) r#type: TypeName<'a>,
152 #[serde(flatten)]
154 pub(crate) attributes: Attributes<'a>,
155}
156
157#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
162#[serde(untagged)]
163pub(crate) enum Schema<'a> {
164 #[serde(borrow)]
166 TypeName(TypeName<'a>),
167 #[serde(borrow)]
169 Union(Vec<Schema<'a>>),
170 #[serde(borrow)]
172 Complex(ComplexType<'a>),
173 #[serde(borrow)]
175 Type(Type<'a>),
176}
177
178#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
182#[serde(tag = "type", rename_all = "camelCase")]
183pub(crate) enum ComplexType<'a> {
184 #[serde(borrow)]
186 Record(Record<'a>),
187 #[serde(borrow)]
189 Enum(Enum<'a>),
190 #[serde(borrow)]
192 Array(Array<'a>),
193 #[serde(borrow)]
195 Map(Map<'a>),
196 #[serde(borrow)]
198 Fixed(Fixed<'a>),
199}
200
201#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
205pub(crate) struct Record<'a> {
206 #[serde(borrow)]
208 pub(crate) name: &'a str,
209 #[serde(borrow, default)]
211 pub(crate) namespace: Option<&'a str>,
212 #[serde(borrow, default)]
214 pub(crate) doc: Option<Cow<'a, str>>,
215 #[serde(borrow, default)]
217 pub(crate) aliases: Vec<&'a str>,
218 #[serde(borrow)]
220 pub(crate) fields: Vec<Field<'a>>,
221 #[serde(flatten)]
223 pub(crate) attributes: Attributes<'a>,
224}
225
226fn deserialize_default<'de, D>(deserializer: D) -> Result<Option<Value>, D::Error>
227where
228 D: serde::Deserializer<'de>,
229{
230 Value::deserialize(deserializer).map(Some)
231}
232
233#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
235pub(crate) struct Field<'a> {
236 #[serde(borrow)]
238 pub(crate) name: &'a str,
239 #[serde(borrow, default)]
241 pub(crate) doc: Option<Cow<'a, str>>,
242 #[serde(borrow)]
244 pub(crate) r#type: Schema<'a>,
245 #[serde(deserialize_with = "deserialize_default", default)]
247 pub(crate) default: Option<Value>,
248 #[serde(borrow, default)]
251 pub(crate) aliases: Vec<&'a str>,
252}
253
254#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
258pub(crate) struct Enum<'a> {
259 #[serde(borrow)]
261 pub(crate) name: &'a str,
262 #[serde(borrow, default)]
264 pub(crate) namespace: Option<&'a str>,
265 #[serde(borrow, default)]
267 pub(crate) doc: Option<Cow<'a, str>>,
268 #[serde(borrow, default)]
270 pub(crate) aliases: Vec<&'a str>,
271 #[serde(borrow)]
273 pub(crate) symbols: Vec<&'a str>,
274 #[serde(borrow, default)]
276 pub(crate) default: Option<&'a str>,
277 #[serde(flatten)]
279 pub(crate) attributes: Attributes<'a>,
280}
281
282#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
286pub(crate) struct Array<'a> {
287 #[serde(borrow)]
289 pub(crate) items: Box<Schema<'a>>,
290 #[serde(flatten)]
292 pub(crate) attributes: Attributes<'a>,
293}
294
295#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
299pub(crate) struct Map<'a> {
300 #[serde(borrow)]
302 pub(crate) values: Box<Schema<'a>>,
303 #[serde(flatten)]
305 pub(crate) attributes: Attributes<'a>,
306}
307
308#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
312pub(crate) struct Fixed<'a> {
313 #[serde(borrow)]
315 pub(crate) name: &'a str,
316 #[serde(borrow, default)]
318 pub(crate) namespace: Option<&'a str>,
319 #[serde(borrow, default)]
321 pub(crate) aliases: Vec<&'a str>,
322 pub(crate) size: usize,
324 #[serde(flatten)]
326 pub(crate) attributes: Attributes<'a>,
327}
328
329#[derive(Debug, Copy, Clone, PartialEq, Default)]
330pub(crate) struct AvroSchemaOptions {
331 pub(crate) null_order: Option<Nullability>,
332 pub(crate) strip_metadata: bool,
333}
334
335#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
337pub struct AvroSchema {
338 pub json_string: String,
340}
341
342impl TryFrom<&ArrowSchema> for AvroSchema {
343 type Error = ArrowError;
344
345 fn try_from(schema: &ArrowSchema) -> Result<Self, Self::Error> {
349 AvroSchema::from_arrow_with_options(schema, None)
350 }
351}
352
353impl AvroSchema {
354 pub fn new(json_string: String) -> Self {
356 Self { json_string }
357 }
358
359 pub(crate) fn schema(&self) -> Result<Schema<'_>, ArrowError> {
360 serde_json::from_str(self.json_string.as_str())
361 .map_err(|e| ArrowError::ParseError(format!("Invalid Avro schema JSON: {e}")))
362 }
363
364 pub fn fingerprint(&self, hash_type: FingerprintAlgorithm) -> Result<Fingerprint, ArrowError> {
392 Self::generate_fingerprint(&self.schema()?, hash_type)
393 }
394
395 pub(crate) fn project(&self, projection: &[usize]) -> Result<Self, ArrowError> {
396 let mut value: Value = serde_json::from_str(&self.json_string)
397 .map_err(|e| ArrowError::AvroError(format!("Invalid Avro schema JSON: {e}")))?;
398 let obj = value.as_object_mut().ok_or_else(|| {
399 ArrowError::AvroError(
400 "Projected schema must be a JSON object Avro record schema".to_string(),
401 )
402 })?;
403 match obj.get("type").and_then(|v| v.as_str()) {
404 Some("record") => {}
405 Some(other) => {
406 return Err(ArrowError::AvroError(format!(
407 "Projected schema must be an Avro record, found type '{other}'"
408 )));
409 }
410 None => {
411 return Err(ArrowError::AvroError(
412 "Projected schema missing required 'type' field".to_string(),
413 ));
414 }
415 }
416 let fields_val = obj.get_mut("fields").ok_or_else(|| {
417 ArrowError::AvroError("Avro record schema missing required 'fields'".to_string())
418 })?;
419 let projected_fields = {
420 let mut original_fields = match fields_val {
421 Value::Array(arr) => std::mem::take(arr),
422 _ => {
423 return Err(ArrowError::AvroError(
424 "Avro record schema 'fields' must be an array".to_string(),
425 ));
426 }
427 };
428 let len = original_fields.len();
429 let mut seen: HashSet<usize> = HashSet::with_capacity(projection.len());
430 let mut out: Vec<Value> = Vec::with_capacity(projection.len());
431 for &i in projection {
432 if i >= len {
433 return Err(ArrowError::AvroError(format!(
434 "Projection index {i} out of bounds for record with {len} fields"
435 )));
436 }
437 if !seen.insert(i) {
438 return Err(ArrowError::AvroError(format!(
439 "Duplicate projection index {i}"
440 )));
441 }
442 out.push(std::mem::replace(&mut original_fields[i], Value::Null));
443 }
444 out
445 };
446 *fields_val = Value::Array(projected_fields);
447 let json_string = serde_json::to_string(&value).map_err(|e| {
448 ArrowError::AvroError(format!(
449 "Failed to serialize projected Avro schema JSON: {e}"
450 ))
451 })?;
452 Ok(Self::new(json_string))
453 }
454
455 pub(crate) fn generate_fingerprint(
456 schema: &Schema,
457 hash_type: FingerprintAlgorithm,
458 ) -> Result<Fingerprint, ArrowError> {
459 let canonical = Self::generate_canonical_form(schema).map_err(|e| {
460 ArrowError::ComputeError(format!("Failed to generate canonical form for schema: {e}"))
461 })?;
462 match hash_type {
463 FingerprintAlgorithm::Rabin => {
464 Ok(Fingerprint::Rabin(compute_fingerprint_rabin(&canonical)))
465 }
466 FingerprintAlgorithm::Id | FingerprintAlgorithm::Id64 => Err(ArrowError::SchemaError(
467 "FingerprintAlgorithm of Id or Id64 cannot be used to generate a fingerprint; \
468 if using Fingerprint::Id, pass the registry ID in instead using the set method."
469 .to_string(),
470 )),
471 #[cfg(feature = "md5")]
472 FingerprintAlgorithm::MD5 => Ok(Fingerprint::MD5(compute_fingerprint_md5(&canonical))),
473 #[cfg(feature = "sha256")]
474 FingerprintAlgorithm::SHA256 => {
475 Ok(Fingerprint::SHA256(compute_fingerprint_sha256(&canonical)))
476 }
477 }
478 }
479
480 pub(crate) fn generate_canonical_form(schema: &Schema) -> Result<String, ArrowError> {
491 build_canonical(schema, None)
492 }
493
494 pub(crate) fn from_arrow_with_options(
501 schema: &ArrowSchema,
502 options: Option<AvroSchemaOptions>,
503 ) -> Result<AvroSchema, ArrowError> {
504 let opts = options.unwrap_or_default();
505 let order = opts.null_order.unwrap_or_default();
506 let strip = opts.strip_metadata;
507 if !strip && let Some(json) = schema.metadata.get(SCHEMA_METADATA_KEY) {
508 return Ok(AvroSchema::new(json.clone()));
509 }
510 let mut name_gen = NameGenerator::default();
511 let fields_json = schema
512 .fields()
513 .iter()
514 .map(|f| arrow_field_to_avro(f, &mut name_gen, order, strip))
515 .collect::<Result<Vec<_>, _>>()?;
516 let record_name = schema
517 .metadata
518 .get(AVRO_NAME_METADATA_KEY)
519 .map_or(AVRO_ROOT_RECORD_DEFAULT_NAME, |s| s.as_str());
520 let mut record = JsonMap::with_capacity(schema.metadata.len() + 4);
521 record.insert("type".into(), Value::String("record".into()));
522 record.insert(
523 "name".into(),
524 Value::String(sanitise_avro_name(record_name)),
525 );
526 if let Some(ns) = schema.metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
527 record.insert("namespace".into(), Value::String(ns.clone()));
528 }
529 if let Some(doc) = schema.metadata.get(AVRO_DOC_METADATA_KEY) {
530 record.insert("doc".into(), Value::String(doc.clone()));
531 }
532 record.insert("fields".into(), Value::Array(fields_json));
533 extend_with_passthrough_metadata(&mut record, &schema.metadata);
534 let json_string = serde_json::to_string(&Value::Object(record))
535 .map_err(|e| ArrowError::SchemaError(format!("Serializing Avro JSON failed: {e}")))?;
536 Ok(AvroSchema::new(json_string))
537 }
538}
539
540#[derive(Debug, Copy, Clone)]
542pub(crate) struct Prefix {
543 buf: [u8; MAX_PREFIX_LEN],
544 len: u8,
545}
546
547impl Prefix {
548 #[inline]
549 pub(crate) fn as_slice(&self) -> &[u8] {
550 &self.buf[..self.len as usize]
551 }
552}
553
554#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
556pub enum FingerprintStrategy {
557 #[default]
559 Rabin,
560 Id(u32),
562 Id64(u64),
564 #[cfg(feature = "md5")]
565 MD5,
567 #[cfg(feature = "sha256")]
568 SHA256,
570}
571
572impl From<Fingerprint> for FingerprintStrategy {
573 fn from(f: Fingerprint) -> Self {
574 Self::from(&f)
575 }
576}
577
578impl From<FingerprintAlgorithm> for FingerprintStrategy {
579 fn from(f: FingerprintAlgorithm) -> Self {
580 match f {
581 FingerprintAlgorithm::Rabin => FingerprintStrategy::Rabin,
582 FingerprintAlgorithm::Id => FingerprintStrategy::Id(0),
583 FingerprintAlgorithm::Id64 => FingerprintStrategy::Id64(0),
584 #[cfg(feature = "md5")]
585 FingerprintAlgorithm::MD5 => FingerprintStrategy::MD5,
586 #[cfg(feature = "sha256")]
587 FingerprintAlgorithm::SHA256 => FingerprintStrategy::SHA256,
588 }
589 }
590}
591
592impl From<&Fingerprint> for FingerprintStrategy {
593 fn from(f: &Fingerprint) -> Self {
594 match f {
595 Fingerprint::Rabin(_) => FingerprintStrategy::Rabin,
596 Fingerprint::Id(_) => FingerprintStrategy::Id(0),
597 Fingerprint::Id64(_) => FingerprintStrategy::Id64(0),
598 #[cfg(feature = "md5")]
599 Fingerprint::MD5(_) => FingerprintStrategy::MD5,
600 #[cfg(feature = "sha256")]
601 Fingerprint::SHA256(_) => FingerprintStrategy::SHA256,
602 }
603 }
604}
605
606#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)]
609pub enum FingerprintAlgorithm {
610 #[default]
612 Rabin,
613 Id,
615 Id64,
617 #[cfg(feature = "md5")]
618 MD5,
620 #[cfg(feature = "sha256")]
621 SHA256,
623}
624
625impl From<&Fingerprint> for FingerprintAlgorithm {
627 fn from(fp: &Fingerprint) -> Self {
628 match fp {
629 Fingerprint::Rabin(_) => FingerprintAlgorithm::Rabin,
630 Fingerprint::Id(_) => FingerprintAlgorithm::Id,
631 Fingerprint::Id64(_) => FingerprintAlgorithm::Id64,
632 #[cfg(feature = "md5")]
633 Fingerprint::MD5(_) => FingerprintAlgorithm::MD5,
634 #[cfg(feature = "sha256")]
635 Fingerprint::SHA256(_) => FingerprintAlgorithm::SHA256,
636 }
637 }
638}
639
640impl From<FingerprintStrategy> for FingerprintAlgorithm {
641 fn from(s: FingerprintStrategy) -> Self {
642 Self::from(&s)
643 }
644}
645
646impl From<&FingerprintStrategy> for FingerprintAlgorithm {
647 fn from(s: &FingerprintStrategy) -> Self {
648 match s {
649 FingerprintStrategy::Rabin => FingerprintAlgorithm::Rabin,
650 FingerprintStrategy::Id(_) => FingerprintAlgorithm::Id,
651 FingerprintStrategy::Id64(_) => FingerprintAlgorithm::Id64,
652 #[cfg(feature = "md5")]
653 FingerprintStrategy::MD5 => FingerprintAlgorithm::MD5,
654 #[cfg(feature = "sha256")]
655 FingerprintStrategy::SHA256 => FingerprintAlgorithm::SHA256,
656 }
657 }
658}
659
660#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
669pub enum Fingerprint {
670 Rabin(u64),
672 Id(u32),
674 Id64(u64),
676 #[cfg(feature = "md5")]
677 MD5([u8; 16]),
679 #[cfg(feature = "sha256")]
680 SHA256([u8; 32]),
682}
683
684impl From<FingerprintStrategy> for Fingerprint {
685 fn from(s: FingerprintStrategy) -> Self {
686 Self::from(&s)
687 }
688}
689
690impl From<&FingerprintStrategy> for Fingerprint {
691 fn from(s: &FingerprintStrategy) -> Self {
692 match s {
693 FingerprintStrategy::Rabin => Fingerprint::Rabin(0),
694 FingerprintStrategy::Id(id) => Fingerprint::Id(*id),
695 FingerprintStrategy::Id64(id) => Fingerprint::Id64(*id),
696 #[cfg(feature = "md5")]
697 FingerprintStrategy::MD5 => Fingerprint::MD5([0; 16]),
698 #[cfg(feature = "sha256")]
699 FingerprintStrategy::SHA256 => Fingerprint::SHA256([0; 32]),
700 }
701 }
702}
703
704impl From<FingerprintAlgorithm> for Fingerprint {
705 fn from(s: FingerprintAlgorithm) -> Self {
706 match s {
707 FingerprintAlgorithm::Rabin => Fingerprint::Rabin(0),
708 FingerprintAlgorithm::Id => Fingerprint::Id(0),
709 FingerprintAlgorithm::Id64 => Fingerprint::Id64(0),
710 #[cfg(feature = "md5")]
711 FingerprintAlgorithm::MD5 => Fingerprint::MD5([0; 16]),
712 #[cfg(feature = "sha256")]
713 FingerprintAlgorithm::SHA256 => Fingerprint::SHA256([0; 32]),
714 }
715 }
716}
717
718impl Fingerprint {
719 pub fn load_fingerprint_id(id: u32) -> Self {
727 Fingerprint::Id(u32::from_be(id))
728 }
729
730 pub fn load_fingerprint_id64(id: u64) -> Self {
738 Fingerprint::Id64(u64::from_be(id))
739 }
740
741 pub(crate) fn make_prefix(&self) -> Prefix {
765 let mut buf = [0u8; MAX_PREFIX_LEN];
766 let len = match self {
767 Self::Id(val) => write_prefix(&mut buf, &CONFLUENT_MAGIC, &val.to_be_bytes()),
768 Self::Id64(val) => write_prefix(&mut buf, &CONFLUENT_MAGIC, &val.to_be_bytes()),
769 Self::Rabin(val) => write_prefix(&mut buf, &SINGLE_OBJECT_MAGIC, &val.to_le_bytes()),
770 #[cfg(feature = "md5")]
771 Self::MD5(val) => write_prefix(&mut buf, &SINGLE_OBJECT_MAGIC, val),
772 #[cfg(feature = "sha256")]
773 Self::SHA256(val) => write_prefix(&mut buf, &SINGLE_OBJECT_MAGIC, val),
774 };
775 Prefix { buf, len }
776 }
777}
778
779fn write_prefix<const MAGIC_LEN: usize, const PAYLOAD_LEN: usize>(
780 buf: &mut [u8; MAX_PREFIX_LEN],
781 magic: &[u8; MAGIC_LEN],
782 payload: &[u8; PAYLOAD_LEN],
783) -> u8 {
784 debug_assert!(MAGIC_LEN + PAYLOAD_LEN <= MAX_PREFIX_LEN);
785 let total = MAGIC_LEN + PAYLOAD_LEN;
786 let prefix_slice = &mut buf[..total];
787 prefix_slice[..MAGIC_LEN].copy_from_slice(magic);
788 prefix_slice[MAGIC_LEN..total].copy_from_slice(payload);
789 total as u8
790}
791
792#[derive(Debug, Clone, Default)]
819pub struct SchemaStore {
820 fingerprint_algorithm: FingerprintAlgorithm,
822 schemas: HashMap<Fingerprint, AvroSchema>,
824}
825
826impl TryFrom<HashMap<Fingerprint, AvroSchema>> for SchemaStore {
827 type Error = ArrowError;
828
829 fn try_from(schemas: HashMap<Fingerprint, AvroSchema>) -> Result<Self, Self::Error> {
832 Ok(Self {
833 schemas,
834 ..Self::default()
835 })
836 }
837}
838
839impl SchemaStore {
840 pub fn new() -> Self {
842 Self::default()
843 }
844
845 pub fn new_with_type(fingerprint_algorithm: FingerprintAlgorithm) -> Self {
847 Self {
848 fingerprint_algorithm,
849 ..Self::default()
850 }
851 }
852
853 pub fn set(
870 &mut self,
871 fingerprint: Fingerprint,
872 schema: AvroSchema,
873 ) -> Result<Fingerprint, ArrowError> {
874 match self.schemas.entry(fingerprint) {
875 Entry::Occupied(entry) => {
876 if entry.get() != &schema {
877 return Err(ArrowError::ComputeError(format!(
878 "Schema fingerprint collision detected for fingerprint {fingerprint:?}"
879 )));
880 }
881 }
882 Entry::Vacant(entry) => {
883 entry.insert(schema);
884 }
885 }
886 Ok(fingerprint)
887 }
888
889 pub fn register(&mut self, schema: AvroSchema) -> Result<Fingerprint, ArrowError> {
907 if self.fingerprint_algorithm == FingerprintAlgorithm::Id
908 || self.fingerprint_algorithm == FingerprintAlgorithm::Id64
909 {
910 return Err(ArrowError::SchemaError(
911 "Invalid FingerprintAlgorithm; unable to generate fingerprint. \
912 Use the set method directly instead, providing a valid fingerprint"
913 .to_string(),
914 ));
915 }
916 let fingerprint =
917 AvroSchema::generate_fingerprint(&schema.schema()?, self.fingerprint_algorithm)?;
918 self.set(fingerprint, schema)?;
919 Ok(fingerprint)
920 }
921
922 pub fn lookup(&self, fingerprint: &Fingerprint) -> Option<&AvroSchema> {
932 self.schemas.get(fingerprint)
933 }
934
935 pub fn fingerprints(&self) -> Vec<Fingerprint> {
941 self.schemas.keys().copied().collect()
942 }
943
944 pub(crate) fn fingerprint_algorithm(&self) -> FingerprintAlgorithm {
946 self.fingerprint_algorithm
947 }
948}
949
950fn quote(s: &str) -> Result<String, ArrowError> {
951 serde_json::to_string(s)
952 .map_err(|e| ArrowError::ComputeError(format!("Failed to quote string: {e}")))
953}
954
955pub(crate) fn make_full_name(
972 name: &str,
973 namespace_attr: Option<&str>,
974 enclosing_ns: Option<&str>,
975) -> (String, Option<String>) {
976 if let Some((ns, _)) = name.rsplit_once('.') {
978 return (name.to_string(), Some(ns.to_string()));
979 }
980 match namespace_attr.or(enclosing_ns) {
981 Some(ns) => (format!("{ns}.{name}"), Some(ns.to_string())),
982 None => (name.to_string(), None),
983 }
984}
985
986fn build_canonical(schema: &Schema, enclosing_ns: Option<&str>) -> Result<String, ArrowError> {
987 Ok(match schema {
988 Schema::TypeName(tn) | Schema::Type(Type { r#type: tn, .. }) => match tn {
989 TypeName::Primitive(pt) => quote(pt.as_ref())?,
990 TypeName::Ref(name) => {
991 let (full_name, _) = make_full_name(name, None, enclosing_ns);
992 quote(&full_name)?
993 }
994 },
995 Schema::Union(branches) => format!(
996 "[{}]",
997 branches
998 .iter()
999 .map(|b| build_canonical(b, enclosing_ns))
1000 .collect::<Result<Vec<_>, _>>()?
1001 .join(",")
1002 ),
1003 Schema::Complex(ct) => match ct {
1004 ComplexType::Record(r) => {
1005 let (full_name, child_ns) = make_full_name(r.name, r.namespace, enclosing_ns);
1006 let fields = r
1007 .fields
1008 .iter()
1009 .map(|f| {
1010 let field_type =
1015 build_canonical(&f.r#type, child_ns.as_deref().or(enclosing_ns))?;
1016 Ok(format!(
1017 r#"{{"name":{},"type":{}}}"#,
1018 quote(f.name)?,
1019 field_type
1020 ))
1021 })
1022 .collect::<Result<Vec<_>, ArrowError>>()?
1023 .join(",");
1024 format!(
1025 r#"{{"name":{},"type":"record","fields":[{fields}]}}"#,
1026 quote(&full_name)?,
1027 )
1028 }
1029 ComplexType::Enum(e) => {
1030 let (full_name, _) = make_full_name(e.name, e.namespace, enclosing_ns);
1031 let symbols = e
1032 .symbols
1033 .iter()
1034 .map(|s| quote(s))
1035 .collect::<Result<Vec<_>, _>>()?
1036 .join(",");
1037 format!(
1038 r#"{{"name":{},"type":"enum","symbols":[{symbols}]}}"#,
1039 quote(&full_name)?
1040 )
1041 }
1042 ComplexType::Array(arr) => format!(
1043 r#"{{"type":"array","items":{}}}"#,
1044 build_canonical(&arr.items, enclosing_ns)?
1045 ),
1046 ComplexType::Map(map) => format!(
1047 r#"{{"type":"map","values":{}}}"#,
1048 build_canonical(&map.values, enclosing_ns)?
1049 ),
1050 ComplexType::Fixed(f) => {
1051 let (full_name, _) = make_full_name(f.name, f.namespace, enclosing_ns);
1052 format!(
1053 r#"{{"name":{},"type":"fixed","size":{}}}"#,
1054 quote(&full_name)?,
1055 f.size
1056 )
1057 }
1058 },
1059 })
1060}
1061
1062const EMPTY: u64 = 0xc15d_213a_a4d7_a795;
1064
1065const fn one_entry(i: usize) -> u64 {
1072 let mut fp = i as u64;
1073 let mut j = 0;
1074 while j < 8 {
1075 fp = (fp >> 1) ^ (EMPTY & (0u64.wrapping_sub(fp & 1)));
1076 j += 1;
1077 }
1078 fp
1079}
1080
1081const fn build_table() -> [u64; 256] {
1088 let mut table = [0u64; 256];
1089 let mut i = 0;
1090 while i < 256 {
1091 table[i] = one_entry(i);
1092 i += 1;
1093 }
1094 table
1095}
1096
1097static FINGERPRINT_TABLE: [u64; 256] = build_table();
1099
1100pub(crate) fn compute_fingerprint_rabin(canonical_form: &str) -> u64 {
1103 let mut fp = EMPTY;
1104 for &byte in canonical_form.as_bytes() {
1105 let idx = ((fp as u8) ^ byte) as usize;
1106 fp = (fp >> 8) ^ FINGERPRINT_TABLE[idx];
1107 }
1108 fp
1109}
1110
1111#[cfg(feature = "md5")]
1112#[inline]
1117pub(crate) fn compute_fingerprint_md5(canonical_form: &str) -> [u8; 16] {
1118 let digest = md5::compute(canonical_form.as_bytes());
1119 digest.0
1120}
1121
1122#[cfg(feature = "sha256")]
1123#[inline]
1127pub(crate) fn compute_fingerprint_sha256(canonical_form: &str) -> [u8; 32] {
1128 let mut hasher = Sha256::new();
1129 hasher.update(canonical_form.as_bytes());
1130 let digest = hasher.finalize();
1131 digest.into()
1132}
1133
1134#[inline]
1135fn is_internal_arrow_key(key: &str) -> bool {
1136 key.starts_with("ARROW:") || key == SCHEMA_METADATA_KEY
1137}
1138
1139fn extend_with_passthrough_metadata(target: &mut JsonMap<String, Value>, metadata: &Metadata) {
1144 for (meta_key, meta_val) in metadata {
1145 if meta_key.starts_with("avro.") || is_internal_arrow_key(meta_key) {
1146 continue;
1147 }
1148 let json_val =
1149 serde_json::from_str(meta_val).unwrap_or_else(|_| Value::String(meta_val.clone()));
1150 target.insert(meta_key.clone(), json_val);
1151 }
1152}
1153
1154fn sanitise_avro_name(base_name: &str) -> String {
1156 if base_name.is_empty() {
1157 return "_".to_owned();
1158 }
1159 let mut out: String = base_name
1160 .chars()
1161 .map(|char| {
1162 if char.is_ascii_alphanumeric() || char == '_' {
1163 char
1164 } else {
1165 '_'
1166 }
1167 })
1168 .collect();
1169 if out.as_bytes()[0].is_ascii_digit() {
1170 out.insert(0, '_');
1171 }
1172 out
1173}
1174
1175#[derive(Default)]
1176struct NameGenerator {
1177 used: HashSet<String>,
1178 counters: HashMap<String, usize>,
1179}
1180
1181impl NameGenerator {
1182 fn make_unique(&mut self, field_name: &str) -> String {
1183 let field_name = sanitise_avro_name(field_name);
1184 if self.used.insert(field_name.clone()) {
1185 self.counters.insert(field_name.clone(), 1);
1186 return field_name;
1187 }
1188 let counter = self.counters.entry(field_name.clone()).or_insert(1);
1189 loop {
1190 let candidate = format!("{field_name}_{}", *counter);
1191 if self.used.insert(candidate.clone()) {
1192 return candidate;
1193 }
1194 *counter += 1;
1195 }
1196 }
1197}
1198
1199fn merge_extras(schema: Value, extras: JsonMap<String, Value>) -> Value {
1200 if extras.is_empty() {
1201 return schema;
1202 }
1203 match schema {
1204 Value::Object(mut map) => {
1205 map.extend(extras);
1206 Value::Object(map)
1207 }
1208 Value::Array(mut union) => {
1209 if let Some(non_null) = union.iter_mut().find(|val| val.as_str() != Some("null")) {
1212 let original = std::mem::take(non_null);
1213 *non_null = merge_extras(original, extras);
1214 }
1215 Value::Array(union)
1216 }
1217 primitive => {
1218 let mut map = JsonMap::with_capacity(extras.len() + 1);
1219 map.insert("type".into(), primitive);
1220 map.extend(extras);
1221 Value::Object(map)
1222 }
1223 }
1224}
1225
1226#[inline]
1227fn is_avro_json_null(v: &Value) -> bool {
1228 matches!(v, Value::String(s) if s == "null")
1229}
1230
1231fn wrap_nullable(inner: Value, null_order: Nullability) -> Value {
1232 let null = Value::String("null".into());
1233 match inner {
1234 Value::Array(mut union) => {
1235 if union.iter().any(is_avro_json_null) {
1240 return Value::Array(union);
1241 }
1242 match null_order {
1244 Nullability::NullFirst => union.insert(0, null),
1245 Nullability::NullSecond => union.push(null),
1246 }
1247 Value::Array(union)
1248 }
1249 other => match null_order {
1250 Nullability::NullFirst => Value::Array(vec![null, other]),
1251 Nullability::NullSecond => Value::Array(vec![other, null]),
1252 },
1253 }
1254}
1255
1256fn min_fixed_bytes_for_precision(p: usize) -> usize {
1257 const MAX_P: [usize; 32] = [
1260 2, 4, 6, 9, 11, 14, 16, 18, 21, 23, 26, 28, 31, 33, 35, 38, 40, 43, 45, 47, 50, 52, 55, 57,
1261 59, 62, 64, 67, 69, 71, 74, 76,
1262 ];
1263 for (i, &max_p) in MAX_P.iter().enumerate() {
1264 if p <= max_p {
1265 return i + 1;
1266 }
1267 }
1268 32 }
1270
1271fn union_branch_signature(branch: &Value) -> Result<String, ArrowError> {
1272 match branch {
1273 Value::String(t) => Ok(format!("P:{t}")),
1274 Value::Object(map) => {
1275 let t = map.get("type").and_then(|v| v.as_str()).ok_or_else(|| {
1276 ArrowError::SchemaError("Union branch object missing string 'type'".into())
1277 })?;
1278 match t {
1279 "record" | "enum" | "fixed" => {
1280 let name = map.get("name").and_then(|v| v.as_str()).ok_or_else(|| {
1281 ArrowError::SchemaError(format!(
1282 "Union branch '{t}' missing required 'name'"
1283 ))
1284 })?;
1285 Ok(format!("N:{t}:{name}"))
1286 }
1287 "array" | "map" => Ok(format!("C:{t}")),
1288 other => Ok(format!("P:{other}")),
1289 }
1290 }
1291 Value::Array(_) => Err(ArrowError::SchemaError(
1292 "Avro union may not immediately contain another union".into(),
1293 )),
1294 _ => Err(ArrowError::SchemaError(
1295 "Invalid JSON for Avro union branch".into(),
1296 )),
1297 }
1298}
1299
1300fn datatype_to_avro(
1301 dt: &DataType,
1302 field_name: &str,
1303 metadata: &Metadata,
1304 name_gen: &mut NameGenerator,
1305 null_order: Nullability,
1306 strip: bool,
1307) -> Result<(Value, JsonMap<String, Value>), ArrowError> {
1308 let mut extras = JsonMap::new();
1309 let mut handle_decimal = |precision: &u8, scale: &i8| -> Result<Value, ArrowError> {
1310 if *scale < 0 {
1311 return Err(ArrowError::SchemaError(format!(
1312 "Invalid Avro decimal for field '{field_name}': scale ({scale}) must be >= 0"
1313 )));
1314 }
1315 if (*scale as usize) > (*precision as usize) {
1316 return Err(ArrowError::SchemaError(format!(
1317 "Invalid Avro decimal for field '{field_name}': scale ({scale}) \
1318 must be <= precision ({precision})"
1319 )));
1320 }
1321 let mut meta = JsonMap::from_iter([
1322 ("logicalType".into(), json!("decimal")),
1323 ("precision".into(), json!(*precision)),
1324 ("scale".into(), json!(*scale)),
1325 ]);
1326 let mut fixed_size = metadata.get("size").and_then(|v| v.parse::<usize>().ok());
1327 let carries_name = metadata.contains_key(AVRO_NAME_METADATA_KEY)
1328 || metadata.contains_key(AVRO_NAMESPACE_METADATA_KEY);
1329 if fixed_size.is_none() && carries_name {
1330 fixed_size = Some(min_fixed_bytes_for_precision(*precision as usize));
1331 }
1332 if let Some(size) = fixed_size {
1333 meta.insert("type".into(), json!("fixed"));
1334 meta.insert("size".into(), json!(size));
1335 let chosen_name = metadata
1336 .get(AVRO_NAME_METADATA_KEY)
1337 .map(|s| sanitise_avro_name(s))
1338 .unwrap_or_else(|| name_gen.make_unique(field_name));
1339 meta.insert("name".into(), json!(chosen_name));
1340 if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1341 meta.insert("namespace".into(), json!(ns));
1342 }
1343 } else {
1344 meta.insert("type".into(), json!("bytes"));
1346 }
1347 Ok(Value::Object(meta))
1348 };
1349 let val = match dt {
1350 DataType::Null => Value::String("null".into()),
1351 DataType::Boolean => Value::String("boolean".into()),
1352 #[cfg(not(feature = "avro_custom_types"))]
1353 DataType::Int8 | DataType::Int16 | DataType::UInt8 | DataType::UInt16 => {
1354 Value::String("int".into())
1355 }
1356 DataType::Int32 => Value::String("int".into()),
1357 #[cfg(feature = "avro_custom_types")]
1358 DataType::Int8 => json!({ "type": "int", "logicalType": "arrow.int8" }),
1359 #[cfg(feature = "avro_custom_types")]
1360 DataType::Int16 => json!({ "type": "int", "logicalType": "arrow.int16" }),
1361 #[cfg(feature = "avro_custom_types")]
1362 DataType::UInt8 => json!({ "type": "int", "logicalType": "arrow.uint8" }),
1363 #[cfg(feature = "avro_custom_types")]
1364 DataType::UInt16 => json!({ "type": "int", "logicalType": "arrow.uint16" }),
1365 #[cfg(not(feature = "avro_custom_types"))]
1366 DataType::UInt32 => Value::String("long".into()),
1367 #[cfg(feature = "avro_custom_types")]
1368 DataType::UInt32 => json!({ "type": "long", "logicalType": "arrow.uint32" }),
1369 DataType::Int64 => Value::String("long".into()),
1370 #[cfg(not(feature = "avro_custom_types"))]
1371 DataType::UInt64 => Value::String("long".into()),
1372 #[cfg(feature = "avro_custom_types")]
1373 DataType::UInt64 => {
1374 let chosen_name = metadata
1376 .get(AVRO_NAME_METADATA_KEY)
1377 .map(|s| sanitise_avro_name(s))
1378 .unwrap_or_else(|| name_gen.make_unique(field_name));
1379 let mut obj = JsonMap::from_iter([
1380 ("type".into(), json!("fixed")),
1381 ("name".into(), json!(chosen_name)),
1382 ("size".into(), json!(8)),
1383 ("logicalType".into(), json!("arrow.uint64")),
1384 ]);
1385 if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1386 obj.insert("namespace".into(), json!(ns));
1387 }
1388 json!(obj)
1389 }
1390 #[cfg(not(feature = "avro_custom_types"))]
1391 DataType::Float16 => Value::String("float".into()),
1392 #[cfg(feature = "avro_custom_types")]
1393 DataType::Float16 => {
1394 let chosen_name = metadata
1396 .get(AVRO_NAME_METADATA_KEY)
1397 .map(|s| sanitise_avro_name(s))
1398 .unwrap_or_else(|| name_gen.make_unique(field_name));
1399 let mut obj = JsonMap::from_iter([
1400 ("type".into(), json!("fixed")),
1401 ("name".into(), json!(chosen_name)),
1402 ("size".into(), json!(2)),
1403 ("logicalType".into(), json!("arrow.float16")),
1404 ]);
1405 if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1406 obj.insert("namespace".into(), json!(ns));
1407 }
1408 json!(obj)
1409 }
1410 DataType::Float32 => Value::String("float".into()),
1411 DataType::Float64 => Value::String("double".into()),
1412 DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Value::String("string".into()),
1413 DataType::Binary | DataType::LargeBinary => Value::String("bytes".into()),
1414 DataType::BinaryView => {
1415 if !strip {
1416 extras.insert("arrowBinaryView".into(), Value::Bool(true));
1417 }
1418 Value::String("bytes".into())
1419 }
1420 DataType::FixedSizeBinary(len) => {
1421 let md_is_uuid = metadata
1422 .get("logicalType")
1423 .map(|s| s.trim_matches('"') == "uuid")
1424 .unwrap_or(false);
1425 #[cfg(feature = "canonical_extension_types")]
1426 let ext_is_uuid = metadata
1427 .get(arrow_schema::extension::EXTENSION_TYPE_NAME_KEY)
1428 .map(|v| v == arrow_schema::extension::Uuid::NAME || v == "uuid")
1429 .unwrap_or(false);
1430 #[cfg(not(feature = "canonical_extension_types"))]
1431 let ext_is_uuid = false;
1432 let is_uuid = (*len == 16) && (md_is_uuid || ext_is_uuid);
1433 if is_uuid {
1434 json!({ "type": "string", "logicalType": "uuid" })
1435 } else {
1436 let chosen_name = metadata
1437 .get(AVRO_NAME_METADATA_KEY)
1438 .map(|s| sanitise_avro_name(s))
1439 .unwrap_or_else(|| name_gen.make_unique(field_name));
1440 let mut obj = JsonMap::from_iter([
1441 ("type".into(), json!("fixed")),
1442 ("name".into(), json!(chosen_name)),
1443 ("size".into(), json!(len)),
1444 ]);
1445 if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1446 obj.insert("namespace".into(), json!(ns));
1447 }
1448 Value::Object(obj)
1449 }
1450 }
1451 #[cfg(feature = "small_decimals")]
1452 DataType::Decimal32(precision, scale) | DataType::Decimal64(precision, scale) => {
1453 handle_decimal(precision, scale)?
1454 }
1455 DataType::Decimal128(precision, scale) | DataType::Decimal256(precision, scale) => {
1456 handle_decimal(precision, scale)?
1457 }
1458 DataType::Date32 => json!({ "type": "int", "logicalType": "date" }),
1459 #[cfg(not(feature = "avro_custom_types"))]
1460 DataType::Date64 => json!({ "type": "long", "logicalType": "local-timestamp-millis" }),
1461 #[cfg(feature = "avro_custom_types")]
1462 DataType::Date64 => json!({ "type": "long", "logicalType": "arrow.date64" }),
1463 DataType::Time32(unit) => match unit {
1464 TimeUnit::Millisecond => json!({ "type": "int", "logicalType": "time-millis" }),
1465 #[cfg(not(feature = "avro_custom_types"))]
1466 TimeUnit::Second => {
1467 if !strip {
1469 extras.insert("arrowTimeUnit".into(), Value::String("second".into()));
1470 }
1471 json!({ "type": "int", "logicalType": "time-millis" })
1472 }
1473 #[cfg(feature = "avro_custom_types")]
1474 TimeUnit::Second => {
1475 json!({ "type": "int", "logicalType": "arrow.time32-second" })
1476 }
1477 _ => Value::String("int".into()),
1478 },
1479 DataType::Time64(unit) => match unit {
1480 TimeUnit::Microsecond => json!({ "type": "long", "logicalType": "time-micros" }),
1481 #[cfg(not(feature = "avro_custom_types"))]
1482 TimeUnit::Nanosecond => {
1483 if !strip {
1485 extras.insert("arrowTimeUnit".into(), Value::String("nanosecond".into()));
1486 }
1487 json!({ "type": "long", "logicalType": "time-micros" })
1488 }
1489 #[cfg(feature = "avro_custom_types")]
1490 TimeUnit::Nanosecond => {
1491 json!({ "type": "long", "logicalType": "arrow.time64-nanosecond" })
1492 }
1493 _ => Value::String("long".into()),
1494 },
1495 DataType::Timestamp(unit, tz) => {
1496 #[cfg(feature = "avro_custom_types")]
1497 if matches!(unit, TimeUnit::Second) {
1498 let logical_type = if tz.is_some() {
1499 "arrow.timestamp-second"
1500 } else {
1501 "arrow.local-timestamp-second"
1502 };
1503 return Ok((
1504 json!({ "type": "long", "logicalType": logical_type }),
1505 extras,
1506 ));
1507 }
1508 let logical_type = match (unit, tz.is_some()) {
1509 (TimeUnit::Millisecond, true) => "timestamp-millis",
1510 (TimeUnit::Millisecond, false) => "local-timestamp-millis",
1511 (TimeUnit::Microsecond, true) => "timestamp-micros",
1512 (TimeUnit::Microsecond, false) => "local-timestamp-micros",
1513 (TimeUnit::Nanosecond, true) => "timestamp-nanos",
1514 (TimeUnit::Nanosecond, false) => "local-timestamp-nanos",
1515 (TimeUnit::Second, has_tz) => {
1516 if !strip {
1518 extras.insert("arrowTimeUnit".into(), Value::String("second".into()));
1519 }
1520 let ts_logical_type = if has_tz {
1521 "timestamp-millis"
1522 } else {
1523 "local-timestamp-millis"
1524 };
1525 return Ok((
1526 json!({ "type": "long", "logicalType": ts_logical_type }),
1527 extras,
1528 ));
1529 }
1530 };
1531 if !strip && matches!(unit, TimeUnit::Nanosecond) {
1532 extras.insert("arrowTimeUnit".into(), Value::String("nanosecond".into()));
1533 }
1534 json!({ "type": "long", "logicalType": logical_type })
1535 }
1536 #[cfg(not(feature = "avro_custom_types"))]
1537 DataType::Duration(_unit) => Value::String("long".into()),
1538 #[cfg(feature = "avro_custom_types")]
1539 DataType::Duration(unit) => {
1540 let logical_type = match unit {
1543 TimeUnit::Second => "arrow.duration-seconds",
1544 TimeUnit::Millisecond => "arrow.duration-millis",
1545 TimeUnit::Microsecond => "arrow.duration-micros",
1546 TimeUnit::Nanosecond => "arrow.duration-nanos",
1547 };
1548 json!({ "type": "long", "logicalType": logical_type })
1549 }
1550 #[cfg(not(feature = "avro_custom_types"))]
1551 DataType::Interval(IntervalUnit::MonthDayNano) => {
1552 let chosen_name = metadata
1554 .get(AVRO_NAME_METADATA_KEY)
1555 .map(|s| sanitise_avro_name(s))
1556 .unwrap_or_else(|| name_gen.make_unique(field_name));
1557 let mut obj = JsonMap::from_iter([
1558 ("type".into(), json!("fixed")),
1559 ("name".into(), json!(chosen_name)),
1560 ("size".into(), json!(12)),
1561 ("logicalType".into(), json!("duration")),
1562 ]);
1563 if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1564 obj.insert("namespace".into(), json!(ns));
1565 }
1566 json!(obj)
1567 }
1568 #[cfg(feature = "avro_custom_types")]
1569 DataType::Interval(IntervalUnit::MonthDayNano) => {
1570 let chosen_name = metadata
1573 .get(AVRO_NAME_METADATA_KEY)
1574 .map(|s| sanitise_avro_name(s))
1575 .unwrap_or_else(|| name_gen.make_unique(field_name));
1576 let mut obj = JsonMap::from_iter([
1577 ("type".into(), json!("fixed")),
1578 ("name".into(), json!(chosen_name)),
1579 ("size".into(), json!(16)),
1580 ("logicalType".into(), json!("arrow.interval-month-day-nano")),
1581 ]);
1582 if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1583 obj.insert("namespace".into(), json!(ns));
1584 }
1585 json!(obj)
1586 }
1587 #[cfg(not(feature = "avro_custom_types"))]
1588 DataType::Interval(IntervalUnit::YearMonth) => {
1589 let chosen_name = metadata
1591 .get(AVRO_NAME_METADATA_KEY)
1592 .map(|s| sanitise_avro_name(s))
1593 .unwrap_or_else(|| name_gen.make_unique(field_name));
1594 let mut extras = JsonMap::from_iter([
1595 ("type".into(), json!("fixed")),
1596 ("name".into(), json!(chosen_name)),
1597 ("size".into(), json!(12)),
1598 ("logicalType".into(), json!("duration")),
1599 ]);
1600 if !strip {
1601 extras.insert(
1602 "arrowIntervalUnit".into(),
1603 Value::String("yearmonth".into()),
1604 );
1605 }
1606 if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1607 extras.insert("namespace".into(), json!(ns));
1608 }
1609 json!(extras)
1610 }
1611 #[cfg(feature = "avro_custom_types")]
1612 DataType::Interval(IntervalUnit::YearMonth) => {
1613 let chosen_name = metadata
1614 .get(AVRO_NAME_METADATA_KEY)
1615 .map(|s| sanitise_avro_name(s))
1616 .unwrap_or_else(|| name_gen.make_unique(field_name));
1617 let mut obj = JsonMap::from_iter([
1618 ("type".into(), json!("fixed")),
1619 ("name".into(), json!(chosen_name)),
1620 ("size".into(), json!(4)),
1621 ("logicalType".into(), json!("arrow.interval-year-month")),
1622 ]);
1623 if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1624 obj.insert("namespace".into(), json!(ns));
1625 }
1626 json!(obj)
1627 }
1628 #[cfg(not(feature = "avro_custom_types"))]
1629 DataType::Interval(IntervalUnit::DayTime) => {
1630 let chosen_name = metadata
1632 .get(AVRO_NAME_METADATA_KEY)
1633 .map(|s| sanitise_avro_name(s))
1634 .unwrap_or_else(|| name_gen.make_unique(field_name));
1635 let mut obj = JsonMap::from_iter([
1636 ("type".into(), json!("fixed")),
1637 ("name".into(), json!(chosen_name)),
1638 ("size".into(), json!(12)),
1639 ("logicalType".into(), json!("duration")),
1640 ]);
1641 if !strip {
1642 obj.insert("arrowIntervalUnit".into(), Value::String("daytime".into()));
1643 }
1644 if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1645 obj.insert("namespace".into(), json!(ns));
1646 }
1647 json!(obj)
1648 }
1649 #[cfg(feature = "avro_custom_types")]
1650 DataType::Interval(IntervalUnit::DayTime) => {
1651 let chosen_name = metadata
1652 .get(AVRO_NAME_METADATA_KEY)
1653 .map(|s| sanitise_avro_name(s))
1654 .unwrap_or_else(|| name_gen.make_unique(field_name));
1655 let mut obj = JsonMap::from_iter([
1656 ("type".into(), json!("fixed")),
1657 ("name".into(), json!(chosen_name)),
1658 ("size".into(), json!(8)),
1659 ("logicalType".into(), json!("arrow.interval-day-time")),
1660 ]);
1661 if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1662 obj.insert("namespace".into(), json!(ns));
1663 }
1664 json!(obj)
1665 }
1666 DataType::List(child) | DataType::LargeList(child) => {
1667 if matches!(dt, DataType::LargeList(_)) && !strip {
1668 extras.insert("arrowLargeList".into(), Value::Bool(true));
1669 }
1670 let items_schema = process_datatype(
1671 child.data_type(),
1672 child.name(),
1673 child.metadata(),
1674 name_gen,
1675 null_order,
1676 child.is_nullable(),
1677 strip,
1678 )?;
1679 json!({
1680 "type": "array",
1681 "items": items_schema
1682 })
1683 }
1684 DataType::ListView(child) | DataType::LargeListView(child) => {
1685 if matches!(dt, DataType::LargeListView(_)) && !strip {
1686 extras.insert("arrowLargeList".into(), Value::Bool(true));
1687 }
1688 if !strip {
1689 extras.insert("arrowListView".into(), Value::Bool(true));
1690 }
1691 let items_schema = process_datatype(
1692 child.data_type(),
1693 child.name(),
1694 child.metadata(),
1695 name_gen,
1696 null_order,
1697 child.is_nullable(),
1698 strip,
1699 )?;
1700 json!({
1701 "type": "array",
1702 "items": items_schema
1703 })
1704 }
1705 DataType::FixedSizeList(child, len) => {
1706 if !strip {
1707 extras.insert("arrowFixedSize".into(), json!(len));
1708 }
1709 let items_schema = process_datatype(
1710 child.data_type(),
1711 child.name(),
1712 child.metadata(),
1713 name_gen,
1714 null_order,
1715 child.is_nullable(),
1716 strip,
1717 )?;
1718 json!({
1719 "type": "array",
1720 "items": items_schema
1721 })
1722 }
1723 DataType::Map(entries, _) => {
1724 let value_field = match entries.data_type() {
1725 DataType::Struct(fs) => &fs[1],
1726 _ => {
1727 return Err(ArrowError::SchemaError(
1728 "Map 'entries' field must be Struct(key,value)".into(),
1729 ));
1730 }
1731 };
1732 let values_schema = process_datatype(
1733 value_field.data_type(),
1734 value_field.name(),
1735 value_field.metadata(),
1736 name_gen,
1737 null_order,
1738 value_field.is_nullable(),
1739 strip,
1740 )?;
1741 json!({
1742 "type": "map",
1743 "values": values_schema
1744 })
1745 }
1746 DataType::Struct(fields) => {
1747 let avro_fields = fields
1748 .iter()
1749 .map(|field| arrow_field_to_avro(field, name_gen, null_order, strip))
1750 .collect::<Result<Vec<_>, _>>()?;
1751 let chosen_name = metadata
1753 .get(AVRO_NAME_METADATA_KEY)
1754 .map(|s| sanitise_avro_name(s))
1755 .unwrap_or_else(|| name_gen.make_unique(field_name));
1756 let mut obj = JsonMap::from_iter([
1757 ("type".into(), json!("record")),
1758 ("name".into(), json!(chosen_name)),
1759 ("fields".into(), Value::Array(avro_fields)),
1760 ]);
1761 if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1762 obj.insert("namespace".into(), json!(ns));
1763 }
1764 Value::Object(obj)
1765 }
1766 DataType::Dictionary(_, value) => {
1767 if let Some(j) = metadata.get(AVRO_ENUM_SYMBOLS_METADATA_KEY) {
1768 let symbols: Vec<&str> =
1769 serde_json::from_str(j).map_err(|e| ArrowError::ParseError(e.to_string()))?;
1770 let chosen_name = metadata
1772 .get(AVRO_NAME_METADATA_KEY)
1773 .map(|s| sanitise_avro_name(s))
1774 .unwrap_or_else(|| name_gen.make_unique(field_name));
1775 let mut obj = JsonMap::from_iter([
1776 ("type".into(), json!("enum")),
1777 ("name".into(), json!(chosen_name)),
1778 ("symbols".into(), json!(symbols)),
1779 ]);
1780 if let Some(ns) = metadata.get(AVRO_NAMESPACE_METADATA_KEY) {
1781 obj.insert("namespace".into(), json!(ns));
1782 }
1783 Value::Object(obj)
1784 } else {
1785 process_datatype(
1786 value.as_ref(),
1787 field_name,
1788 metadata,
1789 name_gen,
1790 null_order,
1791 false,
1792 strip,
1793 )?
1794 }
1795 }
1796 #[cfg(feature = "avro_custom_types")]
1797 DataType::RunEndEncoded(run_ends, values) => {
1798 let bits = match run_ends.data_type() {
1799 DataType::Int16 => 16,
1800 DataType::Int32 => 32,
1801 DataType::Int64 => 64,
1802 other => {
1803 return Err(ArrowError::SchemaError(format!(
1804 "RunEndEncoded requires Int16/Int32/Int64 for run_ends, found: {other:?}"
1805 )));
1806 }
1807 };
1808 let (value_schema, value_extras) = datatype_to_avro(
1810 values.data_type(),
1811 values.name(),
1812 values.metadata(),
1813 name_gen,
1814 null_order,
1815 strip,
1816 )?;
1817 let mut merged = merge_extras(value_schema, value_extras);
1818 if values.is_nullable() {
1819 merged = wrap_nullable(merged, null_order);
1820 }
1821 let mut extras = JsonMap::new();
1822 extras.insert("logicalType".into(), json!("arrow.run-end-encoded"));
1823 extras.insert("arrow.runEndIndexBits".into(), json!(bits));
1824 return Ok((merged, extras));
1825 }
1826 #[cfg(not(feature = "avro_custom_types"))]
1827 DataType::RunEndEncoded(_run_ends, values) => {
1828 let (value_schema, _extras) = datatype_to_avro(
1829 values.data_type(),
1830 values.name(),
1831 values.metadata(),
1832 name_gen,
1833 null_order,
1834 strip,
1835 )?;
1836 return Ok((value_schema, JsonMap::new()));
1837 }
1838 DataType::Union(fields, mode) => {
1839 let mut branches: Vec<Value> = Vec::with_capacity(fields.len());
1840 let mut type_ids: Vec<i32> = Vec::with_capacity(fields.len());
1841 for (type_id, field_ref) in fields.iter() {
1842 let (branch_schema, _branch_extras) = datatype_to_avro(
1844 field_ref.data_type(),
1845 field_ref.name(),
1846 field_ref.metadata(),
1847 name_gen,
1848 null_order,
1849 strip,
1850 )?;
1851 if matches!(branch_schema, Value::Array(_)) {
1853 return Err(ArrowError::SchemaError(
1854 "Avro union may not immediately contain another union".into(),
1855 ));
1856 }
1857 branches.push(branch_schema);
1858 type_ids.push(type_id as i32);
1859 }
1860 let mut seen: HashSet<String> = HashSet::with_capacity(branches.len());
1861 for b in &branches {
1862 let sig = union_branch_signature(b)?;
1863 if !seen.insert(sig) {
1864 return Err(ArrowError::SchemaError(
1865 "Avro union contains duplicate branch types (disallowed by spec)".into(),
1866 ));
1867 }
1868 }
1869 if !strip {
1870 extras.insert(
1871 "arrowUnionMode".into(),
1872 Value::String(
1873 match mode {
1874 UnionMode::Sparse => "sparse",
1875 UnionMode::Dense => "dense",
1876 }
1877 .to_string(),
1878 ),
1879 );
1880 extras.insert(
1881 "arrowUnionTypeIds".into(),
1882 Value::Array(type_ids.into_iter().map(|id| json!(id)).collect()),
1883 );
1884 }
1885 Value::Array(branches)
1886 }
1887 #[cfg(not(feature = "small_decimals"))]
1888 other => {
1889 return Err(ArrowError::NotYetImplemented(format!(
1890 "Arrow type {other:?} has no Avro representation"
1891 )));
1892 }
1893 };
1894 Ok((val, extras))
1895}
1896
1897fn process_datatype(
1898 dt: &DataType,
1899 field_name: &str,
1900 metadata: &Metadata,
1901 name_gen: &mut NameGenerator,
1902 null_order: Nullability,
1903 is_nullable: bool,
1904 strip: bool,
1905) -> Result<Value, ArrowError> {
1906 let (schema, extras) = datatype_to_avro(dt, field_name, metadata, name_gen, null_order, strip)?;
1907 let mut merged = merge_extras(schema, extras);
1908 if is_nullable {
1909 merged = wrap_nullable(merged, null_order)
1910 }
1911 Ok(merged)
1912}
1913
1914fn arrow_field_to_avro(
1915 field: &ArrowField,
1916 name_gen: &mut NameGenerator,
1917 null_order: Nullability,
1918 strip: bool,
1919) -> Result<Value, ArrowError> {
1920 let avro_name = sanitise_avro_name(field.name());
1921 let schema_value = process_datatype(
1922 field.data_type(),
1923 &avro_name,
1924 field.metadata(),
1925 name_gen,
1926 null_order,
1927 field.is_nullable(),
1928 strip,
1929 )?;
1930 let mut map = JsonMap::with_capacity(field.metadata().len() + 3);
1932 map.insert("name".into(), Value::String(avro_name));
1933 map.insert("type".into(), schema_value);
1934 for (meta_key, meta_val) in field.metadata() {
1936 if is_internal_arrow_key(meta_key) {
1937 continue;
1938 }
1939 match meta_key.as_str() {
1940 AVRO_DOC_METADATA_KEY => {
1941 map.insert("doc".into(), Value::String(meta_val.clone()));
1942 }
1943 AVRO_FIELD_DEFAULT_METADATA_KEY => {
1944 let default_value = serde_json::from_str(meta_val)
1945 .unwrap_or_else(|_| Value::String(meta_val.clone()));
1946 map.insert("default".into(), default_value);
1947 }
1948 _ => {
1949 let json_val = serde_json::from_str(meta_val)
1950 .unwrap_or_else(|_| Value::String(meta_val.clone()));
1951 map.insert(meta_key.clone(), json_val);
1952 }
1953 }
1954 }
1955 Ok(Value::Object(map))
1956}
1957
1958#[cfg(test)]
1959mod tests {
1960 use super::*;
1961 use crate::codec::{AvroField, AvroFieldBuilder};
1962 use arrow_schema::{DataType, Fields, SchemaBuilder, TimeUnit, UnionFields};
1963 use serde_json::json;
1964 use std::sync::Arc;
1965
1966 fn int_schema() -> Schema<'static> {
1967 Schema::TypeName(TypeName::Primitive(PrimitiveType::Int))
1968 }
1969
1970 fn record_schema() -> Schema<'static> {
1971 Schema::Complex(ComplexType::Record(Record {
1972 name: "record1",
1973 namespace: Some("test.namespace"),
1974 doc: Some(Cow::from("A test record")),
1975 aliases: vec![],
1976 fields: vec![
1977 Field {
1978 name: "field1",
1979 doc: Some(Cow::from("An integer field")),
1980 r#type: int_schema(),
1981 default: None,
1982 aliases: vec![],
1983 },
1984 Field {
1985 name: "field2",
1986 doc: None,
1987 r#type: Schema::TypeName(TypeName::Primitive(PrimitiveType::String)),
1988 default: None,
1989 aliases: vec![],
1990 },
1991 ],
1992 attributes: Attributes::default(),
1993 }))
1994 }
1995
1996 fn single_field_schema(field: ArrowField) -> arrow_schema::Schema {
1997 let mut sb = SchemaBuilder::new();
1998 sb.push(field);
1999 sb.finish()
2000 }
2001
2002 fn assert_json_contains(avro_json: &str, needle: &str) {
2003 assert!(
2004 avro_json.contains(needle),
2005 "JSON did not contain `{needle}` : {avro_json}"
2006 )
2007 }
2008
2009 #[test]
2010 fn test_deserialize() {
2011 let t: Schema = serde_json::from_str("\"string\"").unwrap();
2012 assert_eq!(
2013 t,
2014 Schema::TypeName(TypeName::Primitive(PrimitiveType::String))
2015 );
2016
2017 let t: Schema = serde_json::from_str("[\"int\", \"null\"]").unwrap();
2018 assert_eq!(
2019 t,
2020 Schema::Union(vec![
2021 Schema::TypeName(TypeName::Primitive(PrimitiveType::Int)),
2022 Schema::TypeName(TypeName::Primitive(PrimitiveType::Null)),
2023 ])
2024 );
2025
2026 let t: Type = serde_json::from_str(
2027 r#"{
2028 "type":"long",
2029 "logicalType":"timestamp-micros"
2030 }"#,
2031 )
2032 .unwrap();
2033
2034 let timestamp = Type {
2035 r#type: TypeName::Primitive(PrimitiveType::Long),
2036 attributes: Attributes {
2037 logical_type: Some("timestamp-micros"),
2038 additional: Default::default(),
2039 },
2040 };
2041
2042 assert_eq!(t, timestamp);
2043
2044 let t: ComplexType = serde_json::from_str(
2045 r#"{
2046 "type":"fixed",
2047 "name":"fixed",
2048 "namespace":"topLevelRecord.value",
2049 "size":11,
2050 "logicalType":"decimal",
2051 "precision":25,
2052 "scale":2
2053 }"#,
2054 )
2055 .unwrap();
2056
2057 let decimal = ComplexType::Fixed(Fixed {
2058 name: "fixed",
2059 namespace: Some("topLevelRecord.value"),
2060 aliases: vec![],
2061 size: 11,
2062 attributes: Attributes {
2063 logical_type: Some("decimal"),
2064 additional: vec![("precision", json!(25)), ("scale", json!(2))]
2065 .into_iter()
2066 .collect(),
2067 },
2068 });
2069
2070 assert_eq!(t, decimal);
2071
2072 let schema: Schema = serde_json::from_str(
2073 r#"{
2074 "type":"record",
2075 "name":"topLevelRecord",
2076 "fields":[
2077 {
2078 "name":"value",
2079 "type":[
2080 {
2081 "type":"fixed",
2082 "name":"fixed",
2083 "namespace":"topLevelRecord.value",
2084 "size":11,
2085 "logicalType":"decimal",
2086 "precision":25,
2087 "scale":2
2088 },
2089 "null"
2090 ]
2091 }
2092 ]
2093 }"#,
2094 )
2095 .unwrap();
2096
2097 assert_eq!(
2098 schema,
2099 Schema::Complex(ComplexType::Record(Record {
2100 name: "topLevelRecord",
2101 namespace: None,
2102 doc: None,
2103 aliases: vec![],
2104 fields: vec![Field {
2105 name: "value",
2106 doc: None,
2107 r#type: Schema::Union(vec![
2108 Schema::Complex(decimal),
2109 Schema::TypeName(TypeName::Primitive(PrimitiveType::Null)),
2110 ]),
2111 default: None,
2112 aliases: vec![],
2113 },],
2114 attributes: Default::default(),
2115 }))
2116 );
2117
2118 let schema: Schema = serde_json::from_str(
2119 r#"{
2120 "type": "record",
2121 "name": "LongList",
2122 "aliases": ["LinkedLongs"],
2123 "fields" : [
2124 {"name": "value", "type": "long"},
2125 {"name": "next", "type": ["null", "LongList"]}
2126 ]
2127 }"#,
2128 )
2129 .unwrap();
2130
2131 assert_eq!(
2132 schema,
2133 Schema::Complex(ComplexType::Record(Record {
2134 name: "LongList",
2135 namespace: None,
2136 doc: None,
2137 aliases: vec!["LinkedLongs"],
2138 fields: vec![
2139 Field {
2140 name: "value",
2141 doc: None,
2142 r#type: Schema::TypeName(TypeName::Primitive(PrimitiveType::Long)),
2143 default: None,
2144 aliases: vec![],
2145 },
2146 Field {
2147 name: "next",
2148 doc: None,
2149 r#type: Schema::Union(vec![
2150 Schema::TypeName(TypeName::Primitive(PrimitiveType::Null)),
2151 Schema::TypeName(TypeName::Ref("LongList")),
2152 ]),
2153 default: None,
2154 aliases: vec![],
2155 }
2156 ],
2157 attributes: Attributes::default(),
2158 }))
2159 );
2160
2161 let err = AvroField::try_from(&schema).unwrap_err().to_string();
2163 assert_eq!(err, "Parser error: Failed to resolve .LongList");
2164
2165 let schema: Schema = serde_json::from_str(
2166 r#"{
2167 "type":"record",
2168 "name":"topLevelRecord",
2169 "fields":[
2170 {
2171 "name":"id",
2172 "type":[
2173 "int",
2174 "null"
2175 ]
2176 },
2177 {
2178 "name":"timestamp_col",
2179 "type":[
2180 {
2181 "type":"long",
2182 "logicalType":"timestamp-micros"
2183 },
2184 "null"
2185 ]
2186 }
2187 ]
2188 }"#,
2189 )
2190 .unwrap();
2191
2192 assert_eq!(
2193 schema,
2194 Schema::Complex(ComplexType::Record(Record {
2195 name: "topLevelRecord",
2196 namespace: None,
2197 doc: None,
2198 aliases: vec![],
2199 fields: vec![
2200 Field {
2201 name: "id",
2202 doc: None,
2203 r#type: Schema::Union(vec![
2204 Schema::TypeName(TypeName::Primitive(PrimitiveType::Int)),
2205 Schema::TypeName(TypeName::Primitive(PrimitiveType::Null)),
2206 ]),
2207 default: None,
2208 aliases: vec![],
2209 },
2210 Field {
2211 name: "timestamp_col",
2212 doc: None,
2213 r#type: Schema::Union(vec![
2214 Schema::Type(timestamp),
2215 Schema::TypeName(TypeName::Primitive(PrimitiveType::Null)),
2216 ]),
2217 default: None,
2218 aliases: vec![],
2219 }
2220 ],
2221 attributes: Default::default(),
2222 }))
2223 );
2224 let codec = AvroField::try_from(&schema).unwrap();
2225 let expected_arrow_field = arrow_schema::Field::new(
2226 "topLevelRecord",
2227 DataType::Struct(Fields::from(vec![
2228 arrow_schema::Field::new("id", DataType::Int32, true),
2229 arrow_schema::Field::new(
2230 "timestamp_col",
2231 DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
2232 true,
2233 ),
2234 ])),
2235 false,
2236 )
2237 .with_metadata(std::collections::HashMap::from([(
2238 AVRO_NAME_METADATA_KEY.to_string(),
2239 "topLevelRecord".to_string(),
2240 )]));
2241
2242 assert_eq!(codec.field(), expected_arrow_field);
2243
2244 let schema: Schema = serde_json::from_str(
2245 r#"{
2246 "type": "record",
2247 "name": "HandshakeRequest", "namespace":"org.apache.avro.ipc",
2248 "fields": [
2249 {"name": "clientHash", "type": {"type": "fixed", "name": "MD5", "size": 16}},
2250 {"name": "clientProtocol", "type": ["null", "string"]},
2251 {"name": "serverHash", "type": "MD5"},
2252 {"name": "meta", "type": ["null", {"type": "map", "values": "bytes"}]}
2253 ]
2254 }"#,
2255 )
2256 .unwrap();
2257
2258 assert_eq!(
2259 schema,
2260 Schema::Complex(ComplexType::Record(Record {
2261 name: "HandshakeRequest",
2262 namespace: Some("org.apache.avro.ipc"),
2263 doc: None,
2264 aliases: vec![],
2265 fields: vec![
2266 Field {
2267 name: "clientHash",
2268 doc: None,
2269 r#type: Schema::Complex(ComplexType::Fixed(Fixed {
2270 name: "MD5",
2271 namespace: None,
2272 aliases: vec![],
2273 size: 16,
2274 attributes: Default::default(),
2275 })),
2276 default: None,
2277 aliases: vec![],
2278 },
2279 Field {
2280 name: "clientProtocol",
2281 doc: None,
2282 r#type: Schema::Union(vec![
2283 Schema::TypeName(TypeName::Primitive(PrimitiveType::Null)),
2284 Schema::TypeName(TypeName::Primitive(PrimitiveType::String)),
2285 ]),
2286 default: None,
2287 aliases: vec![],
2288 },
2289 Field {
2290 name: "serverHash",
2291 doc: None,
2292 r#type: Schema::TypeName(TypeName::Ref("MD5")),
2293 default: None,
2294 aliases: vec![],
2295 },
2296 Field {
2297 name: "meta",
2298 doc: None,
2299 r#type: Schema::Union(vec![
2300 Schema::TypeName(TypeName::Primitive(PrimitiveType::Null)),
2301 Schema::Complex(ComplexType::Map(Map {
2302 values: Box::new(Schema::TypeName(TypeName::Primitive(
2303 PrimitiveType::Bytes
2304 ))),
2305 attributes: Default::default(),
2306 })),
2307 ]),
2308 default: None,
2309 aliases: vec![],
2310 }
2311 ],
2312 attributes: Default::default(),
2313 }))
2314 );
2315 }
2316
2317 #[test]
2318 fn test_canonical_form_generation_comprehensive_record() {
2319 let json_str = r#"{
2321 "type": "record",
2322 "name": "E2eComprehensive",
2323 "namespace": "org.apache.arrow.avrotests.v1",
2324 "doc": "Comprehensive Avro writer schema to exercise arrow-avro Reader/Decoder paths.",
2325 "fields": [
2326 {"name": "id", "type": "long", "doc": "Primary row id", "aliases": ["identifier"]},
2327 {"name": "flag", "type": "boolean", "default": true, "doc": "A sample boolean with default true"},
2328 {"name": "ratio_f32", "type": "float", "default": 0.0, "doc": "Float32 example"},
2329 {"name": "ratio_f64", "type": "double", "default": 0.0, "doc": "Float64 example"},
2330 {"name": "count_i32", "type": "int", "default": 0, "doc": "Int32 example"},
2331 {"name": "count_i64", "type": "long", "default": 0, "doc": "Int64 example"},
2332 {"name": "opt_i32_nullfirst", "type": ["null", "int"], "default": null, "doc": "Nullable int (null-first)"},
2333 {"name": "opt_str_nullsecond", "type": ["string", "null"], "default": "", "aliases": ["old_opt_str"], "doc": "Nullable string (null-second). Default is empty string."},
2334 {"name": "tri_union_prim", "type": ["int", "string", "boolean"], "default": 0, "doc": "Union[int, string, boolean] with default on first branch (int=0)."},
2335 {"name": "str_utf8", "type": "string", "default": "default", "doc": "Plain Utf8 string (Reader may use Utf8View)."},
2336 {"name": "raw_bytes", "type": "bytes", "default": "", "doc": "Raw bytes field"},
2337 {"name": "fx16_plain", "type": {"type": "fixed", "name": "Fx16", "namespace": "org.apache.arrow.avrotests.v1.types", "aliases": ["Fixed16Old"], "size": 16}, "doc": "Plain fixed(16)"},
2338 {"name": "dec_bytes_s10_2", "type": {"type": "bytes", "logicalType": "decimal", "precision": 10, "scale": 2}, "doc": "Decimal encoded on bytes, precision 10, scale 2"},
2339 {"name": "dec_fix_s20_4", "type": {"type": "fixed", "name": "DecFix20", "namespace": "org.apache.arrow.avrotests.v1.types", "size": 20, "logicalType": "decimal", "precision": 20, "scale": 4}, "doc": "Decimal encoded on fixed(20), precision 20, scale 4"},
2340 {"name": "uuid_str", "type": {"type": "string", "logicalType": "uuid"}, "doc": "UUID logical type on string"},
2341 {"name": "d_date", "type": {"type": "int", "logicalType": "date"}, "doc": "Date32: days since 1970-01-01"},
2342 {"name": "t_millis", "type": {"type": "int", "logicalType": "time-millis"}, "doc": "Time32-millis"},
2343 {"name": "t_micros", "type": {"type": "long", "logicalType": "time-micros"}, "doc": "Time64-micros"},
2344 {"name": "ts_millis_utc", "type": {"type": "long", "logicalType": "timestamp-millis"}, "doc": "Timestamp ms (UTC)"},
2345 {"name": "ts_micros_utc", "type": {"type": "long", "logicalType": "timestamp-micros"}, "doc": "Timestamp µs (UTC)"},
2346 {"name": "ts_millis_local", "type": {"type": "long", "logicalType": "local-timestamp-millis"}, "doc": "Local timestamp ms"},
2347 {"name": "ts_micros_local", "type": {"type": "long", "logicalType": "local-timestamp-micros"}, "doc": "Local timestamp µs"},
2348 {"name": "interval_mdn", "type": {"type": "fixed", "name": "Dur12", "namespace": "org.apache.arrow.avrotests.v1.types", "size": 12, "logicalType": "duration"}, "doc": "Duration: fixed(12) little-endian (months, days, millis)"},
2349 {"name": "status", "type": {"type": "enum", "name": "Status", "namespace": "org.apache.arrow.avrotests.v1.types", "symbols": ["UNKNOWN", "NEW", "PROCESSING", "DONE"], "aliases": ["State"], "doc": "Processing status enum with default"}, "default": "UNKNOWN", "doc": "Enum field using default when resolving"},
2350 {"name": "arr_union", "type": {"type": "array", "items": ["long", "string", "null"]}, "default": [], "doc": "Array whose items are a union[long,string,null]"},
2351 {"name": "map_union", "type": {"type": "map", "values": ["null", "double", "string"]}, "default": {}, "doc": "Map whose values are a union[null,double,string]"},
2352 {"name": "address", "type": {"type": "record", "name": "Address", "namespace": "org.apache.arrow.avrotests.v1.types", "doc": "Postal address with defaults and field alias", "fields": [
2353 {"name": "street", "type": "string", "default": "", "aliases": ["street_name"], "doc": "Street (field alias = street_name)"},
2354 {"name": "zip", "type": "int", "default": 0, "doc": "ZIP/postal code"},
2355 {"name": "country", "type": "string", "default": "US", "doc": "Country code"}
2356 ]}, "doc": "Embedded Address record"},
2357 {"name": "maybe_auth", "type": {"type": "record", "name": "MaybeAuth", "namespace": "org.apache.arrow.avrotests.v1.types", "doc": "Optional auth token model", "fields": [
2358 {"name": "user", "type": "string", "doc": "Username"},
2359 {"name": "token", "type": ["null", "bytes"], "default": null, "doc": "Nullable auth token"}
2360 ]}},
2361 {"name": "union_enum_record_array_map", "type": [
2362 {"type": "enum", "name": "Color", "namespace": "org.apache.arrow.avrotests.v1.types", "symbols": ["RED", "GREEN", "BLUE"], "doc": "Color enum"},
2363 {"type": "record", "name": "RecA", "namespace": "org.apache.arrow.avrotests.v1.types", "fields": [{"name": "a", "type": "int"}, {"name": "b", "type": "string"}]},
2364 {"type": "record", "name": "RecB", "namespace": "org.apache.arrow.avrotests.v1.types", "fields": [{"name": "x", "type": "long"}, {"name": "y", "type": "bytes"}]},
2365 {"type": "array", "items": "long"},
2366 {"type": "map", "values": "string"}
2367 ], "doc": "Union of enum, two records, array, and map"},
2368 {"name": "union_date_or_fixed4", "type": [
2369 {"type": "int", "logicalType": "date"},
2370 {"type": "fixed", "name": "Fx4", "size": 4}
2371 ], "doc": "Union of date(int) or fixed(4)"},
2372 {"name": "union_interval_or_string", "type": [
2373 {"type": "fixed", "name": "Dur12U", "size": 12, "logicalType": "duration"},
2374 "string"
2375 ], "doc": "Union of duration(fixed12) or string"},
2376 {"name": "union_uuid_or_fixed10", "type": [
2377 {"type": "string", "logicalType": "uuid"},
2378 {"type": "fixed", "name": "Fx10", "size": 10}
2379 ], "doc": "Union of UUID string or fixed(10)"},
2380 {"name": "array_records_with_union", "type": {"type": "array", "items": {
2381 "type": "record", "name": "KV", "namespace": "org.apache.arrow.avrotests.v1.types",
2382 "fields": [
2383 {"name": "key", "type": "string"},
2384 {"name": "val", "type": ["null", "int", "long"], "default": null}
2385 ]
2386 }}, "doc": "Array<record{key, val: union[null,int,long]}>", "default": []},
2387 {"name": "union_map_or_array_int", "type": [
2388 {"type": "map", "values": "int"},
2389 {"type": "array", "items": "int"}
2390 ], "doc": "Union[map<string,int>, array<int>]"},
2391 {"name": "renamed_with_default", "type": "int", "default": 42, "aliases": ["old_count"], "doc": "Field with alias and default"},
2392 {"name": "person", "type": {"type": "record", "name": "PersonV2", "namespace": "com.example.v2", "aliases": ["com.example.Person"], "doc": "Person record with alias pointing to previous namespace/name", "fields": [
2393 {"name": "name", "type": "string"},
2394 {"name": "age", "type": "int", "default": 0}
2395 ]}, "doc": "Record using type alias for schema evolution tests"}
2396 ]
2397 }"#;
2398 let avro = AvroSchema::new(json_str.to_string());
2399 let parsed = avro.schema().expect("schema should deserialize");
2400 let expected_canonical_form = r#"{"name":"org.apache.arrow.avrotests.v1.E2eComprehensive","type":"record","fields":[{"name":"id","type":"long"},{"name":"flag","type":"boolean"},{"name":"ratio_f32","type":"float"},{"name":"ratio_f64","type":"double"},{"name":"count_i32","type":"int"},{"name":"count_i64","type":"long"},{"name":"opt_i32_nullfirst","type":["null","int"]},{"name":"opt_str_nullsecond","type":["string","null"]},{"name":"tri_union_prim","type":["int","string","boolean"]},{"name":"str_utf8","type":"string"},{"name":"raw_bytes","type":"bytes"},{"name":"fx16_plain","type":{"name":"org.apache.arrow.avrotests.v1.types.Fx16","type":"fixed","size":16}},{"name":"dec_bytes_s10_2","type":"bytes"},{"name":"dec_fix_s20_4","type":{"name":"org.apache.arrow.avrotests.v1.types.DecFix20","type":"fixed","size":20}},{"name":"uuid_str","type":"string"},{"name":"d_date","type":"int"},{"name":"t_millis","type":"int"},{"name":"t_micros","type":"long"},{"name":"ts_millis_utc","type":"long"},{"name":"ts_micros_utc","type":"long"},{"name":"ts_millis_local","type":"long"},{"name":"ts_micros_local","type":"long"},{"name":"interval_mdn","type":{"name":"org.apache.arrow.avrotests.v1.types.Dur12","type":"fixed","size":12}},{"name":"status","type":{"name":"org.apache.arrow.avrotests.v1.types.Status","type":"enum","symbols":["UNKNOWN","NEW","PROCESSING","DONE"]}},{"name":"arr_union","type":{"type":"array","items":["long","string","null"]}},{"name":"map_union","type":{"type":"map","values":["null","double","string"]}},{"name":"address","type":{"name":"org.apache.arrow.avrotests.v1.types.Address","type":"record","fields":[{"name":"street","type":"string"},{"name":"zip","type":"int"},{"name":"country","type":"string"}]}},{"name":"maybe_auth","type":{"name":"org.apache.arrow.avrotests.v1.types.MaybeAuth","type":"record","fields":[{"name":"user","type":"string"},{"name":"token","type":["null","bytes"]}]}},{"name":"union_enum_record_array_map","type":[{"name":"org.apache.arrow.avrotests.v1.types.Color","type":"enum","symbols":["RED","GREEN","BLUE"]},{"name":"org.apache.arrow.avrotests.v1.types.RecA","type":"record","fields":[{"name":"a","type":"int"},{"name":"b","type":"string"}]},{"name":"org.apache.arrow.avrotests.v1.types.RecB","type":"record","fields":[{"name":"x","type":"long"},{"name":"y","type":"bytes"}]},{"type":"array","items":"long"},{"type":"map","values":"string"}]},{"name":"union_date_or_fixed4","type":["int",{"name":"org.apache.arrow.avrotests.v1.Fx4","type":"fixed","size":4}]},{"name":"union_interval_or_string","type":[{"name":"org.apache.arrow.avrotests.v1.Dur12U","type":"fixed","size":12},"string"]},{"name":"union_uuid_or_fixed10","type":["string",{"name":"org.apache.arrow.avrotests.v1.Fx10","type":"fixed","size":10}]},{"name":"array_records_with_union","type":{"type":"array","items":{"name":"org.apache.arrow.avrotests.v1.types.KV","type":"record","fields":[{"name":"key","type":"string"},{"name":"val","type":["null","int","long"]}]}}},{"name":"union_map_or_array_int","type":[{"type":"map","values":"int"},{"type":"array","items":"int"}]},{"name":"renamed_with_default","type":"int"},{"name":"person","type":{"name":"com.example.v2.PersonV2","type":"record","fields":[{"name":"name","type":"string"},{"name":"age","type":"int"}]}}]}"#;
2401 let canonical_form =
2402 AvroSchema::generate_canonical_form(&parsed).expect("canonical form should be built");
2403 assert_eq!(
2404 canonical_form, expected_canonical_form,
2405 "Canonical form must match Avro spec PCF exactly"
2406 );
2407 }
2408
2409 #[test]
2410 fn test_new_schema_store() {
2411 let store = SchemaStore::new();
2412 assert!(store.schemas.is_empty());
2413 }
2414
2415 #[test]
2416 fn test_try_from_schemas_rabin() {
2417 let int_avro_schema = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2418 let record_avro_schema = AvroSchema::new(serde_json::to_string(&record_schema()).unwrap());
2419 let mut schemas: HashMap<Fingerprint, AvroSchema> = HashMap::new();
2420 schemas.insert(
2421 int_avro_schema
2422 .fingerprint(FingerprintAlgorithm::Rabin)
2423 .unwrap(),
2424 int_avro_schema.clone(),
2425 );
2426 schemas.insert(
2427 record_avro_schema
2428 .fingerprint(FingerprintAlgorithm::Rabin)
2429 .unwrap(),
2430 record_avro_schema.clone(),
2431 );
2432 let store = SchemaStore::try_from(schemas).unwrap();
2433 let int_fp = int_avro_schema
2434 .fingerprint(FingerprintAlgorithm::Rabin)
2435 .unwrap();
2436 assert_eq!(store.lookup(&int_fp).cloned(), Some(int_avro_schema));
2437 let rec_fp = record_avro_schema
2438 .fingerprint(FingerprintAlgorithm::Rabin)
2439 .unwrap();
2440 assert_eq!(store.lookup(&rec_fp).cloned(), Some(record_avro_schema));
2441 }
2442
2443 #[test]
2444 fn test_try_from_with_duplicates() {
2445 let int_avro_schema = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2446 let record_avro_schema = AvroSchema::new(serde_json::to_string(&record_schema()).unwrap());
2447 let mut schemas: HashMap<Fingerprint, AvroSchema> = HashMap::new();
2448 schemas.insert(
2449 int_avro_schema
2450 .fingerprint(FingerprintAlgorithm::Rabin)
2451 .unwrap(),
2452 int_avro_schema.clone(),
2453 );
2454 schemas.insert(
2455 record_avro_schema
2456 .fingerprint(FingerprintAlgorithm::Rabin)
2457 .unwrap(),
2458 record_avro_schema.clone(),
2459 );
2460 schemas.insert(
2462 int_avro_schema
2463 .fingerprint(FingerprintAlgorithm::Rabin)
2464 .unwrap(),
2465 int_avro_schema.clone(),
2466 );
2467 let store = SchemaStore::try_from(schemas).unwrap();
2468 assert_eq!(store.schemas.len(), 2);
2469 let int_fp = int_avro_schema
2470 .fingerprint(FingerprintAlgorithm::Rabin)
2471 .unwrap();
2472 assert_eq!(store.lookup(&int_fp).cloned(), Some(int_avro_schema));
2473 }
2474
2475 #[test]
2476 fn test_register_and_lookup_rabin() {
2477 let mut store = SchemaStore::new();
2478 let schema = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2479 let fp_enum = store.register(schema.clone()).unwrap();
2480 match fp_enum {
2481 Fingerprint::Rabin(fp_val) => {
2482 assert_eq!(
2483 store.lookup(&Fingerprint::Rabin(fp_val)).cloned(),
2484 Some(schema.clone())
2485 );
2486 assert!(
2487 store
2488 .lookup(&Fingerprint::Rabin(fp_val.wrapping_add(1)))
2489 .is_none()
2490 );
2491 }
2492 Fingerprint::Id(_id) => {
2493 unreachable!("This test should only generate Rabin fingerprints")
2494 }
2495 Fingerprint::Id64(_id) => {
2496 unreachable!("This test should only generate Rabin fingerprints")
2497 }
2498 #[cfg(feature = "md5")]
2499 Fingerprint::MD5(_id) => {
2500 unreachable!("This test should only generate Rabin fingerprints")
2501 }
2502 #[cfg(feature = "sha256")]
2503 Fingerprint::SHA256(_id) => {
2504 unreachable!("This test should only generate Rabin fingerprints")
2505 }
2506 }
2507 }
2508
2509 #[test]
2510 fn test_set_and_lookup_id() {
2511 let mut store = SchemaStore::new();
2512 let schema = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2513 let id = 42u32;
2514 let fp = Fingerprint::Id(id);
2515 let out_fp = store.set(fp, schema.clone()).unwrap();
2516 assert_eq!(out_fp, fp);
2517 assert_eq!(store.lookup(&fp).cloned(), Some(schema.clone()));
2518 assert!(store.lookup(&Fingerprint::Id(id.wrapping_add(1))).is_none());
2519 }
2520
2521 #[test]
2522 fn test_set_and_lookup_id64() {
2523 let mut store = SchemaStore::new();
2524 let schema = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2525 let id64: u64 = 0xDEAD_BEEF_DEAD_BEEF;
2526 let fp = Fingerprint::Id64(id64);
2527 let out_fp = store.set(fp, schema.clone()).unwrap();
2528 assert_eq!(out_fp, fp, "set should return the same Id64 fingerprint");
2529 assert_eq!(
2530 store.lookup(&fp).cloned(),
2531 Some(schema.clone()),
2532 "lookup should find the schema by Id64"
2533 );
2534 assert!(
2535 store
2536 .lookup(&Fingerprint::Id64(id64.wrapping_add(1)))
2537 .is_none(),
2538 "lookup with a different Id64 must return None"
2539 );
2540 }
2541
2542 #[test]
2543 fn test_fingerprint_id64_conversions() {
2544 let algo_from_fp = FingerprintAlgorithm::from(&Fingerprint::Id64(123));
2545 assert_eq!(algo_from_fp, FingerprintAlgorithm::Id64);
2546 let fp_from_algo = Fingerprint::from(FingerprintAlgorithm::Id64);
2547 assert!(matches!(fp_from_algo, Fingerprint::Id64(0)));
2548 let strategy_from_fp = FingerprintStrategy::from(Fingerprint::Id64(5));
2549 assert!(matches!(strategy_from_fp, FingerprintStrategy::Id64(0)));
2550 let algo_from_strategy = FingerprintAlgorithm::from(strategy_from_fp);
2551 assert_eq!(algo_from_strategy, FingerprintAlgorithm::Id64);
2552 }
2553
2554 #[test]
2555 fn test_register_duplicate_schema() {
2556 let mut store = SchemaStore::new();
2557 let schema1 = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2558 let schema2 = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2559 let fingerprint1 = store.register(schema1).unwrap();
2560 let fingerprint2 = store.register(schema2).unwrap();
2561 assert_eq!(fingerprint1, fingerprint2);
2562 assert_eq!(store.schemas.len(), 1);
2563 }
2564
2565 #[test]
2566 fn test_set_and_lookup_with_provided_fingerprint() {
2567 let mut store = SchemaStore::new();
2568 let schema = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2569 let fp = schema.fingerprint(FingerprintAlgorithm::Rabin).unwrap();
2570 let out_fp = store.set(fp, schema.clone()).unwrap();
2571 assert_eq!(out_fp, fp);
2572 assert_eq!(store.lookup(&fp).cloned(), Some(schema));
2573 }
2574
2575 #[test]
2576 fn test_set_duplicate_same_schema_ok() {
2577 let mut store = SchemaStore::new();
2578 let schema = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2579 let fp = schema.fingerprint(FingerprintAlgorithm::Rabin).unwrap();
2580 let _ = store.set(fp, schema.clone()).unwrap();
2581 let _ = store.set(fp, schema.clone()).unwrap();
2582 assert_eq!(store.schemas.len(), 1);
2583 }
2584
2585 #[test]
2586 fn test_set_duplicate_different_schema_collision_error() {
2587 let mut store = SchemaStore::new();
2588 let schema1 = AvroSchema::new(serde_json::to_string(&int_schema()).unwrap());
2589 let schema2 = AvroSchema::new(serde_json::to_string(&record_schema()).unwrap());
2590 let fp = Fingerprint::Id(123);
2592 let _ = store.set(fp, schema1).unwrap();
2593 let err = store.set(fp, schema2).unwrap_err();
2594 let msg = format!("{err}");
2595 assert!(msg.contains("Schema fingerprint collision"));
2596 }
2597
2598 #[test]
2599 fn test_canonical_form_generation_primitive() {
2600 let schema = int_schema();
2601 let canonical_form = AvroSchema::generate_canonical_form(&schema).unwrap();
2602 assert_eq!(canonical_form, r#""int""#);
2603 }
2604
2605 #[test]
2606 fn test_canonical_form_generation_record() {
2607 let schema = record_schema();
2608 let expected_canonical_form = r#"{"name":"test.namespace.record1","type":"record","fields":[{"name":"field1","type":"int"},{"name":"field2","type":"string"}]}"#;
2609 let canonical_form = AvroSchema::generate_canonical_form(&schema).unwrap();
2610 assert_eq!(canonical_form, expected_canonical_form);
2611 }
2612
2613 #[test]
2614 fn test_fingerprint_calculation() {
2615 let canonical_form = r#"{"fields":[{"name":"a","type":"long"},{"name":"b","type":"string"}],"name":"test","type":"record"}"#;
2616 let expected_fingerprint = 10505236152925314060;
2617 let fingerprint = compute_fingerprint_rabin(canonical_form);
2618 assert_eq!(fingerprint, expected_fingerprint);
2619 }
2620
2621 #[test]
2622 fn test_register_and_lookup_complex_schema() {
2623 let mut store = SchemaStore::new();
2624 let schema = AvroSchema::new(serde_json::to_string(&record_schema()).unwrap());
2625 let canonical_form = r#"{"name":"test.namespace.record1","type":"record","fields":[{"name":"field1","type":"int"},{"name":"field2","type":"string"}]}"#;
2626 let expected_fingerprint = Fingerprint::Rabin(compute_fingerprint_rabin(canonical_form));
2627 let fingerprint = store.register(schema.clone()).unwrap();
2628 assert_eq!(fingerprint, expected_fingerprint);
2629 let looked_up = store.lookup(&fingerprint).cloned();
2630 assert_eq!(looked_up, Some(schema));
2631 }
2632
2633 #[test]
2634 fn test_fingerprints_returns_all_keys() {
2635 let mut store = SchemaStore::new();
2636 let fp_int = store
2637 .register(AvroSchema::new(
2638 serde_json::to_string(&int_schema()).unwrap(),
2639 ))
2640 .unwrap();
2641 let fp_record = store
2642 .register(AvroSchema::new(
2643 serde_json::to_string(&record_schema()).unwrap(),
2644 ))
2645 .unwrap();
2646 let fps = store.fingerprints();
2647 assert_eq!(fps.len(), 2);
2648 assert!(fps.contains(&fp_int));
2649 assert!(fps.contains(&fp_record));
2650 }
2651
2652 #[test]
2653 fn test_canonical_form_strips_attributes() {
2654 let schema_with_attrs = Schema::Complex(ComplexType::Record(Record {
2655 name: "record_with_attrs",
2656 namespace: None,
2657 doc: Some(Cow::from("This doc should be stripped")),
2658 aliases: vec!["alias1", "alias2"],
2659 fields: vec![Field {
2660 name: "f1",
2661 doc: Some(Cow::from("field doc")),
2662 r#type: Schema::Type(Type {
2663 r#type: TypeName::Primitive(PrimitiveType::Bytes),
2664 attributes: Attributes {
2665 logical_type: None,
2666 additional: HashMap::from([("precision", json!(4))]),
2667 },
2668 }),
2669 default: None,
2670 aliases: vec![],
2671 }],
2672 attributes: Attributes {
2673 logical_type: None,
2674 additional: HashMap::from([("custom_attr", json!("value"))]),
2675 },
2676 }));
2677 let expected_canonical_form = r#"{"name":"record_with_attrs","type":"record","fields":[{"name":"f1","type":"bytes"}]}"#;
2678 let canonical_form = AvroSchema::generate_canonical_form(&schema_with_attrs).unwrap();
2679 assert_eq!(canonical_form, expected_canonical_form);
2680 }
2681
2682 #[cfg(not(feature = "avro_custom_types"))]
2683 #[test]
2684 fn test_primitive_mappings() {
2685 let cases = vec![
2686 (DataType::Boolean, "\"boolean\""),
2687 (DataType::Int8, "\"int\""),
2688 (DataType::Int16, "\"int\""),
2689 (DataType::Int32, "\"int\""),
2690 (DataType::Int64, "\"long\""),
2691 (DataType::UInt8, "\"int\""),
2692 (DataType::UInt16, "\"int\""),
2693 (DataType::UInt32, "\"long\""),
2694 (DataType::UInt64, "\"long\""),
2695 (DataType::Float16, "\"float\""),
2696 (DataType::Float32, "\"float\""),
2697 (DataType::Float64, "\"double\""),
2698 (DataType::Utf8, "\"string\""),
2699 (DataType::Binary, "\"bytes\""),
2700 ];
2701 for (dt, avro_token) in cases {
2702 let field = ArrowField::new("col", dt.clone(), false);
2703 let arrow_schema = single_field_schema(field);
2704 let avro = AvroSchema::try_from(&arrow_schema).unwrap();
2705 assert_json_contains(&avro.json_string, avro_token);
2706 }
2707 }
2708
2709 #[cfg(feature = "avro_custom_types")]
2710 #[test]
2711 fn test_primitive_mappings() {
2712 let cases = vec![
2713 (DataType::Boolean, "\"boolean\""),
2714 (DataType::Int8, "\"logicalType\":\"arrow.int8\""),
2715 (DataType::Int16, "\"logicalType\":\"arrow.int16\""),
2716 (DataType::Int32, "\"int\""),
2717 (DataType::Int64, "\"long\""),
2718 (DataType::UInt8, "\"logicalType\":\"arrow.uint8\""),
2719 (DataType::UInt16, "\"logicalType\":\"arrow.uint16\""),
2720 (DataType::UInt32, "\"logicalType\":\"arrow.uint32\""),
2721 (DataType::UInt64, "\"logicalType\":\"arrow.uint64\""),
2722 (DataType::Float16, "\"logicalType\":\"arrow.float16\""),
2723 (DataType::Float32, "\"float\""),
2724 (DataType::Float64, "\"double\""),
2725 (DataType::Utf8, "\"string\""),
2726 (DataType::Binary, "\"bytes\""),
2727 ];
2728 for (dt, avro_token) in cases {
2729 let field = ArrowField::new("col", dt.clone(), false);
2730 let arrow_schema = single_field_schema(field);
2731 let avro = AvroSchema::try_from(&arrow_schema).unwrap();
2732 assert_json_contains(&avro.json_string, avro_token);
2733 }
2734 }
2735
2736 #[cfg(feature = "avro_custom_types")]
2737 #[test]
2738 fn test_custom_fixed_logical_types_preserve_namespace_metadata() {
2739 let namespace = "com.example.types";
2740
2741 let mut md_u64 = HashMap::new();
2742 md_u64.insert(AVRO_NAME_METADATA_KEY.to_string(), "U64Type".to_string());
2743 md_u64.insert(
2744 AVRO_NAMESPACE_METADATA_KEY.to_string(),
2745 namespace.to_string(),
2746 );
2747
2748 let mut md_f16 = HashMap::new();
2749 md_f16.insert(AVRO_NAME_METADATA_KEY.to_string(), "F16Type".to_string());
2750 md_f16.insert(
2751 AVRO_NAMESPACE_METADATA_KEY.to_string(),
2752 namespace.to_string(),
2753 );
2754
2755 let mut md_iv_ym = HashMap::new();
2756 md_iv_ym.insert(AVRO_NAME_METADATA_KEY.to_string(), "IvYmType".to_string());
2757 md_iv_ym.insert(
2758 AVRO_NAMESPACE_METADATA_KEY.to_string(),
2759 namespace.to_string(),
2760 );
2761
2762 let mut md_iv_dt = HashMap::new();
2763 md_iv_dt.insert(AVRO_NAME_METADATA_KEY.to_string(), "IvDtType".to_string());
2764 md_iv_dt.insert(
2765 AVRO_NAMESPACE_METADATA_KEY.to_string(),
2766 namespace.to_string(),
2767 );
2768
2769 let arrow_schema = ArrowSchema::new(vec![
2770 ArrowField::new("u64_col", DataType::UInt64, false).with_metadata(md_u64),
2771 ArrowField::new("f16_col", DataType::Float16, false).with_metadata(md_f16),
2772 ArrowField::new(
2773 "iv_ym_col",
2774 DataType::Interval(IntervalUnit::YearMonth),
2775 false,
2776 )
2777 .with_metadata(md_iv_ym),
2778 ArrowField::new(
2779 "iv_dt_col",
2780 DataType::Interval(IntervalUnit::DayTime),
2781 false,
2782 )
2783 .with_metadata(md_iv_dt),
2784 ]);
2785
2786 let avro = AvroSchema::try_from(&arrow_schema).unwrap();
2787 let root: Value = serde_json::from_str(&avro.json_string).unwrap();
2788 let fields = root
2789 .get("fields")
2790 .and_then(|f| f.as_array())
2791 .expect("record fields array");
2792
2793 let expected = [
2794 ("u64_col", "arrow.uint64"),
2795 ("f16_col", "arrow.float16"),
2796 ("iv_ym_col", "arrow.interval-year-month"),
2797 ("iv_dt_col", "arrow.interval-day-time"),
2798 ];
2799
2800 for (field_name, logical_type) in expected {
2801 let field = fields
2802 .iter()
2803 .find(|f| f.get("name").and_then(Value::as_str) == Some(field_name))
2804 .unwrap_or_else(|| panic!("missing field {field_name}"));
2805 let ty = field
2806 .get("type")
2807 .and_then(Value::as_object)
2808 .unwrap_or_else(|| panic!("field {field_name} type must be object"));
2809
2810 assert_eq!(ty.get("type").and_then(Value::as_str), Some("fixed"));
2811 assert_eq!(
2812 ty.get("logicalType").and_then(Value::as_str),
2813 Some(logical_type)
2814 );
2815 assert_eq!(
2816 ty.get("namespace").and_then(Value::as_str),
2817 Some(namespace),
2818 "field {field_name} must preserve avro.namespace metadata"
2819 );
2820 }
2821 }
2822
2823 #[cfg(feature = "avro_custom_types")]
2824 #[test]
2825 fn test_custom_fixed_logical_types_omit_namespace_without_metadata() {
2826 let mut md_u64 = HashMap::new();
2827 md_u64.insert(AVRO_NAME_METADATA_KEY.to_string(), "U64Type".to_string());
2828
2829 let mut md_f16 = HashMap::new();
2830 md_f16.insert(AVRO_NAME_METADATA_KEY.to_string(), "F16Type".to_string());
2831
2832 let mut md_iv_ym = HashMap::new();
2833 md_iv_ym.insert(AVRO_NAME_METADATA_KEY.to_string(), "IvYmType".to_string());
2834
2835 let mut md_iv_dt = HashMap::new();
2836 md_iv_dt.insert(AVRO_NAME_METADATA_KEY.to_string(), "IvDtType".to_string());
2837
2838 let arrow_schema = ArrowSchema::new(vec![
2839 ArrowField::new("u64_col", DataType::UInt64, false).with_metadata(md_u64),
2840 ArrowField::new("f16_col", DataType::Float16, false).with_metadata(md_f16),
2841 ArrowField::new(
2842 "iv_ym_col",
2843 DataType::Interval(IntervalUnit::YearMonth),
2844 false,
2845 )
2846 .with_metadata(md_iv_ym),
2847 ArrowField::new(
2848 "iv_dt_col",
2849 DataType::Interval(IntervalUnit::DayTime),
2850 false,
2851 )
2852 .with_metadata(md_iv_dt),
2853 ]);
2854
2855 let avro = AvroSchema::try_from(&arrow_schema).unwrap();
2856 let root: Value = serde_json::from_str(&avro.json_string).unwrap();
2857 let fields = root
2858 .get("fields")
2859 .and_then(|f| f.as_array())
2860 .expect("record fields array");
2861
2862 for field_name in ["u64_col", "f16_col", "iv_ym_col", "iv_dt_col"] {
2863 let field = fields
2864 .iter()
2865 .find(|f| f.get("name").and_then(Value::as_str) == Some(field_name))
2866 .unwrap_or_else(|| panic!("missing field {field_name}"));
2867 let ty = field
2868 .get("type")
2869 .and_then(Value::as_object)
2870 .unwrap_or_else(|| panic!("field {field_name} type must be object"));
2871
2872 assert_eq!(ty.get("type").and_then(Value::as_str), Some("fixed"));
2873 assert!(
2874 !ty.contains_key("namespace"),
2875 "field {field_name} should not include namespace when metadata lacks avro.namespace"
2876 );
2877 }
2878 }
2879
2880 #[test]
2881 fn test_temporal_mappings() {
2882 let cases = vec![
2883 (DataType::Date32, "\"logicalType\":\"date\""),
2884 (
2885 DataType::Time32(TimeUnit::Millisecond),
2886 "\"logicalType\":\"time-millis\"",
2887 ),
2888 (
2889 DataType::Time64(TimeUnit::Microsecond),
2890 "\"logicalType\":\"time-micros\"",
2891 ),
2892 (
2893 DataType::Timestamp(TimeUnit::Millisecond, None),
2894 "\"logicalType\":\"local-timestamp-millis\"",
2895 ),
2896 (
2897 DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
2898 "\"logicalType\":\"timestamp-micros\"",
2899 ),
2900 ];
2901 for (dt, needle) in cases {
2902 let field = ArrowField::new("ts", dt.clone(), true);
2903 let arrow_schema = single_field_schema(field);
2904 let avro = AvroSchema::try_from(&arrow_schema).unwrap();
2905 assert_json_contains(&avro.json_string, needle);
2906 }
2907 }
2908
2909 #[test]
2910 fn test_decimal_and_uuid() {
2911 let decimal_field = ArrowField::new("amount", DataType::Decimal128(25, 2), false);
2912 let dec_schema = single_field_schema(decimal_field);
2913 let avro_dec = AvroSchema::try_from(&dec_schema).unwrap();
2914 assert_json_contains(&avro_dec.json_string, "\"logicalType\":\"decimal\"");
2915 assert_json_contains(&avro_dec.json_string, "\"precision\":25");
2916 assert_json_contains(&avro_dec.json_string, "\"scale\":2");
2917 let mut md = HashMap::new();
2918 md.insert("logicalType".into(), "uuid".into());
2919 let uuid_field =
2920 ArrowField::new("id", DataType::FixedSizeBinary(16), false).with_metadata(md);
2921 let uuid_schema = single_field_schema(uuid_field);
2922 let avro_uuid = AvroSchema::try_from(&uuid_schema).unwrap();
2923 assert_json_contains(&avro_uuid.json_string, "\"logicalType\":\"uuid\"");
2924 }
2925
2926 #[cfg(not(feature = "avro_custom_types"))]
2927 #[test]
2928 fn test_interval_month_day_nano_duration_schema() {
2929 let interval_field = ArrowField::new(
2930 "span",
2931 DataType::Interval(IntervalUnit::MonthDayNano),
2932 false,
2933 );
2934 let s = single_field_schema(interval_field);
2935 let avro = AvroSchema::try_from(&s).unwrap();
2936 assert_json_contains(&avro.json_string, "\"logicalType\":\"duration\"");
2937 assert_json_contains(&avro.json_string, "\"size\":12");
2938 }
2939
2940 #[cfg(feature = "avro_custom_types")]
2941 #[test]
2942 fn test_interval_month_day_nano_custom_schema() {
2943 let interval_field = ArrowField::new(
2944 "span",
2945 DataType::Interval(IntervalUnit::MonthDayNano),
2946 false,
2947 );
2948 let s = single_field_schema(interval_field);
2949 let avro = AvroSchema::try_from(&s).unwrap();
2950 assert_json_contains(
2951 &avro.json_string,
2952 "\"logicalType\":\"arrow.interval-month-day-nano\"",
2953 );
2954 assert_json_contains(&avro.json_string, "\"size\":16");
2955 }
2956
2957 #[cfg(feature = "avro_custom_types")]
2958 #[test]
2959 fn test_duration_custom_logical_type() {
2960 let dur_field = ArrowField::new("latency", DataType::Duration(TimeUnit::Nanosecond), false);
2961 let s2 = single_field_schema(dur_field);
2962 let avro2 = AvroSchema::try_from(&s2).unwrap();
2963 assert_json_contains(
2964 &avro2.json_string,
2965 "\"logicalType\":\"arrow.duration-nanos\"",
2966 );
2967 }
2968
2969 #[test]
2970 fn test_complex_types() {
2971 let list_dt = DataType::List(Arc::new(ArrowField::new("item", DataType::Int32, true)));
2972 let list_schema = single_field_schema(ArrowField::new("numbers", list_dt, false));
2973 let avro_list = AvroSchema::try_from(&list_schema).unwrap();
2974 assert_json_contains(&avro_list.json_string, "\"type\":\"array\"");
2975 assert_json_contains(&avro_list.json_string, "\"items\"");
2976 let value_field = ArrowField::new(
2977 arrow_schema::Field::MAP_VALUE_FIELD_DEFAULT_NAME,
2978 DataType::Boolean,
2979 true,
2980 );
2981 let entries_struct = ArrowField::new(
2982 arrow_schema::Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
2983 DataType::Struct(Fields::from(vec![
2984 ArrowField::new(
2985 arrow_schema::Field::MAP_KEY_FIELD_DEFAULT_NAME,
2986 DataType::Utf8,
2987 false,
2988 ),
2989 value_field.clone(),
2990 ])),
2991 false,
2992 );
2993 let map_dt = DataType::Map(Arc::new(entries_struct), false);
2994 let map_schema = single_field_schema(ArrowField::new("props", map_dt, false));
2995 let avro_map = AvroSchema::try_from(&map_schema).unwrap();
2996 assert_json_contains(&avro_map.json_string, "\"type\":\"map\"");
2997 assert_json_contains(&avro_map.json_string, "\"values\"");
2998 let struct_dt = DataType::Struct(Fields::from(vec![
2999 ArrowField::new("f1", DataType::Int64, false),
3000 ArrowField::new("f2", DataType::Utf8, true),
3001 ]));
3002 let struct_schema = single_field_schema(ArrowField::new("person", struct_dt, true));
3003 let avro_struct = AvroSchema::try_from(&struct_schema).unwrap();
3004 assert_json_contains(&avro_struct.json_string, "\"type\":\"record\"");
3005 assert_json_contains(&avro_struct.json_string, "\"null\"");
3006 }
3007
3008 #[test]
3009 fn test_enum_dictionary() {
3010 let mut md = HashMap::new();
3011 md.insert(
3012 AVRO_ENUM_SYMBOLS_METADATA_KEY.into(),
3013 "[\"OPEN\",\"CLOSED\"]".into(),
3014 );
3015 let enum_dt = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
3016 let field = ArrowField::new("status", enum_dt, false).with_metadata(md);
3017 let schema = single_field_schema(field);
3018 let avro = AvroSchema::try_from(&schema).unwrap();
3019 assert_json_contains(&avro.json_string, "\"type\":\"enum\"");
3020 assert_json_contains(&avro.json_string, "\"symbols\":[\"OPEN\",\"CLOSED\"]");
3021 }
3022
3023 #[test]
3024 fn test_run_end_encoded() {
3025 let ree_dt = DataType::RunEndEncoded(
3026 Arc::new(ArrowField::new(
3027 ArrowField::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
3028 DataType::Int32,
3029 false,
3030 )),
3031 Arc::new(ArrowField::new(
3032 ArrowField::REE_VALUES_FIELD_DEFAULT_NAME,
3033 DataType::Utf8,
3034 false,
3035 )),
3036 );
3037 let s = single_field_schema(ArrowField::new("text", ree_dt, false));
3038 let avro = AvroSchema::try_from(&s).unwrap();
3039 assert_json_contains(&avro.json_string, "\"string\"");
3040 }
3041
3042 #[test]
3043 fn test_dense_union() {
3044 let uf: UnionFields = vec![
3045 (2i8, Arc::new(ArrowField::new("a", DataType::Int32, false))),
3046 (7i8, Arc::new(ArrowField::new("b", DataType::Utf8, true))),
3047 ]
3048 .into_iter()
3049 .collect();
3050 let union_dt = DataType::Union(uf, UnionMode::Dense);
3051 let s = single_field_schema(ArrowField::new("u", union_dt, false));
3052 let avro =
3053 AvroSchema::try_from(&s).expect("Arrow Union -> Avro union conversion should succeed");
3054 let v: serde_json::Value = serde_json::from_str(&avro.json_string).unwrap();
3055 let fields = v
3056 .get("fields")
3057 .and_then(|x| x.as_array())
3058 .expect("fields array");
3059 let u_field = fields
3060 .iter()
3061 .find(|f| f.get("name").and_then(|n| n.as_str()) == Some("u"))
3062 .expect("field 'u'");
3063 let union = u_field.get("type").expect("u.type");
3064 let arr = union.as_array().expect("u.type must be Avro union array");
3065 assert_eq!(arr.len(), 2, "expected two union branches");
3066 let first = &arr[0];
3067 let obj = first
3068 .as_object()
3069 .expect("first branch should be an object with metadata");
3070 assert_eq!(obj.get("type").and_then(|t| t.as_str()), Some("int"));
3071 assert_eq!(
3072 obj.get("arrowUnionMode").and_then(|m| m.as_str()),
3073 Some("dense")
3074 );
3075 let type_ids: Vec<i64> = obj
3076 .get("arrowUnionTypeIds")
3077 .and_then(|a| a.as_array())
3078 .expect("arrowUnionTypeIds array")
3079 .iter()
3080 .map(|n| n.as_i64().expect("i64"))
3081 .collect();
3082 assert_eq!(type_ids, vec![2, 7], "type id ordering should be preserved");
3083 assert_eq!(arr[1], Value::String("string".into()));
3084 }
3085
3086 #[test]
3087 fn round_trip_primitive() {
3088 let arrow_schema = ArrowSchema::new(vec![ArrowField::new("f1", DataType::Int32, false)]);
3089 let avro_schema = AvroSchema::try_from(&arrow_schema).unwrap();
3090 let decoded = avro_schema.schema().unwrap();
3091 assert!(matches!(decoded, Schema::Complex(_)));
3092 }
3093
3094 #[test]
3095 fn test_name_generator_sanitization_and_uniqueness() {
3096 let f1 = ArrowField::new("weird-name", DataType::FixedSizeBinary(8), false);
3097 let f2 = ArrowField::new("weird name", DataType::FixedSizeBinary(8), false);
3098 let f3 = ArrowField::new("123bad", DataType::FixedSizeBinary(8), false);
3099 let arrow_schema = ArrowSchema::new(vec![f1, f2, f3]);
3100 let avro = AvroSchema::try_from(&arrow_schema).unwrap();
3101 assert_json_contains(&avro.json_string, "\"name\":\"weird_name\"");
3102 assert_json_contains(&avro.json_string, "\"name\":\"weird_name_1\"");
3103 assert_json_contains(&avro.json_string, "\"name\":\"_123bad\"");
3104 }
3105
3106 #[cfg(not(feature = "avro_custom_types"))]
3107 #[test]
3108 fn test_date64_logical_type_mapping() {
3109 let field = ArrowField::new("d", DataType::Date64, true);
3110 let schema = single_field_schema(field);
3111 let avro = AvroSchema::try_from(&schema).unwrap();
3112 assert_json_contains(
3113 &avro.json_string,
3114 "\"logicalType\":\"local-timestamp-millis\"",
3115 );
3116 }
3117
3118 #[cfg(feature = "avro_custom_types")]
3119 #[test]
3120 fn test_date64_logical_type_mapping_custom() {
3121 let field = ArrowField::new("d", DataType::Date64, true);
3122 let schema = single_field_schema(field);
3123 let avro = AvroSchema::try_from(&schema).unwrap();
3124 assert_json_contains(&avro.json_string, "\"logicalType\":\"arrow.date64\"");
3125 }
3126
3127 #[cfg(feature = "avro_custom_types")]
3128 #[test]
3129 fn test_duration_list_extras_propagated() {
3130 let child = ArrowField::new("lat", DataType::Duration(TimeUnit::Microsecond), false);
3131 let list_dt = DataType::List(Arc::new(child));
3132 let arrow_schema = single_field_schema(ArrowField::new("durations", list_dt, false));
3133 let avro = AvroSchema::try_from(&arrow_schema).unwrap();
3134 assert_json_contains(
3135 &avro.json_string,
3136 "\"logicalType\":\"arrow.duration-micros\"",
3137 );
3138 }
3139
3140 #[cfg(not(feature = "avro_custom_types"))]
3141 #[test]
3142 fn test_interval_yearmonth_extra() {
3143 let field = ArrowField::new("iv", DataType::Interval(IntervalUnit::YearMonth), false);
3144 let schema = single_field_schema(field);
3145 let avro = AvroSchema::try_from(&schema).unwrap();
3146 assert_json_contains(&avro.json_string, "\"arrowIntervalUnit\":\"yearmonth\"");
3147 }
3148
3149 #[cfg(not(feature = "avro_custom_types"))]
3150 #[test]
3151 fn test_interval_daytime_extra() {
3152 let field = ArrowField::new("iv_dt", DataType::Interval(IntervalUnit::DayTime), false);
3153 let schema = single_field_schema(field);
3154 let avro = AvroSchema::try_from(&schema).unwrap();
3155 assert_json_contains(&avro.json_string, "\"arrowIntervalUnit\":\"daytime\"");
3156 }
3157
3158 #[cfg(feature = "avro_custom_types")]
3159 #[test]
3160 fn test_interval_yearmonth_custom() {
3161 let field = ArrowField::new("iv", DataType::Interval(IntervalUnit::YearMonth), false);
3162 let schema = single_field_schema(field);
3163 let avro = AvroSchema::try_from(&schema).unwrap();
3164 assert_json_contains(
3165 &avro.json_string,
3166 "\"logicalType\":\"arrow.interval-year-month\"",
3167 );
3168 }
3169
3170 #[cfg(feature = "avro_custom_types")]
3171 #[test]
3172 fn test_interval_daytime_custom() {
3173 let field = ArrowField::new("iv_dt", DataType::Interval(IntervalUnit::DayTime), false);
3174 let schema = single_field_schema(field);
3175 let avro = AvroSchema::try_from(&schema).unwrap();
3176 assert_json_contains(
3177 &avro.json_string,
3178 "\"logicalType\":\"arrow.interval-day-time\"",
3179 );
3180 }
3181
3182 #[test]
3183 fn test_fixed_size_list_extra() {
3184 let child = ArrowField::new("item", DataType::Int32, false);
3185 let dt = DataType::FixedSizeList(Arc::new(child), 3);
3186 let schema = single_field_schema(ArrowField::new("triples", dt, false));
3187 let avro = AvroSchema::try_from(&schema).unwrap();
3188 assert_json_contains(&avro.json_string, "\"arrowFixedSize\":3");
3189 }
3190
3191 #[cfg(feature = "avro_custom_types")]
3192 #[test]
3193 fn test_map_duration_value_extra() {
3194 let val_field = ArrowField::new(
3195 ArrowField::MAP_VALUE_FIELD_DEFAULT_NAME,
3196 DataType::Duration(TimeUnit::Second),
3197 true,
3198 );
3199 let entries_struct = ArrowField::new(
3200 ArrowField::MAP_ENTRIES_FIELD_DEFAULT_NAME,
3201 DataType::Struct(Fields::from(vec![
3202 ArrowField::new(
3203 ArrowField::MAP_KEY_FIELD_DEFAULT_NAME,
3204 DataType::Utf8,
3205 false,
3206 ),
3207 val_field,
3208 ])),
3209 false,
3210 );
3211 let map_dt = DataType::Map(Arc::new(entries_struct), false);
3212 let schema = single_field_schema(ArrowField::new("metrics", map_dt, false));
3213 let avro = AvroSchema::try_from(&schema).unwrap();
3214 assert_json_contains(
3215 &avro.json_string,
3216 "\"logicalType\":\"arrow.duration-seconds\"",
3217 );
3218 }
3219
3220 #[test]
3221 fn test_schema_with_non_string_defaults_decodes_successfully() {
3222 let schema_json = r#"{
3223 "type": "record",
3224 "name": "R",
3225 "fields": [
3226 {"name": "a", "type": "int", "default": 0},
3227 {"name": "b", "type": {"type": "array", "items": "long"}, "default": [1, 2, 3]},
3228 {"name": "c", "type": {"type": "map", "values": "double"}, "default": {"x": 1.5, "y": 2.5}},
3229 {"name": "inner", "type": {"type": "record", "name": "Inner", "fields": [
3230 {"name": "flag", "type": "boolean", "default": true},
3231 {"name": "name", "type": "string", "default": "hi"}
3232 ]}, "default": {"flag": false, "name": "d"}},
3233 {"name": "u", "type": ["int", "null"], "default": 42}
3234 ]
3235 }"#;
3236 let schema: Schema = serde_json::from_str(schema_json).expect("schema should parse");
3237 match &schema {
3238 Schema::Complex(ComplexType::Record(_)) => {}
3239 other => panic!("expected record schema, got: {other:?}"),
3240 }
3241 let field = crate::codec::AvroField::try_from(&schema)
3243 .expect("Avro->Arrow conversion should succeed");
3244 let arrow_field = field.field();
3245 let expected_list_item = ArrowField::new(
3247 arrow_schema::Field::LIST_FIELD_DEFAULT_NAME,
3248 DataType::Int64,
3249 false,
3250 );
3251 let expected_b = ArrowField::new("b", DataType::List(Arc::new(expected_list_item)), false);
3252
3253 let expected_map_value = ArrowField::new(
3254 arrow_schema::Field::MAP_VALUE_FIELD_DEFAULT_NAME,
3255 DataType::Float64,
3256 false,
3257 );
3258 let expected_entries = ArrowField::new(
3259 arrow_schema::Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
3260 DataType::Struct(Fields::from(vec![
3261 ArrowField::new(
3262 arrow_schema::Field::MAP_KEY_FIELD_DEFAULT_NAME,
3263 DataType::Utf8,
3264 false,
3265 ),
3266 expected_map_value,
3267 ])),
3268 false,
3269 );
3270 let expected_c =
3271 ArrowField::new("c", DataType::Map(Arc::new(expected_entries), false), false);
3272 let mut inner_md = std::collections::HashMap::new();
3273 inner_md.insert(AVRO_NAME_METADATA_KEY.to_string(), "Inner".to_string());
3274 let expected_inner = ArrowField::new(
3275 "inner",
3276 DataType::Struct(Fields::from(vec![
3277 ArrowField::new("flag", DataType::Boolean, false),
3278 ArrowField::new("name", DataType::Utf8, false),
3279 ])),
3280 false,
3281 )
3282 .with_metadata(inner_md);
3283 let mut root_md = std::collections::HashMap::new();
3284 root_md.insert(AVRO_NAME_METADATA_KEY.to_string(), "R".to_string());
3285 let expected = ArrowField::new(
3286 "R",
3287 DataType::Struct(Fields::from(vec![
3288 ArrowField::new("a", DataType::Int32, false),
3289 expected_b,
3290 expected_c,
3291 expected_inner,
3292 ArrowField::new("u", DataType::Int32, true),
3293 ])),
3294 false,
3295 )
3296 .with_metadata(root_md);
3297 assert_eq!(arrow_field, expected);
3298 }
3299
3300 #[test]
3301 fn default_order_is_consistent() {
3302 let arrow_schema = ArrowSchema::new(vec![ArrowField::new("s", DataType::Utf8, true)]);
3303 let a = AvroSchema::try_from(&arrow_schema).unwrap().json_string;
3304 let b = AvroSchema::from_arrow_with_options(&arrow_schema, None);
3305 assert_eq!(a, b.unwrap().json_string);
3306 }
3307
3308 #[test]
3309 fn test_union_branch_missing_name_errors() {
3310 for t in ["record", "enum", "fixed"] {
3311 let branch = json!({ "type": t });
3312 let err = union_branch_signature(&branch).unwrap_err().to_string();
3313 assert!(
3314 err.contains(&format!("Union branch '{t}' missing required 'name'")),
3315 "expected missing-name error for {t}, got: {err}"
3316 );
3317 }
3318 }
3319
3320 #[test]
3321 fn test_union_branch_named_type_signature_includes_name() {
3322 let rec = json!({ "type": "record", "name": "Foo" });
3323 assert_eq!(union_branch_signature(&rec).unwrap(), "N:record:Foo");
3324 let en = json!({ "type": "enum", "name": "Color", "symbols": ["R", "G", "B"] });
3325 assert_eq!(union_branch_signature(&en).unwrap(), "N:enum:Color");
3326 let fx = json!({ "type": "fixed", "name": "Bytes16", "size": 16 });
3327 assert_eq!(union_branch_signature(&fx).unwrap(), "N:fixed:Bytes16");
3328 }
3329
3330 #[test]
3331 fn test_record_field_alias_resolution_without_default() {
3332 let writer_json = r#"{
3333 "type":"record",
3334 "name":"R",
3335 "fields":[{"name":"old","type":"int"}]
3336 }"#;
3337 let reader_json = r#"{
3338 "type":"record",
3339 "name":"R",
3340 "fields":[{"name":"new","aliases":["old"],"type":"int"}]
3341 }"#;
3342 let writer: Schema = serde_json::from_str(writer_json).unwrap();
3343 let reader: Schema = serde_json::from_str(reader_json).unwrap();
3344 let resolved = AvroFieldBuilder::new(&writer)
3345 .with_reader_schema(&reader)
3346 .with_utf8view(false)
3347 .with_strict_mode(false)
3348 .build()
3349 .unwrap();
3350 let expected = ArrowField::new(
3351 "R",
3352 DataType::Struct(Fields::from(vec![ArrowField::new(
3353 "new",
3354 DataType::Int32,
3355 false,
3356 )])),
3357 false,
3358 )
3359 .with_metadata(HashMap::from_iter([(
3360 "avro.name".to_owned(),
3361 "R".to_owned(),
3362 )]));
3363 assert_eq!(resolved.field(), expected);
3364 }
3365
3366 #[test]
3367 fn test_record_field_alias_ambiguous_in_strict_mode_errors() {
3368 let writer_json = r#"{
3369 "type":"record",
3370 "name":"R",
3371 "fields":[
3372 {"name":"a","type":"int","aliases":["old"]},
3373 {"name":"b","type":"int","aliases":["old"]}
3374 ]
3375 }"#;
3376 let reader_json = r#"{
3377 "type":"record",
3378 "name":"R",
3379 "fields":[{"name":"target","type":"int","aliases":["old"]}]
3380 }"#;
3381 let writer: Schema = serde_json::from_str(writer_json).unwrap();
3382 let reader: Schema = serde_json::from_str(reader_json).unwrap();
3383 let err = AvroFieldBuilder::new(&writer)
3384 .with_reader_schema(&reader)
3385 .with_utf8view(false)
3386 .with_strict_mode(true)
3387 .build()
3388 .unwrap_err()
3389 .to_string();
3390 assert!(
3391 err.contains("Ambiguous alias 'old'"),
3392 "expected ambiguous-alias error, got: {err}"
3393 );
3394 }
3395
3396 #[test]
3397 fn test_pragmatic_writer_field_alias_mapping_non_strict() {
3398 let writer_json = r#"{
3399 "type":"record",
3400 "name":"R",
3401 "fields":[{"name":"before","type":"int","aliases":["now"]}]
3402 }"#;
3403 let reader_json = r#"{
3404 "type":"record",
3405 "name":"R",
3406 "fields":[{"name":"now","type":"int"}]
3407 }"#;
3408 let writer: Schema = serde_json::from_str(writer_json).unwrap();
3409 let reader: Schema = serde_json::from_str(reader_json).unwrap();
3410 let resolved = AvroFieldBuilder::new(&writer)
3411 .with_reader_schema(&reader)
3412 .with_utf8view(false)
3413 .with_strict_mode(false)
3414 .build()
3415 .unwrap();
3416 let expected = ArrowField::new(
3417 "R",
3418 DataType::Struct(Fields::from(vec![ArrowField::new(
3419 "now",
3420 DataType::Int32,
3421 false,
3422 )])),
3423 false,
3424 )
3425 .with_metadata(HashMap::from_iter([(
3426 "avro.name".to_owned(),
3427 "R".to_owned(),
3428 )]));
3429 assert_eq!(resolved.field(), expected);
3430 }
3431
3432 #[test]
3433 fn test_missing_reader_field_null_first_no_default_is_ok() {
3434 let writer_json = r#"{
3435 "type":"record",
3436 "name":"R",
3437 "fields":[{"name":"a","type":"int"}]
3438 }"#;
3439 let reader_json = r#"{
3440 "type":"record",
3441 "name":"R",
3442 "fields":[
3443 {"name":"a","type":"int"},
3444 {"name":"b","type":["null","int"]}
3445 ]
3446 }"#;
3447 let writer: Schema = serde_json::from_str(writer_json).unwrap();
3448 let reader: Schema = serde_json::from_str(reader_json).unwrap();
3449 let resolved = AvroFieldBuilder::new(&writer)
3450 .with_reader_schema(&reader)
3451 .with_utf8view(false)
3452 .with_strict_mode(false)
3453 .build()
3454 .unwrap();
3455 let expected = ArrowField::new(
3456 "R",
3457 DataType::Struct(Fields::from(vec![
3458 ArrowField::new("a", DataType::Int32, false),
3459 ArrowField::new("b", DataType::Int32, true).with_metadata(HashMap::from([(
3460 AVRO_FIELD_DEFAULT_METADATA_KEY.to_string(),
3461 "null".to_string(),
3462 )])),
3463 ])),
3464 false,
3465 )
3466 .with_metadata(HashMap::from_iter([(
3467 "avro.name".to_owned(),
3468 "R".to_owned(),
3469 )]));
3470 assert_eq!(resolved.field(), expected);
3471 }
3472
3473 #[test]
3474 fn test_missing_reader_field_null_second_without_default_errors() {
3475 let writer_json = r#"{
3476 "type":"record",
3477 "name":"R",
3478 "fields":[{"name":"a","type":"int"}]
3479 }"#;
3480 let reader_json = r#"{
3481 "type":"record",
3482 "name":"R",
3483 "fields":[
3484 {"name":"a","type":"int"},
3485 {"name":"b","type":["int","null"]}
3486 ]
3487 }"#;
3488 let writer: Schema = serde_json::from_str(writer_json).unwrap();
3489 let reader: Schema = serde_json::from_str(reader_json).unwrap();
3490 let err = AvroFieldBuilder::new(&writer)
3491 .with_reader_schema(&reader)
3492 .with_utf8view(false)
3493 .with_strict_mode(false)
3494 .build()
3495 .unwrap_err()
3496 .to_string();
3497 assert!(
3498 err.contains("must have a default value"),
3499 "expected missing-default error, got: {err}"
3500 );
3501 }
3502
3503 #[test]
3504 fn test_from_arrow_with_options_respects_schema_metadata_when_not_stripping() {
3505 let field = ArrowField::new("x", DataType::Int32, true);
3506 let injected_json =
3507 r#"{"type":"record","name":"Injected","fields":[{"name":"ignored","type":"int"}]}"#
3508 .to_string();
3509 let mut md = HashMap::new();
3510 md.insert(SCHEMA_METADATA_KEY.to_string(), injected_json.clone());
3511 md.insert("custom".to_string(), "123".to_string());
3512 let arrow_schema = ArrowSchema::new_with_metadata(vec![field], md);
3513 let opts = AvroSchemaOptions {
3514 null_order: Some(Nullability::NullSecond),
3515 strip_metadata: false,
3516 };
3517 let out = AvroSchema::from_arrow_with_options(&arrow_schema, Some(opts)).unwrap();
3518 assert_eq!(
3519 out.json_string, injected_json,
3520 "When strip_metadata=false and avro.schema is present, return the embedded JSON verbatim"
3521 );
3522 let v: Value = serde_json::from_str(&out.json_string).unwrap();
3523 assert_eq!(v.get("type").and_then(|t| t.as_str()), Some("record"));
3524 assert_eq!(v.get("name").and_then(|n| n.as_str()), Some("Injected"));
3525 }
3526
3527 #[test]
3528 fn test_from_arrow_with_options_ignores_schema_metadata_when_stripping_and_keeps_passthrough() {
3529 let field = ArrowField::new("x", DataType::Int32, true);
3530 let injected_json =
3531 r#"{"type":"record","name":"Injected","fields":[{"name":"ignored","type":"int"}]}"#
3532 .to_string();
3533 let mut md = HashMap::new();
3534 md.insert(SCHEMA_METADATA_KEY.to_string(), injected_json);
3535 md.insert("custom_meta".to_string(), "7".to_string());
3536 let arrow_schema = ArrowSchema::new_with_metadata(vec![field], md);
3537 let opts = AvroSchemaOptions {
3538 null_order: Some(Nullability::NullFirst),
3539 strip_metadata: true,
3540 };
3541 let out = AvroSchema::from_arrow_with_options(&arrow_schema, Some(opts)).unwrap();
3542 assert_json_contains(&out.json_string, "\"type\":\"record\"");
3543 assert_json_contains(&out.json_string, "\"name\":\"topLevelRecord\"");
3544 assert_json_contains(&out.json_string, "\"custom_meta\":7");
3545 }
3546
3547 #[test]
3548 fn test_from_arrow_with_options_null_first_for_nullable_primitive() {
3549 let field = ArrowField::new("s", DataType::Utf8, true);
3550 let arrow_schema = single_field_schema(field);
3551 let opts = AvroSchemaOptions {
3552 null_order: Some(Nullability::NullFirst),
3553 strip_metadata: true,
3554 };
3555 let out = AvroSchema::from_arrow_with_options(&arrow_schema, Some(opts)).unwrap();
3556 let v: Value = serde_json::from_str(&out.json_string).unwrap();
3557 let arr = v["fields"][0]["type"]
3558 .as_array()
3559 .expect("nullable primitive should be Avro union array");
3560 assert_eq!(arr[0], Value::String("null".into()));
3561 assert_eq!(arr[1], Value::String("string".into()));
3562 }
3563
3564 #[test]
3565 fn test_from_arrow_with_options_null_second_for_nullable_primitive() {
3566 let field = ArrowField::new("s", DataType::Utf8, true);
3567 let arrow_schema = single_field_schema(field);
3568 let opts = AvroSchemaOptions {
3569 null_order: Some(Nullability::NullSecond),
3570 strip_metadata: true,
3571 };
3572 let out = AvroSchema::from_arrow_with_options(&arrow_schema, Some(opts)).unwrap();
3573 let v: Value = serde_json::from_str(&out.json_string).unwrap();
3574 let arr = v["fields"][0]["type"]
3575 .as_array()
3576 .expect("nullable primitive should be Avro union array");
3577 assert_eq!(arr[0], Value::String("string".into()));
3578 assert_eq!(arr[1], Value::String("null".into()));
3579 }
3580
3581 #[test]
3582 fn test_from_arrow_with_options_union_extras_respected_by_strip_metadata() {
3583 let uf: UnionFields = vec![
3584 (2i8, Arc::new(ArrowField::new("a", DataType::Int32, false))),
3585 (7i8, Arc::new(ArrowField::new("b", DataType::Utf8, false))),
3586 ]
3587 .into_iter()
3588 .collect();
3589 let union_dt = DataType::Union(uf, UnionMode::Dense);
3590 let arrow_schema = single_field_schema(ArrowField::new("u", union_dt, true));
3591 let with_extras = AvroSchema::from_arrow_with_options(
3592 &arrow_schema,
3593 Some(AvroSchemaOptions {
3594 null_order: Some(Nullability::NullFirst),
3595 strip_metadata: false,
3596 }),
3597 )
3598 .unwrap();
3599 let v_with: Value = serde_json::from_str(&with_extras.json_string).unwrap();
3600 let union_arr = v_with["fields"][0]["type"].as_array().expect("union array");
3601 let first_obj = union_arr
3602 .iter()
3603 .find(|b| b.is_object())
3604 .expect("expected an object branch with extras");
3605 let obj = first_obj.as_object().unwrap();
3606 assert_eq!(obj.get("type").and_then(|t| t.as_str()), Some("int"));
3607 assert_eq!(
3608 obj.get("arrowUnionMode").and_then(|m| m.as_str()),
3609 Some("dense")
3610 );
3611 let type_ids: Vec<i64> = obj["arrowUnionTypeIds"]
3612 .as_array()
3613 .expect("arrowUnionTypeIds array")
3614 .iter()
3615 .map(|n| n.as_i64().expect("i64"))
3616 .collect();
3617 assert_eq!(type_ids, vec![2, 7]);
3618 let stripped = AvroSchema::from_arrow_with_options(
3619 &arrow_schema,
3620 Some(AvroSchemaOptions {
3621 null_order: Some(Nullability::NullFirst),
3622 strip_metadata: true,
3623 }),
3624 )
3625 .unwrap();
3626 let v_stripped: Value = serde_json::from_str(&stripped.json_string).unwrap();
3627 let union_arr2 = v_stripped["fields"][0]["type"]
3628 .as_array()
3629 .expect("union array");
3630 assert!(
3631 !union_arr2.iter().any(|b| b
3632 .as_object()
3633 .is_some_and(|m| m.contains_key("arrowUnionMode"))),
3634 "extras must be removed when strip_metadata=true"
3635 );
3636 assert_eq!(union_arr2[0], Value::String("null".into()));
3637 assert_eq!(union_arr2[1], Value::String("int".into()));
3638 assert_eq!(union_arr2[2], Value::String("string".into()));
3639 }
3640
3641 #[test]
3642 fn test_project_empty_projection() {
3643 let schema_json = r#"{
3644 "type": "record",
3645 "name": "Test",
3646 "fields": [
3647 {"name": "a", "type": "int"},
3648 {"name": "b", "type": "string"}
3649 ]
3650 }"#;
3651 let schema = AvroSchema::new(schema_json.to_string());
3652 let projected = schema.project(&[]).unwrap();
3653 let v: Value = serde_json::from_str(&projected.json_string).unwrap();
3654 let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
3655 assert!(
3656 fields.is_empty(),
3657 "Empty projection should yield empty fields"
3658 );
3659 }
3660
3661 #[test]
3662 fn test_project_single_field() {
3663 let schema_json = r#"{
3664 "type": "record",
3665 "name": "Test",
3666 "fields": [
3667 {"name": "a", "type": "int"},
3668 {"name": "b", "type": "string"},
3669 {"name": "c", "type": "long"}
3670 ]
3671 }"#;
3672 let schema = AvroSchema::new(schema_json.to_string());
3673 let projected = schema.project(&[1]).unwrap();
3674 let v: Value = serde_json::from_str(&projected.json_string).unwrap();
3675 let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
3676 assert_eq!(fields.len(), 1);
3677 assert_eq!(fields[0].get("name").and_then(|n| n.as_str()), Some("b"));
3678 }
3679
3680 #[test]
3681 fn test_project_multiple_fields() {
3682 let schema_json = r#"{
3683 "type": "record",
3684 "name": "Test",
3685 "fields": [
3686 {"name": "a", "type": "int"},
3687 {"name": "b", "type": "string"},
3688 {"name": "c", "type": "long"},
3689 {"name": "d", "type": "boolean"}
3690 ]
3691 }"#;
3692 let schema = AvroSchema::new(schema_json.to_string());
3693 let projected = schema.project(&[0, 2, 3]).unwrap();
3694 let v: Value = serde_json::from_str(&projected.json_string).unwrap();
3695 let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
3696 assert_eq!(fields.len(), 3);
3697 assert_eq!(fields[0].get("name").and_then(|n| n.as_str()), Some("a"));
3698 assert_eq!(fields[1].get("name").and_then(|n| n.as_str()), Some("c"));
3699 assert_eq!(fields[2].get("name").and_then(|n| n.as_str()), Some("d"));
3700 }
3701
3702 #[test]
3703 fn test_project_all_fields() {
3704 let schema_json = r#"{
3705 "type": "record",
3706 "name": "Test",
3707 "fields": [
3708 {"name": "a", "type": "int"},
3709 {"name": "b", "type": "string"}
3710 ]
3711 }"#;
3712 let schema = AvroSchema::new(schema_json.to_string());
3713 let projected = schema.project(&[0, 1]).unwrap();
3714 let v: Value = serde_json::from_str(&projected.json_string).unwrap();
3715 let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
3716 assert_eq!(fields.len(), 2);
3717 assert_eq!(fields[0].get("name").and_then(|n| n.as_str()), Some("a"));
3718 assert_eq!(fields[1].get("name").and_then(|n| n.as_str()), Some("b"));
3719 }
3720
3721 #[test]
3722 fn test_project_reorder_fields() {
3723 let schema_json = r#"{
3724 "type": "record",
3725 "name": "Test",
3726 "fields": [
3727 {"name": "a", "type": "int"},
3728 {"name": "b", "type": "string"},
3729 {"name": "c", "type": "long"}
3730 ]
3731 }"#;
3732 let schema = AvroSchema::new(schema_json.to_string());
3733 let projected = schema.project(&[2, 0, 1]).unwrap();
3735 let v: Value = serde_json::from_str(&projected.json_string).unwrap();
3736 let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
3737 assert_eq!(fields.len(), 3);
3738 assert_eq!(fields[0].get("name").and_then(|n| n.as_str()), Some("c"));
3739 assert_eq!(fields[1].get("name").and_then(|n| n.as_str()), Some("a"));
3740 assert_eq!(fields[2].get("name").and_then(|n| n.as_str()), Some("b"));
3741 }
3742
3743 #[test]
3744 fn test_project_preserves_record_metadata() {
3745 let schema_json = r#"{
3746 "type": "record",
3747 "name": "MyRecord",
3748 "namespace": "com.example",
3749 "doc": "A test record",
3750 "aliases": ["OldRecord"],
3751 "fields": [
3752 {"name": "a", "type": "int"},
3753 {"name": "b", "type": "string"}
3754 ]
3755 }"#;
3756 let schema = AvroSchema::new(schema_json.to_string());
3757 let projected = schema.project(&[0]).unwrap();
3758 let v: Value = serde_json::from_str(&projected.json_string).unwrap();
3759 assert_eq!(v.get("name").and_then(|n| n.as_str()), Some("MyRecord"));
3760 assert_eq!(
3761 v.get("namespace").and_then(|n| n.as_str()),
3762 Some("com.example")
3763 );
3764 assert_eq!(v.get("doc").and_then(|n| n.as_str()), Some("A test record"));
3765 assert!(v.get("aliases").is_some());
3766 }
3767
3768 #[test]
3769 fn test_project_preserves_field_metadata() {
3770 let schema_json = r#"{
3771 "type": "record",
3772 "name": "Test",
3773 "fields": [
3774 {"name": "a", "type": "int", "doc": "Field A", "default": 0},
3775 {"name": "b", "type": "string"}
3776 ]
3777 }"#;
3778 let schema = AvroSchema::new(schema_json.to_string());
3779 let projected = schema.project(&[0]).unwrap();
3780 let v: Value = serde_json::from_str(&projected.json_string).unwrap();
3781 let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
3782 assert_eq!(
3783 fields[0].get("doc").and_then(|d| d.as_str()),
3784 Some("Field A")
3785 );
3786 assert_eq!(fields[0].get("default").and_then(|d| d.as_i64()), Some(0));
3787 }
3788
3789 #[test]
3790 fn test_project_with_nested_record() {
3791 let schema_json = r#"{
3792 "type": "record",
3793 "name": "Outer",
3794 "fields": [
3795 {"name": "id", "type": "int"},
3796 {"name": "inner", "type": {
3797 "type": "record",
3798 "name": "Inner",
3799 "fields": [
3800 {"name": "x", "type": "int"},
3801 {"name": "y", "type": "string"}
3802 ]
3803 }},
3804 {"name": "value", "type": "double"}
3805 ]
3806 }"#;
3807 let schema = AvroSchema::new(schema_json.to_string());
3808 let projected = schema.project(&[1]).unwrap();
3809 let v: Value = serde_json::from_str(&projected.json_string).unwrap();
3810 let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
3811 assert_eq!(fields.len(), 1);
3812 assert_eq!(
3813 fields[0].get("name").and_then(|n| n.as_str()),
3814 Some("inner")
3815 );
3816 let inner_type = fields[0].get("type").unwrap();
3818 assert_eq!(
3819 inner_type.get("type").and_then(|t| t.as_str()),
3820 Some("record")
3821 );
3822 assert_eq!(
3823 inner_type.get("name").and_then(|n| n.as_str()),
3824 Some("Inner")
3825 );
3826 }
3827
3828 #[test]
3829 fn test_project_with_complex_field_types() {
3830 let schema_json = r#"{
3831 "type": "record",
3832 "name": "Test",
3833 "fields": [
3834 {"name": "arr", "type": {"type": "array", "items": "int"}},
3835 {"name": "map", "type": {"type": "map", "values": "string"}},
3836 {"name": "union", "type": ["null", "int"]}
3837 ]
3838 }"#;
3839 let schema = AvroSchema::new(schema_json.to_string());
3840 let projected = schema.project(&[0, 2]).unwrap();
3841 let v: Value = serde_json::from_str(&projected.json_string).unwrap();
3842 let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
3843 assert_eq!(fields.len(), 2);
3844 let arr_type = fields[0].get("type").unwrap();
3846 assert_eq!(arr_type.get("type").and_then(|t| t.as_str()), Some("array"));
3847 let union_type = fields[1].get("type").unwrap();
3849 assert!(union_type.is_array());
3850 }
3851
3852 #[test]
3853 fn test_project_error_invalid_json() {
3854 let schema = AvroSchema::new("not valid json".to_string());
3855 let err = schema.project(&[0]).unwrap_err();
3856 let msg = err.to_string();
3857 assert!(
3858 msg.contains("Invalid Avro schema JSON"),
3859 "Expected parse error, got: {msg}"
3860 );
3861 }
3862
3863 #[test]
3864 fn test_project_error_not_object() {
3865 let schema = AvroSchema::new(r#""string""#.to_string());
3867 let err = schema.project(&[0]).unwrap_err();
3868 let msg = err.to_string();
3869 assert!(
3870 msg.contains("must be a JSON object"),
3871 "Expected object error, got: {msg}"
3872 );
3873 }
3874
3875 #[test]
3876 fn test_project_error_array_schema() {
3877 let schema = AvroSchema::new(r#"["null", "int"]"#.to_string());
3879 let err = schema.project(&[0]).unwrap_err();
3880 let msg = err.to_string();
3881 assert!(
3882 msg.contains("must be a JSON object"),
3883 "Expected object error for array schema, got: {msg}"
3884 );
3885 }
3886
3887 #[test]
3888 fn test_project_error_type_not_record() {
3889 let schema_json = r#"{
3890 "type": "enum",
3891 "name": "Color",
3892 "symbols": ["RED", "GREEN", "BLUE"]
3893 }"#;
3894 let schema = AvroSchema::new(schema_json.to_string());
3895 let err = schema.project(&[0]).unwrap_err();
3896 let msg = err.to_string();
3897 assert!(
3898 msg.contains("must be an Avro record") && msg.contains("'enum'"),
3899 "Expected type mismatch error, got: {msg}"
3900 );
3901 }
3902
3903 #[test]
3904 fn test_project_error_type_array() {
3905 let schema_json = r#"{
3906 "type": "array",
3907 "items": "int"
3908 }"#;
3909 let schema = AvroSchema::new(schema_json.to_string());
3910 let err = schema.project(&[0]).unwrap_err();
3911 let msg = err.to_string();
3912 assert!(
3913 msg.contains("must be an Avro record") && msg.contains("'array'"),
3914 "Expected type mismatch error for array type, got: {msg}"
3915 );
3916 }
3917
3918 #[test]
3919 fn test_project_error_type_fixed() {
3920 let schema_json = r#"{
3921 "type": "fixed",
3922 "name": "MD5",
3923 "size": 16
3924 }"#;
3925 let schema = AvroSchema::new(schema_json.to_string());
3926 let err = schema.project(&[0]).unwrap_err();
3927 let msg = err.to_string();
3928 assert!(
3929 msg.contains("must be an Avro record") && msg.contains("'fixed'"),
3930 "Expected type mismatch error for fixed type, got: {msg}"
3931 );
3932 }
3933
3934 #[test]
3935 fn test_project_error_type_map() {
3936 let schema_json = r#"{
3937 "type": "map",
3938 "values": "string"
3939 }"#;
3940 let schema = AvroSchema::new(schema_json.to_string());
3941 let err = schema.project(&[0]).unwrap_err();
3942 let msg = err.to_string();
3943 assert!(
3944 msg.contains("must be an Avro record") && msg.contains("'map'"),
3945 "Expected type mismatch error for map type, got: {msg}"
3946 );
3947 }
3948
3949 #[test]
3950 fn test_project_error_missing_type_field() {
3951 let schema_json = r#"{
3952 "name": "Test",
3953 "fields": [{"name": "a", "type": "int"}]
3954 }"#;
3955 let schema = AvroSchema::new(schema_json.to_string());
3956 let err = schema.project(&[0]).unwrap_err();
3957 let msg = err.to_string();
3958 assert!(
3959 msg.contains("missing required 'type' field"),
3960 "Expected missing type error, got: {msg}"
3961 );
3962 }
3963
3964 #[test]
3965 fn test_project_error_missing_fields() {
3966 let schema_json = r#"{
3967 "type": "record",
3968 "name": "Test"
3969 }"#;
3970 let schema = AvroSchema::new(schema_json.to_string());
3971 let err = schema.project(&[0]).unwrap_err();
3972 let msg = err.to_string();
3973 assert!(
3974 msg.contains("missing required 'fields'"),
3975 "Expected missing fields error, got: {msg}"
3976 );
3977 }
3978
3979 #[test]
3980 fn test_project_error_fields_not_array() {
3981 let schema_json = r#"{
3982 "type": "record",
3983 "name": "Test",
3984 "fields": "not an array"
3985 }"#;
3986 let schema = AvroSchema::new(schema_json.to_string());
3987 let err = schema.project(&[0]).unwrap_err();
3988 let msg = err.to_string();
3989 assert!(
3990 msg.contains("'fields' must be an array"),
3991 "Expected fields array error, got: {msg}"
3992 );
3993 }
3994
3995 #[test]
3996 fn test_project_error_index_out_of_bounds() {
3997 let schema_json = r#"{
3998 "type": "record",
3999 "name": "Test",
4000 "fields": [
4001 {"name": "a", "type": "int"},
4002 {"name": "b", "type": "string"}
4003 ]
4004 }"#;
4005 let schema = AvroSchema::new(schema_json.to_string());
4006 let err = schema.project(&[5]).unwrap_err();
4007 let msg = err.to_string();
4008 assert!(
4009 msg.contains("out of bounds") && msg.contains('5') && msg.contains('2'),
4010 "Expected out of bounds error, got: {msg}"
4011 );
4012 }
4013
4014 #[test]
4015 fn test_project_error_index_out_of_bounds_edge() {
4016 let schema_json = r#"{
4017 "type": "record",
4018 "name": "Test",
4019 "fields": [
4020 {"name": "a", "type": "int"}
4021 ]
4022 }"#;
4023 let schema = AvroSchema::new(schema_json.to_string());
4024 let err = schema.project(&[1]).unwrap_err();
4026 let msg = err.to_string();
4027 assert!(
4028 msg.contains("out of bounds") && msg.contains('1'),
4029 "Expected out of bounds error for edge case, got: {msg}"
4030 );
4031 }
4032
4033 #[test]
4034 fn test_project_error_duplicate_index() {
4035 let schema_json = r#"{
4036 "type": "record",
4037 "name": "Test",
4038 "fields": [
4039 {"name": "a", "type": "int"},
4040 {"name": "b", "type": "string"},
4041 {"name": "c", "type": "long"}
4042 ]
4043 }"#;
4044 let schema = AvroSchema::new(schema_json.to_string());
4045 let err = schema.project(&[0, 1, 0]).unwrap_err();
4046 let msg = err.to_string();
4047 assert!(
4048 msg.contains("Duplicate projection index") && msg.contains('0'),
4049 "Expected duplicate index error, got: {msg}"
4050 );
4051 }
4052
4053 #[test]
4054 fn test_project_error_duplicate_index_consecutive() {
4055 let schema_json = r#"{
4056 "type": "record",
4057 "name": "Test",
4058 "fields": [
4059 {"name": "a", "type": "int"},
4060 {"name": "b", "type": "string"}
4061 ]
4062 }"#;
4063 let schema = AvroSchema::new(schema_json.to_string());
4064 let err = schema.project(&[1, 1]).unwrap_err();
4065 let msg = err.to_string();
4066 assert!(
4067 msg.contains("Duplicate projection index") && msg.contains('1'),
4068 "Expected duplicate index error for consecutive duplicates, got: {msg}"
4069 );
4070 }
4071
4072 #[test]
4073 fn test_project_with_empty_fields() {
4074 let schema_json = r#"{
4075 "type": "record",
4076 "name": "EmptyRecord",
4077 "fields": []
4078 }"#;
4079 let schema = AvroSchema::new(schema_json.to_string());
4080 let projected = schema.project(&[]).unwrap();
4082 let v: Value = serde_json::from_str(&projected.json_string).unwrap();
4083 let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
4084 assert!(fields.is_empty());
4085 }
4086
4087 #[test]
4088 fn test_project_empty_fields_index_out_of_bounds() {
4089 let schema_json = r#"{
4090 "type": "record",
4091 "name": "EmptyRecord",
4092 "fields": []
4093 }"#;
4094 let schema = AvroSchema::new(schema_json.to_string());
4095 let err = schema.project(&[0]).unwrap_err();
4096 let msg = err.to_string();
4097 assert!(
4098 msg.contains("out of bounds") && msg.contains("0 fields"),
4099 "Expected out of bounds error for empty record, got: {msg}"
4100 );
4101 }
4102
4103 #[test]
4104 fn test_project_result_is_valid_avro_schema() {
4105 let schema_json = r#"{
4106 "type": "record",
4107 "name": "Test",
4108 "namespace": "com.example",
4109 "fields": [
4110 {"name": "id", "type": "long"},
4111 {"name": "name", "type": "string"},
4112 {"name": "active", "type": "boolean"}
4113 ]
4114 }"#;
4115 let schema = AvroSchema::new(schema_json.to_string());
4116 let projected = schema.project(&[0, 2]).unwrap();
4117 let parsed = projected.schema();
4119 assert!(parsed.is_ok(), "Projected schema should be valid Avro");
4120 match parsed.unwrap() {
4121 Schema::Complex(ComplexType::Record(r)) => {
4122 assert_eq!(r.name, "Test");
4123 assert_eq!(r.namespace, Some("com.example"));
4124 assert_eq!(r.fields.len(), 2);
4125 assert_eq!(r.fields[0].name, "id");
4126 assert_eq!(r.fields[1].name, "active");
4127 }
4128 _ => panic!("Expected Record schema"),
4129 }
4130 }
4131
4132 #[test]
4133 fn test_project_non_contiguous_indices() {
4134 let schema_json = r#"{
4135 "type": "record",
4136 "name": "Test",
4137 "fields": [
4138 {"name": "f0", "type": "int"},
4139 {"name": "f1", "type": "int"},
4140 {"name": "f2", "type": "int"},
4141 {"name": "f3", "type": "int"},
4142 {"name": "f4", "type": "int"}
4143 ]
4144 }"#;
4145 let schema = AvroSchema::new(schema_json.to_string());
4146 let projected = schema.project(&[0, 2, 4]).unwrap();
4148 let v: Value = serde_json::from_str(&projected.json_string).unwrap();
4149 let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
4150 assert_eq!(fields.len(), 3);
4151 assert_eq!(fields[0].get("name").and_then(|n| n.as_str()), Some("f0"));
4152 assert_eq!(fields[1].get("name").and_then(|n| n.as_str()), Some("f2"));
4153 assert_eq!(fields[2].get("name").and_then(|n| n.as_str()), Some("f4"));
4154 }
4155
4156 #[test]
4157 fn test_project_single_field_from_many() {
4158 let schema_json = r#"{
4159 "type": "record",
4160 "name": "BigRecord",
4161 "fields": [
4162 {"name": "f0", "type": "int"},
4163 {"name": "f1", "type": "int"},
4164 {"name": "f2", "type": "int"},
4165 {"name": "f3", "type": "int"},
4166 {"name": "f4", "type": "int"},
4167 {"name": "f5", "type": "int"},
4168 {"name": "f6", "type": "int"},
4169 {"name": "f7", "type": "int"},
4170 {"name": "f8", "type": "int"},
4171 {"name": "f9", "type": "int"}
4172 ]
4173 }"#;
4174 let schema = AvroSchema::new(schema_json.to_string());
4175 let projected = schema.project(&[9]).unwrap();
4177 let v: Value = serde_json::from_str(&projected.json_string).unwrap();
4178 let fields = v.get("fields").and_then(|f| f.as_array()).unwrap();
4179 assert_eq!(fields.len(), 1);
4180 assert_eq!(fields[0].get("name").and_then(|n| n.as_str()), Some("f9"));
4181 }
4182}