arrow_schema/
schema.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use std::fmt;
6use std::hash::Hash;
7use std::sync::Arc;
8
9use crate::error::ArrowError;
10use crate::field::Field;
11use crate::{DataType, FieldRef, Fields, Metadata};
12
13/// A builder to facilitate building a [`Schema`] iteratively from [`FieldRef`]
14///
15/// # Examples
16///
17/// Build a schema from scratch:
18///
19/// ```
20/// # use arrow_schema::*;
21/// let schema = {
22///     let mut builder = SchemaBuilder::new();
23///     builder.push(Field::new("id", DataType::Int64, false));
24///     builder.push(Field::new("name", DataType::Utf8, true));
25///     builder.finish()
26/// };
27/// assert_eq!(schema.fields().len(), 2);
28/// ```
29///
30/// Derive a new schema from an existing one, keeping all fields and metadata
31/// while appending new columns:
32///
33/// ```
34/// # use arrow_schema::*;
35/// let base = Schema::new_with_metadata(
36///     vec![
37///         Field::new("id", DataType::Int64, false),
38///         Field::new("name", DataType::Utf8, true),
39///     ],
40///     [("created_by", "myapp")],
41/// );
42///
43/// // Build a new schema that extends `base` with an extra field.
44/// let mut builder = SchemaBuilder::from(&base); // copies all fields *and* metadata.
45/// builder.push(Field::new("score", DataType::Float64, true));
46/// let extended = builder.finish();
47///
48/// assert_eq!(extended.fields().len(), 3);
49/// assert_eq!(extended.field(0).name(), "id");     // original fields preserved
50/// assert_eq!(extended.field(2).name(), "score");  // new field appended
51/// assert_eq!(extended.metadata()["created_by"], "myapp"); // metadata carried over
52/// ```
53#[derive(Debug, Default)]
54pub struct SchemaBuilder {
55    fields: Vec<FieldRef>,
56    metadata: Metadata,
57}
58
59impl SchemaBuilder {
60    /// Creates a new empty [`SchemaBuilder`]
61    pub fn new() -> Self {
62        Self::default()
63    }
64
65    /// Creates a new empty [`SchemaBuilder`] with space for `capacity` fields
66    pub fn with_capacity(capacity: usize) -> Self {
67        Self {
68            fields: Vec::with_capacity(capacity),
69            metadata: Default::default(),
70        }
71    }
72
73    /// Appends a [`FieldRef`] to this [`SchemaBuilder`] without checking for collision
74    pub fn push(&mut self, field: impl Into<FieldRef>) {
75        self.fields.push(field.into())
76    }
77
78    /// Removes and returns the [`FieldRef`] as index `idx`
79    ///
80    /// # Panics
81    ///
82    /// Panics if index out of bounds
83    pub fn remove(&mut self, idx: usize) -> FieldRef {
84        self.fields.remove(idx)
85    }
86
87    /// Returns an immutable reference to the [`FieldRef`] at index `idx`
88    ///
89    /// # Panics
90    ///
91    /// Panics if index out of bounds
92    pub fn field(&mut self, idx: usize) -> &FieldRef {
93        &mut self.fields[idx]
94    }
95
96    /// Returns a mutable reference to the [`FieldRef`] at index `idx`
97    ///
98    /// # Example
99    ///
100    /// ```
101    /// # use std::sync::Arc;
102    /// # use arrow_schema::*;
103    /// let original = Schema::new(vec![
104    ///     Field::new("id", DataType::Int32, false),
105    ///     Field::new("value", DataType::Utf8, true),
106    /// ]);
107    ///
108    /// let mut builder = SchemaBuilder::from(&original);
109    /// // Widen the "id" column from Int32 to Int64
110    /// *builder.field_mut(0) = Arc::new(Field::new("id", DataType::Int64, false));
111    /// let widened = builder.finish();
112    ///
113    /// assert_eq!(widened.field(0).data_type(), &DataType::Int64);
114    /// assert_eq!(widened.field(1).name(), "value"); // unchanged
115    /// ```
116    ///
117    /// # Panics
118    ///
119    /// Panics if index out of bounds
120    pub fn field_mut(&mut self, idx: usize) -> &mut FieldRef {
121        &mut self.fields[idx]
122    }
123
124    /// Returns an immutable reference to the Map of custom metadata key-value pairs.
125    pub fn metadata(&mut self) -> &Metadata {
126        &self.metadata
127    }
128
129    /// Returns a mutable reference to the Map of custom metadata key-value pairs.
130    pub fn metadata_mut(&mut self) -> &mut Metadata {
131        &mut self.metadata
132    }
133
134    /// Reverse the fields
135    pub fn reverse(&mut self) {
136        self.fields.reverse();
137    }
138
139    /// Appends a [`FieldRef`] to this [`SchemaBuilder`] checking for collision
140    ///
141    /// If an existing field exists with the same name, calls [`Field::try_merge`]
142    pub fn try_merge(&mut self, field: &FieldRef) -> Result<(), ArrowError> {
143        // This could potentially be sped up with a HashMap or similar
144        let existing = self.fields.iter_mut().find(|f| f.name() == field.name());
145        match existing {
146            Some(e) if Arc::ptr_eq(e, field) => {} // Nothing to do
147            Some(e) => match Arc::get_mut(e) {
148                Some(e) => e.try_merge(field.as_ref())?,
149                None => {
150                    let mut t = e.as_ref().clone();
151                    t.try_merge(field)?;
152                    *e = Arc::new(t)
153                }
154            },
155            None => self.fields.push(field.clone()),
156        }
157        Ok(())
158    }
159
160    /// Consume this [`SchemaBuilder`] yielding the final [`Schema`]
161    pub fn finish(self) -> Schema {
162        Schema {
163            fields: self.fields.into(),
164            metadata: self.metadata,
165        }
166    }
167
168    /// Consume this [`SchemaBuilder`] yielding a [`Schema`] with fields reordered
169    /// or subsetted according to `indices`.
170    ///
171    /// Fields appear in the output in the order given by `indices`. Metadata is
172    /// carried over from the builder unchanged.
173    ///
174    /// # Errors
175    ///
176    /// Returns an error if any index is out of bounds or if any index is repeated.
177    ///
178    /// # Example: reorder fields
179    ///
180    /// ```
181    /// # use arrow_schema::*;
182    /// let schema = Schema::new(vec![
183    ///     Field::new("id", DataType::Int64, false),
184    ///     Field::new("name", DataType::Utf8, true),
185    ///     Field::new("score", DataType::Float64, true),
186    /// ]);
187    ///
188    /// let reordered = SchemaBuilder::from(&schema).project(&[2, 0, 1]).unwrap();
189    /// assert_eq!(reordered.field(0).name(), "score");
190    /// assert_eq!(reordered.field(1).name(), "id");
191    /// assert_eq!(reordered.field(2).name(), "name");
192    /// ```
193    ///
194    /// # Example: select a subset of fields
195    ///
196    /// ```
197    /// # use arrow_schema::*;
198    /// let schema = Schema::new(vec![
199    ///     Field::new("id", DataType::Int64, false),
200    ///     Field::new("name", DataType::Utf8, true),
201    ///     Field::new("score", DataType::Float64, true),
202    /// ]);
203    ///
204    /// let subset = SchemaBuilder::from(&schema).project(&[0, 2]).unwrap();
205    /// assert_eq!(subset.fields().len(), 2);
206    /// assert_eq!(subset.field(0).name(), "id");
207    /// assert_eq!(subset.field(1).name(), "score");
208    /// ```
209    pub fn project(self, indices: &[usize]) -> Result<Schema, ArrowError> {
210        let num_fields = self.fields.len();
211        let mut seen = std::collections::HashSet::new();
212        for &idx in indices {
213            if idx >= num_fields {
214                return Err(ArrowError::SchemaError(format!(
215                    "project index {idx} out of bounds, schema has {num_fields} fields"
216                )));
217            }
218            if !seen.insert(idx) {
219                return Err(ArrowError::SchemaError(format!(
220                    "project index {idx} is repeated"
221                )));
222            }
223        }
224        let fields: Vec<FieldRef> = indices
225            .iter()
226            .map(|&idx| self.fields[idx].clone())
227            .collect();
228        Ok(Schema {
229            fields: fields.into(),
230            metadata: self.metadata,
231        })
232    }
233}
234
235impl From<&Fields> for SchemaBuilder {
236    fn from(value: &Fields) -> Self {
237        Self {
238            fields: value.to_vec(),
239            metadata: Default::default(),
240        }
241    }
242}
243
244impl From<Fields> for SchemaBuilder {
245    fn from(value: Fields) -> Self {
246        Self {
247            fields: value.to_vec(),
248            metadata: Default::default(),
249        }
250    }
251}
252
253impl From<&Schema> for SchemaBuilder {
254    fn from(value: &Schema) -> Self {
255        Self::from(value.clone())
256    }
257}
258
259impl From<Schema> for SchemaBuilder {
260    fn from(value: Schema) -> Self {
261        Self {
262            fields: value.fields.to_vec(),
263            metadata: value.metadata,
264        }
265    }
266}
267
268impl Extend<FieldRef> for SchemaBuilder {
269    fn extend<T: IntoIterator<Item = FieldRef>>(&mut self, iter: T) {
270        let iter = iter.into_iter();
271        self.fields.reserve(iter.size_hint().0);
272        for f in iter {
273            self.push(f)
274        }
275    }
276}
277
278impl Extend<Field> for SchemaBuilder {
279    fn extend<T: IntoIterator<Item = Field>>(&mut self, iter: T) {
280        let iter = iter.into_iter();
281        self.fields.reserve(iter.size_hint().0);
282        for f in iter {
283            self.push(f)
284        }
285    }
286}
287
288/// A reference-counted reference to a [`Schema`].
289pub type SchemaRef = Arc<Schema>;
290
291/// Describes the meta-data of an ordered sequence of relative types.
292///
293/// Note that this information is only part of the meta-data and not part of the physical
294/// memory layout.
295#[derive(Debug, Clone, PartialEq, Eq, Hash)]
296#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
297pub struct Schema {
298    /// A sequence of fields that describe the schema.
299    pub fields: Fields,
300    /// A map of key-value pairs containing additional metadata.
301    pub metadata: Metadata,
302}
303
304impl Schema {
305    /// Creates an empty `Schema`
306    pub fn empty() -> Self {
307        Self {
308            fields: Default::default(),
309            metadata: Default::default(),
310        }
311    }
312
313    /// Creates a new [`Schema`] from a sequence of [`Field`] values.
314    ///
315    /// # Example
316    ///
317    /// ```
318    /// # use arrow_schema::*;
319    /// let field_a = Field::new("a", DataType::Int64, false);
320    /// let field_b = Field::new("b", DataType::Boolean, false);
321    ///
322    /// let schema = Schema::new(vec![field_a, field_b]);
323    /// ```
324    pub fn new(fields: impl Into<Fields>) -> Self {
325        Self::new_with_metadata(fields, Metadata::new())
326    }
327
328    /// Creates a new [`Schema`] from a sequence of [`Field`] values
329    /// and adds additional metadata in form of key value pairs.
330    ///
331    /// # Example
332    ///
333    /// ```
334    /// # use arrow_schema::*;
335    /// let field_a = Field::new("a", DataType::Int64, false);
336    /// let field_b = Field::new("b", DataType::Boolean, false);
337    ///
338    /// let schema = Schema::new_with_metadata(vec![field_a, field_b], [("row_count", "100")]);
339    /// ```
340    #[inline]
341    pub fn new_with_metadata(fields: impl Into<Fields>, metadata: impl Into<Metadata>) -> Self {
342        Self {
343            fields: fields.into(),
344            metadata: metadata.into(),
345        }
346    }
347
348    /// Sets the metadata of this `Schema` to be `metadata` and returns self
349    pub fn with_metadata(mut self, metadata: impl Into<Metadata>) -> Self {
350        self.metadata = metadata.into();
351        self
352    }
353
354    /// Returns a new schema with only the specified columns in the new schema
355    /// This carries metadata from the parent schema over as well
356    pub fn project(&self, indices: &[usize]) -> Result<Schema, ArrowError> {
357        let new_fields = indices
358            .iter()
359            .map(|i| {
360                self.fields.get(*i).cloned().ok_or_else(|| {
361                    ArrowError::SchemaError(format!(
362                        "project index {} out of bounds, max field {}",
363                        i,
364                        self.fields().len()
365                    ))
366                })
367            })
368            .collect::<Result<Vec<_>, _>>()?;
369        Ok(Self::new_with_metadata(new_fields, self.metadata.clone()))
370    }
371
372    /// Merge schema into self if it is compatible. Struct fields will be merged recursively.
373    ///
374    /// Example:
375    ///
376    /// ```
377    /// # use arrow_schema::*;
378    ///
379    /// let merged = Schema::try_merge(vec![
380    ///     Schema::new(vec![
381    ///         Field::new("c1", DataType::Int64, false),
382    ///         Field::new("c2", DataType::Utf8, false),
383    ///     ]),
384    ///     Schema::new(vec![
385    ///         Field::new("c1", DataType::Int64, true),
386    ///         Field::new("c2", DataType::Utf8, false),
387    ///         Field::new("c3", DataType::Utf8, false),
388    ///     ]),
389    /// ]).unwrap();
390    ///
391    /// assert_eq!(
392    ///     merged,
393    ///     Schema::new(vec![
394    ///         Field::new("c1", DataType::Int64, true),
395    ///         Field::new("c2", DataType::Utf8, false),
396    ///         Field::new("c3", DataType::Utf8, false),
397    ///     ]),
398    /// );
399    /// ```
400    pub fn try_merge(schemas: impl IntoIterator<Item = Self>) -> Result<Self, ArrowError> {
401        let mut out_meta = Metadata::new();
402        let mut out_fields = SchemaBuilder::new();
403        for schema in schemas {
404            let Schema { metadata, fields } = schema;
405
406            // merge metadata
407            for (key, value) in metadata {
408                if let Some(old_val) = out_meta.get(&key)
409                    && old_val != &value
410                {
411                    return Err(ArrowError::SchemaError(format!(
412                        "Fail to merge schema due to conflicting metadata. \
413                                     Key '{key}' has different values '{old_val}' and '{value}'"
414                    )));
415                }
416                out_meta.insert(key, value);
417            }
418
419            // merge fields
420            fields.iter().try_for_each(|x| out_fields.try_merge(x))?
421        }
422
423        Ok(out_fields.finish().with_metadata(out_meta))
424    }
425
426    /// Returns an immutable reference of the vector of `Field` instances.
427    #[inline]
428    pub const fn fields(&self) -> &Fields {
429        &self.fields
430    }
431
432    /// Returns a vector with references to all fields (including nested fields)
433    ///
434    /// # Example
435    ///
436    /// ```
437    /// use std::sync::Arc;
438    /// use arrow_schema::{DataType, Field, Fields, Schema};
439    ///
440    /// let f1 = Arc::new(Field::new("a", DataType::Boolean, false));
441    ///
442    /// let f2_inner = Arc::new(Field::new("b_inner", DataType::Int8, false));
443    /// let f2 = Arc::new(Field::new("b", DataType::List(f2_inner.clone()), false));
444    ///
445    /// let f3_inner1 = Arc::new(Field::new("c_inner1", DataType::Int8, false));
446    /// let f3_inner2 = Arc::new(Field::new("c_inner2", DataType::Int8, false));
447    /// let f3 = Arc::new(Field::new(
448    ///     "c",
449    ///     DataType::Struct(vec![f3_inner1.clone(), f3_inner2.clone()].into()),
450    ///     false
451    /// ));
452    ///
453    /// let mut schema = Schema::new(vec![
454    ///   f1.clone(), f2.clone(), f3.clone()
455    /// ]);
456    /// assert_eq!(
457    ///     schema.flattened_fields(),
458    ///     vec![
459    ///         f1.as_ref(),
460    ///         f2.as_ref(),
461    ///         f2_inner.as_ref(),
462    ///         f3.as_ref(),
463    ///         f3_inner1.as_ref(),
464    ///         f3_inner2.as_ref()
465    ///    ]
466    /// );
467    /// ```
468    #[inline]
469    pub fn flattened_fields(&self) -> Vec<&Field> {
470        self.fields.iter().flat_map(|f| f.fields()).collect()
471    }
472
473    /// Returns an immutable reference of a specific [`Field`] instance selected using an
474    /// offset within the internal `fields` vector.
475    ///
476    /// # Panics
477    ///
478    /// Panics if index out of bounds
479    pub fn field(&self, i: usize) -> &Field {
480        &self.fields[i]
481    }
482
483    /// Returns an immutable reference of a specific [`Field`] instance selected by name.
484    pub fn field_with_name(&self, name: &str) -> Result<&Field, ArrowError> {
485        Ok(&self.fields[self.index_of(name)?])
486    }
487
488    /// Returns a vector of immutable references to all [`Field`] instances selected by
489    /// the dictionary ID they use.
490    #[deprecated(
491        since = "54.0.0",
492        note = "The ability to preserve dictionary IDs will be removed. With it, all functions related to it."
493    )]
494    pub fn fields_with_dict_id(&self, dict_id: i64) -> Vec<&Field> {
495        #[expect(deprecated)]
496        self.fields
497            .iter()
498            .flat_map(|f| f.fields_with_dict_id(dict_id))
499            .collect()
500    }
501
502    /// Find the index of the column with the given name.
503    pub fn index_of(&self, name: &str) -> Result<usize, ArrowError> {
504        let (idx, _) = self.fields().find(name).ok_or_else(|| {
505            let valid_fields: Vec<_> = self.fields.iter().map(|f| f.name()).collect();
506            ArrowError::SchemaError(format!(
507                "Unable to get field named \"{name}\". Valid fields: {valid_fields:?}"
508            ))
509        })?;
510        Ok(idx)
511    }
512
513    /// Returns an immutable reference to the Map of custom metadata key-value pairs.
514    #[inline]
515    pub const fn metadata(&self) -> &Metadata {
516        &self.metadata
517    }
518
519    /// Normalize a [`Schema`] into a flat table.
520    ///
521    /// Nested [`Field`]s will generate names separated by `separator`, up to a depth of `max_level`
522    /// (unlimited if `None`).
523    ///
524    /// e.g. given a [`Schema`]:
525    ///
526    /// ```text
527    ///     "foo": StructArray<"bar": Utf8>
528    /// ```
529    ///
530    /// A separator of `"."` would generate a batch with the schema:
531    ///
532    /// ```text
533    ///     "foo.bar": Utf8
534    /// ```
535    ///
536    /// Note that giving a depth of `Some(0)` to `max_level` is the same as passing in `None`;
537    /// it will be treated as unlimited.
538    ///
539    /// # Example
540    ///
541    /// ```
542    /// # use std::sync::Arc;
543    /// # use arrow_schema::{DataType, Field, Fields, Schema};
544    /// let schema = Schema::new(vec![
545    ///     Field::new(
546    ///         "a",
547    ///         DataType::Struct(Fields::from(vec![
548    ///             Arc::new(Field::new("animals", DataType::Utf8, true)),
549    ///             Arc::new(Field::new("n_legs", DataType::Int64, true)),
550    ///         ])),
551    ///         false,
552    ///     ),
553    /// ])
554    /// .normalize(".", None)
555    /// .expect("valid normalization");
556    /// let expected = Schema::new(vec![
557    ///     Field::new("a.animals", DataType::Utf8, true),
558    ///     Field::new("a.n_legs", DataType::Int64, true),
559    /// ]);
560    /// assert_eq!(schema, expected);
561    /// ```
562    pub fn normalize(&self, separator: &str, max_level: Option<usize>) -> Result<Self, ArrowError> {
563        let max_level = match max_level.unwrap_or(usize::MAX) {
564            0 => usize::MAX,
565            val => val,
566        };
567        let mut stack: Vec<(usize, Vec<&str>, &FieldRef)> = self
568            .fields()
569            .iter()
570            .rev()
571            .map(|f| {
572                let name_vec: Vec<&str> = vec![f.name()];
573                (0, name_vec, f)
574            })
575            .collect();
576        let mut fields: Vec<FieldRef> = Vec::new();
577
578        while let Some((depth, name, field_ref)) = stack.pop() {
579            match field_ref.data_type() {
580                DataType::Struct(ff) if depth < max_level => {
581                    // Need to zip these in reverse to maintain original order
582                    for fff in ff.into_iter().rev() {
583                        let mut name = name.clone();
584                        name.push(separator);
585                        name.push(fff.name());
586                        stack.push((depth + 1, name, fff))
587                    }
588                }
589                _ => {
590                    let updated_field = Field::new(
591                        name.concat(),
592                        field_ref.data_type().clone(),
593                        field_ref.is_nullable(),
594                    );
595                    fields.push(Arc::new(updated_field));
596                }
597            }
598        }
599        Ok(Schema::new(fields))
600    }
601
602    /// Look up a column by name and return a immutable reference to the column along with
603    /// its index.
604    pub fn column_with_name(&self, name: &str) -> Option<(usize, &Field)> {
605        let (idx, field) = self.fields.find(name)?;
606        Some((idx, field.as_ref()))
607    }
608
609    /// Check to see if `self` is a superset of `other` schema.
610    ///
611    /// In particular returns true if `self.metadata` is a superset of `other.metadata`
612    /// and [`Fields::contains`] for `self.fields` and `other.fields`
613    ///
614    /// In other words, any record that conforms to `other` should also conform to `self`.
615    pub fn contains(&self, other: &Schema) -> bool {
616        // make sure self.metadata is a superset of other.metadata
617        self.fields.contains(&other.fields)
618            && other
619                .metadata
620                .iter()
621                .all(|(k, v1)| self.metadata.get(k).is_some_and(|v2| v1 == v2))
622    }
623}
624
625impl fmt::Display for Schema {
626    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
627        f.write_str(
628            &self
629                .fields
630                .iter()
631                .map(|c| c.to_string())
632                .collect::<Vec<String>>()
633                .join(", "),
634        )
635    }
636}
637
638impl AsRef<Schema> for Schema {
639    fn as_ref(&self) -> &Schema {
640        self
641    }
642}
643
644#[cfg(test)]
645mod tests {
646    use crate::datatype::DataType;
647    use crate::{TimeUnit, UnionMode};
648    use std::collections::HashMap;
649
650    use super::*;
651
652    #[test]
653    #[expect(clippy::needless_borrows_for_generic_args)] // intentional to exercise various references
654    fn test_schema_as_ref() {
655        fn accept_ref(_: impl AsRef<Schema>) {}
656
657        let schema = Schema::new(vec![
658            Field::new("name", DataType::Utf8, false),
659            Field::new("address", DataType::Utf8, false),
660            Field::new("priority", DataType::UInt8, false),
661        ]);
662
663        accept_ref(schema.clone());
664        accept_ref(&schema.clone());
665        accept_ref(&&schema.clone());
666        accept_ref(Arc::new(schema.clone()));
667        accept_ref(&Arc::new(schema.clone()));
668        accept_ref(&&Arc::new(schema.clone()));
669    }
670
671    #[test]
672    #[cfg(feature = "serde")]
673    fn test_ser_de_metadata() {
674        // ser/de with empty metadata
675        let schema = Schema::new(vec![
676            Field::new("name", DataType::Utf8, false),
677            Field::new("address", DataType::Utf8, false),
678            Field::new("priority", DataType::UInt8, false),
679        ]);
680
681        let json = serde_json::to_string(&schema).unwrap();
682        let de_schema = serde_json::from_str(&json).unwrap();
683
684        assert_eq!(schema, de_schema);
685
686        // ser/de with non-empty metadata
687        let schema = schema.with_metadata([("key", "val")]);
688        let json = serde_json::to_string(&schema).unwrap();
689        let de_schema = serde_json::from_str(&json).unwrap();
690
691        assert_eq!(schema, de_schema);
692    }
693
694    #[test]
695    fn test_projection() {
696        let mut metadata = HashMap::new();
697        metadata.insert("meta".to_string(), "data".to_string());
698
699        let schema = Schema::new(vec![
700            Field::new("name", DataType::Utf8, false),
701            Field::new("address", DataType::Utf8, false),
702            Field::new("priority", DataType::UInt8, false),
703        ])
704        .with_metadata(metadata);
705
706        let projected: Schema = schema.project(&[0, 2]).unwrap();
707
708        assert_eq!(projected.fields().len(), 2);
709        assert_eq!(projected.fields()[0].name(), "name");
710        assert_eq!(projected.fields()[1].name(), "priority");
711        assert_eq!(projected.metadata.get("meta").unwrap(), "data")
712    }
713
714    #[test]
715    fn test_oob_projection() {
716        let mut metadata = HashMap::new();
717        metadata.insert("meta".to_string(), "data".to_string());
718
719        let schema = Schema::new(vec![
720            Field::new("name", DataType::Utf8, false),
721            Field::new("address", DataType::Utf8, false),
722            Field::new("priority", DataType::UInt8, false),
723        ])
724        .with_metadata(metadata);
725
726        let projected = schema.project(&[0, 3]);
727
728        assert!(projected.is_err());
729        if let Err(e) = projected {
730            assert_eq!(
731                e.to_string(),
732                "Schema error: project index 3 out of bounds, max field 3".to_string()
733            )
734        }
735    }
736
737    #[test]
738    fn test_schema_contains() {
739        let mut metadata1 = HashMap::new();
740        metadata1.insert("meta".to_string(), "data".to_string());
741
742        let schema1 = Schema::new(vec![
743            Field::new("name", DataType::Utf8, false),
744            Field::new("address", DataType::Utf8, false),
745            Field::new("priority", DataType::UInt8, false),
746        ])
747        .with_metadata(metadata1.clone());
748
749        let mut metadata2 = HashMap::new();
750        metadata2.insert("meta".to_string(), "data".to_string());
751        metadata2.insert("meta2".to_string(), "data".to_string());
752        let schema2 = Schema::new(vec![
753            Field::new("name", DataType::Utf8, false),
754            Field::new("address", DataType::Utf8, false),
755            Field::new("priority", DataType::UInt8, false),
756        ])
757        .with_metadata(metadata2);
758
759        // reflexivity
760        assert!(schema1.contains(&schema1));
761        assert!(schema2.contains(&schema2));
762
763        assert!(!schema1.contains(&schema2));
764        assert!(schema2.contains(&schema1));
765    }
766
767    #[test]
768    fn schema_equality() {
769        let schema1 = Schema::new(vec![
770            Field::new("c1", DataType::Utf8, false),
771            Field::new("c2", DataType::Float64, true),
772            Field::new("c3", DataType::LargeBinary, true),
773        ]);
774        let schema2 = Schema::new(vec![
775            Field::new("c1", DataType::Utf8, false),
776            Field::new("c2", DataType::Float64, true),
777            Field::new("c3", DataType::LargeBinary, true),
778        ]);
779
780        assert_eq!(schema1, schema2);
781
782        let schema3 = Schema::new(vec![
783            Field::new("c1", DataType::Utf8, false),
784            Field::new("c2", DataType::Float32, true),
785        ]);
786        let schema4 = Schema::new(vec![
787            Field::new("C1", DataType::Utf8, false),
788            Field::new("C2", DataType::Float64, true),
789        ]);
790
791        assert_ne!(schema1, schema3);
792        assert_ne!(schema1, schema4);
793        assert_ne!(schema2, schema3);
794        assert_ne!(schema2, schema4);
795        assert_ne!(schema3, schema4);
796
797        let f = Field::new("c1", DataType::Utf8, false).with_metadata([("foo", "bar")]);
798        let schema5 = Schema::new(vec![
799            f,
800            Field::new("c2", DataType::Float64, true),
801            Field::new("c3", DataType::LargeBinary, true),
802        ]);
803        assert_ne!(schema1, schema5);
804    }
805
806    #[test]
807    fn create_schema_string() {
808        let schema = person_schema();
809        assert_eq!(
810            schema.to_string(),
811            "Field { \"first_name\": Utf8, metadata: {\"k\": \"v\"} }, \
812             Field { \"last_name\": Utf8 }, \
813             Field { \"address\": Struct(\"street\": non-null Utf8, \"zip\": non-null UInt16) }, \
814             Field { \"interests\": nullable Dictionary(Int32, Utf8), dict_id: 123, dict_is_ordered }"
815        )
816    }
817
818    #[test]
819    fn schema_field_accessors() {
820        let schema = person_schema();
821
822        // test schema accessors
823        assert_eq!(schema.fields().len(), 4);
824
825        // test field accessors
826        let first_name = &schema.fields()[0];
827        assert_eq!(first_name.name(), "first_name");
828        assert_eq!(first_name.data_type(), &DataType::Utf8);
829        assert!(!first_name.is_nullable());
830        #[expect(deprecated)]
831        let dict_id = first_name.dict_id();
832        assert_eq!(dict_id, None);
833        assert_eq!(first_name.dict_is_ordered(), None);
834
835        let metadata = first_name.metadata();
836        assert!(!metadata.is_empty());
837        let md = &metadata;
838        assert_eq!(md.len(), 1);
839        let key = md.get("k");
840        assert!(key.is_some());
841        assert_eq!(key.unwrap(), "v");
842
843        let interests = &schema.fields()[3];
844        assert_eq!(interests.name(), "interests");
845        assert_eq!(
846            interests.data_type(),
847            &DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8))
848        );
849        #[expect(deprecated)]
850        let dict_id = interests.dict_id();
851        assert_eq!(dict_id, Some(123));
852        assert_eq!(interests.dict_is_ordered(), Some(true));
853    }
854
855    #[test]
856    #[should_panic(
857        expected = "Unable to get field named \\\"nickname\\\". Valid fields: [\\\"first_name\\\", \\\"last_name\\\", \\\"address\\\", \\\"interests\\\"]"
858    )]
859    fn schema_index_of() {
860        let schema = person_schema();
861        assert_eq!(schema.index_of("first_name").unwrap(), 0);
862        assert_eq!(schema.index_of("last_name").unwrap(), 1);
863        schema.index_of("nickname").unwrap();
864    }
865
866    #[test]
867    fn normalize_simple() {
868        let schema = Schema::new(vec![
869            Field::new(
870                "a",
871                DataType::Struct(Fields::from(vec![
872                    Arc::new(Field::new("animals", DataType::Utf8, true)),
873                    Arc::new(Field::new("n_legs", DataType::Int64, true)),
874                    Arc::new(Field::new("year", DataType::Int64, true)),
875                ])),
876                false,
877            ),
878            Field::new("month", DataType::Int64, true),
879        ])
880        .normalize(".", Some(0))
881        .expect("valid normalization");
882
883        let expected = Schema::new(vec![
884            Field::new("a.animals", DataType::Utf8, true),
885            Field::new("a.n_legs", DataType::Int64, true),
886            Field::new("a.year", DataType::Int64, true),
887            Field::new("month", DataType::Int64, true),
888        ]);
889
890        assert_eq!(schema, expected);
891
892        // Check that 0, None have the same result
893        let schema = Schema::new(vec![
894            Field::new(
895                "a",
896                DataType::Struct(Fields::from(vec![
897                    Arc::new(Field::new("animals", DataType::Utf8, true)),
898                    Arc::new(Field::new("n_legs", DataType::Int64, true)),
899                    Arc::new(Field::new("year", DataType::Int64, true)),
900                ])),
901                false,
902            ),
903            Field::new("month", DataType::Int64, true),
904        ])
905        .normalize(".", None)
906        .expect("valid normalization");
907
908        assert_eq!(schema, expected);
909    }
910
911    #[test]
912    fn normalize_nested() {
913        let a = Arc::new(Field::new("a", DataType::Utf8, true));
914        let b = Arc::new(Field::new("b", DataType::Int64, false));
915        let c = Arc::new(Field::new("c", DataType::Int64, true));
916
917        let d = Arc::new(Field::new("d", DataType::Utf8, true));
918        let e = Arc::new(Field::new("e", DataType::Int64, false));
919        let f = Arc::new(Field::new("f", DataType::Int64, true));
920
921        let one = Arc::new(Field::new(
922            "1",
923            DataType::Struct(Fields::from(vec![a.clone(), b.clone(), c.clone()])),
924            false,
925        ));
926        let two = Arc::new(Field::new(
927            "2",
928            DataType::Struct(Fields::from(vec![d.clone(), e.clone(), f.clone()])),
929            true,
930        ));
931
932        let exclamation = Arc::new(Field::new(
933            "!",
934            DataType::Struct(Fields::from(vec![one, two])),
935            false,
936        ));
937
938        let normalize_all = Schema::new(vec![exclamation.clone()])
939            .normalize(".", Some(0))
940            .expect("valid normalization");
941
942        let expected = Schema::new(vec![
943            Field::new("!.1.a", DataType::Utf8, true),
944            Field::new("!.1.b", DataType::Int64, false),
945            Field::new("!.1.c", DataType::Int64, true),
946            Field::new("!.2.d", DataType::Utf8, true),
947            Field::new("!.2.e", DataType::Int64, false),
948            Field::new("!.2.f", DataType::Int64, true),
949        ]);
950
951        assert_eq!(normalize_all, expected);
952
953        let normalize_depth_one = Schema::new(vec![exclamation])
954            .normalize(".", Some(1))
955            .expect("valid normalization");
956
957        let expected = Schema::new(vec![
958            Field::new("!.1", DataType::Struct(Fields::from(vec![a, b, c])), false),
959            Field::new("!.2", DataType::Struct(Fields::from(vec![d, e, f])), true),
960        ]);
961
962        assert_eq!(normalize_depth_one, expected);
963    }
964
965    #[test]
966    fn normalize_list() {
967        // Only the Struct type field should be unwrapped
968        let a = Arc::new(Field::new("a", DataType::Utf8, true));
969        let b = Arc::new(Field::new("b", DataType::Int64, false));
970        let c = Arc::new(Field::new("c", DataType::Int64, true));
971        let d = Arc::new(Field::new("d", DataType::Utf8, true));
972        let e = Arc::new(Field::new("e", DataType::Int64, false));
973        let f = Arc::new(Field::new("f", DataType::Int64, true));
974
975        let one = Arc::new(Field::new(
976            "1",
977            DataType::Struct(Fields::from(vec![a.clone(), b.clone(), c.clone()])),
978            true,
979        ));
980
981        let two = Arc::new(Field::new(
982            "2",
983            DataType::List(Arc::new(Field::new_list_field(
984                DataType::Struct(Fields::from(vec![d.clone(), e.clone(), f.clone()])),
985                true,
986            ))),
987            false,
988        ));
989
990        let exclamation = Arc::new(Field::new(
991            "!",
992            DataType::Struct(Fields::from(vec![one.clone(), two.clone()])),
993            false,
994        ));
995
996        let normalize_all = Schema::new(vec![exclamation.clone()])
997            .normalize(".", None)
998            .expect("valid normalization");
999
1000        // List shouldn't be affected
1001        let expected = Schema::new(vec![
1002            Field::new("!.1.a", DataType::Utf8, true),
1003            Field::new("!.1.b", DataType::Int64, false),
1004            Field::new("!.1.c", DataType::Int64, true),
1005            Field::new(
1006                "!.2",
1007                DataType::List(Arc::new(Field::new_list_field(
1008                    DataType::Struct(Fields::from(vec![d.clone(), e.clone(), f.clone()])),
1009                    true,
1010                ))),
1011                false,
1012            ),
1013        ]);
1014
1015        assert_eq!(normalize_all, expected);
1016        assert_eq!(normalize_all.fields().len(), 4);
1017
1018        // FixedSizeList
1019        let two = Arc::new(Field::new(
1020            "2",
1021            DataType::FixedSizeList(
1022                Arc::new(Field::new_fixed_size_list(
1023                    "3",
1024                    Arc::new(Field::new_list_field(
1025                        DataType::Struct(Fields::from(vec![d.clone(), e.clone(), f.clone()])),
1026                        true,
1027                    )),
1028                    1,
1029                    true,
1030                )),
1031                1,
1032            ),
1033            false,
1034        ));
1035
1036        let exclamation = Arc::new(Field::new(
1037            "!",
1038            DataType::Struct(Fields::from(vec![one.clone(), two])),
1039            false,
1040        ));
1041
1042        let normalize_all = Schema::new(vec![exclamation.clone()])
1043            .normalize(".", None)
1044            .expect("valid normalization");
1045
1046        // FixedSizeList shouldn't be affected
1047        let expected = Schema::new(vec![
1048            Field::new("!.1.a", DataType::Utf8, true),
1049            Field::new("!.1.b", DataType::Int64, false),
1050            Field::new("!.1.c", DataType::Int64, true),
1051            Field::new(
1052                "!.2",
1053                DataType::FixedSizeList(
1054                    Arc::new(Field::new_fixed_size_list(
1055                        "3",
1056                        Arc::new(Field::new_list_field(
1057                            DataType::Struct(Fields::from(vec![d.clone(), e.clone(), f.clone()])),
1058                            true,
1059                        )),
1060                        1,
1061                        true,
1062                    )),
1063                    1,
1064                ),
1065                false,
1066            ),
1067        ]);
1068
1069        assert_eq!(normalize_all, expected);
1070        assert_eq!(normalize_all.fields().len(), 4);
1071
1072        // LargeList
1073        let two = Arc::new(Field::new(
1074            "2",
1075            DataType::FixedSizeList(
1076                Arc::new(Field::new_large_list(
1077                    "3",
1078                    Arc::new(Field::new_list_field(
1079                        DataType::Struct(Fields::from(vec![d.clone(), e.clone(), f.clone()])),
1080                        true,
1081                    )),
1082                    true,
1083                )),
1084                1,
1085            ),
1086            false,
1087        ));
1088
1089        let exclamation = Arc::new(Field::new(
1090            "!",
1091            DataType::Struct(Fields::from(vec![one.clone(), two])),
1092            false,
1093        ));
1094
1095        let normalize_all = Schema::new(vec![exclamation.clone()])
1096            .normalize(".", None)
1097            .expect("valid normalization");
1098
1099        // LargeList shouldn't be affected
1100        let expected = Schema::new(vec![
1101            Field::new("!.1.a", DataType::Utf8, true),
1102            Field::new("!.1.b", DataType::Int64, false),
1103            Field::new("!.1.c", DataType::Int64, true),
1104            Field::new(
1105                "!.2",
1106                DataType::FixedSizeList(
1107                    Arc::new(Field::new_large_list(
1108                        "3",
1109                        Arc::new(Field::new_list_field(
1110                            DataType::Struct(Fields::from(vec![d.clone(), e.clone(), f.clone()])),
1111                            true,
1112                        )),
1113                        true,
1114                    )),
1115                    1,
1116                ),
1117                false,
1118            ),
1119        ]);
1120
1121        assert_eq!(normalize_all, expected);
1122        assert_eq!(normalize_all.fields().len(), 4);
1123    }
1124
1125    #[test]
1126    fn normalize_deep_nested() {
1127        // No unwrapping expected
1128        let a = Arc::new(Field::new("a", DataType::Utf8, true));
1129        let b = Arc::new(Field::new("b", DataType::Int64, false));
1130        let c = Arc::new(Field::new("c", DataType::Int64, true));
1131        let d = Arc::new(Field::new("d", DataType::Utf8, true));
1132        let e = Arc::new(Field::new("e", DataType::Int64, false));
1133        let f = Arc::new(Field::new("f", DataType::Int64, true));
1134
1135        let one = Arc::new(Field::new(
1136            "1",
1137            DataType::Struct(Fields::from(vec![a.clone(), b.clone(), c.clone()])),
1138            true,
1139        ));
1140
1141        let two = Arc::new(Field::new(
1142            "2",
1143            DataType::List(Arc::new(Field::new_list_field(
1144                DataType::Struct(Fields::from(vec![d.clone(), e.clone(), f.clone()])),
1145                true,
1146            ))),
1147            false,
1148        ));
1149
1150        let l10 = Arc::new(Field::new(
1151            "l10",
1152            DataType::List(Arc::new(Field::new_list_field(
1153                DataType::Struct(Fields::from(vec![one, two])),
1154                true,
1155            ))),
1156            false,
1157        ));
1158
1159        let l9 = Arc::new(Field::new(
1160            "l9",
1161            DataType::List(Arc::new(Field::new_list_field(
1162                DataType::Struct(Fields::from(vec![l10])),
1163                true,
1164            ))),
1165            false,
1166        ));
1167
1168        let l8 = Arc::new(Field::new(
1169            "l8",
1170            DataType::List(Arc::new(Field::new_list_field(
1171                DataType::Struct(Fields::from(vec![l9])),
1172                true,
1173            ))),
1174            false,
1175        ));
1176        let l7 = Arc::new(Field::new(
1177            "l7",
1178            DataType::List(Arc::new(Field::new_list_field(
1179                DataType::Struct(Fields::from(vec![l8])),
1180                true,
1181            ))),
1182            false,
1183        ));
1184        let l6 = Arc::new(Field::new(
1185            "l6",
1186            DataType::List(Arc::new(Field::new_list_field(
1187                DataType::Struct(Fields::from(vec![l7])),
1188                true,
1189            ))),
1190            false,
1191        ));
1192        let l5 = Arc::new(Field::new(
1193            "l5",
1194            DataType::List(Arc::new(Field::new_list_field(
1195                DataType::Struct(Fields::from(vec![l6])),
1196                true,
1197            ))),
1198            false,
1199        ));
1200        let l4 = Arc::new(Field::new(
1201            "l4",
1202            DataType::List(Arc::new(Field::new_list_field(
1203                DataType::Struct(Fields::from(vec![l5])),
1204                true,
1205            ))),
1206            false,
1207        ));
1208        let l3 = Arc::new(Field::new(
1209            "l3",
1210            DataType::List(Arc::new(Field::new_list_field(
1211                DataType::Struct(Fields::from(vec![l4])),
1212                true,
1213            ))),
1214            false,
1215        ));
1216        let l2 = Arc::new(Field::new(
1217            "l2",
1218            DataType::List(Arc::new(Field::new_list_field(
1219                DataType::Struct(Fields::from(vec![l3])),
1220                true,
1221            ))),
1222            false,
1223        ));
1224        let l1 = Arc::new(Field::new(
1225            "l1",
1226            DataType::List(Arc::new(Field::new_list_field(
1227                DataType::Struct(Fields::from(vec![l2])),
1228                true,
1229            ))),
1230            false,
1231        ));
1232
1233        let normalize_all = Schema::new(vec![l1])
1234            .normalize(".", None)
1235            .expect("valid normalization");
1236
1237        assert_eq!(normalize_all.fields().len(), 1);
1238    }
1239
1240    #[test]
1241    fn normalize_dictionary() {
1242        let a = Arc::new(Field::new("a", DataType::Utf8, true));
1243        let b = Arc::new(Field::new("b", DataType::Int64, false));
1244
1245        let one = Arc::new(Field::new(
1246            "1",
1247            DataType::Dictionary(
1248                Box::new(DataType::Int32),
1249                Box::new(DataType::Struct(Fields::from(vec![a.clone(), b.clone()]))),
1250            ),
1251            false,
1252        ));
1253
1254        let normalize_all = Schema::new(vec![one.clone()])
1255            .normalize(".", None)
1256            .expect("valid normalization");
1257
1258        let expected = Schema::new(vec![Field::new(
1259            "1",
1260            DataType::Dictionary(
1261                Box::new(DataType::Int32),
1262                Box::new(DataType::Struct(Fields::from(vec![a.clone(), b.clone()]))),
1263            ),
1264            false,
1265        )]);
1266
1267        assert_eq!(normalize_all, expected);
1268    }
1269
1270    #[test]
1271    #[should_panic(
1272        expected = "Unable to get field named \\\"nickname\\\". Valid fields: [\\\"first_name\\\", \\\"last_name\\\", \\\"address\\\", \\\"interests\\\"]"
1273    )]
1274    fn schema_field_with_name() {
1275        let schema = person_schema();
1276        assert_eq!(
1277            schema.field_with_name("first_name").unwrap().name(),
1278            "first_name"
1279        );
1280        assert_eq!(
1281            schema.field_with_name("last_name").unwrap().name(),
1282            "last_name"
1283        );
1284        schema.field_with_name("nickname").unwrap();
1285    }
1286
1287    #[test]
1288    fn schema_field_with_dict_id() {
1289        let schema = person_schema();
1290
1291        #[expect(deprecated)]
1292        let fields_dict_123: Vec<_> = schema
1293            .fields_with_dict_id(123)
1294            .iter()
1295            .map(|f| f.name())
1296            .collect();
1297        assert_eq!(fields_dict_123, vec!["interests"]);
1298
1299        #[expect(deprecated)]
1300        let is_empty = schema.fields_with_dict_id(456).is_empty();
1301        assert!(is_empty);
1302    }
1303
1304    fn person_schema() -> Schema {
1305        let kv_array = [("k".to_string(), "v".to_string())];
1306        let field_metadata: HashMap<String, String> = kv_array.iter().cloned().collect();
1307        let first_name =
1308            Field::new("first_name", DataType::Utf8, false).with_metadata(field_metadata);
1309
1310        Schema::new(vec![
1311            first_name,
1312            Field::new("last_name", DataType::Utf8, false),
1313            Field::new(
1314                "address",
1315                DataType::Struct(Fields::from(vec![
1316                    Field::new("street", DataType::Utf8, false),
1317                    Field::new("zip", DataType::UInt16, false),
1318                ])),
1319                false,
1320            ),
1321            #[expect(deprecated)]
1322            Field::new_dict(
1323                "interests",
1324                DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
1325                true,
1326                123,
1327                true,
1328            ),
1329        ])
1330    }
1331
1332    #[test]
1333    fn test_try_merge_field_with_metadata() {
1334        // 1. Different values for the same key should cause error.
1335        let metadata1 = HashMap::from([("foo".to_string(), "bar".to_string())]);
1336        let f1 = Field::new("first_name", DataType::Utf8, false).with_metadata(metadata1);
1337
1338        let metadata2 = HashMap::from([("foo".to_string(), "baz".to_string())]);
1339        let f2 = Field::new("first_name", DataType::Utf8, false).with_metadata(metadata2);
1340
1341        assert!(Schema::try_merge(vec![Schema::new(vec![f1]), Schema::new(vec![f2])]).is_err());
1342
1343        // 2. None + Some
1344        let mut f1 = Field::new("first_name", DataType::Utf8, false);
1345        let metadata2 = HashMap::from([("missing".to_string(), "value".to_string())]);
1346        let f2 = Field::new("first_name", DataType::Utf8, false).with_metadata(metadata2);
1347
1348        assert!(f1.try_merge(&f2).is_ok());
1349        assert!(!f1.metadata().is_empty());
1350        assert_eq!(f1.metadata(), f2.metadata());
1351
1352        // 3. Some + Some
1353        let mut f1 =
1354            Field::new("first_name", DataType::Utf8, false).with_metadata([("foo", "bar")]);
1355        let f2 = Field::new("first_name", DataType::Utf8, false).with_metadata([("foo2", "bar2")]);
1356
1357        assert!(f1.try_merge(&f2).is_ok());
1358        assert!(!f1.metadata().is_empty());
1359        assert_eq!(
1360            f1.metadata(),
1361            &Metadata::from([("foo", "bar"), ("foo2", "bar2")])
1362        );
1363
1364        // 4. Some + None.
1365        let mut f1 =
1366            Field::new("first_name", DataType::Utf8, false).with_metadata([("foo", "bar")]);
1367        let f2 = Field::new("first_name", DataType::Utf8, false);
1368        assert!(f1.try_merge(&f2).is_ok());
1369        assert!(!f1.metadata().is_empty());
1370        assert_eq!(f1.metadata(), &Metadata::from([("foo", "bar")]));
1371
1372        // 5. None + None.
1373        let mut f1 = Field::new("first_name", DataType::Utf8, false);
1374        let f2 = Field::new("first_name", DataType::Utf8, false);
1375        assert!(f1.try_merge(&f2).is_ok());
1376        assert!(f1.metadata().is_empty());
1377    }
1378
1379    #[test]
1380    fn test_schema_merge() {
1381        let merged = Schema::try_merge(vec![
1382            Schema::new(vec![
1383                Field::new("first_name", DataType::Utf8, false),
1384                Field::new("last_name", DataType::Utf8, false),
1385                Field::new(
1386                    "address",
1387                    DataType::Struct(vec![Field::new("zip", DataType::UInt16, false)].into()),
1388                    false,
1389                ),
1390            ]),
1391            Schema::new_with_metadata(
1392                vec![
1393                    // nullable merge
1394                    Field::new("last_name", DataType::Utf8, true),
1395                    Field::new(
1396                        "address",
1397                        DataType::Struct(Fields::from(vec![
1398                            // add new nested field
1399                            Field::new("street", DataType::Utf8, false),
1400                            // nullable merge on nested field
1401                            Field::new("zip", DataType::UInt16, true),
1402                        ])),
1403                        false,
1404                    ),
1405                    // new field
1406                    Field::new("number", DataType::Utf8, true),
1407                ],
1408                HashMap::from([("foo".to_string(), "bar".to_string())]),
1409            ),
1410        ])
1411        .unwrap();
1412
1413        assert_eq!(
1414            merged,
1415            Schema::new_with_metadata(
1416                vec![
1417                    Field::new("first_name", DataType::Utf8, false),
1418                    Field::new("last_name", DataType::Utf8, true),
1419                    Field::new(
1420                        "address",
1421                        DataType::Struct(Fields::from(vec![
1422                            Field::new("zip", DataType::UInt16, true),
1423                            Field::new("street", DataType::Utf8, false),
1424                        ])),
1425                        false,
1426                    ),
1427                    Field::new("number", DataType::Utf8, true),
1428                ],
1429                HashMap::from([("foo".to_string(), "bar".to_string())])
1430            )
1431        );
1432
1433        // support merge union fields
1434        assert_eq!(
1435            Schema::try_merge(vec![
1436                Schema::new(vec![Field::new_union(
1437                    "c1",
1438                    vec![0, 1],
1439                    vec![
1440                        Field::new("c11", DataType::Utf8, true),
1441                        Field::new("c12", DataType::Utf8, true),
1442                    ],
1443                    UnionMode::Dense
1444                ),]),
1445                Schema::new(vec![Field::new_union(
1446                    "c1",
1447                    vec![1, 2],
1448                    vec![
1449                        Field::new("c12", DataType::Utf8, true),
1450                        Field::new("c13", DataType::Time64(TimeUnit::Second), true),
1451                    ],
1452                    UnionMode::Dense
1453                ),])
1454            ])
1455            .unwrap(),
1456            Schema::new(vec![Field::new_union(
1457                "c1",
1458                vec![0, 1, 2],
1459                vec![
1460                    Field::new("c11", DataType::Utf8, true),
1461                    Field::new("c12", DataType::Utf8, true),
1462                    Field::new("c13", DataType::Time64(TimeUnit::Second), true),
1463                ],
1464                UnionMode::Dense
1465            ),]),
1466        );
1467
1468        // union members with the same type id must be the same field
1469        let res = Schema::try_merge(vec![
1470            Schema::new(vec![Field::new_union(
1471                "c1",
1472                vec![0],
1473                vec![Field::new("c11", DataType::Utf8, true)],
1474                UnionMode::Dense,
1475            )]),
1476            Schema::new(vec![Field::new_union(
1477                "c1",
1478                vec![0],
1479                vec![Field::new("c12", DataType::Int64, true)],
1480                UnionMode::Dense,
1481            )]),
1482        ])
1483        .unwrap_err();
1484        assert_eq!(
1485            res.to_string(),
1486            "Schema error: Fail to merge schema field 'c12' because type id 0 is already used by field 'c11'"
1487        );
1488
1489        // incompatible field should throw error
1490        assert!(
1491            Schema::try_merge(vec![
1492                Schema::new(vec![
1493                    Field::new("first_name", DataType::Utf8, false),
1494                    Field::new("last_name", DataType::Utf8, false),
1495                ]),
1496                Schema::new(vec![Field::new("last_name", DataType::Int64, false),])
1497            ])
1498            .is_err()
1499        );
1500
1501        // incompatible metadata should throw error
1502        let res = Schema::try_merge(vec![
1503            Schema::new_with_metadata(
1504                vec![Field::new("first_name", DataType::Utf8, false)],
1505                HashMap::from([("foo".to_string(), "bar".to_string())]),
1506            ),
1507            Schema::new_with_metadata(
1508                vec![Field::new("last_name", DataType::Utf8, false)],
1509                HashMap::from([("foo".to_string(), "baz".to_string())]),
1510            ),
1511        ])
1512        .unwrap_err();
1513
1514        let expected = "Fail to merge schema due to conflicting metadata. Key 'foo' has different values 'bar' and 'baz'";
1515        assert!(
1516            res.to_string().contains(expected),
1517            "Could not find expected string '{expected}' in '{res}'"
1518        );
1519    }
1520
1521    #[test]
1522    fn test_schema_builder_change_field() {
1523        let mut builder = SchemaBuilder::new();
1524        builder.push(Field::new("a", DataType::Int32, false));
1525        builder.push(Field::new("b", DataType::Utf8, false));
1526        *builder.field_mut(1) = Arc::new(Field::new("c", DataType::Int32, false));
1527        assert_eq!(
1528            builder.fields,
1529            vec![
1530                Arc::new(Field::new("a", DataType::Int32, false)),
1531                Arc::new(Field::new("c", DataType::Int32, false))
1532            ]
1533        );
1534    }
1535
1536    #[test]
1537    fn test_schema_builder_reverse() {
1538        let mut builder = SchemaBuilder::new();
1539        builder.push(Field::new("a", DataType::Int32, false));
1540        builder.push(Field::new("b", DataType::Utf8, true));
1541        builder.reverse();
1542        assert_eq!(
1543            builder.fields,
1544            vec![
1545                Arc::new(Field::new("b", DataType::Utf8, true)),
1546                Arc::new(Field::new("a", DataType::Int32, false))
1547            ]
1548        );
1549    }
1550
1551    #[test]
1552    fn test_schema_builder_metadata() {
1553        let mut metadata: HashMap<String, String> = HashMap::with_capacity(1);
1554        metadata.insert("key".to_string(), "value".to_string());
1555
1556        let fields = vec![Field::new("test", DataType::Int8, true)];
1557        let mut builder: SchemaBuilder = Schema::new(fields).with_metadata(metadata).into();
1558        builder.metadata_mut().insert("k", "v");
1559        let out = builder.finish();
1560        assert_eq!(out.metadata.len(), 2);
1561        assert_eq!(out.metadata["k"], "v");
1562        assert_eq!(out.metadata["key"], "value");
1563    }
1564
1565    #[test]
1566    fn test_schema_builder_project() {
1567        let schema = Schema::new_with_metadata(
1568            vec![
1569                Field::new("a", DataType::Int32, false),
1570                Field::new("b", DataType::Utf8, true),
1571                Field::new("c", DataType::Float64, true),
1572                Field::new("d", DataType::Boolean, false),
1573            ],
1574            [("meta", "data")],
1575        );
1576
1577        // Reorder: reverse field order, keeping all fields.
1578        let reordered = SchemaBuilder::from(&schema).project(&[3, 2, 1, 0]).unwrap();
1579        assert_eq!(reordered.fields().len(), 4);
1580        assert_eq!(reordered.field(0).name(), "d");
1581        assert_eq!(reordered.field(1).name(), "c");
1582        assert_eq!(reordered.field(2).name(), "b");
1583        assert_eq!(reordered.field(3).name(), "a");
1584        assert_eq!(reordered.metadata()["meta"], "data"); // metadata carried over
1585
1586        // Subset: keep only the two middle fields.
1587        let subset = SchemaBuilder::from(&schema).project(&[1, 2]).unwrap();
1588        assert_eq!(subset.fields().len(), 2);
1589        assert_eq!(subset.field(0).name(), "b");
1590        assert_eq!(subset.field(1).name(), "c");
1591    }
1592
1593    #[test]
1594    fn test_schema_builder_project_errors() {
1595        let schema = Schema::new(vec![
1596            Field::new("a", DataType::Int32, false),
1597            Field::new("b", DataType::Utf8, true),
1598        ]);
1599
1600        // Out of bounds.
1601        let err = SchemaBuilder::from(&schema).project(&[0, 5]).unwrap_err();
1602        assert!(
1603            err.to_string().contains("out of bounds"),
1604            "unexpected error: {err}"
1605        );
1606
1607        // Repeated index.
1608        let err = SchemaBuilder::from(&schema)
1609            .project(&[0, 1, 0])
1610            .unwrap_err();
1611        assert!(
1612            err.to_string().contains("repeated"),
1613            "unexpected error: {err}"
1614        );
1615    }
1616}
arrow_schema/schema.rs

arrow_schema/
schema.rs
