arrow_schema/
schema.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::fmt;
19use std::hash::Hash;
20use std::sync::Arc;
21
22use crate::error::ArrowError;
23use crate::field::Field;
24use crate::{DataType, FieldRef, Fields, Metadata};
25
26/// A builder to facilitate building a [`Schema`] iteratively from [`FieldRef`]
27///
28/// # Examples
29///
30/// Build a schema from scratch:
31///
32/// ```
33/// # use arrow_schema::*;
34/// let schema = {
35///     let mut builder = SchemaBuilder::new();
36///     builder.push(Field::new("id", DataType::Int64, false));
37///     builder.push(Field::new("name", DataType::Utf8, true));
38///     builder.finish()
39/// };
40/// assert_eq!(schema.fields().len(), 2);
41/// ```
42///
43/// Derive a new schema from an existing one, keeping all fields and metadata
44/// while appending new columns:
45///
46/// ```
47/// # use arrow_schema::*;
48/// let base = Schema::new_with_metadata(
49///     vec![
50///         Field::new("id", DataType::Int64, false),
51///         Field::new("name", DataType::Utf8, true),
52///     ],
53///     [("created_by", "myapp")],
54/// );
55///
56/// // Build a new schema that extends `base` with an extra field.
57/// let mut builder = SchemaBuilder::from(&base); // copies all fields *and* metadata.
58/// builder.push(Field::new("score", DataType::Float64, true));
59/// let extended = builder.finish();
60///
61/// assert_eq!(extended.fields().len(), 3);
62/// assert_eq!(extended.field(0).name(), "id");     // original fields preserved
63/// assert_eq!(extended.field(2).name(), "score");  // new field appended
64/// assert_eq!(extended.metadata()["created_by"], "myapp"); // metadata carried over
65/// ```
66#[derive(Debug, Default)]
67pub struct SchemaBuilder {
68    fields: Vec<FieldRef>,
69    metadata: Metadata,
70}
71
72impl SchemaBuilder {
73    /// Creates a new empty [`SchemaBuilder`]
74    pub fn new() -> Self {
75        Self::default()
76    }
77
78    /// Creates a new empty [`SchemaBuilder`] with space for `capacity` fields
79    pub fn with_capacity(capacity: usize) -> Self {
80        Self {
81            fields: Vec::with_capacity(capacity),
82            metadata: Default::default(),
83        }
84    }
85
86    /// Appends a [`FieldRef`] to this [`SchemaBuilder`] without checking for collision
87    pub fn push(&mut self, field: impl Into<FieldRef>) {
88        self.fields.push(field.into())
89    }
90
91    /// Removes and returns the [`FieldRef`] as index `idx`
92    ///
93    /// # Panics
94    ///
95    /// Panics if index out of bounds
96    pub fn remove(&mut self, idx: usize) -> FieldRef {
97        self.fields.remove(idx)
98    }
99
100    /// Returns an immutable reference to the [`FieldRef`] at index `idx`
101    ///
102    /// # Panics
103    ///
104    /// Panics if index out of bounds
105    pub fn field(&mut self, idx: usize) -> &FieldRef {
106        &mut self.fields[idx]
107    }
108
109    /// Returns a mutable reference to the [`FieldRef`] at index `idx`
110    ///
111    /// # Example
112    ///
113    /// ```
114    /// # use std::sync::Arc;
115    /// # use arrow_schema::*;
116    /// let original = Schema::new(vec![
117    ///     Field::new("id", DataType::Int32, false),
118    ///     Field::new("value", DataType::Utf8, true),
119    /// ]);
120    ///
121    /// let mut builder = SchemaBuilder::from(&original);
122    /// // Widen the "id" column from Int32 to Int64
123    /// *builder.field_mut(0) = Arc::new(Field::new("id", DataType::Int64, false));
124    /// let widened = builder.finish();
125    ///
126    /// assert_eq!(widened.field(0).data_type(), &DataType::Int64);
127    /// assert_eq!(widened.field(1).name(), "value"); // unchanged
128    /// ```
129    ///
130    /// # Panics
131    ///
132    /// Panics if index out of bounds
133    pub fn field_mut(&mut self, idx: usize) -> &mut FieldRef {
134        &mut self.fields[idx]
135    }
136
137    /// Returns an immutable reference to the Map of custom metadata key-value pairs.
138    pub fn metadata(&mut self) -> &Metadata {
139        &self.metadata
140    }
141
142    /// Returns a mutable reference to the Map of custom metadata key-value pairs.
143    pub fn metadata_mut(&mut self) -> &mut Metadata {
144        &mut self.metadata
145    }
146
147    /// Reverse the fields
148    pub fn reverse(&mut self) {
149        self.fields.reverse();
150    }
151
152    /// Appends a [`FieldRef`] to this [`SchemaBuilder`] checking for collision
153    ///
154    /// If an existing field exists with the same name, calls [`Field::try_merge`]
155    pub fn try_merge(&mut self, field: &FieldRef) -> Result<(), ArrowError> {
156        // This could potentially be sped up with a HashMap or similar
157        let existing = self.fields.iter_mut().find(|f| f.name() == field.name());
158        match existing {
159            Some(e) if Arc::ptr_eq(e, field) => {} // Nothing to do
160            Some(e) => match Arc::get_mut(e) {
161                Some(e) => e.try_merge(field.as_ref())?,
162                None => {
163                    let mut t = e.as_ref().clone();
164                    t.try_merge(field)?;
165                    *e = Arc::new(t)
166                }
167            },
168            None => self.fields.push(field.clone()),
169        }
170        Ok(())
171    }
172
173    /// Consume this [`SchemaBuilder`] yielding the final [`Schema`]
174    pub fn finish(self) -> Schema {
175        Schema {
176            fields: self.fields.into(),
177            metadata: self.metadata,
178        }
179    }
180
181    /// Consume this [`SchemaBuilder`] yielding a [`Schema`] with fields reordered
182    /// or subsetted according to `indices`.
183    ///
184    /// Fields appear in the output in the order given by `indices`. Metadata is
185    /// carried over from the builder unchanged.
186    ///
187    /// # Errors
188    ///
189    /// Returns an error if any index is out of bounds or if any index is repeated.
190    ///
191    /// # Example: reorder fields
192    ///
193    /// ```
194    /// # use arrow_schema::*;
195    /// let schema = Schema::new(vec![
196    ///     Field::new("id", DataType::Int64, false),
197    ///     Field::new("name", DataType::Utf8, true),
198    ///     Field::new("score", DataType::Float64, true),
199    /// ]);
200    ///
201    /// let reordered = SchemaBuilder::from(&schema).project(&[2, 0, 1]).unwrap();
202    /// assert_eq!(reordered.field(0).name(), "score");
203    /// assert_eq!(reordered.field(1).name(), "id");
204    /// assert_eq!(reordered.field(2).name(), "name");
205    /// ```
206    ///
207    /// # Example: select a subset of fields
208    ///
209    /// ```
210    /// # use arrow_schema::*;
211    /// let schema = Schema::new(vec![
212    ///     Field::new("id", DataType::Int64, false),
213    ///     Field::new("name", DataType::Utf8, true),
214    ///     Field::new("score", DataType::Float64, true),
215    /// ]);
216    ///
217    /// let subset = SchemaBuilder::from(&schema).project(&[0, 2]).unwrap();
218    /// assert_eq!(subset.fields().len(), 2);
219    /// assert_eq!(subset.field(0).name(), "id");
220    /// assert_eq!(subset.field(1).name(), "score");
221    /// ```
222    pub fn project(self, indices: &[usize]) -> Result<Schema, ArrowError> {
223        let num_fields = self.fields.len();
224        let mut seen = std::collections::HashSet::new();
225        for &idx in indices {
226            if idx >= num_fields {
227                return Err(ArrowError::SchemaError(format!(
228                    "project index {idx} out of bounds, schema has {num_fields} fields"
229                )));
230            }
231            if !seen.insert(idx) {
232                return Err(ArrowError::SchemaError(format!(
233                    "project index {idx} is repeated"
234                )));
235            }
236        }
237        let fields: Vec<FieldRef> = indices
238            .iter()
239            .map(|&idx| self.fields[idx].clone())
240            .collect();
241        Ok(Schema {
242            fields: fields.into(),
243            metadata: self.metadata,
244        })
245    }
246}
247
248impl From<&Fields> for SchemaBuilder {
249    fn from(value: &Fields) -> Self {
250        Self {
251            fields: value.to_vec(),
252            metadata: Default::default(),
253        }
254    }
255}
256
257impl From<Fields> for SchemaBuilder {
258    fn from(value: Fields) -> Self {
259        Self {
260            fields: value.to_vec(),
261            metadata: Default::default(),
262        }
263    }
264}
265
266impl From<&Schema> for SchemaBuilder {
267    fn from(value: &Schema) -> Self {
268        Self::from(value.clone())
269    }
270}
271
272impl From<Schema> for SchemaBuilder {
273    fn from(value: Schema) -> Self {
274        Self {
275            fields: value.fields.to_vec(),
276            metadata: value.metadata,
277        }
278    }
279}
280
281impl Extend<FieldRef> for SchemaBuilder {
282    fn extend<T: IntoIterator<Item = FieldRef>>(&mut self, iter: T) {
283        let iter = iter.into_iter();
284        self.fields.reserve(iter.size_hint().0);
285        for f in iter {
286            self.push(f)
287        }
288    }
289}
290
291impl Extend<Field> for SchemaBuilder {
292    fn extend<T: IntoIterator<Item = Field>>(&mut self, iter: T) {
293        let iter = iter.into_iter();
294        self.fields.reserve(iter.size_hint().0);
295        for f in iter {
296            self.push(f)
297        }
298    }
299}
300
301/// A reference-counted reference to a [`Schema`].
302pub type SchemaRef = Arc<Schema>;
303
304/// Describes the meta-data of an ordered sequence of relative types.
305///
306/// Note that this information is only part of the meta-data and not part of the physical
307/// memory layout.
308#[derive(Debug, Clone, PartialEq, Eq, Hash)]
309#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
310pub struct Schema {
311    /// A sequence of fields that describe the schema.
312    pub fields: Fields,
313    /// A map of key-value pairs containing additional metadata.
314    pub metadata: Metadata,
315}
316
317impl Schema {
318    /// Creates an empty `Schema`
319    pub fn empty() -> Self {
320        Self {
321            fields: Default::default(),
322            metadata: Default::default(),
323        }
324    }
325
326    /// Creates a new [`Schema`] from a sequence of [`Field`] values.
327    ///
328    /// # Example
329    ///
330    /// ```
331    /// # use arrow_schema::*;
332    /// let field_a = Field::new("a", DataType::Int64, false);
333    /// let field_b = Field::new("b", DataType::Boolean, false);
334    ///
335    /// let schema = Schema::new(vec![field_a, field_b]);
336    /// ```
337    pub fn new(fields: impl Into<Fields>) -> Self {
338        Self::new_with_metadata(fields, Metadata::new())
339    }
340
341    /// Creates a new [`Schema`] from a sequence of [`Field`] values
342    /// and adds additional metadata in form of key value pairs.
343    ///
344    /// # Example
345    ///
346    /// ```
347    /// # use arrow_schema::*;
348    /// let field_a = Field::new("a", DataType::Int64, false);
349    /// let field_b = Field::new("b", DataType::Boolean, false);
350    ///
351    /// let schema = Schema::new_with_metadata(vec![field_a, field_b], [("row_count", "100")]);
352    /// ```
353    #[inline]
354    pub fn new_with_metadata(fields: impl Into<Fields>, metadata: impl Into<Metadata>) -> Self {
355        Self {
356            fields: fields.into(),
357            metadata: metadata.into(),
358        }
359    }
360
361    /// Sets the metadata of this `Schema` to be `metadata` and returns self
362    pub fn with_metadata(mut self, metadata: impl Into<Metadata>) -> Self {
363        self.metadata = metadata.into();
364        self
365    }
366
367    /// Returns a new schema with only the specified columns in the new schema
368    /// This carries metadata from the parent schema over as well
369    pub fn project(&self, indices: &[usize]) -> Result<Schema, ArrowError> {
370        let new_fields = indices
371            .iter()
372            .map(|i| {
373                self.fields.get(*i).cloned().ok_or_else(|| {
374                    ArrowError::SchemaError(format!(
375                        "project index {} out of bounds, max field {}",
376                        i,
377                        self.fields().len()
378                    ))
379                })
380            })
381            .collect::<Result<Vec<_>, _>>()?;
382        Ok(Self::new_with_metadata(new_fields, self.metadata.clone()))
383    }
384
385    /// Merge schema into self if it is compatible. Struct fields will be merged recursively.
386    ///
387    /// Example:
388    ///
389    /// ```
390    /// # use arrow_schema::*;
391    ///
392    /// let merged = Schema::try_merge(vec![
393    ///     Schema::new(vec![
394    ///         Field::new("c1", DataType::Int64, false),
395    ///         Field::new("c2", DataType::Utf8, false),
396    ///     ]),
397    ///     Schema::new(vec![
398    ///         Field::new("c1", DataType::Int64, true),
399    ///         Field::new("c2", DataType::Utf8, false),
400    ///         Field::new("c3", DataType::Utf8, false),
401    ///     ]),
402    /// ]).unwrap();
403    ///
404    /// assert_eq!(
405    ///     merged,
406    ///     Schema::new(vec![
407    ///         Field::new("c1", DataType::Int64, true),
408    ///         Field::new("c2", DataType::Utf8, false),
409    ///         Field::new("c3", DataType::Utf8, false),
410    ///     ]),
411    /// );
412    /// ```
413    pub fn try_merge(schemas: impl IntoIterator<Item = Self>) -> Result<Self, ArrowError> {
414        let mut out_meta = Metadata::new();
415        let mut out_fields = SchemaBuilder::new();
416        for schema in schemas {
417            let Schema { metadata, fields } = schema;
418
419            // merge metadata
420            for (key, value) in metadata {
421                if let Some(old_val) = out_meta.get(&key)
422                    && old_val != &value
423                {
424                    return Err(ArrowError::SchemaError(format!(
425                        "Fail to merge schema due to conflicting metadata. \
426                                     Key '{key}' has different values '{old_val}' and '{value}'"
427                    )));
428                }
429                out_meta.insert(key, value);
430            }
431
432            // merge fields
433            fields.iter().try_for_each(|x| out_fields.try_merge(x))?
434        }
435
436        Ok(out_fields.finish().with_metadata(out_meta))
437    }
438
439    /// Returns an immutable reference of the vector of `Field` instances.
440    #[inline]
441    pub const fn fields(&self) -> &Fields {
442        &self.fields
443    }
444
445    /// Returns a vector with references to all fields (including nested fields)
446    ///
447    /// # Example
448    ///
449    /// ```
450    /// use std::sync::Arc;
451    /// use arrow_schema::{DataType, Field, Fields, Schema};
452    ///
453    /// let f1 = Arc::new(Field::new("a", DataType::Boolean, false));
454    ///
455    /// let f2_inner = Arc::new(Field::new("b_inner", DataType::Int8, false));
456    /// let f2 = Arc::new(Field::new("b", DataType::List(f2_inner.clone()), false));
457    ///
458    /// let f3_inner1 = Arc::new(Field::new("c_inner1", DataType::Int8, false));
459    /// let f3_inner2 = Arc::new(Field::new("c_inner2", DataType::Int8, false));
460    /// let f3 = Arc::new(Field::new(
461    ///     "c",
462    ///     DataType::Struct(vec![f3_inner1.clone(), f3_inner2.clone()].into()),
463    ///     false
464    /// ));
465    ///
466    /// let mut schema = Schema::new(vec![
467    ///   f1.clone(), f2.clone(), f3.clone()
468    /// ]);
469    /// assert_eq!(
470    ///     schema.flattened_fields(),
471    ///     vec![
472    ///         f1.as_ref(),
473    ///         f2.as_ref(),
474    ///         f2_inner.as_ref(),
475    ///         f3.as_ref(),
476    ///         f3_inner1.as_ref(),
477    ///         f3_inner2.as_ref()
478    ///    ]
479    /// );
480    /// ```
481    #[inline]
482    pub fn flattened_fields(&self) -> Vec<&Field> {
483        self.fields.iter().flat_map(|f| f.fields()).collect()
484    }
485
486    /// Returns an immutable reference of a specific [`Field`] instance selected using an
487    /// offset within the internal `fields` vector.
488    ///
489    /// # Panics
490    ///
491    /// Panics if index out of bounds
492    pub fn field(&self, i: usize) -> &Field {
493        &self.fields[i]
494    }
495
496    /// Returns an immutable reference of a specific [`Field`] instance selected by name.
497    pub fn field_with_name(&self, name: &str) -> Result<&Field, ArrowError> {
498        Ok(&self.fields[self.index_of(name)?])
499    }
500
501    /// Returns a vector of immutable references to all [`Field`] instances selected by
502    /// the dictionary ID they use.
503    #[deprecated(
504        since = "54.0.0",
505        note = "The ability to preserve dictionary IDs will be removed. With it, all functions related to it."
506    )]
507    pub fn fields_with_dict_id(&self, dict_id: i64) -> Vec<&Field> {
508        #[expect(deprecated)]
509        self.fields
510            .iter()
511            .flat_map(|f| f.fields_with_dict_id(dict_id))
512            .collect()
513    }
514
515    /// Find the index of the column with the given name.
516    pub fn index_of(&self, name: &str) -> Result<usize, ArrowError> {
517        let (idx, _) = self.fields().find(name).ok_or_else(|| {
518            let valid_fields: Vec<_> = self.fields.iter().map(|f| f.name()).collect();
519            ArrowError::SchemaError(format!(
520                "Unable to get field named \"{name}\". Valid fields: {valid_fields:?}"
521            ))
522        })?;
523        Ok(idx)
524    }
525
526    /// Returns an immutable reference to the Map of custom metadata key-value pairs.
527    #[inline]
528    pub const fn metadata(&self) -> &Metadata {
529        &self.metadata
530    }
531
532    /// Normalize a [`Schema`] into a flat table.
533    ///
534    /// Nested [`Field`]s will generate names separated by `separator`, up to a depth of `max_level`
535    /// (unlimited if `None`).
536    ///
537    /// e.g. given a [`Schema`]:
538    ///
539    /// ```text
540    ///     "foo": StructArray<"bar": Utf8>
541    /// ```
542    ///
543    /// A separator of `"."` would generate a batch with the schema:
544    ///
545    /// ```text
546    ///     "foo.bar": Utf8
547    /// ```
548    ///
549    /// Note that giving a depth of `Some(0)` to `max_level` is the same as passing in `None`;
550    /// it will be treated as unlimited.
551    ///
552    /// # Example
553    ///
554    /// ```
555    /// # use std::sync::Arc;
556    /// # use arrow_schema::{DataType, Field, Fields, Schema};
557    /// let schema = Schema::new(vec![
558    ///     Field::new(
559    ///         "a",
560    ///         DataType::Struct(Fields::from(vec![
561    ///             Arc::new(Field::new("animals", DataType::Utf8, true)),
562    ///             Arc::new(Field::new("n_legs", DataType::Int64, true)),
563    ///         ])),
564    ///         false,
565    ///     ),
566    /// ])
567    /// .normalize(".", None)
568    /// .expect("valid normalization");
569    /// let expected = Schema::new(vec![
570    ///     Field::new("a.animals", DataType::Utf8, true),
571    ///     Field::new("a.n_legs", DataType::Int64, true),
572    /// ]);
573    /// assert_eq!(schema, expected);
574    /// ```
575    pub fn normalize(&self, separator: &str, max_level: Option<usize>) -> Result<Self, ArrowError> {
576        let max_level = match max_level.unwrap_or(usize::MAX) {
577            0 => usize::MAX,
578            val => val,
579        };
580        let mut stack: Vec<(usize, Vec<&str>, &FieldRef)> = self
581            .fields()
582            .iter()
583            .rev()
584            .map(|f| {
585                let name_vec: Vec<&str> = vec![f.name()];
586                (0, name_vec, f)
587            })
588            .collect();
589        let mut fields: Vec<FieldRef> = Vec::new();
590
591        while let Some((depth, name, field_ref)) = stack.pop() {
592            match field_ref.data_type() {
593                DataType::Struct(ff) if depth < max_level => {
594                    // Need to zip these in reverse to maintain original order
595                    for fff in ff.into_iter().rev() {
596                        let mut name = name.clone();
597                        name.push(separator);
598                        name.push(fff.name());
599                        stack.push((depth + 1, name, fff))
600                    }
601                }
602                _ => {
603                    let updated_field = Field::new(
604                        name.concat(),
605                        field_ref.data_type().clone(),
606                        field_ref.is_nullable(),
607                    );
608                    fields.push(Arc::new(updated_field));
609                }
610            }
611        }
612        Ok(Schema::new(fields))
613    }
614
615    /// Look up a column by name and return a immutable reference to the column along with
616    /// its index.
617    pub fn column_with_name(&self, name: &str) -> Option<(usize, &Field)> {
618        let (idx, field) = self.fields.find(name)?;
619        Some((idx, field.as_ref()))
620    }
621
622    /// Check to see if `self` is a superset of `other` schema.
623    ///
624    /// In particular returns true if `self.metadata` is a superset of `other.metadata`
625    /// and [`Fields::contains`] for `self.fields` and `other.fields`
626    ///
627    /// In other words, any record that conforms to `other` should also conform to `self`.
628    pub fn contains(&self, other: &Schema) -> bool {
629        // make sure self.metadata is a superset of other.metadata
630        self.fields.contains(&other.fields)
631            && other
632                .metadata
633                .iter()
634                .all(|(k, v1)| self.metadata.get(k).is_some_and(|v2| v1 == v2))
635    }
636}
637
638impl fmt::Display for Schema {
639    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
640        f.write_str(
641            &self
642                .fields
643                .iter()
644                .map(|c| c.to_string())
645                .collect::<Vec<String>>()
646                .join(", "),
647        )
648    }
649}
650
651impl AsRef<Schema> for Schema {
652    fn as_ref(&self) -> &Schema {
653        self
654    }
655}
656
657#[cfg(test)]
658mod tests {
659    use crate::datatype::DataType;
660    use crate::{TimeUnit, UnionMode};
661    use std::collections::HashMap;
662
663    use super::*;
664
665    #[test]
666    #[expect(clippy::needless_borrows_for_generic_args)] // intentional to exercise various references
667    fn test_schema_as_ref() {
668        fn accept_ref(_: impl AsRef<Schema>) {}
669
670        let schema = Schema::new(vec![
671            Field::new("name", DataType::Utf8, false),
672            Field::new("address", DataType::Utf8, false),
673            Field::new("priority", DataType::UInt8, false),
674        ]);
675
676        accept_ref(schema.clone());
677        accept_ref(&schema.clone());
678        accept_ref(&&schema.clone());
679        accept_ref(Arc::new(schema.clone()));
680        accept_ref(&Arc::new(schema.clone()));
681        accept_ref(&&Arc::new(schema.clone()));
682    }
683
684    #[test]
685    #[cfg(feature = "serde")]
686    fn test_ser_de_metadata() {
687        // ser/de with empty metadata
688        let schema = Schema::new(vec![
689            Field::new("name", DataType::Utf8, false),
690            Field::new("address", DataType::Utf8, false),
691            Field::new("priority", DataType::UInt8, false),
692        ]);
693
694        let json = serde_json::to_string(&schema).unwrap();
695        let de_schema = serde_json::from_str(&json).unwrap();
696
697        assert_eq!(schema, de_schema);
698
699        // ser/de with non-empty metadata
700        let schema = schema.with_metadata([("key", "val")]);
701        let json = serde_json::to_string(&schema).unwrap();
702        let de_schema = serde_json::from_str(&json).unwrap();
703
704        assert_eq!(schema, de_schema);
705    }
706
707    #[test]
708    fn test_projection() {
709        let mut metadata = HashMap::new();
710        metadata.insert("meta".to_string(), "data".to_string());
711
712        let schema = Schema::new(vec![
713            Field::new("name", DataType::Utf8, false),
714            Field::new("address", DataType::Utf8, false),
715            Field::new("priority", DataType::UInt8, false),
716        ])
717        .with_metadata(metadata);
718
719        let projected: Schema = schema.project(&[0, 2]).unwrap();
720
721        assert_eq!(projected.fields().len(), 2);
722        assert_eq!(projected.fields()[0].name(), "name");
723        assert_eq!(projected.fields()[1].name(), "priority");
724        assert_eq!(projected.metadata.get("meta").unwrap(), "data")
725    }
726
727    #[test]
728    fn test_oob_projection() {
729        let mut metadata = HashMap::new();
730        metadata.insert("meta".to_string(), "data".to_string());
731
732        let schema = Schema::new(vec![
733            Field::new("name", DataType::Utf8, false),
734            Field::new("address", DataType::Utf8, false),
735            Field::new("priority", DataType::UInt8, false),
736        ])
737        .with_metadata(metadata);
738
739        let projected = schema.project(&[0, 3]);
740
741        assert!(projected.is_err());
742        if let Err(e) = projected {
743            assert_eq!(
744                e.to_string(),
745                "Schema error: project index 3 out of bounds, max field 3".to_string()
746            )
747        }
748    }
749
750    #[test]
751    fn test_schema_contains() {
752        let mut metadata1 = HashMap::new();
753        metadata1.insert("meta".to_string(), "data".to_string());
754
755        let schema1 = Schema::new(vec![
756            Field::new("name", DataType::Utf8, false),
757            Field::new("address", DataType::Utf8, false),
758            Field::new("priority", DataType::UInt8, false),
759        ])
760        .with_metadata(metadata1.clone());
761
762        let mut metadata2 = HashMap::new();
763        metadata2.insert("meta".to_string(), "data".to_string());
764        metadata2.insert("meta2".to_string(), "data".to_string());
765        let schema2 = Schema::new(vec![
766            Field::new("name", DataType::Utf8, false),
767            Field::new("address", DataType::Utf8, false),
768            Field::new("priority", DataType::UInt8, false),
769        ])
770        .with_metadata(metadata2);
771
772        // reflexivity
773        assert!(schema1.contains(&schema1));
774        assert!(schema2.contains(&schema2));
775
776        assert!(!schema1.contains(&schema2));
777        assert!(schema2.contains(&schema1));
778    }
779
780    #[test]
781    fn schema_equality() {
782        let schema1 = Schema::new(vec![
783            Field::new("c1", DataType::Utf8, false),
784            Field::new("c2", DataType::Float64, true),
785            Field::new("c3", DataType::LargeBinary, true),
786        ]);
787        let schema2 = Schema::new(vec![
788            Field::new("c1", DataType::Utf8, false),
789            Field::new("c2", DataType::Float64, true),
790            Field::new("c3", DataType::LargeBinary, true),
791        ]);
792
793        assert_eq!(schema1, schema2);
794
795        let schema3 = Schema::new(vec![
796            Field::new("c1", DataType::Utf8, false),
797            Field::new("c2", DataType::Float32, true),
798        ]);
799        let schema4 = Schema::new(vec![
800            Field::new("C1", DataType::Utf8, false),
801            Field::new("C2", DataType::Float64, true),
802        ]);
803
804        assert_ne!(schema1, schema3);
805        assert_ne!(schema1, schema4);
806        assert_ne!(schema2, schema3);
807        assert_ne!(schema2, schema4);
808        assert_ne!(schema3, schema4);
809
810        let f = Field::new("c1", DataType::Utf8, false).with_metadata([("foo", "bar")]);
811        let schema5 = Schema::new(vec![
812            f,
813            Field::new("c2", DataType::Float64, true),
814            Field::new("c3", DataType::LargeBinary, true),
815        ]);
816        assert_ne!(schema1, schema5);
817    }
818
819    #[test]
820    fn create_schema_string() {
821        let schema = person_schema();
822        assert_eq!(
823            schema.to_string(),
824            "Field { \"first_name\": Utf8, metadata: {\"k\": \"v\"} }, \
825             Field { \"last_name\": Utf8 }, \
826             Field { \"address\": Struct(\"street\": non-null Utf8, \"zip\": non-null UInt16) }, \
827             Field { \"interests\": nullable Dictionary(Int32, Utf8), dict_id: 123, dict_is_ordered }"
828        )
829    }
830
831    #[test]
832    fn schema_field_accessors() {
833        let schema = person_schema();
834
835        // test schema accessors
836        assert_eq!(schema.fields().len(), 4);
837
838        // test field accessors
839        let first_name = &schema.fields()[0];
840        assert_eq!(first_name.name(), "first_name");
841        assert_eq!(first_name.data_type(), &DataType::Utf8);
842        assert!(!first_name.is_nullable());
843        #[expect(deprecated)]
844        let dict_id = first_name.dict_id();
845        assert_eq!(dict_id, None);
846        assert_eq!(first_name.dict_is_ordered(), None);
847
848        let metadata = first_name.metadata();
849        assert!(!metadata.is_empty());
850        let md = &metadata;
851        assert_eq!(md.len(), 1);
852        let key = md.get("k");
853        assert!(key.is_some());
854        assert_eq!(key.unwrap(), "v");
855
856        let interests = &schema.fields()[3];
857        assert_eq!(interests.name(), "interests");
858        assert_eq!(
859            interests.data_type(),
860            &DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8))
861        );
862        #[expect(deprecated)]
863        let dict_id = interests.dict_id();
864        assert_eq!(dict_id, Some(123));
865        assert_eq!(interests.dict_is_ordered(), Some(true));
866    }
867
868    #[test]
869    #[should_panic(
870        expected = "Unable to get field named \\\"nickname\\\". Valid fields: [\\\"first_name\\\", \\\"last_name\\\", \\\"address\\\", \\\"interests\\\"]"
871    )]
872    fn schema_index_of() {
873        let schema = person_schema();
874        assert_eq!(schema.index_of("first_name").unwrap(), 0);
875        assert_eq!(schema.index_of("last_name").unwrap(), 1);
876        schema.index_of("nickname").unwrap();
877    }
878
879    #[test]
880    fn normalize_simple() {
881        let schema = Schema::new(vec![
882            Field::new(
883                "a",
884                DataType::Struct(Fields::from(vec![
885                    Arc::new(Field::new("animals", DataType::Utf8, true)),
886                    Arc::new(Field::new("n_legs", DataType::Int64, true)),
887                    Arc::new(Field::new("year", DataType::Int64, true)),
888                ])),
889                false,
890            ),
891            Field::new("month", DataType::Int64, true),
892        ])
893        .normalize(".", Some(0))
894        .expect("valid normalization");
895
896        let expected = Schema::new(vec![
897            Field::new("a.animals", DataType::Utf8, true),
898            Field::new("a.n_legs", DataType::Int64, true),
899            Field::new("a.year", DataType::Int64, true),
900            Field::new("month", DataType::Int64, true),
901        ]);
902
903        assert_eq!(schema, expected);
904
905        // Check that 0, None have the same result
906        let schema = Schema::new(vec![
907            Field::new(
908                "a",
909                DataType::Struct(Fields::from(vec![
910                    Arc::new(Field::new("animals", DataType::Utf8, true)),
911                    Arc::new(Field::new("n_legs", DataType::Int64, true)),
912                    Arc::new(Field::new("year", DataType::Int64, true)),
913                ])),
914                false,
915            ),
916            Field::new("month", DataType::Int64, true),
917        ])
918        .normalize(".", None)
919        .expect("valid normalization");
920
921        assert_eq!(schema, expected);
922    }
923
924    #[test]
925    fn normalize_nested() {
926        let a = Arc::new(Field::new("a", DataType::Utf8, true));
927        let b = Arc::new(Field::new("b", DataType::Int64, false));
928        let c = Arc::new(Field::new("c", DataType::Int64, true));
929
930        let d = Arc::new(Field::new("d", DataType::Utf8, true));
931        let e = Arc::new(Field::new("e", DataType::Int64, false));
932        let f = Arc::new(Field::new("f", DataType::Int64, true));
933
934        let one = Arc::new(Field::new(
935            "1",
936            DataType::Struct(Fields::from(vec![a.clone(), b.clone(), c.clone()])),
937            false,
938        ));
939        let two = Arc::new(Field::new(
940            "2",
941            DataType::Struct(Fields::from(vec![d.clone(), e.clone(), f.clone()])),
942            true,
943        ));
944
945        let exclamation = Arc::new(Field::new(
946            "!",
947            DataType::Struct(Fields::from(vec![one, two])),
948            false,
949        ));
950
951        let normalize_all = Schema::new(vec![exclamation.clone()])
952            .normalize(".", Some(0))
953            .expect("valid normalization");
954
955        let expected = Schema::new(vec![
956            Field::new("!.1.a", DataType::Utf8, true),
957            Field::new("!.1.b", DataType::Int64, false),
958            Field::new("!.1.c", DataType::Int64, true),
959            Field::new("!.2.d", DataType::Utf8, true),
960            Field::new("!.2.e", DataType::Int64, false),
961            Field::new("!.2.f", DataType::Int64, true),
962        ]);
963
964        assert_eq!(normalize_all, expected);
965
966        let normalize_depth_one = Schema::new(vec![exclamation])
967            .normalize(".", Some(1))
968            .expect("valid normalization");
969
970        let expected = Schema::new(vec![
971            Field::new("!.1", DataType::Struct(Fields::from(vec![a, b, c])), false),
972            Field::new("!.2", DataType::Struct(Fields::from(vec![d, e, f])), true),
973        ]);
974
975        assert_eq!(normalize_depth_one, expected);
976    }
977
978    #[test]
979    fn normalize_list() {
980        // Only the Struct type field should be unwrapped
981        let a = Arc::new(Field::new("a", DataType::Utf8, true));
982        let b = Arc::new(Field::new("b", DataType::Int64, false));
983        let c = Arc::new(Field::new("c", DataType::Int64, true));
984        let d = Arc::new(Field::new("d", DataType::Utf8, true));
985        let e = Arc::new(Field::new("e", DataType::Int64, false));
986        let f = Arc::new(Field::new("f", DataType::Int64, true));
987
988        let one = Arc::new(Field::new(
989            "1",
990            DataType::Struct(Fields::from(vec![a.clone(), b.clone(), c.clone()])),
991            true,
992        ));
993
994        let two = Arc::new(Field::new(
995            "2",
996            DataType::List(Arc::new(Field::new_list_field(
997                DataType::Struct(Fields::from(vec![d.clone(), e.clone(), f.clone()])),
998                true,
999            ))),
1000            false,
1001        ));
1002
1003        let exclamation = Arc::new(Field::new(
1004            "!",
1005            DataType::Struct(Fields::from(vec![one.clone(), two.clone()])),
1006            false,
1007        ));
1008
1009        let normalize_all = Schema::new(vec![exclamation.clone()])
1010            .normalize(".", None)
1011            .expect("valid normalization");
1012
1013        // List shouldn't be affected
1014        let expected = Schema::new(vec![
1015            Field::new("!.1.a", DataType::Utf8, true),
1016            Field::new("!.1.b", DataType::Int64, false),
1017            Field::new("!.1.c", DataType::Int64, true),
1018            Field::new(
1019                "!.2",
1020                DataType::List(Arc::new(Field::new_list_field(
1021                    DataType::Struct(Fields::from(vec![d.clone(), e.clone(), f.clone()])),
1022                    true,
1023                ))),
1024                false,
1025            ),
1026        ]);
1027
1028        assert_eq!(normalize_all, expected);
1029        assert_eq!(normalize_all.fields().len(), 4);
1030
1031        // FixedSizeList
1032        let two = Arc::new(Field::new(
1033            "2",
1034            DataType::FixedSizeList(
1035                Arc::new(Field::new_fixed_size_list(
1036                    "3",
1037                    Arc::new(Field::new_list_field(
1038                        DataType::Struct(Fields::from(vec![d.clone(), e.clone(), f.clone()])),
1039                        true,
1040                    )),
1041                    1,
1042                    true,
1043                )),
1044                1,
1045            ),
1046            false,
1047        ));
1048
1049        let exclamation = Arc::new(Field::new(
1050            "!",
1051            DataType::Struct(Fields::from(vec![one.clone(), two])),
1052            false,
1053        ));
1054
1055        let normalize_all = Schema::new(vec![exclamation.clone()])
1056            .normalize(".", None)
1057            .expect("valid normalization");
1058
1059        // FixedSizeList shouldn't be affected
1060        let expected = Schema::new(vec![
1061            Field::new("!.1.a", DataType::Utf8, true),
1062            Field::new("!.1.b", DataType::Int64, false),
1063            Field::new("!.1.c", DataType::Int64, true),
1064            Field::new(
1065                "!.2",
1066                DataType::FixedSizeList(
1067                    Arc::new(Field::new_fixed_size_list(
1068                        "3",
1069                        Arc::new(Field::new_list_field(
1070                            DataType::Struct(Fields::from(vec![d.clone(), e.clone(), f.clone()])),
1071                            true,
1072                        )),
1073                        1,
1074                        true,
1075                    )),
1076                    1,
1077                ),
1078                false,
1079            ),
1080        ]);
1081
1082        assert_eq!(normalize_all, expected);
1083        assert_eq!(normalize_all.fields().len(), 4);
1084
1085        // LargeList
1086        let two = Arc::new(Field::new(
1087            "2",
1088            DataType::FixedSizeList(
1089                Arc::new(Field::new_large_list(
1090                    "3",
1091                    Arc::new(Field::new_list_field(
1092                        DataType::Struct(Fields::from(vec![d.clone(), e.clone(), f.clone()])),
1093                        true,
1094                    )),
1095                    true,
1096                )),
1097                1,
1098            ),
1099            false,
1100        ));
1101
1102        let exclamation = Arc::new(Field::new(
1103            "!",
1104            DataType::Struct(Fields::from(vec![one.clone(), two])),
1105            false,
1106        ));
1107
1108        let normalize_all = Schema::new(vec![exclamation.clone()])
1109            .normalize(".", None)
1110            .expect("valid normalization");
1111
1112        // LargeList shouldn't be affected
1113        let expected = Schema::new(vec![
1114            Field::new("!.1.a", DataType::Utf8, true),
1115            Field::new("!.1.b", DataType::Int64, false),
1116            Field::new("!.1.c", DataType::Int64, true),
1117            Field::new(
1118                "!.2",
1119                DataType::FixedSizeList(
1120                    Arc::new(Field::new_large_list(
1121                        "3",
1122                        Arc::new(Field::new_list_field(
1123                            DataType::Struct(Fields::from(vec![d.clone(), e.clone(), f.clone()])),
1124                            true,
1125                        )),
1126                        true,
1127                    )),
1128                    1,
1129                ),
1130                false,
1131            ),
1132        ]);
1133
1134        assert_eq!(normalize_all, expected);
1135        assert_eq!(normalize_all.fields().len(), 4);
1136    }
1137
1138    #[test]
1139    fn normalize_deep_nested() {
1140        // No unwrapping expected
1141        let a = Arc::new(Field::new("a", DataType::Utf8, true));
1142        let b = Arc::new(Field::new("b", DataType::Int64, false));
1143        let c = Arc::new(Field::new("c", DataType::Int64, true));
1144        let d = Arc::new(Field::new("d", DataType::Utf8, true));
1145        let e = Arc::new(Field::new("e", DataType::Int64, false));
1146        let f = Arc::new(Field::new("f", DataType::Int64, true));
1147
1148        let one = Arc::new(Field::new(
1149            "1",
1150            DataType::Struct(Fields::from(vec![a.clone(), b.clone(), c.clone()])),
1151            true,
1152        ));
1153
1154        let two = Arc::new(Field::new(
1155            "2",
1156            DataType::List(Arc::new(Field::new_list_field(
1157                DataType::Struct(Fields::from(vec![d.clone(), e.clone(), f.clone()])),
1158                true,
1159            ))),
1160            false,
1161        ));
1162
1163        let l10 = Arc::new(Field::new(
1164            "l10",
1165            DataType::List(Arc::new(Field::new_list_field(
1166                DataType::Struct(Fields::from(vec![one, two])),
1167                true,
1168            ))),
1169            false,
1170        ));
1171
1172        let l9 = Arc::new(Field::new(
1173            "l9",
1174            DataType::List(Arc::new(Field::new_list_field(
1175                DataType::Struct(Fields::from(vec![l10])),
1176                true,
1177            ))),
1178            false,
1179        ));
1180
1181        let l8 = Arc::new(Field::new(
1182            "l8",
1183            DataType::List(Arc::new(Field::new_list_field(
1184                DataType::Struct(Fields::from(vec![l9])),
1185                true,
1186            ))),
1187            false,
1188        ));
1189        let l7 = Arc::new(Field::new(
1190            "l7",
1191            DataType::List(Arc::new(Field::new_list_field(
1192                DataType::Struct(Fields::from(vec![l8])),
1193                true,
1194            ))),
1195            false,
1196        ));
1197        let l6 = Arc::new(Field::new(
1198            "l6",
1199            DataType::List(Arc::new(Field::new_list_field(
1200                DataType::Struct(Fields::from(vec![l7])),
1201                true,
1202            ))),
1203            false,
1204        ));
1205        let l5 = Arc::new(Field::new(
1206            "l5",
1207            DataType::List(Arc::new(Field::new_list_field(
1208                DataType::Struct(Fields::from(vec![l6])),
1209                true,
1210            ))),
1211            false,
1212        ));
1213        let l4 = Arc::new(Field::new(
1214            "l4",
1215            DataType::List(Arc::new(Field::new_list_field(
1216                DataType::Struct(Fields::from(vec![l5])),
1217                true,
1218            ))),
1219            false,
1220        ));
1221        let l3 = Arc::new(Field::new(
1222            "l3",
1223            DataType::List(Arc::new(Field::new_list_field(
1224                DataType::Struct(Fields::from(vec![l4])),
1225                true,
1226            ))),
1227            false,
1228        ));
1229        let l2 = Arc::new(Field::new(
1230            "l2",
1231            DataType::List(Arc::new(Field::new_list_field(
1232                DataType::Struct(Fields::from(vec![l3])),
1233                true,
1234            ))),
1235            false,
1236        ));
1237        let l1 = Arc::new(Field::new(
1238            "l1",
1239            DataType::List(Arc::new(Field::new_list_field(
1240                DataType::Struct(Fields::from(vec![l2])),
1241                true,
1242            ))),
1243            false,
1244        ));
1245
1246        let normalize_all = Schema::new(vec![l1])
1247            .normalize(".", None)
1248            .expect("valid normalization");
1249
1250        assert_eq!(normalize_all.fields().len(), 1);
1251    }
1252
1253    #[test]
1254    fn normalize_dictionary() {
1255        let a = Arc::new(Field::new("a", DataType::Utf8, true));
1256        let b = Arc::new(Field::new("b", DataType::Int64, false));
1257
1258        let one = Arc::new(Field::new(
1259            "1",
1260            DataType::Dictionary(
1261                Box::new(DataType::Int32),
1262                Box::new(DataType::Struct(Fields::from(vec![a.clone(), b.clone()]))),
1263            ),
1264            false,
1265        ));
1266
1267        let normalize_all = Schema::new(vec![one.clone()])
1268            .normalize(".", None)
1269            .expect("valid normalization");
1270
1271        let expected = Schema::new(vec![Field::new(
1272            "1",
1273            DataType::Dictionary(
1274                Box::new(DataType::Int32),
1275                Box::new(DataType::Struct(Fields::from(vec![a.clone(), b.clone()]))),
1276            ),
1277            false,
1278        )]);
1279
1280        assert_eq!(normalize_all, expected);
1281    }
1282
1283    #[test]
1284    #[should_panic(
1285        expected = "Unable to get field named \\\"nickname\\\". Valid fields: [\\\"first_name\\\", \\\"last_name\\\", \\\"address\\\", \\\"interests\\\"]"
1286    )]
1287    fn schema_field_with_name() {
1288        let schema = person_schema();
1289        assert_eq!(
1290            schema.field_with_name("first_name").unwrap().name(),
1291            "first_name"
1292        );
1293        assert_eq!(
1294            schema.field_with_name("last_name").unwrap().name(),
1295            "last_name"
1296        );
1297        schema.field_with_name("nickname").unwrap();
1298    }
1299
1300    #[test]
1301    fn schema_field_with_dict_id() {
1302        let schema = person_schema();
1303
1304        #[expect(deprecated)]
1305        let fields_dict_123: Vec<_> = schema
1306            .fields_with_dict_id(123)
1307            .iter()
1308            .map(|f| f.name())
1309            .collect();
1310        assert_eq!(fields_dict_123, vec!["interests"]);
1311
1312        #[expect(deprecated)]
1313        let is_empty = schema.fields_with_dict_id(456).is_empty();
1314        assert!(is_empty);
1315    }
1316
1317    fn person_schema() -> Schema {
1318        let kv_array = [("k".to_string(), "v".to_string())];
1319        let field_metadata: HashMap<String, String> = kv_array.iter().cloned().collect();
1320        let first_name =
1321            Field::new("first_name", DataType::Utf8, false).with_metadata(field_metadata);
1322
1323        Schema::new(vec![
1324            first_name,
1325            Field::new("last_name", DataType::Utf8, false),
1326            Field::new(
1327                "address",
1328                DataType::Struct(Fields::from(vec![
1329                    Field::new("street", DataType::Utf8, false),
1330                    Field::new("zip", DataType::UInt16, false),
1331                ])),
1332                false,
1333            ),
1334            #[expect(deprecated)]
1335            Field::new_dict(
1336                "interests",
1337                DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
1338                true,
1339                123,
1340                true,
1341            ),
1342        ])
1343    }
1344
1345    #[test]
1346    fn test_try_merge_field_with_metadata() {
1347        // 1. Different values for the same key should cause error.
1348        let metadata1 = HashMap::from([("foo".to_string(), "bar".to_string())]);
1349        let f1 = Field::new("first_name", DataType::Utf8, false).with_metadata(metadata1);
1350
1351        let metadata2 = HashMap::from([("foo".to_string(), "baz".to_string())]);
1352        let f2 = Field::new("first_name", DataType::Utf8, false).with_metadata(metadata2);
1353
1354        assert!(Schema::try_merge(vec![Schema::new(vec![f1]), Schema::new(vec![f2])]).is_err());
1355
1356        // 2. None + Some
1357        let mut f1 = Field::new("first_name", DataType::Utf8, false);
1358        let metadata2 = HashMap::from([("missing".to_string(), "value".to_string())]);
1359        let f2 = Field::new("first_name", DataType::Utf8, false).with_metadata(metadata2);
1360
1361        assert!(f1.try_merge(&f2).is_ok());
1362        assert!(!f1.metadata().is_empty());
1363        assert_eq!(f1.metadata(), f2.metadata());
1364
1365        // 3. Some + Some
1366        let mut f1 =
1367            Field::new("first_name", DataType::Utf8, false).with_metadata([("foo", "bar")]);
1368        let f2 = Field::new("first_name", DataType::Utf8, false).with_metadata([("foo2", "bar2")]);
1369
1370        assert!(f1.try_merge(&f2).is_ok());
1371        assert!(!f1.metadata().is_empty());
1372        assert_eq!(
1373            f1.metadata(),
1374            &Metadata::from([("foo", "bar"), ("foo2", "bar2")])
1375        );
1376
1377        // 4. Some + None.
1378        let mut f1 =
1379            Field::new("first_name", DataType::Utf8, false).with_metadata([("foo", "bar")]);
1380        let f2 = Field::new("first_name", DataType::Utf8, false);
1381        assert!(f1.try_merge(&f2).is_ok());
1382        assert!(!f1.metadata().is_empty());
1383        assert_eq!(f1.metadata(), &Metadata::from([("foo", "bar")]));
1384
1385        // 5. None + None.
1386        let mut f1 = Field::new("first_name", DataType::Utf8, false);
1387        let f2 = Field::new("first_name", DataType::Utf8, false);
1388        assert!(f1.try_merge(&f2).is_ok());
1389        assert!(f1.metadata().is_empty());
1390    }
1391
1392    #[test]
1393    fn test_schema_merge() {
1394        let merged = Schema::try_merge(vec![
1395            Schema::new(vec![
1396                Field::new("first_name", DataType::Utf8, false),
1397                Field::new("last_name", DataType::Utf8, false),
1398                Field::new(
1399                    "address",
1400                    DataType::Struct(vec![Field::new("zip", DataType::UInt16, false)].into()),
1401                    false,
1402                ),
1403            ]),
1404            Schema::new_with_metadata(
1405                vec![
1406                    // nullable merge
1407                    Field::new("last_name", DataType::Utf8, true),
1408                    Field::new(
1409                        "address",
1410                        DataType::Struct(Fields::from(vec![
1411                            // add new nested field
1412                            Field::new("street", DataType::Utf8, false),
1413                            // nullable merge on nested field
1414                            Field::new("zip", DataType::UInt16, true),
1415                        ])),
1416                        false,
1417                    ),
1418                    // new field
1419                    Field::new("number", DataType::Utf8, true),
1420                ],
1421                HashMap::from([("foo".to_string(), "bar".to_string())]),
1422            ),
1423        ])
1424        .unwrap();
1425
1426        assert_eq!(
1427            merged,
1428            Schema::new_with_metadata(
1429                vec![
1430                    Field::new("first_name", DataType::Utf8, false),
1431                    Field::new("last_name", DataType::Utf8, true),
1432                    Field::new(
1433                        "address",
1434                        DataType::Struct(Fields::from(vec![
1435                            Field::new("zip", DataType::UInt16, true),
1436                            Field::new("street", DataType::Utf8, false),
1437                        ])),
1438                        false,
1439                    ),
1440                    Field::new("number", DataType::Utf8, true),
1441                ],
1442                HashMap::from([("foo".to_string(), "bar".to_string())])
1443            )
1444        );
1445
1446        // support merge union fields
1447        assert_eq!(
1448            Schema::try_merge(vec![
1449                Schema::new(vec![Field::new_union(
1450                    "c1",
1451                    vec![0, 1],
1452                    vec![
1453                        Field::new("c11", DataType::Utf8, true),
1454                        Field::new("c12", DataType::Utf8, true),
1455                    ],
1456                    UnionMode::Dense
1457                ),]),
1458                Schema::new(vec![Field::new_union(
1459                    "c1",
1460                    vec![1, 2],
1461                    vec![
1462                        Field::new("c12", DataType::Utf8, true),
1463                        Field::new("c13", DataType::Time64(TimeUnit::Second), true),
1464                    ],
1465                    UnionMode::Dense
1466                ),])
1467            ])
1468            .unwrap(),
1469            Schema::new(vec![Field::new_union(
1470                "c1",
1471                vec![0, 1, 2],
1472                vec![
1473                    Field::new("c11", DataType::Utf8, true),
1474                    Field::new("c12", DataType::Utf8, true),
1475                    Field::new("c13", DataType::Time64(TimeUnit::Second), true),
1476                ],
1477                UnionMode::Dense
1478            ),]),
1479        );
1480
1481        // incompatible field should throw error
1482        assert!(
1483            Schema::try_merge(vec![
1484                Schema::new(vec![
1485                    Field::new("first_name", DataType::Utf8, false),
1486                    Field::new("last_name", DataType::Utf8, false),
1487                ]),
1488                Schema::new(vec![Field::new("last_name", DataType::Int64, false),])
1489            ])
1490            .is_err()
1491        );
1492
1493        // incompatible metadata should throw error
1494        let res = Schema::try_merge(vec![
1495            Schema::new_with_metadata(
1496                vec![Field::new("first_name", DataType::Utf8, false)],
1497                HashMap::from([("foo".to_string(), "bar".to_string())]),
1498            ),
1499            Schema::new_with_metadata(
1500                vec![Field::new("last_name", DataType::Utf8, false)],
1501                HashMap::from([("foo".to_string(), "baz".to_string())]),
1502            ),
1503        ])
1504        .unwrap_err();
1505
1506        let expected = "Fail to merge schema due to conflicting metadata. Key 'foo' has different values 'bar' and 'baz'";
1507        assert!(
1508            res.to_string().contains(expected),
1509            "Could not find expected string '{expected}' in '{res}'"
1510        );
1511    }
1512
1513    #[test]
1514    fn test_schema_builder_change_field() {
1515        let mut builder = SchemaBuilder::new();
1516        builder.push(Field::new("a", DataType::Int32, false));
1517        builder.push(Field::new("b", DataType::Utf8, false));
1518        *builder.field_mut(1) = Arc::new(Field::new("c", DataType::Int32, false));
1519        assert_eq!(
1520            builder.fields,
1521            vec![
1522                Arc::new(Field::new("a", DataType::Int32, false)),
1523                Arc::new(Field::new("c", DataType::Int32, false))
1524            ]
1525        );
1526    }
1527
1528    #[test]
1529    fn test_schema_builder_reverse() {
1530        let mut builder = SchemaBuilder::new();
1531        builder.push(Field::new("a", DataType::Int32, false));
1532        builder.push(Field::new("b", DataType::Utf8, true));
1533        builder.reverse();
1534        assert_eq!(
1535            builder.fields,
1536            vec![
1537                Arc::new(Field::new("b", DataType::Utf8, true)),
1538                Arc::new(Field::new("a", DataType::Int32, false))
1539            ]
1540        );
1541    }
1542
1543    #[test]
1544    fn test_schema_builder_metadata() {
1545        let mut metadata: HashMap<String, String> = HashMap::with_capacity(1);
1546        metadata.insert("key".to_string(), "value".to_string());
1547
1548        let fields = vec![Field::new("test", DataType::Int8, true)];
1549        let mut builder: SchemaBuilder = Schema::new(fields).with_metadata(metadata).into();
1550        builder.metadata_mut().insert("k", "v");
1551        let out = builder.finish();
1552        assert_eq!(out.metadata.len(), 2);
1553        assert_eq!(out.metadata["k"], "v");
1554        assert_eq!(out.metadata["key"], "value");
1555    }
1556
1557    #[test]
1558    fn test_schema_builder_project() {
1559        let schema = Schema::new_with_metadata(
1560            vec![
1561                Field::new("a", DataType::Int32, false),
1562                Field::new("b", DataType::Utf8, true),
1563                Field::new("c", DataType::Float64, true),
1564                Field::new("d", DataType::Boolean, false),
1565            ],
1566            [("meta", "data")],
1567        );
1568
1569        // Reorder: reverse field order, keeping all fields.
1570        let reordered = SchemaBuilder::from(&schema).project(&[3, 2, 1, 0]).unwrap();
1571        assert_eq!(reordered.fields().len(), 4);
1572        assert_eq!(reordered.field(0).name(), "d");
1573        assert_eq!(reordered.field(1).name(), "c");
1574        assert_eq!(reordered.field(2).name(), "b");
1575        assert_eq!(reordered.field(3).name(), "a");
1576        assert_eq!(reordered.metadata()["meta"], "data"); // metadata carried over
1577
1578        // Subset: keep only the two middle fields.
1579        let subset = SchemaBuilder::from(&schema).project(&[1, 2]).unwrap();
1580        assert_eq!(subset.fields().len(), 2);
1581        assert_eq!(subset.field(0).name(), "b");
1582        assert_eq!(subset.field(1).name(), "c");
1583    }
1584
1585    #[test]
1586    fn test_schema_builder_project_errors() {
1587        let schema = Schema::new(vec![
1588            Field::new("a", DataType::Int32, false),
1589            Field::new("b", DataType::Utf8, true),
1590        ]);
1591
1592        // Out of bounds.
1593        let err = SchemaBuilder::from(&schema).project(&[0, 5]).unwrap_err();
1594        assert!(
1595            err.to_string().contains("out of bounds"),
1596            "unexpected error: {err}"
1597        );
1598
1599        // Repeated index.
1600        let err = SchemaBuilder::from(&schema)
1601            .project(&[0, 1, 0])
1602            .unwrap_err();
1603        assert!(
1604            err.to_string().contains("repeated"),
1605            "unexpected error: {err}"
1606        );
1607    }
1608}
arrow_schema/schema.rs

arrow_schema/
schema.rs
