arrow_ipc/
convert.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Utilities for converting between IPC types and native Arrow types
19
20use arrow_buffer::Buffer;
21use arrow_schema::*;
22use core::panic;
23use flatbuffers::{
24    FlatBufferBuilder, ForwardsUOffset, UnionWIPOffset, Vector, Verifiable, Verifier,
25    VerifierOptions, WIPOffset,
26};
27use std::collections::HashMap;
28use std::fmt::{Debug, Formatter};
29use std::sync::Arc;
30
31use crate::writer::DictionaryTracker;
32use crate::{CONTINUATION_MARKER, KeyValue, Message};
33use DataType::*;
34
35/// Low level Arrow [Schema] to IPC bytes converter
36///
37/// See also [`fb_to_schema`] for the reverse operation
38///
39/// # Example
40/// ```
41/// # use arrow_ipc::convert::{fb_to_schema, IpcSchemaEncoder};
42/// # use arrow_ipc::root_as_schema;
43/// # use arrow_ipc::writer::DictionaryTracker;
44/// # use arrow_schema::{DataType, Field, Schema};
45/// // given an arrow schema to serialize
46/// let schema = Schema::new(vec![
47///    Field::new("a", DataType::Int32, false),
48/// ]);
49///
50/// // Use a dictionary tracker to track dictionary id if needed
51///  let mut dictionary_tracker = DictionaryTracker::new(true);
52/// // create a FlatBuffersBuilder that contains the encoded bytes
53///  let fb = IpcSchemaEncoder::new()
54///    .with_dictionary_tracker(&mut dictionary_tracker)
55///    .schema_to_fb(&schema);
56///
57/// // the bytes are in `fb.finished_data()`
58/// let ipc_bytes = fb.finished_data();
59///
60///  // convert the IPC bytes back to an Arrow schema
61///  let ipc_schema = root_as_schema(ipc_bytes).unwrap();
62///  let schema2 = fb_to_schema(ipc_schema);
63/// assert_eq!(schema, schema2);
64/// ```
65#[derive(Debug)]
66pub struct IpcSchemaEncoder<'a> {
67    dictionary_tracker: Option<&'a mut DictionaryTracker>,
68}
69
70impl Default for IpcSchemaEncoder<'_> {
71    fn default() -> Self {
72        Self::new()
73    }
74}
75
76impl<'a> IpcSchemaEncoder<'a> {
77    /// Create a new schema encoder
78    pub fn new() -> IpcSchemaEncoder<'a> {
79        IpcSchemaEncoder {
80            dictionary_tracker: None,
81        }
82    }
83
84    /// Specify a dictionary tracker to use
85    pub fn with_dictionary_tracker(
86        mut self,
87        dictionary_tracker: &'a mut DictionaryTracker,
88    ) -> Self {
89        self.dictionary_tracker = Some(dictionary_tracker);
90        self
91    }
92
93    /// Serialize a schema in IPC format, returning a completed [`FlatBufferBuilder`]
94    ///
95    /// Note: Call [`FlatBufferBuilder::finished_data`] to get the serialized bytes
96    pub fn schema_to_fb<'b>(&mut self, schema: &Schema) -> FlatBufferBuilder<'b> {
97        let mut fbb = FlatBufferBuilder::new();
98
99        let root = self.schema_to_fb_offset(&mut fbb, schema);
100
101        fbb.finish(root, None);
102
103        fbb
104    }
105
106    /// Serialize a schema to an in progress [`FlatBufferBuilder`], returning the in progress offset.
107    pub fn schema_to_fb_offset<'b>(
108        &mut self,
109        fbb: &mut FlatBufferBuilder<'b>,
110        schema: &Schema,
111    ) -> WIPOffset<crate::Schema<'b>> {
112        let fields = schema
113            .fields()
114            .iter()
115            .map(|field| build_field(fbb, &mut self.dictionary_tracker, field))
116            .collect::<Vec<_>>();
117        let fb_field_list = fbb.create_vector(&fields);
118
119        let fb_metadata_list =
120            (!schema.metadata().is_empty()).then(|| metadata_to_fb(fbb, schema.metadata()));
121
122        let mut builder = crate::SchemaBuilder::new(fbb);
123        builder.add_fields(fb_field_list);
124        if let Some(fb_metadata_list) = fb_metadata_list {
125            builder.add_custom_metadata(fb_metadata_list);
126        }
127        builder.finish()
128    }
129}
130
131/// Push a key-value metadata into a FlatBufferBuilder and return [WIPOffset]
132pub fn metadata_to_fb<'a>(
133    fbb: &mut FlatBufferBuilder<'a>,
134    metadata: &Metadata,
135) -> WIPOffset<Vector<'a, ForwardsUOffset<KeyValue<'a>>>> {
136    // `Metadata` iterates in deterministic (sorted) key order
137    let custom_metadata = metadata
138        .iter()
139        .map(|(k, v)| {
140            let fb_key_name = fbb.create_string(k);
141            let fb_val_name = fbb.create_string(v);
142
143            let mut kv_builder = crate::KeyValueBuilder::new(fbb);
144            kv_builder.add_key(fb_key_name);
145            kv_builder.add_value(fb_val_name);
146            kv_builder.finish()
147        })
148        .collect::<Vec<_>>();
149    fbb.create_vector(&custom_metadata)
150}
151
152/// Adds a [Schema] to a flatbuffer and returns the offset
153pub fn schema_to_fb_offset<'a>(
154    fbb: &mut FlatBufferBuilder<'a>,
155    schema: &Schema,
156) -> WIPOffset<crate::Schema<'a>> {
157    IpcSchemaEncoder::new().schema_to_fb_offset(fbb, schema)
158}
159
160/// Convert an IPC Field to Arrow Field
161impl From<crate::Field<'_>> for Field {
162    fn from(field: crate::Field) -> Field {
163        let arrow_field = if let Some(dictionary) = field.dictionary() {
164            #[allow(deprecated)]
165            Field::new_dict(
166                field.name().unwrap_or_default(),
167                get_data_type(field, true),
168                field.nullable(),
169                dictionary.id(),
170                dictionary.isOrdered(),
171            )
172        } else {
173            Field::new(
174                field.name().unwrap_or_default(),
175                get_data_type(field, true),
176                field.nullable(),
177            )
178        };
179
180        let mut metadata_map = HashMap::default();
181        if let Some(list) = field.custom_metadata() {
182            for kv in list {
183                if let (Some(k), Some(v)) = (kv.key(), kv.value()) {
184                    metadata_map.insert(k.to_string(), v.to_string());
185                }
186            }
187        }
188
189        arrow_field.with_metadata(metadata_map)
190    }
191}
192
193/// Deserialize an ipc [`crate::Schema`] from flat buffers to an arrow [Schema].
194pub fn fb_to_schema(fb: crate::Schema) -> Schema {
195    let mut fields: Vec<Field> = vec![];
196    let c_fields = fb.fields().unwrap();
197    let len = c_fields.len();
198    for i in 0..len {
199        let c_field: crate::Field = c_fields.get(i);
200        match c_field.type_type() {
201            crate::Type::Decimal if fb.endianness() == crate::Endianness::Big => {
202                unimplemented!("Big Endian is not supported for Decimal!")
203            }
204            _ => (),
205        };
206        fields.push(c_field.into());
207    }
208
209    let mut metadata: HashMap<String, String> = HashMap::default();
210    if let Some(md_fields) = fb.custom_metadata() {
211        let len = md_fields.len();
212        for i in 0..len {
213            let kv = md_fields.get(i);
214            let k_str = kv.key();
215            let v_str = kv.value();
216            if let Some(k) = k_str
217                && let Some(v) = v_str
218            {
219                metadata.insert(k.to_string(), v.to_string());
220            }
221        }
222    }
223    Schema::new_with_metadata(fields, metadata)
224}
225
226/// Try deserialize flat buffer format bytes into a schema
227pub fn try_schema_from_flatbuffer_bytes(bytes: &[u8]) -> Result<Schema, ArrowError> {
228    if let Ok(ipc) = crate::root_as_message(bytes) {
229        if let Some(schema) = ipc.header_as_schema().map(fb_to_schema) {
230            Ok(schema)
231        } else {
232            Err(ArrowError::ParseError(
233                "Unable to get head as schema".to_string(),
234            ))
235        }
236    } else {
237        Err(ArrowError::ParseError(
238            "Unable to get root as message".to_string(),
239        ))
240    }
241}
242
243/// Try deserialize the IPC format bytes into a schema
244pub fn try_schema_from_ipc_buffer(buffer: &[u8]) -> Result<Schema, ArrowError> {
245    // There are two protocol types: https://issues.apache.org/jira/browse/ARROW-6313
246    // The original protocol is:
247    //   4 bytes - the byte length of the payload
248    //   a flatbuffer Message whose header is the Schema
249    // The latest version of protocol is:
250    // The schema of the dataset in its IPC form:
251    //   4 bytes - an optional IPC_CONTINUATION_TOKEN prefix
252    //   4 bytes - the byte length of the payload
253    //   a flatbuffer Message whose header is the Schema
254    if buffer.len() < 4 {
255        return Err(ArrowError::ParseError(
256            "The buffer length is less than 4 and missing the continuation marker or length of buffer".to_string()
257        ));
258    }
259
260    let (len, buffer) = if buffer[..4] == CONTINUATION_MARKER {
261        if buffer.len() < 8 {
262            return Err(ArrowError::ParseError(
263                "The buffer length is less than 8 and missing the length of buffer".to_string(),
264            ));
265        }
266        buffer[4..].split_at(4)
267    } else {
268        buffer.split_at(4)
269    };
270
271    let len = <i32>::from_le_bytes(len.try_into().unwrap());
272    if len < 0 {
273        return Err(ArrowError::ParseError(format!(
274            "The encapsulated message's reported length is negative ({len})"
275        )));
276    }
277
278    if buffer.len() < len as usize {
279        let actual_len = buffer.len();
280        return Err(ArrowError::ParseError(format!(
281            "The buffer length ({actual_len}) is less than the encapsulated message's reported length ({len})"
282        )));
283    }
284
285    let msg = crate::root_as_message(buffer)
286        .map_err(|err| ArrowError::ParseError(format!("Unable to get root as message: {err:?}")))?;
287    let ipc_schema = msg.header_as_schema().ok_or_else(|| {
288        ArrowError::ParseError("Unable to convert flight info to a schema".to_string())
289    })?;
290    Ok(fb_to_schema(ipc_schema))
291}
292
293/// Get the Arrow data type from the flatbuffer Field table
294pub(crate) fn get_data_type(field: crate::Field, may_be_dictionary: bool) -> DataType {
295    if let Some(dictionary) = field.dictionary()
296        && may_be_dictionary
297    {
298        let int = dictionary.indexType().unwrap();
299        let index_type = match (int.bitWidth(), int.is_signed()) {
300            (8, true) => DataType::Int8,
301            (8, false) => DataType::UInt8,
302            (16, true) => DataType::Int16,
303            (16, false) => DataType::UInt16,
304            (32, true) => DataType::Int32,
305            (32, false) => DataType::UInt32,
306            (64, true) => DataType::Int64,
307            (64, false) => DataType::UInt64,
308            _ => panic!("Unexpected bitwidth and signed"),
309        };
310        return DataType::Dictionary(Box::new(index_type), Box::new(get_data_type(field, false)));
311    }
312
313    match field.type_type() {
314        crate::Type::Null => DataType::Null,
315        crate::Type::Bool => DataType::Boolean,
316        crate::Type::Int => {
317            let int = field.type_as_int().unwrap();
318            match (int.bitWidth(), int.is_signed()) {
319                (8, true) => DataType::Int8,
320                (8, false) => DataType::UInt8,
321                (16, true) => DataType::Int16,
322                (16, false) => DataType::UInt16,
323                (32, true) => DataType::Int32,
324                (32, false) => DataType::UInt32,
325                (64, true) => DataType::Int64,
326                (64, false) => DataType::UInt64,
327                z => panic!(
328                    "Int type with bit width of {} and signed of {} not supported",
329                    z.0, z.1
330                ),
331            }
332        }
333        crate::Type::Binary => DataType::Binary,
334        crate::Type::BinaryView => DataType::BinaryView,
335        crate::Type::LargeBinary => DataType::LargeBinary,
336        crate::Type::Utf8 => DataType::Utf8,
337        crate::Type::Utf8View => DataType::Utf8View,
338        crate::Type::LargeUtf8 => DataType::LargeUtf8,
339        crate::Type::FixedSizeBinary => {
340            let fsb = field.type_as_fixed_size_binary().unwrap();
341            DataType::FixedSizeBinary(fsb.byteWidth())
342        }
343        crate::Type::FloatingPoint => {
344            let float = field.type_as_floating_point().unwrap();
345            match float.precision() {
346                crate::Precision::HALF => DataType::Float16,
347                crate::Precision::SINGLE => DataType::Float32,
348                crate::Precision::DOUBLE => DataType::Float64,
349                z => panic!("FloatingPoint type with precision of {z:?} not supported"),
350            }
351        }
352        crate::Type::Date => {
353            let date = field.type_as_date().unwrap();
354            match date.unit() {
355                crate::DateUnit::DAY => DataType::Date32,
356                crate::DateUnit::MILLISECOND => DataType::Date64,
357                z => panic!("Date type with unit of {z:?} not supported"),
358            }
359        }
360        crate::Type::Time => {
361            let time = field.type_as_time().unwrap();
362            match (time.bitWidth(), time.unit()) {
363                (32, crate::TimeUnit::SECOND) => DataType::Time32(TimeUnit::Second),
364                (32, crate::TimeUnit::MILLISECOND) => DataType::Time32(TimeUnit::Millisecond),
365                (64, crate::TimeUnit::MICROSECOND) => DataType::Time64(TimeUnit::Microsecond),
366                (64, crate::TimeUnit::NANOSECOND) => DataType::Time64(TimeUnit::Nanosecond),
367                z => panic!(
368                    "Time type with bit width of {} and unit of {:?} not supported",
369                    z.0, z.1
370                ),
371            }
372        }
373        crate::Type::Timestamp => {
374            let timestamp = field.type_as_timestamp().unwrap();
375            let timezone: Option<_> = timestamp.timezone().map(|tz| tz.into());
376            match timestamp.unit() {
377                crate::TimeUnit::SECOND => DataType::Timestamp(TimeUnit::Second, timezone),
378                crate::TimeUnit::MILLISECOND => {
379                    DataType::Timestamp(TimeUnit::Millisecond, timezone)
380                }
381                crate::TimeUnit::MICROSECOND => {
382                    DataType::Timestamp(TimeUnit::Microsecond, timezone)
383                }
384                crate::TimeUnit::NANOSECOND => DataType::Timestamp(TimeUnit::Nanosecond, timezone),
385                z => panic!("Timestamp type with unit of {z:?} not supported"),
386            }
387        }
388        crate::Type::Interval => {
389            let interval = field.type_as_interval().unwrap();
390            match interval.unit() {
391                crate::IntervalUnit::YEAR_MONTH => DataType::Interval(IntervalUnit::YearMonth),
392                crate::IntervalUnit::DAY_TIME => DataType::Interval(IntervalUnit::DayTime),
393                crate::IntervalUnit::MONTH_DAY_NANO => {
394                    DataType::Interval(IntervalUnit::MonthDayNano)
395                }
396                z => panic!("Interval type with unit of {z:?} unsupported"),
397            }
398        }
399        crate::Type::Duration => {
400            let duration = field.type_as_duration().unwrap();
401            match duration.unit() {
402                crate::TimeUnit::SECOND => DataType::Duration(TimeUnit::Second),
403                crate::TimeUnit::MILLISECOND => DataType::Duration(TimeUnit::Millisecond),
404                crate::TimeUnit::MICROSECOND => DataType::Duration(TimeUnit::Microsecond),
405                crate::TimeUnit::NANOSECOND => DataType::Duration(TimeUnit::Nanosecond),
406                z => panic!("Duration type with unit of {z:?} unsupported"),
407            }
408        }
409        crate::Type::List => {
410            let children = field.children().unwrap();
411            if children.len() != 1 {
412                panic!("expect a list to have one child")
413            }
414            DataType::List(Arc::new(children.get(0).into()))
415        }
416        crate::Type::LargeList => {
417            let children = field.children().unwrap();
418            if children.len() != 1 {
419                panic!("expect a large list to have one child")
420            }
421            DataType::LargeList(Arc::new(children.get(0).into()))
422        }
423        crate::Type::ListView => {
424            let children = field.children().unwrap();
425            if children.len() != 1 {
426                panic!("expect a listview to have one child")
427            }
428            DataType::ListView(Arc::new(children.get(0).into()))
429        }
430        crate::Type::LargeListView => {
431            let children = field.children().unwrap();
432            if children.len() != 1 {
433                panic!("expect a large listview to have one child")
434            }
435            DataType::LargeListView(Arc::new(children.get(0).into()))
436        }
437        crate::Type::FixedSizeList => {
438            let children = field.children().unwrap();
439            if children.len() != 1 {
440                panic!("expect a list to have one child")
441            }
442            let fsl = field.type_as_fixed_size_list().unwrap();
443            DataType::FixedSizeList(Arc::new(children.get(0).into()), fsl.listSize())
444        }
445        crate::Type::Struct_ => {
446            let fields = match field.children() {
447                Some(children) => children.iter().map(Field::from).collect(),
448                None => Fields::empty(),
449            };
450            DataType::Struct(fields)
451        }
452        crate::Type::RunEndEncoded => {
453            let children = field.children().unwrap();
454            if children.len() != 2 {
455                panic!(
456                    "RunEndEncoded type should have exactly two children. Found {}",
457                    children.len()
458                )
459            }
460            let run_ends_field = children.get(0).into();
461            let values_field = children.get(1).into();
462            DataType::RunEndEncoded(Arc::new(run_ends_field), Arc::new(values_field))
463        }
464        crate::Type::Map => {
465            let map = field.type_as_map().unwrap();
466            let children = field.children().unwrap();
467            if children.len() != 1 {
468                panic!("expect a map to have one child")
469            }
470            DataType::Map(Arc::new(children.get(0).into()), map.keysSorted())
471        }
472        crate::Type::Decimal => {
473            let fsb = field.type_as_decimal().unwrap();
474            let bit_width = fsb.bitWidth();
475            let precision: u8 = fsb.precision().try_into().unwrap();
476            let scale: i8 = fsb.scale().try_into().unwrap();
477            match bit_width {
478                32 => DataType::Decimal32(precision, scale),
479                64 => DataType::Decimal64(precision, scale),
480                128 => DataType::Decimal128(precision, scale),
481                256 => DataType::Decimal256(precision, scale),
482                _ => panic!("Unexpected decimal bit width {bit_width}"),
483            }
484        }
485        crate::Type::Union => {
486            let union = field.type_as_union().unwrap();
487
488            let union_mode = match union.mode() {
489                crate::UnionMode::Dense => UnionMode::Dense,
490                crate::UnionMode::Sparse => UnionMode::Sparse,
491                mode => panic!("Unexpected union mode: {mode:?}"),
492            };
493
494            let mut fields = vec![];
495            if let Some(children) = field.children() {
496                for i in 0..children.len() {
497                    fields.push(Field::from(children.get(i)));
498                }
499            };
500
501            let fields = match union.typeIds() {
502                None => UnionFields::from_fields(fields),
503                Some(ids) => UnionFields::try_new(ids.iter().map(|i| i as i8), fields)
504                    .expect("invalid union field"),
505            };
506
507            DataType::Union(fields, union_mode)
508        }
509        t => unimplemented!("Type {:?} not supported", t),
510    }
511}
512
513pub(crate) struct FBFieldType<'b> {
514    pub(crate) type_type: crate::Type,
515    pub(crate) type_: WIPOffset<UnionWIPOffset>,
516    pub(crate) children: Option<WIPOffset<Vector<'b, ForwardsUOffset<crate::Field<'b>>>>>,
517}
518
519/// Create an IPC Field from an Arrow Field
520pub(crate) fn build_field<'a>(
521    fbb: &mut FlatBufferBuilder<'a>,
522    dictionary_tracker: &mut Option<&mut DictionaryTracker>,
523    field: &Field,
524) -> WIPOffset<crate::Field<'a>> {
525    // Optional custom metadata.
526    let mut fb_metadata = None;
527    if !field.metadata().is_empty() {
528        fb_metadata = Some(metadata_to_fb(fbb, field.metadata()));
529    };
530
531    let fb_field_name = fbb.create_string(field.name().as_str());
532    let field_type = get_fb_field_type(field.data_type(), dictionary_tracker, fbb);
533
534    let fb_dictionary = if let Dictionary(index_type, _) = field.data_type() {
535        match dictionary_tracker {
536            Some(tracker) => Some(get_fb_dictionary(
537                index_type,
538                tracker.next_dict_id(),
539                field
540                    .dict_is_ordered()
541                    .expect("All Dictionary types have `dict_is_ordered`"),
542                fbb,
543            )),
544            None => panic!("IPC must no longer be used without dictionary tracker"),
545        }
546    } else {
547        None
548    };
549
550    let mut field_builder = crate::FieldBuilder::new(fbb);
551    field_builder.add_name(fb_field_name);
552    if let Some(dictionary) = fb_dictionary {
553        field_builder.add_dictionary(dictionary)
554    }
555    field_builder.add_type_type(field_type.type_type);
556    field_builder.add_nullable(field.is_nullable());
557    match field_type.children {
558        None => {}
559        Some(children) => field_builder.add_children(children),
560    };
561    field_builder.add_type_(field_type.type_);
562
563    if let Some(fb_metadata) = fb_metadata {
564        field_builder.add_custom_metadata(fb_metadata);
565    }
566
567    field_builder.finish()
568}
569
570/// Get the IPC type of a data type
571pub(crate) fn get_fb_field_type<'a>(
572    data_type: &DataType,
573    dictionary_tracker: &mut Option<&mut DictionaryTracker>,
574    fbb: &mut FlatBufferBuilder<'a>,
575) -> FBFieldType<'a> {
576    // some IPC implementations expect an empty list for child data, instead of a null value.
577    // An empty field list is thus returned for primitive types
578    let empty_fields: Vec<WIPOffset<crate::Field>> = vec![];
579    match data_type {
580        Null => FBFieldType {
581            type_type: crate::Type::Null,
582            type_: crate::NullBuilder::new(fbb).finish().as_union_value(),
583            children: Some(fbb.create_vector(&empty_fields[..])),
584        },
585        Boolean => FBFieldType {
586            type_type: crate::Type::Bool,
587            type_: crate::BoolBuilder::new(fbb).finish().as_union_value(),
588            children: Some(fbb.create_vector(&empty_fields[..])),
589        },
590        UInt8 | UInt16 | UInt32 | UInt64 => {
591            let children = fbb.create_vector(&empty_fields[..]);
592            let mut builder = crate::IntBuilder::new(fbb);
593            builder.add_is_signed(false);
594            match data_type {
595                UInt8 => builder.add_bitWidth(8),
596                UInt16 => builder.add_bitWidth(16),
597                UInt32 => builder.add_bitWidth(32),
598                UInt64 => builder.add_bitWidth(64),
599                _ => {}
600            };
601            FBFieldType {
602                type_type: crate::Type::Int,
603                type_: builder.finish().as_union_value(),
604                children: Some(children),
605            }
606        }
607        Int8 | Int16 | Int32 | Int64 => {
608            let children = fbb.create_vector(&empty_fields[..]);
609            let mut builder = crate::IntBuilder::new(fbb);
610            builder.add_is_signed(true);
611            match data_type {
612                Int8 => builder.add_bitWidth(8),
613                Int16 => builder.add_bitWidth(16),
614                Int32 => builder.add_bitWidth(32),
615                Int64 => builder.add_bitWidth(64),
616                _ => {}
617            };
618            FBFieldType {
619                type_type: crate::Type::Int,
620                type_: builder.finish().as_union_value(),
621                children: Some(children),
622            }
623        }
624        Float16 | Float32 | Float64 => {
625            let children = fbb.create_vector(&empty_fields[..]);
626            let mut builder = crate::FloatingPointBuilder::new(fbb);
627            match data_type {
628                Float16 => builder.add_precision(crate::Precision::HALF),
629                Float32 => builder.add_precision(crate::Precision::SINGLE),
630                Float64 => builder.add_precision(crate::Precision::DOUBLE),
631                _ => {}
632            };
633            FBFieldType {
634                type_type: crate::Type::FloatingPoint,
635                type_: builder.finish().as_union_value(),
636                children: Some(children),
637            }
638        }
639        Binary => FBFieldType {
640            type_type: crate::Type::Binary,
641            type_: crate::BinaryBuilder::new(fbb).finish().as_union_value(),
642            children: Some(fbb.create_vector(&empty_fields[..])),
643        },
644        LargeBinary => FBFieldType {
645            type_type: crate::Type::LargeBinary,
646            type_: crate::LargeBinaryBuilder::new(fbb)
647                .finish()
648                .as_union_value(),
649            children: Some(fbb.create_vector(&empty_fields[..])),
650        },
651        BinaryView => FBFieldType {
652            type_type: crate::Type::BinaryView,
653            type_: crate::BinaryViewBuilder::new(fbb).finish().as_union_value(),
654            children: Some(fbb.create_vector(&empty_fields[..])),
655        },
656        Utf8View => FBFieldType {
657            type_type: crate::Type::Utf8View,
658            type_: crate::Utf8ViewBuilder::new(fbb).finish().as_union_value(),
659            children: Some(fbb.create_vector(&empty_fields[..])),
660        },
661        Utf8 => FBFieldType {
662            type_type: crate::Type::Utf8,
663            type_: crate::Utf8Builder::new(fbb).finish().as_union_value(),
664            children: Some(fbb.create_vector(&empty_fields[..])),
665        },
666        LargeUtf8 => FBFieldType {
667            type_type: crate::Type::LargeUtf8,
668            type_: crate::LargeUtf8Builder::new(fbb).finish().as_union_value(),
669            children: Some(fbb.create_vector(&empty_fields[..])),
670        },
671        FixedSizeBinary(len) => {
672            let mut builder = crate::FixedSizeBinaryBuilder::new(fbb);
673            builder.add_byteWidth(*len);
674            FBFieldType {
675                type_type: crate::Type::FixedSizeBinary,
676                type_: builder.finish().as_union_value(),
677                children: Some(fbb.create_vector(&empty_fields[..])),
678            }
679        }
680        Date32 => {
681            let mut builder = crate::DateBuilder::new(fbb);
682            builder.add_unit(crate::DateUnit::DAY);
683            FBFieldType {
684                type_type: crate::Type::Date,
685                type_: builder.finish().as_union_value(),
686                children: Some(fbb.create_vector(&empty_fields[..])),
687            }
688        }
689        Date64 => {
690            let mut builder = crate::DateBuilder::new(fbb);
691            builder.add_unit(crate::DateUnit::MILLISECOND);
692            FBFieldType {
693                type_type: crate::Type::Date,
694                type_: builder.finish().as_union_value(),
695                children: Some(fbb.create_vector(&empty_fields[..])),
696            }
697        }
698        Time32(unit) | Time64(unit) => {
699            let mut builder = crate::TimeBuilder::new(fbb);
700            match unit {
701                TimeUnit::Second => {
702                    builder.add_bitWidth(32);
703                    builder.add_unit(crate::TimeUnit::SECOND);
704                }
705                TimeUnit::Millisecond => {
706                    builder.add_bitWidth(32);
707                    builder.add_unit(crate::TimeUnit::MILLISECOND);
708                }
709                TimeUnit::Microsecond => {
710                    builder.add_bitWidth(64);
711                    builder.add_unit(crate::TimeUnit::MICROSECOND);
712                }
713                TimeUnit::Nanosecond => {
714                    builder.add_bitWidth(64);
715                    builder.add_unit(crate::TimeUnit::NANOSECOND);
716                }
717            }
718            FBFieldType {
719                type_type: crate::Type::Time,
720                type_: builder.finish().as_union_value(),
721                children: Some(fbb.create_vector(&empty_fields[..])),
722            }
723        }
724        Timestamp(unit, tz) => {
725            let tz = tz.as_deref().unwrap_or_default();
726            let tz_str = fbb.create_string(tz);
727            let mut builder = crate::TimestampBuilder::new(fbb);
728            let time_unit = match unit {
729                TimeUnit::Second => crate::TimeUnit::SECOND,
730                TimeUnit::Millisecond => crate::TimeUnit::MILLISECOND,
731                TimeUnit::Microsecond => crate::TimeUnit::MICROSECOND,
732                TimeUnit::Nanosecond => crate::TimeUnit::NANOSECOND,
733            };
734            builder.add_unit(time_unit);
735            if !tz.is_empty() {
736                builder.add_timezone(tz_str);
737            }
738            FBFieldType {
739                type_type: crate::Type::Timestamp,
740                type_: builder.finish().as_union_value(),
741                children: Some(fbb.create_vector(&empty_fields[..])),
742            }
743        }
744        Interval(unit) => {
745            let mut builder = crate::IntervalBuilder::new(fbb);
746            let interval_unit = match unit {
747                IntervalUnit::YearMonth => crate::IntervalUnit::YEAR_MONTH,
748                IntervalUnit::DayTime => crate::IntervalUnit::DAY_TIME,
749                IntervalUnit::MonthDayNano => crate::IntervalUnit::MONTH_DAY_NANO,
750            };
751            builder.add_unit(interval_unit);
752            FBFieldType {
753                type_type: crate::Type::Interval,
754                type_: builder.finish().as_union_value(),
755                children: Some(fbb.create_vector(&empty_fields[..])),
756            }
757        }
758        Duration(unit) => {
759            let mut builder = crate::DurationBuilder::new(fbb);
760            let time_unit = match unit {
761                TimeUnit::Second => crate::TimeUnit::SECOND,
762                TimeUnit::Millisecond => crate::TimeUnit::MILLISECOND,
763                TimeUnit::Microsecond => crate::TimeUnit::MICROSECOND,
764                TimeUnit::Nanosecond => crate::TimeUnit::NANOSECOND,
765            };
766            builder.add_unit(time_unit);
767            FBFieldType {
768                type_type: crate::Type::Duration,
769                type_: builder.finish().as_union_value(),
770                children: Some(fbb.create_vector(&empty_fields[..])),
771            }
772        }
773        List(list_type) => {
774            let child = build_field(fbb, dictionary_tracker, list_type);
775            FBFieldType {
776                type_type: crate::Type::List,
777                type_: crate::ListBuilder::new(fbb).finish().as_union_value(),
778                children: Some(fbb.create_vector(&[child])),
779            }
780        }
781        ListView(list_type) => {
782            let child = build_field(fbb, dictionary_tracker, list_type);
783            FBFieldType {
784                type_type: crate::Type::ListView,
785                type_: crate::ListViewBuilder::new(fbb).finish().as_union_value(),
786                children: Some(fbb.create_vector(&[child])),
787            }
788        }
789        LargeListView(list_type) => {
790            let child = build_field(fbb, dictionary_tracker, list_type);
791            FBFieldType {
792                type_type: crate::Type::LargeListView,
793                type_: crate::LargeListViewBuilder::new(fbb)
794                    .finish()
795                    .as_union_value(),
796                children: Some(fbb.create_vector(&[child])),
797            }
798        }
799        LargeList(list_type) => {
800            let child = build_field(fbb, dictionary_tracker, list_type);
801            FBFieldType {
802                type_type: crate::Type::LargeList,
803                type_: crate::LargeListBuilder::new(fbb).finish().as_union_value(),
804                children: Some(fbb.create_vector(&[child])),
805            }
806        }
807        FixedSizeList(list_type, len) => {
808            let child = build_field(fbb, dictionary_tracker, list_type);
809            let mut builder = crate::FixedSizeListBuilder::new(fbb);
810            builder.add_listSize(*len);
811            FBFieldType {
812                type_type: crate::Type::FixedSizeList,
813                type_: builder.finish().as_union_value(),
814                children: Some(fbb.create_vector(&[child])),
815            }
816        }
817        Struct(fields) => {
818            // struct's fields are children
819            let mut children = vec![];
820            for field in fields {
821                children.push(build_field(fbb, dictionary_tracker, field));
822            }
823            FBFieldType {
824                type_type: crate::Type::Struct_,
825                type_: crate::Struct_Builder::new(fbb).finish().as_union_value(),
826                children: Some(fbb.create_vector(&children[..])),
827            }
828        }
829        RunEndEncoded(run_ends, values) => {
830            let run_ends_field = build_field(fbb, dictionary_tracker, run_ends);
831            let values_field = build_field(fbb, dictionary_tracker, values);
832            let children = [run_ends_field, values_field];
833            FBFieldType {
834                type_type: crate::Type::RunEndEncoded,
835                type_: crate::RunEndEncodedBuilder::new(fbb)
836                    .finish()
837                    .as_union_value(),
838                children: Some(fbb.create_vector(&children[..])),
839            }
840        }
841        Map(map_field, keys_sorted) => {
842            let child = build_field(fbb, dictionary_tracker, map_field);
843            let mut field_type = crate::MapBuilder::new(fbb);
844            field_type.add_keysSorted(*keys_sorted);
845            FBFieldType {
846                type_type: crate::Type::Map,
847                type_: field_type.finish().as_union_value(),
848                children: Some(fbb.create_vector(&[child])),
849            }
850        }
851        Dictionary(_, value_type) => {
852            // In this library, the dictionary "type" is a logical construct. Here we
853            // pass through to the value type, as we've already captured the index
854            // type in the DictionaryEncoding metadata in the parent field
855            get_fb_field_type(value_type, dictionary_tracker, fbb)
856        }
857        Decimal32(precision, scale) => {
858            let mut builder = crate::DecimalBuilder::new(fbb);
859            builder.add_precision(*precision as i32);
860            builder.add_scale(*scale as i32);
861            builder.add_bitWidth(32);
862            FBFieldType {
863                type_type: crate::Type::Decimal,
864                type_: builder.finish().as_union_value(),
865                children: Some(fbb.create_vector(&empty_fields[..])),
866            }
867        }
868        Decimal64(precision, scale) => {
869            let mut builder = crate::DecimalBuilder::new(fbb);
870            builder.add_precision(*precision as i32);
871            builder.add_scale(*scale as i32);
872            builder.add_bitWidth(64);
873            FBFieldType {
874                type_type: crate::Type::Decimal,
875                type_: builder.finish().as_union_value(),
876                children: Some(fbb.create_vector(&empty_fields[..])),
877            }
878        }
879        Decimal128(precision, scale) => {
880            let mut builder = crate::DecimalBuilder::new(fbb);
881            builder.add_precision(*precision as i32);
882            builder.add_scale(*scale as i32);
883            builder.add_bitWidth(128);
884            FBFieldType {
885                type_type: crate::Type::Decimal,
886                type_: builder.finish().as_union_value(),
887                children: Some(fbb.create_vector(&empty_fields[..])),
888            }
889        }
890        Decimal256(precision, scale) => {
891            let mut builder = crate::DecimalBuilder::new(fbb);
892            builder.add_precision(*precision as i32);
893            builder.add_scale(*scale as i32);
894            builder.add_bitWidth(256);
895            FBFieldType {
896                type_type: crate::Type::Decimal,
897                type_: builder.finish().as_union_value(),
898                children: Some(fbb.create_vector(&empty_fields[..])),
899            }
900        }
901        Union(fields, mode) => {
902            let mut children = vec![];
903            for (_, field) in fields.iter() {
904                children.push(build_field(fbb, dictionary_tracker, field));
905            }
906
907            let union_mode = match mode {
908                UnionMode::Sparse => crate::UnionMode::Sparse,
909                UnionMode::Dense => crate::UnionMode::Dense,
910            };
911
912            let fbb_type_ids =
913                fbb.create_vector(&fields.iter().map(|(t, _)| t as i32).collect::<Vec<_>>());
914            let mut builder = crate::UnionBuilder::new(fbb);
915            builder.add_mode(union_mode);
916            builder.add_typeIds(fbb_type_ids);
917
918            FBFieldType {
919                type_type: crate::Type::Union,
920                type_: builder.finish().as_union_value(),
921                children: Some(fbb.create_vector(&children[..])),
922            }
923        }
924    }
925}
926
927/// Create an IPC dictionary encoding
928pub(crate) fn get_fb_dictionary<'a>(
929    index_type: &DataType,
930    dict_id: i64,
931    dict_is_ordered: bool,
932    fbb: &mut FlatBufferBuilder<'a>,
933) -> WIPOffset<crate::DictionaryEncoding<'a>> {
934    // We assume that the dictionary index type (as an integer) has already been
935    // validated elsewhere, and can safely assume we are dealing with integers
936    let mut index_builder = crate::IntBuilder::new(fbb);
937
938    match *index_type {
939        Int8 | Int16 | Int32 | Int64 => index_builder.add_is_signed(true),
940        UInt8 | UInt16 | UInt32 | UInt64 => index_builder.add_is_signed(false),
941        _ => {}
942    }
943
944    match *index_type {
945        Int8 | UInt8 => index_builder.add_bitWidth(8),
946        Int16 | UInt16 => index_builder.add_bitWidth(16),
947        Int32 | UInt32 => index_builder.add_bitWidth(32),
948        Int64 | UInt64 => index_builder.add_bitWidth(64),
949        _ => {}
950    }
951
952    let index_builder = index_builder.finish();
953
954    let mut builder = crate::DictionaryEncodingBuilder::new(fbb);
955    builder.add_id(dict_id);
956    builder.add_indexType(index_builder);
957    builder.add_isOrdered(dict_is_ordered);
958
959    builder.finish()
960}
961
962/// An owned container for a validated [`Message`]
963///
964/// Safely decoding a flatbuffer requires validating the various embedded offsets,
965/// see [`Verifier`]. This is a potentially expensive operation, and it is therefore desirable
966/// to only do this once. [`crate::root_as_message`] performs this validation on construction,
967/// however, it returns a [`Message`] borrowing the provided byte slice. This prevents
968/// storing this [`Message`] in the same data structure that owns the buffer, as this
969/// would require self-referential borrows.
970///
971/// [`MessageBuffer`] solves this problem by providing a safe API for a [`Message`]
972/// without a lifetime bound.
973#[derive(Clone)]
974pub struct MessageBuffer(Buffer);
975
976impl Debug for MessageBuffer {
977    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
978        self.as_ref().fmt(f)
979    }
980}
981
982impl MessageBuffer {
983    /// Try to create a [`MessageBuffer`] from the provided [`Buffer`]
984    pub fn try_new(buf: Buffer) -> Result<Self, ArrowError> {
985        let opts = VerifierOptions::default();
986        let mut v = Verifier::new(&opts, &buf);
987        <ForwardsUOffset<Message>>::run_verifier(&mut v, 0).map_err(|err| {
988            ArrowError::ParseError(format!("Unable to get root as message: {err:?}"))
989        })?;
990        Ok(Self(buf))
991    }
992
993    /// Return the [`Message`]
994    #[inline]
995    pub fn as_ref(&self) -> Message<'_> {
996        // SAFETY: Run verifier on construction
997        unsafe { crate::root_as_message_unchecked(&self.0) }
998    }
999}
1000
1001#[cfg(test)]
1002mod tests {
1003    use super::*;
1004
1005    #[test]
1006    fn convert_schema_round_trip() {
1007        let md: HashMap<String, String> = [("Key".to_string(), "value".to_string())]
1008            .iter()
1009            .cloned()
1010            .collect();
1011        let field_md: HashMap<String, String> = [("k".to_string(), "v".to_string())]
1012            .iter()
1013            .cloned()
1014            .collect();
1015        let schema = Schema::new_with_metadata(
1016            vec![
1017                Field::new("uint8", DataType::UInt8, false).with_metadata(field_md),
1018                Field::new("uint16", DataType::UInt16, true),
1019                Field::new("uint32", DataType::UInt32, false),
1020                Field::new("uint64", DataType::UInt64, true),
1021                Field::new("int8", DataType::Int8, true),
1022                Field::new("int16", DataType::Int16, false),
1023                Field::new("int32", DataType::Int32, true),
1024                Field::new("int64", DataType::Int64, false),
1025                Field::new("float16", DataType::Float16, true),
1026                Field::new("float32", DataType::Float32, false),
1027                Field::new("float64", DataType::Float64, true),
1028                Field::new("null", DataType::Null, false),
1029                Field::new("bool", DataType::Boolean, false),
1030                Field::new("date32", DataType::Date32, false),
1031                Field::new("date64", DataType::Date64, true),
1032                Field::new("time32[s]", DataType::Time32(TimeUnit::Second), true),
1033                Field::new("time32[ms]", DataType::Time32(TimeUnit::Millisecond), false),
1034                Field::new("time64[us]", DataType::Time64(TimeUnit::Microsecond), false),
1035                Field::new("time64[ns]", DataType::Time64(TimeUnit::Nanosecond), true),
1036                Field::new(
1037                    "timestamp[s]",
1038                    DataType::Timestamp(TimeUnit::Second, None),
1039                    false,
1040                ),
1041                Field::new(
1042                    "timestamp[ms]",
1043                    DataType::Timestamp(TimeUnit::Millisecond, None),
1044                    true,
1045                ),
1046                Field::new(
1047                    "timestamp[us]",
1048                    DataType::Timestamp(TimeUnit::Microsecond, Some("Africa/Johannesburg".into())),
1049                    false,
1050                ),
1051                Field::new(
1052                    "timestamp[ns]",
1053                    DataType::Timestamp(TimeUnit::Nanosecond, None),
1054                    true,
1055                ),
1056                Field::new(
1057                    "interval[ym]",
1058                    DataType::Interval(IntervalUnit::YearMonth),
1059                    true,
1060                ),
1061                Field::new(
1062                    "interval[dt]",
1063                    DataType::Interval(IntervalUnit::DayTime),
1064                    true,
1065                ),
1066                Field::new(
1067                    "interval[mdn]",
1068                    DataType::Interval(IntervalUnit::MonthDayNano),
1069                    true,
1070                ),
1071                Field::new("utf8", DataType::Utf8, false),
1072                Field::new("utf8_view", DataType::Utf8View, false),
1073                Field::new("binary", DataType::Binary, false),
1074                Field::new("binary_view", DataType::BinaryView, false),
1075                Field::new_list(
1076                    "list[u8]",
1077                    Field::new_list_field(DataType::UInt8, false),
1078                    true,
1079                ),
1080                Field::new_fixed_size_list(
1081                    "fixed_size_list[u8]",
1082                    Field::new_list_field(DataType::UInt8, false),
1083                    2,
1084                    true,
1085                ),
1086                Field::new_list(
1087                    "list[struct<float32, int32, bool>]",
1088                    Field::new_struct(
1089                        "struct",
1090                        vec![
1091                            Field::new("float32", UInt8, false),
1092                            Field::new("int32", Int32, true),
1093                            Field::new("bool", Boolean, true),
1094                        ],
1095                        true,
1096                    ),
1097                    false,
1098                ),
1099                Field::new_struct(
1100                    "struct<dictionary<int32, utf8>>",
1101                    vec![Field::new(
1102                        "dictionary<int32, utf8>",
1103                        Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
1104                        false,
1105                    )],
1106                    false,
1107                ),
1108                Field::new_struct(
1109                    "struct<int64, list[struct<date32, list[struct<>]>]>",
1110                    vec![
1111                        Field::new("int64", DataType::Int64, true),
1112                        Field::new_list(
1113                            "list[struct<date32, list[struct<>]>]",
1114                            Field::new_struct(
1115                                "struct",
1116                                vec![
1117                                    Field::new("date32", DataType::Date32, true),
1118                                    Field::new_list(
1119                                        "list[struct<>]",
1120                                        Field::new(
1121                                            "struct",
1122                                            DataType::Struct(Fields::empty()),
1123                                            false,
1124                                        ),
1125                                        false,
1126                                    ),
1127                                ],
1128                                false,
1129                            ),
1130                            false,
1131                        ),
1132                    ],
1133                    false,
1134                ),
1135                Field::new_union(
1136                    "union<int64, list[union<date32, list[union<>]>]>",
1137                    vec![0, 1],
1138                    vec![
1139                        Field::new("int64", DataType::Int64, true),
1140                        Field::new_list(
1141                            "list[union<date32, list[union<>]>]",
1142                            Field::new_union(
1143                                "union<date32, list[union<>]>",
1144                                vec![0, 1],
1145                                vec![
1146                                    Field::new("date32", DataType::Date32, true),
1147                                    Field::new_list(
1148                                        "list[union<>]",
1149                                        Field::new(
1150                                            "union",
1151                                            DataType::Union(
1152                                                UnionFields::empty(),
1153                                                UnionMode::Sparse,
1154                                            ),
1155                                            false,
1156                                        ),
1157                                        false,
1158                                    ),
1159                                ],
1160                                UnionMode::Dense,
1161                            ),
1162                            false,
1163                        ),
1164                    ],
1165                    UnionMode::Sparse,
1166                ),
1167                Field::new("struct<>", DataType::Struct(Fields::empty()), true),
1168                Field::new(
1169                    "union<>",
1170                    DataType::Union(UnionFields::empty(), UnionMode::Dense),
1171                    true,
1172                ),
1173                Field::new(
1174                    "union<>",
1175                    DataType::Union(UnionFields::empty(), UnionMode::Sparse),
1176                    true,
1177                ),
1178                Field::new(
1179                    "union<int32, utf8>",
1180                    DataType::Union(
1181                        UnionFields::try_new(
1182                            vec![2, 3], // non-default type ids
1183                            vec![
1184                                Field::new("int32", DataType::Int32, true),
1185                                Field::new("utf8", DataType::Utf8, true),
1186                            ],
1187                        )
1188                        .unwrap(),
1189                        UnionMode::Dense,
1190                    ),
1191                    true,
1192                ),
1193                #[allow(deprecated)]
1194                Field::new_dict(
1195                    "dictionary<int32, utf8>",
1196                    DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
1197                    true,
1198                    123,
1199                    true,
1200                ),
1201                #[allow(deprecated)]
1202                Field::new_dict(
1203                    "dictionary<uint8, uint32>",
1204                    DataType::Dictionary(Box::new(DataType::UInt8), Box::new(DataType::UInt32)),
1205                    true,
1206                    123,
1207                    true,
1208                ),
1209                Field::new("decimal<usize, usize>", DataType::Decimal128(10, 6), false),
1210            ],
1211            md,
1212        );
1213
1214        let mut dictionary_tracker = DictionaryTracker::new(true);
1215        let fb = IpcSchemaEncoder::new()
1216            .with_dictionary_tracker(&mut dictionary_tracker)
1217            .schema_to_fb(&schema);
1218
1219        // read back fields
1220        let ipc = crate::root_as_schema(fb.finished_data()).unwrap();
1221        let schema2 = fb_to_schema(ipc);
1222        assert_eq!(schema, schema2);
1223    }
1224
1225    #[test]
1226    fn schema_from_bytes() {
1227        // Bytes of a schema generated via following python code, using pyarrow 10.0.1:
1228        //
1229        // import pyarrow as pa
1230        // schema = pa.schema([pa.field('field1', pa.uint32(), nullable=False)])
1231        // sink = pa.BufferOutputStream()
1232        // with pa.ipc.new_stream(sink, schema) as writer:
1233        //     pass
1234        // # stripping continuation & length prefix & suffix bytes to get only schema bytes
1235        // [x for x in sink.getvalue().to_pybytes()][8:-8]
1236        let bytes: Vec<u8> = vec![
1237            16, 0, 0, 0, 0, 0, 10, 0, 12, 0, 6, 0, 5, 0, 8, 0, 10, 0, 0, 0, 0, 1, 4, 0, 12, 0, 0,
1238            0, 8, 0, 8, 0, 0, 0, 4, 0, 8, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 20, 0, 0, 0, 16, 0, 20,
1239            0, 8, 0, 0, 0, 7, 0, 12, 0, 0, 0, 16, 0, 16, 0, 0, 0, 0, 0, 0, 2, 16, 0, 0, 0, 32, 0,
1240            0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 102, 105, 101, 108, 100, 49, 0, 0, 0, 0, 6,
1241            0, 8, 0, 4, 0, 6, 0, 0, 0, 32, 0, 0, 0,
1242        ];
1243        let ipc = crate::root_as_message(&bytes).unwrap();
1244        let schema = ipc.header_as_schema().unwrap();
1245
1246        // generate same message with Rust
1247        let data_gen = crate::writer::IpcDataGenerator::default();
1248        let mut dictionary_tracker = DictionaryTracker::new(true);
1249        let arrow_schema = Schema::new(vec![Field::new("field1", DataType::UInt32, false)]);
1250        let bytes = data_gen
1251            .schema_to_bytes_with_dictionary_tracker(
1252                &arrow_schema,
1253                &mut dictionary_tracker,
1254                &crate::writer::IpcWriteOptions::default(),
1255            )
1256            .ipc_message;
1257
1258        let ipc2 = crate::root_as_message(&bytes).unwrap();
1259        let schema2 = ipc2.header_as_schema().unwrap();
1260
1261        // can't compare schema directly as it compares the underlying bytes, which can differ
1262        assert!(schema.custom_metadata().is_none());
1263        assert!(schema2.custom_metadata().is_none());
1264        assert_eq!(schema.endianness(), schema2.endianness());
1265        assert!(schema.features().is_none());
1266        assert!(schema2.features().is_none());
1267        assert_eq!(fb_to_schema(schema), fb_to_schema(schema2));
1268
1269        assert_eq!(ipc.version(), ipc2.version());
1270        assert_eq!(ipc.header_type(), ipc2.header_type());
1271        assert_eq!(ipc.bodyLength(), ipc2.bodyLength());
1272        assert!(ipc.custom_metadata().is_none());
1273        assert!(ipc2.custom_metadata().is_none());
1274    }
1275}
arrow_ipc/convert.rs

arrow_ipc/
convert.rs
