1use crate::{
25 ArrowError, DataType, Field, FieldRef, IntervalUnit, Schema, TimeUnit, UnionFields, UnionMode,
26};
27use bitflags::bitflags;
28use std::borrow::Cow;
29use std::sync::Arc;
30use std::{
31 collections::HashMap,
32 ffi::{CStr, CString, c_char, c_void},
33};
34
35bitflags! {
36 pub struct Flags: i64 {
41 const DICTIONARY_ORDERED = 0b00000001;
43 const NULLABLE = 0b00000010;
45 const MAP_KEYS_SORTED = 0b00000100;
47 }
48}
49
50#[repr(C)]
62#[derive(Debug)]
63pub struct FFI_ArrowSchema {
64 format: *const c_char,
68 name: *const c_char,
70 metadata: *const c_char,
72 flags: i64,
75 n_children: i64,
77 children: *mut *mut FFI_ArrowSchema,
79 dictionary: *mut FFI_ArrowSchema,
81 release: Option<unsafe extern "C" fn(arg1: *mut FFI_ArrowSchema)>,
83 private_data: *mut c_void,
85}
86
87struct SchemaPrivateData {
88 children: Box<[*mut FFI_ArrowSchema]>,
89 dictionary: *mut FFI_ArrowSchema,
90 metadata: Option<Vec<u8>>,
91}
92
93unsafe extern "C" fn release_schema(schema: *mut FFI_ArrowSchema) {
95 if schema.is_null() {
96 return;
97 }
98 let schema = unsafe { &mut *schema };
99
100 drop(unsafe { CString::from_raw(schema.format.cast_mut()) });
102 if !schema.name.is_null() {
103 drop(unsafe { CString::from_raw(schema.name.cast_mut()) });
104 }
105 if !schema.private_data.is_null() {
106 let private_data =
107 unsafe { Box::from_raw(schema.private_data.cast::<SchemaPrivateData>()) };
108 for child in &private_data.children {
109 drop(unsafe { Box::from_raw(*child) })
110 }
111 if !private_data.dictionary.is_null() {
112 drop(unsafe { Box::from_raw(private_data.dictionary) });
113 }
114
115 drop(private_data);
116 }
117
118 schema.release = None;
119}
120
121impl FFI_ArrowSchema {
122 pub fn try_new(
129 format: &str,
130 children: Vec<FFI_ArrowSchema>,
131 dictionary: Option<FFI_ArrowSchema>,
132 ) -> Result<Self, ArrowError> {
133 let format = CString::new(format).map_err(|err| {
136 ArrowError::CDataInterface(format!(
137 "Null byte at position {} not allowed in format",
138 err.nul_position()
139 ))
140 })?;
141
142 let mut this = Self::empty();
143
144 let children_ptr = children
145 .into_iter()
146 .map(Box::new)
147 .map(Box::into_raw)
148 .collect::<Box<_>>();
149
150 this.format = format.into_raw();
151 this.release = Some(release_schema);
152 this.n_children = children_ptr.len() as i64;
153
154 let dictionary_ptr = dictionary
155 .map(|d| Box::into_raw(Box::new(d)))
156 .unwrap_or(std::ptr::null_mut());
157
158 let mut private_data = Box::new(SchemaPrivateData {
159 children: children_ptr,
160 dictionary: dictionary_ptr,
161 metadata: None,
162 });
163
164 this.children = private_data.children.as_mut_ptr();
166
167 this.dictionary = dictionary_ptr;
168
169 this.private_data = Box::into_raw(private_data).cast::<c_void>();
170
171 Ok(this)
172 }
173
174 pub fn with_name(mut self, name: &str) -> Result<Self, ArrowError> {
176 self.name = CString::new(name)
177 .map_err(|e| {
178 ArrowError::CDataInterface(format!(
179 "Null byte at position {} not allowed in name",
180 e.nul_position()
181 ))
182 })?
183 .into_raw();
184 Ok(self)
185 }
186
187 pub fn with_flags(mut self, flags: Flags) -> Result<Self, ArrowError> {
189 self.flags = flags.bits();
190 Ok(self)
191 }
192
193 pub unsafe fn with_metadata<I, S>(mut self, metadata: I) -> Result<Self, ArrowError>
204 where
205 I: IntoIterator<Item = (S, S)>,
206 S: AsRef<str>,
207 {
208 let metadata: Vec<(S, S)> = metadata.into_iter().collect();
209 let new_metadata = if !metadata.is_empty() {
211 let mut metadata_serialized: Vec<u8> = Vec::new();
212 let num_entries: i32 = metadata.len().try_into().map_err(|_| {
213 ArrowError::CDataInterface(format!(
214 "metadata can only have {} entries, but {} were provided",
215 i32::MAX,
216 metadata.len()
217 ))
218 })?;
219 metadata_serialized.extend(num_entries.to_ne_bytes());
220
221 for (key, value) in metadata {
222 let key_len: i32 = key.as_ref().len().try_into().map_err(|_| {
223 ArrowError::CDataInterface(format!(
224 "metadata key can only have {} bytes, but {} were provided",
225 i32::MAX,
226 key.as_ref().len()
227 ))
228 })?;
229 let value_len: i32 = value.as_ref().len().try_into().map_err(|_| {
230 ArrowError::CDataInterface(format!(
231 "metadata value can only have {} bytes, but {} were provided",
232 i32::MAX,
233 value.as_ref().len()
234 ))
235 })?;
236
237 metadata_serialized.extend(key_len.to_ne_bytes());
238 metadata_serialized.extend_from_slice(key.as_ref().as_bytes());
239 metadata_serialized.extend(value_len.to_ne_bytes());
240 metadata_serialized.extend_from_slice(value.as_ref().as_bytes());
241 }
242
243 self.metadata = metadata_serialized.as_ptr().cast::<c_char>();
244 Some(metadata_serialized)
245 } else {
246 self.metadata = std::ptr::null_mut();
247 None
248 };
249
250 unsafe {
254 let mut private_data = Box::from_raw(self.private_data.cast::<SchemaPrivateData>());
255 private_data.metadata = new_metadata;
256 self.private_data = Box::into_raw(private_data).cast::<c_void>();
257 }
258
259 Ok(self)
260 }
261
262 pub unsafe fn from_raw(schema: *mut FFI_ArrowSchema) -> Self {
275 unsafe { std::ptr::replace(schema, Self::empty()) }
276 }
277
278 pub fn empty() -> Self {
280 Self {
281 format: std::ptr::null_mut(),
282 name: std::ptr::null_mut(),
283 metadata: std::ptr::null_mut(),
284 flags: 0,
285 n_children: 0,
286 children: std::ptr::null_mut(),
287 dictionary: std::ptr::null_mut(),
288 release: None,
289 private_data: std::ptr::null_mut(),
290 }
291 }
292
293 pub fn release(&self) -> Option<unsafe extern "C" fn(arg1: *mut FFI_ArrowSchema)> {
295 self.release
296 }
297
298 pub fn private_data(&self) -> *mut c_void {
300 self.private_data
301 }
302
303 pub unsafe fn set_release(
315 &mut self,
316 release: Option<unsafe extern "C" fn(arg1: *mut FFI_ArrowSchema)>,
317 ) -> Option<unsafe extern "C" fn(arg1: *mut FFI_ArrowSchema)> {
318 std::mem::replace(&mut self.release, release)
319 }
320
321 pub unsafe fn set_private_data(&mut self, private_data: *mut c_void) -> *mut c_void {
329 std::mem::replace(&mut self.private_data, private_data)
330 }
331
332 pub fn format(&self) -> &str {
338 assert!(!self.format.is_null());
339 unsafe { CStr::from_ptr(self.format) }
341 .to_str()
342 .expect("The external API has a non-utf8 as format")
343 }
344
345 pub fn name(&self) -> Option<&str> {
351 if self.name.is_null() {
352 None
353 } else {
354 Some(
356 unsafe { CStr::from_ptr(self.name) }
357 .to_str()
358 .expect("The external API has a non-utf8 as name"),
359 )
360 }
361 }
362
363 pub fn flags(&self) -> Option<Flags> {
365 Flags::from_bits(self.flags)
366 }
367
368 pub fn child(&self, index: usize) -> &Self {
376 assert!(index < self.n_children as usize);
377 unsafe { self.children.add(index).as_ref().unwrap().as_ref().unwrap() }
378 }
379
380 pub fn children(&self) -> impl Iterator<Item = &Self> {
382 (0..self.n_children as usize).map(move |i| self.child(i))
383 }
384
385 pub fn nullable(&self) -> bool {
388 (self.flags / 2) & 1 == 1
389 }
390
391 pub fn dictionary(&self) -> Option<&Self> {
396 unsafe { self.dictionary.as_ref() }
399 }
400
401 pub fn map_keys_sorted(&self) -> bool {
405 self.flags & 0b00000100 != 0
406 }
407
408 pub fn dictionary_ordered(&self) -> bool {
410 self.flags & 0b00000001 != 0
411 }
412
413 pub fn metadata(&self) -> Result<HashMap<String, String>, ArrowError> {
415 if self.metadata.is_null() {
416 Ok(HashMap::new())
417 } else {
418 let mut pos = 0;
419
420 let buffer = self.metadata.cast::<u8>();
422
423 fn next_four_bytes(buffer: *const u8, pos: &mut usize) -> [u8; 4] {
424 let out = unsafe {
427 [
428 *buffer.add(*pos),
429 *buffer.add(*pos + 1),
430 *buffer.add(*pos + 2),
431 *buffer.add(*pos + 3),
432 ]
433 };
434 *pos += 4;
435 out
436 }
437
438 fn next_n_bytes(buffer: *const u8, pos: &mut usize, n: usize) -> &[u8] {
439 let out = unsafe { std::slice::from_raw_parts(buffer.add(*pos), n) };
441 *pos += n;
442 out
443 }
444
445 fn checked_length(what: &str, length: i32) -> Result<usize, ArrowError> {
447 usize::try_from(length).map_err(|_| {
448 ArrowError::CDataInterface(format!("Invalid {what} in metadata: {length}"))
449 })
450 }
451
452 let num_entries = i32::from_ne_bytes(next_four_bytes(buffer, &mut pos));
453 let num_entries = checked_length("number of entries", num_entries)?;
454
455 let mut metadata = HashMap::with_capacity(num_entries.min(128));
457
458 for _ in 0..num_entries {
459 let key_length = i32::from_ne_bytes(next_four_bytes(buffer, &mut pos));
460 let key_length = checked_length("key length", key_length)?;
461 let key = String::from_utf8(next_n_bytes(buffer, &mut pos, key_length).to_vec())?;
462 let value_length = i32::from_ne_bytes(next_four_bytes(buffer, &mut pos));
463 let value_length = checked_length("value length", value_length)?;
464 let value =
465 String::from_utf8(next_n_bytes(buffer, &mut pos, value_length).to_vec())?;
466 metadata.insert(key, value);
467 }
468
469 Ok(metadata)
470 }
471 }
472}
473
474impl Drop for FFI_ArrowSchema {
475 fn drop(&mut self) {
476 match self.release {
477 None => (),
478 Some(release) => unsafe { release(self) },
482 }
483 }
484}
485
486unsafe impl Send for FFI_ArrowSchema {}
487
488impl TryFrom<&FFI_ArrowSchema> for DataType {
489 type Error = ArrowError;
490
491 fn try_from(c_schema: &FFI_ArrowSchema) -> Result<Self, ArrowError> {
493 let mut dtype = match c_schema.format() {
494 "n" => DataType::Null,
495 "b" => DataType::Boolean,
496 "c" => DataType::Int8,
497 "C" => DataType::UInt8,
498 "s" => DataType::Int16,
499 "S" => DataType::UInt16,
500 "i" => DataType::Int32,
501 "I" => DataType::UInt32,
502 "l" => DataType::Int64,
503 "L" => DataType::UInt64,
504 "e" => DataType::Float16,
505 "f" => DataType::Float32,
506 "g" => DataType::Float64,
507 "vz" => DataType::BinaryView,
508 "z" => DataType::Binary,
509 "Z" => DataType::LargeBinary,
510 "vu" => DataType::Utf8View,
511 "u" => DataType::Utf8,
512 "U" => DataType::LargeUtf8,
513 "tdD" => DataType::Date32,
514 "tdm" => DataType::Date64,
515 "tts" => DataType::Time32(TimeUnit::Second),
516 "ttm" => DataType::Time32(TimeUnit::Millisecond),
517 "ttu" => DataType::Time64(TimeUnit::Microsecond),
518 "ttn" => DataType::Time64(TimeUnit::Nanosecond),
519 "tDs" => DataType::Duration(TimeUnit::Second),
520 "tDm" => DataType::Duration(TimeUnit::Millisecond),
521 "tDu" => DataType::Duration(TimeUnit::Microsecond),
522 "tDn" => DataType::Duration(TimeUnit::Nanosecond),
523 "tiM" => DataType::Interval(IntervalUnit::YearMonth),
524 "tiD" => DataType::Interval(IntervalUnit::DayTime),
525 "tin" => DataType::Interval(IntervalUnit::MonthDayNano),
526 "+l" => {
527 let c_child = c_schema.child(0);
528 DataType::List(Arc::new(Field::try_from(c_child)?))
529 }
530 "+L" => {
531 let c_child = c_schema.child(0);
532 DataType::LargeList(Arc::new(Field::try_from(c_child)?))
533 }
534 "+vl" => {
535 let c_child = c_schema.child(0);
536 DataType::ListView(Arc::new(Field::try_from(c_child)?))
537 }
538 "+vL" => {
539 let c_child = c_schema.child(0);
540 DataType::LargeListView(Arc::new(Field::try_from(c_child)?))
541 }
542 "+s" => {
543 let fields = c_schema.children().map(Field::try_from);
544 DataType::Struct(fields.collect::<Result<_, ArrowError>>()?)
545 }
546 "+m" => {
547 let c_child = c_schema.child(0);
548 let map_keys_sorted = c_schema.map_keys_sorted();
549 DataType::Map(Arc::new(Field::try_from(c_child)?), map_keys_sorted)
550 }
551 "+r" => {
552 let c_run_ends = c_schema.child(0);
553 let c_values = c_schema.child(1);
554 DataType::RunEndEncoded(
555 Arc::new(Field::try_from(c_run_ends)?),
556 Arc::new(Field::try_from(c_values)?),
557 )
558 }
559 other => {
561 match other.splitn(2, ':').collect::<Vec<&str>>().as_slice() {
562 ["w", num_bytes] => {
564 let parsed_num_bytes = num_bytes.parse::<i32>().map_err(|_| {
565 ArrowError::CDataInterface(
566 "FixedSizeBinary requires an integer parameter representing number of bytes per element".to_string())
567 })?;
568 DataType::FixedSizeBinary(parsed_num_bytes)
569 }
570 ["+w", num_elems] => {
572 let c_child = c_schema.child(0);
573 let parsed_num_elems = num_elems.parse::<i32>().map_err(|_| {
574 ArrowError::CDataInterface(
575 "The FixedSizeList type requires an integer parameter representing number of elements per list".to_string())
576 })?;
577 DataType::FixedSizeList(
578 Arc::new(Field::try_from(c_child)?),
579 parsed_num_elems,
580 )
581 }
582 ["d", extra] => match extra.splitn(3, ',').collect::<Vec<&str>>().as_slice() {
584 [precision, scale] => {
585 let parsed_precision = precision.parse::<u8>().map_err(|_| {
586 ArrowError::CDataInterface(
587 "The decimal type requires an integer precision".to_string(),
588 )
589 })?;
590 let parsed_scale = scale.parse::<i8>().map_err(|_| {
591 ArrowError::CDataInterface(
592 "The decimal type requires an integer scale".to_string(),
593 )
594 })?;
595 DataType::Decimal128(parsed_precision, parsed_scale)
596 }
597 [precision, scale, bits] => {
598 let parsed_precision = precision.parse::<u8>().map_err(|_| {
599 ArrowError::CDataInterface(
600 "The decimal type requires an integer precision".to_string(),
601 )
602 })?;
603 let parsed_scale = scale.parse::<i8>().map_err(|_| {
604 ArrowError::CDataInterface(
605 "The decimal type requires an integer scale".to_string(),
606 )
607 })?;
608 match *bits {
609 "32" => DataType::Decimal32(parsed_precision, parsed_scale),
610 "64" => DataType::Decimal64(parsed_precision, parsed_scale),
611 "128" => DataType::Decimal128(parsed_precision, parsed_scale),
612 "256" => DataType::Decimal256(parsed_precision, parsed_scale),
613 _ => return Err(ArrowError::CDataInterface("Only 32/64/128/256 bit wide decimals are supported in the Rust implementation".to_string())),
614 }
615 }
616 _ => {
617 return Err(ArrowError::CDataInterface(format!(
618 "The decimal pattern \"d:{extra:?}\" is not supported in the Rust implementation"
619 )));
620 }
621 },
622 ["+ud", extra] => {
624 let type_ids = extra
625 .split(',')
626 .map(|t| {
627 t.parse::<i8>().map_err(|_| {
628 ArrowError::CDataInterface(
629 "The Union type requires an integer type id".to_string(),
630 )
631 })
632 })
633 .collect::<Result<Vec<_>, ArrowError>>()?;
634 let mut fields = Vec::with_capacity(type_ids.len());
635 for idx in 0..c_schema.n_children {
636 let c_child = c_schema.child(idx as usize);
637 let field = Field::try_from(c_child)?;
638 fields.push(field);
639 }
640
641 if fields.len() != type_ids.len() {
642 return Err(ArrowError::CDataInterface(
643 "The Union type requires same number of fields and type ids"
644 .to_string(),
645 ));
646 }
647
648 DataType::Union(UnionFields::try_new(type_ids, fields)?, UnionMode::Dense)
649 }
650 ["+us", extra] => {
652 let type_ids = extra
653 .split(',')
654 .map(|t| {
655 t.parse::<i8>().map_err(|_| {
656 ArrowError::CDataInterface(
657 "The Union type requires an integer type id".to_string(),
658 )
659 })
660 })
661 .collect::<Result<Vec<_>, ArrowError>>()?;
662 let mut fields = Vec::with_capacity(type_ids.len());
663 for idx in 0..c_schema.n_children {
664 let c_child = c_schema.child(idx as usize);
665 let field = Field::try_from(c_child)?;
666 fields.push(field);
667 }
668
669 if fields.len() != type_ids.len() {
670 return Err(ArrowError::CDataInterface(
671 "The Union type requires same number of fields and type ids"
672 .to_string(),
673 ));
674 }
675
676 DataType::Union(UnionFields::try_new(type_ids, fields)?, UnionMode::Sparse)
677 }
678
679 ["tss", ""] => DataType::Timestamp(TimeUnit::Second, None),
681 ["tsm", ""] => DataType::Timestamp(TimeUnit::Millisecond, None),
682 ["tsu", ""] => DataType::Timestamp(TimeUnit::Microsecond, None),
683 ["tsn", ""] => DataType::Timestamp(TimeUnit::Nanosecond, None),
684 ["tss", tz] => DataType::Timestamp(TimeUnit::Second, Some(Arc::from(*tz))),
685 ["tsm", tz] => DataType::Timestamp(TimeUnit::Millisecond, Some(Arc::from(*tz))),
686 ["tsu", tz] => DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::from(*tz))),
687 ["tsn", tz] => DataType::Timestamp(TimeUnit::Nanosecond, Some(Arc::from(*tz))),
688 _ => {
689 return Err(ArrowError::CDataInterface(format!(
690 "The datatype \"{other:?}\" is still not supported in Rust implementation"
691 )));
692 }
693 }
694 }
695 };
696
697 if let Some(dict_schema) = c_schema.dictionary() {
698 let value_type = Self::try_from(dict_schema)?;
699 dtype = DataType::Dictionary(Box::new(dtype), Box::new(value_type));
700 }
701
702 Ok(dtype)
703 }
704}
705
706impl TryFrom<&FFI_ArrowSchema> for Field {
707 type Error = ArrowError;
708
709 fn try_from(c_schema: &FFI_ArrowSchema) -> Result<Self, ArrowError> {
710 let dtype = DataType::try_from(c_schema)?;
711 let field = Field::new(c_schema.name().unwrap_or(""), dtype, c_schema.nullable())
712 .with_dict_is_ordered(c_schema.dictionary_ordered())
713 .with_metadata(c_schema.metadata()?);
714 Ok(field)
715 }
716}
717
718impl TryFrom<&FFI_ArrowSchema> for Schema {
719 type Error = ArrowError;
720
721 fn try_from(c_schema: &FFI_ArrowSchema) -> Result<Self, ArrowError> {
722 let dtype = DataType::try_from(c_schema)?;
724 if let DataType::Struct(fields) = dtype {
725 Ok(Schema::new(fields).with_metadata(c_schema.metadata()?))
726 } else {
727 Err(ArrowError::CDataInterface(
728 "Unable to interpret C data struct as a Schema".to_string(),
729 ))
730 }
731 }
732}
733
734impl TryFrom<&DataType> for FFI_ArrowSchema {
735 type Error = ArrowError;
736
737 fn try_from(dtype: &DataType) -> Result<Self, ArrowError> {
739 let format = get_format_string(dtype)?;
740 let children = match dtype {
742 DataType::List(child)
743 | DataType::LargeList(child)
744 | DataType::ListView(child)
745 | DataType::LargeListView(child)
746 | DataType::FixedSizeList(child, _)
747 | DataType::Map(child, _) => {
748 vec![FFI_ArrowSchema::try_from(child.as_ref())?]
749 }
750 DataType::Union(fields, _) => fields
751 .iter()
752 .map(|(_, f)| f.as_ref().try_into())
753 .collect::<Result<Vec<_>, ArrowError>>()?,
754 DataType::Struct(fields) => fields
755 .iter()
756 .map(FFI_ArrowSchema::try_from)
757 .collect::<Result<Vec<_>, ArrowError>>()?,
758 DataType::RunEndEncoded(run_ends, values) => vec![
759 FFI_ArrowSchema::try_from(run_ends.as_ref())?,
760 FFI_ArrowSchema::try_from(values.as_ref())?,
761 ],
762 _ => vec![],
763 };
764 let dictionary = if let DataType::Dictionary(_, value_data_type) = dtype {
765 Some(Self::try_from(value_data_type.as_ref())?)
766 } else {
767 None
768 };
769
770 let flags = match dtype {
771 DataType::Map(_, true) => Flags::MAP_KEYS_SORTED,
772 _ => Flags::empty(),
773 };
774
775 FFI_ArrowSchema::try_new(&format, children, dictionary)?.with_flags(flags)
776 }
777}
778
779fn get_format_string(dtype: &DataType) -> Result<Cow<'static, str>, ArrowError> {
780 match dtype {
781 DataType::Null => Ok("n".into()),
782 DataType::Boolean => Ok("b".into()),
783 DataType::Int8 => Ok("c".into()),
784 DataType::UInt8 => Ok("C".into()),
785 DataType::Int16 => Ok("s".into()),
786 DataType::UInt16 => Ok("S".into()),
787 DataType::Int32 => Ok("i".into()),
788 DataType::UInt32 => Ok("I".into()),
789 DataType::Int64 => Ok("l".into()),
790 DataType::UInt64 => Ok("L".into()),
791 DataType::Float16 => Ok("e".into()),
792 DataType::Float32 => Ok("f".into()),
793 DataType::Float64 => Ok("g".into()),
794 DataType::BinaryView => Ok("vz".into()),
795 DataType::Binary => Ok("z".into()),
796 DataType::LargeBinary => Ok("Z".into()),
797 DataType::Utf8View => Ok("vu".into()),
798 DataType::Utf8 => Ok("u".into()),
799 DataType::LargeUtf8 => Ok("U".into()),
800 DataType::FixedSizeBinary(num_bytes) => Ok(Cow::Owned(format!("w:{num_bytes}"))),
801 DataType::FixedSizeList(_, num_elems) => Ok(Cow::Owned(format!("+w:{num_elems}"))),
802 DataType::Decimal32(precision, scale) => {
803 Ok(Cow::Owned(format!("d:{precision},{scale},32")))
804 }
805 DataType::Decimal64(precision, scale) => {
806 Ok(Cow::Owned(format!("d:{precision},{scale},64")))
807 }
808 DataType::Decimal128(precision, scale) => Ok(Cow::Owned(format!("d:{precision},{scale}"))),
809 DataType::Decimal256(precision, scale) => {
810 Ok(Cow::Owned(format!("d:{precision},{scale},256")))
811 }
812 DataType::Date32 => Ok("tdD".into()),
813 DataType::Date64 => Ok("tdm".into()),
814 DataType::Time32(TimeUnit::Second) => Ok("tts".into()),
815 DataType::Time32(TimeUnit::Millisecond) => Ok("ttm".into()),
816 DataType::Time64(TimeUnit::Microsecond) => Ok("ttu".into()),
817 DataType::Time64(TimeUnit::Nanosecond) => Ok("ttn".into()),
818 DataType::Timestamp(TimeUnit::Second, None) => Ok("tss:".into()),
819 DataType::Timestamp(TimeUnit::Millisecond, None) => Ok("tsm:".into()),
820 DataType::Timestamp(TimeUnit::Microsecond, None) => Ok("tsu:".into()),
821 DataType::Timestamp(TimeUnit::Nanosecond, None) => Ok("tsn:".into()),
822 DataType::Timestamp(TimeUnit::Second, Some(tz)) => Ok(Cow::Owned(format!("tss:{tz}"))),
823 DataType::Timestamp(TimeUnit::Millisecond, Some(tz)) => Ok(Cow::Owned(format!("tsm:{tz}"))),
824 DataType::Timestamp(TimeUnit::Microsecond, Some(tz)) => Ok(Cow::Owned(format!("tsu:{tz}"))),
825 DataType::Timestamp(TimeUnit::Nanosecond, Some(tz)) => Ok(Cow::Owned(format!("tsn:{tz}"))),
826 DataType::Duration(TimeUnit::Second) => Ok("tDs".into()),
827 DataType::Duration(TimeUnit::Millisecond) => Ok("tDm".into()),
828 DataType::Duration(TimeUnit::Microsecond) => Ok("tDu".into()),
829 DataType::Duration(TimeUnit::Nanosecond) => Ok("tDn".into()),
830 DataType::Interval(IntervalUnit::YearMonth) => Ok("tiM".into()),
831 DataType::Interval(IntervalUnit::DayTime) => Ok("tiD".into()),
832 DataType::Interval(IntervalUnit::MonthDayNano) => Ok("tin".into()),
833 DataType::List(_) => Ok("+l".into()),
834 DataType::LargeList(_) => Ok("+L".into()),
835 DataType::ListView(_) => Ok("+vl".into()),
836 DataType::LargeListView(_) => Ok("+vL".into()),
837 DataType::Struct(_) => Ok("+s".into()),
838 DataType::Map(_, _) => Ok("+m".into()),
839 DataType::RunEndEncoded(_, _) => Ok("+r".into()),
840 DataType::Dictionary(key_data_type, _) => get_format_string(key_data_type),
841 DataType::Union(fields, mode) => {
842 let formats = fields
843 .iter()
844 .map(|(t, _)| t.to_string())
845 .collect::<Vec<_>>();
846 match mode {
847 UnionMode::Dense => Ok(Cow::Owned(format!("{}:{}", "+ud", formats.join(",")))),
848 UnionMode::Sparse => Ok(Cow::Owned(format!("{}:{}", "+us", formats.join(",")))),
849 }
850 }
851 other => Err(ArrowError::CDataInterface(format!(
852 "The datatype \"{other:?}\" is still not supported in Rust implementation"
853 ))),
854 }
855}
856
857impl TryFrom<&FieldRef> for FFI_ArrowSchema {
858 type Error = ArrowError;
859
860 fn try_from(value: &FieldRef) -> Result<Self, Self::Error> {
861 value.as_ref().try_into()
862 }
863}
864
865impl TryFrom<&Field> for FFI_ArrowSchema {
866 type Error = ArrowError;
867
868 fn try_from(field: &Field) -> Result<Self, ArrowError> {
869 let mut flags = if field.is_nullable() {
870 Flags::NULLABLE
871 } else {
872 Flags::empty()
873 };
874
875 if field.dict_is_ordered() == Some(true) {
876 flags |= Flags::DICTIONARY_ORDERED;
877 }
878
879 if let DataType::Map(_, true) = field.data_type() {
880 flags |= Flags::MAP_KEYS_SORTED;
881 }
882
883 let schema = FFI_ArrowSchema::try_from(field.data_type())?
884 .with_name(field.name())?
885 .with_flags(flags)?;
886 unsafe { schema.with_metadata(field.metadata()) }
888 }
889}
890
891impl TryFrom<&Schema> for FFI_ArrowSchema {
892 type Error = ArrowError;
893
894 fn try_from(schema: &Schema) -> Result<Self, ArrowError> {
895 let dtype = DataType::Struct(schema.fields().clone());
896 let c_schema = FFI_ArrowSchema::try_from(&dtype)?;
897 unsafe { c_schema.with_metadata(&schema.metadata) }
899 }
900}
901
902impl TryFrom<DataType> for FFI_ArrowSchema {
903 type Error = ArrowError;
904
905 fn try_from(dtype: DataType) -> Result<Self, ArrowError> {
906 FFI_ArrowSchema::try_from(&dtype)
907 }
908}
909
910impl TryFrom<Field> for FFI_ArrowSchema {
911 type Error = ArrowError;
912
913 fn try_from(field: Field) -> Result<Self, ArrowError> {
914 FFI_ArrowSchema::try_from(&field)
915 }
916}
917
918impl TryFrom<Schema> for FFI_ArrowSchema {
919 type Error = ArrowError;
920
921 fn try_from(schema: Schema) -> Result<Self, ArrowError> {
922 FFI_ArrowSchema::try_from(&schema)
923 }
924}
925
926#[cfg(test)]
927mod tests {
928 use super::*;
929 use crate::Fields;
930 use std::sync::atomic::{AtomicBool, Ordering};
931
932 fn round_trip_type(dtype: DataType) {
933 let c_schema = FFI_ArrowSchema::try_from(&dtype).unwrap();
934 let restored = DataType::try_from(&c_schema).unwrap();
935 assert_eq!(restored, dtype);
936 }
937
938 fn round_trip_field(field: Field) {
939 let c_schema = FFI_ArrowSchema::try_from(&field).unwrap();
940 let restored = Field::try_from(&c_schema).unwrap();
941 assert_eq!(restored, field);
942 }
943
944 fn round_trip_schema(schema: Schema) {
945 let c_schema = FFI_ArrowSchema::try_from(&schema).unwrap();
946 let restored = Schema::try_from(&c_schema).unwrap();
947 assert_eq!(restored, schema);
948 }
949
950 #[test]
951 fn test_try_new_with_interior_nul_byte() {
952 let err = FFI_ArrowSchema::try_new("i\0nt", vec![], None).unwrap_err();
953 assert_eq!(
954 err.to_string(),
955 "C Data interface error: Null byte at position 1 not allowed in format"
956 );
957 }
958
959 #[test]
960 fn test_type() {
961 round_trip_type(DataType::Int64);
962 round_trip_type(DataType::UInt64);
963 round_trip_type(DataType::Float64);
964 round_trip_type(DataType::Date64);
965 round_trip_type(DataType::Time64(TimeUnit::Nanosecond));
966 round_trip_type(DataType::FixedSizeBinary(12));
967 round_trip_type(DataType::FixedSizeList(
968 Arc::new(Field::new("a", DataType::Int64, false)),
969 5,
970 ));
971 round_trip_type(DataType::Utf8);
972 round_trip_type(DataType::Utf8View);
973 round_trip_type(DataType::BinaryView);
974 round_trip_type(DataType::Binary);
975 round_trip_type(DataType::LargeBinary);
976 round_trip_type(DataType::List(Arc::new(Field::new(
977 "a",
978 DataType::Int16,
979 false,
980 ))));
981 round_trip_type(DataType::ListView(Arc::new(Field::new(
982 "a",
983 DataType::Int16,
984 false,
985 ))));
986 round_trip_type(DataType::LargeListView(Arc::new(Field::new(
987 "a",
988 DataType::Int16,
989 false,
990 ))));
991 round_trip_type(DataType::Struct(Fields::from(vec![Field::new(
992 "a",
993 DataType::Utf8,
994 true,
995 )])));
996 round_trip_type(DataType::RunEndEncoded(
997 Arc::new(Field::new(
998 Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
999 DataType::Int32,
1000 false,
1001 )),
1002 Arc::new(Field::new(
1003 Field::REE_VALUES_FIELD_DEFAULT_NAME,
1004 DataType::Binary,
1005 true,
1006 )),
1007 ));
1008 }
1009
1010 #[test]
1011 fn test_field() {
1012 let dtype = DataType::Struct(vec![Field::new("a", DataType::Utf8, true)].into());
1013 round_trip_field(Field::new("test", dtype, true));
1014 }
1015
1016 #[test]
1017 fn test_schema() {
1018 let schema = Schema::new(vec![
1019 Field::new("name", DataType::Utf8, false),
1020 Field::new("address", DataType::Utf8, false),
1021 Field::new("priority", DataType::UInt8, false),
1022 ])
1023 .with_metadata([("hello", "world")]);
1024
1025 round_trip_schema(schema);
1026
1027 let dtype = DataType::Struct(Fields::from(vec![
1029 Field::new("a", DataType::Utf8, true),
1030 Field::new("b", DataType::Int16, false),
1031 ]));
1032 let c_schema = FFI_ArrowSchema::try_from(&dtype).unwrap();
1033 let schema = Schema::try_from(&c_schema).unwrap();
1034 assert_eq!(schema.fields().len(), 2);
1035
1036 let c_schema = FFI_ArrowSchema::try_from(&DataType::Float64).unwrap();
1038 let result = Schema::try_from(&c_schema);
1039 assert!(result.is_err());
1040 }
1041
1042 #[test]
1043 fn test_map_keys_sorted() {
1044 let keys = Field::new(Field::MAP_KEY_FIELD_DEFAULT_NAME, DataType::Int32, false);
1045 let values = Field::new(Field::MAP_VALUE_FIELD_DEFAULT_NAME, DataType::UInt32, false);
1046 let entry_struct = DataType::Struct(vec![keys, values].into());
1047
1048 let map_data_type = DataType::Map(
1050 Arc::new(Field::new(
1051 Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
1052 entry_struct,
1053 false,
1054 )),
1055 true,
1056 );
1057
1058 let arrow_schema = FFI_ArrowSchema::try_from(map_data_type).unwrap();
1059 assert!(arrow_schema.map_keys_sorted());
1060 }
1061
1062 fn map_type(keys_sorted: bool) -> DataType {
1063 let entries = Field::new(
1064 Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
1065 DataType::Struct(Fields::from(vec![
1066 Field::new(Field::MAP_KEY_FIELD_DEFAULT_NAME, DataType::Utf8, false),
1067 Field::new(Field::MAP_VALUE_FIELD_DEFAULT_NAME, DataType::Int32, true),
1068 ])),
1069 false,
1070 );
1071 DataType::Map(Arc::new(entries), keys_sorted)
1072 }
1073
1074 #[test]
1075 fn test_map_keys_sorted_field() {
1076 for keys_sorted in [false, true] {
1077 for nullable in [false, true] {
1078 let field = Field::new("m", map_type(keys_sorted), nullable);
1079 let c_schema = FFI_ArrowSchema::try_from(&field).unwrap();
1080 assert_eq!(c_schema.map_keys_sorted(), keys_sorted);
1081 assert_eq!(c_schema.nullable(), nullable);
1082 round_trip_field(field);
1083 }
1084 }
1085 }
1086
1087 #[test]
1088 fn test_map_keys_sorted_nested() {
1089 let map = |keys_sorted| Field::new("m", map_type(keys_sorted), true);
1090
1091 for keys_sorted in [false, true] {
1092 round_trip_type(DataType::Struct(Fields::from(vec![
1093 Field::new("a", DataType::Int32, true),
1094 map(keys_sorted),
1095 ])));
1096 round_trip_type(DataType::List(Arc::new(map(keys_sorted))));
1097 round_trip_type(DataType::Union(
1098 UnionFields::try_new(
1099 vec![0, 1],
1100 vec![Field::new("i", DataType::Int32, true), map(keys_sorted)],
1101 )
1102 .unwrap(),
1103 UnionMode::Dense,
1104 ));
1105 round_trip_schema(Schema::new(vec![
1106 Field::new("id", DataType::Int64, false),
1107 map(keys_sorted),
1108 ]));
1109 }
1110 }
1111
1112 #[test]
1113 fn test_dictionary_ordered() {
1114 #[expect(deprecated)]
1115 let schema = Schema::new(vec![Field::new_dict(
1116 "dict",
1117 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
1118 false,
1119 0,
1120 true,
1121 )]);
1122
1123 let arrow_schema = FFI_ArrowSchema::try_from(schema).unwrap();
1124 assert!(arrow_schema.child(0).dictionary_ordered());
1125
1126 let field = Field::try_from(arrow_schema.child(0)).unwrap();
1128 assert_eq!(field.dict_is_ordered(), Some(true));
1129 }
1130
1131 #[test]
1132 fn test_set_field_metadata() {
1133 let metadata_cases: Vec<HashMap<String, String>> = vec![
1134 [].into(),
1135 [("key".to_string(), "value".to_string())].into(),
1136 [
1137 ("key".to_string(), String::new()),
1138 ("ascii123".to_string(), "你好".to_string()),
1139 (String::new(), "value".to_string()),
1140 ]
1141 .into(),
1142 ];
1143
1144 let mut schema = FFI_ArrowSchema::try_new("b", vec![], None)
1145 .unwrap()
1146 .with_name("test")
1147 .unwrap();
1148
1149 for metadata in metadata_cases {
1150 schema = unsafe { schema.with_metadata(&metadata) }.unwrap();
1152 let field = Field::try_from(&schema).unwrap();
1153 assert_eq!(field.metadata(), &metadata);
1154 }
1155 }
1156
1157 #[test]
1158 fn test_name_with_null_byte() {
1159 let schema = FFI_ArrowSchema::try_new("i", vec![], None).unwrap();
1160 assert!(schema.with_name("ab\0cd").is_err());
1161 }
1162
1163 #[test]
1164 fn test_import_field_with_null_name() {
1165 let dtype = DataType::Int16;
1166 let c_schema = FFI_ArrowSchema::try_from(&dtype).unwrap();
1167 assert!(c_schema.name().is_none());
1168 let field = Field::try_from(&c_schema).unwrap();
1169 assert_eq!(field.name(), "");
1170 }
1171
1172 static WRAPPER_RAN: AtomicBool = AtomicBool::new(false);
1176
1177 struct WrapperData {
1178 original_release: Option<unsafe extern "C" fn(*mut FFI_ArrowSchema)>,
1179 original_private_data: *mut c_void,
1180 }
1181
1182 unsafe extern "C" fn wrapping_release(schema: *mut FFI_ArrowSchema) {
1183 let schema = unsafe { &mut *schema };
1184 let data = unsafe { Box::from_raw(schema.private_data().cast::<WrapperData>()) };
1185 WRAPPER_RAN.store(true, Ordering::SeqCst);
1186 unsafe { schema.set_release(data.original_release) };
1188 unsafe { schema.set_private_data(data.original_private_data) };
1189 if let Some(release) = schema.release() {
1190 unsafe { release(schema) };
1191 }
1192 }
1193
1194 #[test]
1195 fn test_wrap_release_callback() {
1196 let mut schema = FFI_ArrowSchema::try_from(&DataType::Int32).unwrap();
1197
1198 let data = Box::new(WrapperData {
1199 original_release: schema.release(),
1200 original_private_data: schema.private_data(),
1201 });
1202 unsafe { schema.set_release(Some(wrapping_release)) };
1203 unsafe { schema.set_private_data(Box::into_raw(data).cast::<c_void>()) };
1204
1205 drop(schema); assert!(WRAPPER_RAN.load(Ordering::SeqCst));
1207 }
1208}