1#![doc(
27 html_logo_url = "https://arrow.apache.org/img/arrow-logo_chevrons_black-txt_white-bg.svg",
28 html_favicon_url = "https://arrow.apache.org/img/arrow-logo_chevrons_black-txt_transparent-bg.svg"
29)]
30#![cfg_attr(docsrs, feature(doc_cfg))]
31#![warn(missing_docs)]
32use arrow_buffer::{IntervalDayTime, IntervalMonthDayNano, ScalarBuffer};
33use hex::decode;
34use num_bigint::BigInt;
35use num_traits::Signed;
36use serde::{Deserialize, Serialize};
37use serde_json::{Map as SJMap, Value};
38use std::collections::HashMap;
39use std::sync::Arc;
40
41use arrow::array::*;
42use arrow::buffer::{Buffer, MutableBuffer};
43use arrow::datatypes::*;
44use arrow::error::{ArrowError, Result};
45use arrow::util::bit_util;
46
47mod datatype;
48mod field;
49mod schema;
50
51use datatype::{json_int, json_uint};
52
53pub use datatype::*;
54pub use field::*;
55pub use schema::*;
56
57#[derive(Deserialize, Serialize, Debug)]
61pub struct ArrowJson {
62 pub schema: ArrowJsonSchema,
64 pub batches: Vec<ArrowJsonBatch>,
66 #[serde(skip_serializing_if = "Option::is_none")]
68 pub dictionaries: Option<Vec<ArrowJsonDictionaryBatch>>,
69}
70
71#[derive(Deserialize, Serialize, Debug)]
75pub struct ArrowJsonSchema {
76 pub fields: Vec<ArrowJsonField>,
78 #[serde(skip_serializing_if = "Option::is_none")]
80 pub metadata: Option<Vec<HashMap<String, String>>>,
81}
82
83#[derive(Deserialize, Serialize, Debug)]
85pub struct ArrowJsonField {
86 pub name: String,
88 #[serde(rename = "type")]
91 pub field_type: Value,
92 pub nullable: bool,
94 pub children: Vec<ArrowJsonField>,
96 #[serde(skip_serializing_if = "Option::is_none")]
98 pub dictionary: Option<ArrowJsonFieldDictionary>,
99 #[serde(skip_serializing_if = "Option::is_none")]
101 pub metadata: Option<Value>,
102}
103
104impl From<&FieldRef> for ArrowJsonField {
105 fn from(value: &FieldRef) -> Self {
106 Self::from(value.as_ref())
107 }
108}
109
110impl From<&Field> for ArrowJsonField {
111 fn from(field: &Field) -> Self {
112 let metadata_value = match field.metadata().is_empty() {
113 false => {
114 let mut array = Vec::new();
115 for (k, v) in field.metadata() {
116 let mut kv_map = SJMap::new();
117 kv_map.insert(k.clone(), Value::String(v.clone()));
118 array.push(Value::Object(kv_map));
119 }
120 if !array.is_empty() {
121 Some(Value::Array(array))
122 } else {
123 None
124 }
125 }
126 _ => None,
127 };
128
129 Self {
130 name: field.name().clone(),
131 field_type: data_type_to_json(field.data_type()),
132 nullable: field.is_nullable(),
133 children: vec![],
134 dictionary: None, metadata: metadata_value,
136 }
137 }
138}
139
140#[derive(Deserialize, Serialize, Debug)]
142pub struct ArrowJsonFieldDictionary {
143 pub id: i64,
145 #[serde(rename = "indexType")]
147 pub index_type: DictionaryIndexType,
148 #[serde(rename = "isOrdered")]
150 pub is_ordered: bool,
151}
152
153#[derive(Deserialize, Serialize, Debug)]
155pub struct DictionaryIndexType {
156 pub name: String,
158 #[serde(rename = "isSigned")]
160 pub is_signed: bool,
161 #[serde(rename = "bitWidth")]
163 pub bit_width: i64,
164}
165
166#[derive(Deserialize, Serialize, Debug, Clone)]
168pub struct ArrowJsonBatch {
169 count: usize,
170 pub columns: Vec<ArrowJsonColumn>,
172}
173
174#[derive(Deserialize, Serialize, Debug, Clone)]
176pub struct ArrowJsonDictionaryBatch {
177 pub id: i64,
179 pub data: ArrowJsonBatch,
181}
182
183#[derive(Deserialize, Serialize, Clone, Debug)]
185pub struct ArrowJsonColumn {
186 name: String,
187 pub count: usize,
189 #[serde(rename = "VALIDITY")]
191 pub validity: Option<Vec<u8>>,
192 #[serde(rename = "DATA")]
194 pub data: Option<Vec<Value>>,
195 #[serde(rename = "OFFSET")]
197 pub offset: Option<Vec<Value>>, #[serde(rename = "TYPE_ID")]
200 pub type_id: Option<Vec<i8>>,
201 #[serde(rename = "SIZE")]
203 pub size: Option<Vec<Value>>,
204 #[serde(rename = "VIEWS")]
206 pub views: Option<Vec<Value>>,
207 #[serde(rename = "VARIADIC_DATA_BUFFERS")]
209 pub variadic_data_buffers: Option<Vec<String>>,
210 pub children: Option<Vec<ArrowJsonColumn>>,
212}
213
214impl ArrowJson {
215 pub fn equals_reader(&self, reader: &mut dyn RecordBatchReader) -> Result<bool> {
217 if !self.schema.equals_schema(&reader.schema()) {
218 return Ok(false);
219 }
220
221 for json_batch in self.get_record_batches()? {
222 let batch = reader.next();
223 match batch {
224 Some(Ok(batch)) => {
225 if json_batch != batch {
226 println!("json: {json_batch:?}");
227 println!("batch: {batch:?}");
228 return Ok(false);
229 }
230 }
231 Some(Err(e)) => return Err(e),
232 None => return Ok(false),
233 }
234 }
235
236 Ok(true)
237 }
238
239 pub fn get_record_batches(&self) -> Result<Vec<RecordBatch>> {
241 let schema = self.schema.to_arrow_schema()?;
242
243 let mut dictionaries = HashMap::new();
244 self.dictionaries.iter().for_each(|dict_batches| {
245 dict_batches.iter().for_each(|d| {
246 dictionaries.insert(d.id, d.clone());
247 });
248 });
249
250 let batches: Result<Vec<_>> = self
251 .batches
252 .iter()
253 .map(|col| record_batch_from_json(&schema, col.clone(), Some(&dictionaries)))
254 .collect();
255
256 batches
257 }
258}
259
260impl ArrowJsonSchema {
261 fn equals_schema(&self, schema: &Schema) -> bool {
263 if self.fields.len() != schema.fields().len() {
264 return false;
265 }
266 for (i, json_field) in self.fields.iter().enumerate() {
267 let field = schema.field(i);
268 if !json_field.equals_field(field) {
269 return false;
270 }
271 }
272 true
273 }
274
275 fn to_arrow_schema(&self) -> Result<Schema> {
276 let arrow_fields: Result<Vec<_>> = self
277 .fields
278 .iter()
279 .map(|field| field.to_arrow_field())
280 .collect();
281
282 if let Some(metadatas) = &self.metadata {
283 let mut metadata: HashMap<String, String> = HashMap::new();
284
285 metadatas.iter().for_each(|pair| {
286 let key = pair.get("key").unwrap();
287 let value = pair.get("value").unwrap();
288 metadata.insert(key.clone(), value.clone());
289 });
290
291 Ok(Schema::new_with_metadata(arrow_fields?, metadata))
292 } else {
293 Ok(Schema::new(arrow_fields?))
294 }
295 }
296}
297
298impl ArrowJsonField {
299 fn equals_field(&self, field: &Field) -> bool {
301 match self.to_arrow_field() {
303 Ok(self_field) => {
304 assert_eq!(&self_field, field, "Arrow fields not the same");
305 true
306 }
307 Err(e) => {
308 eprintln!("Encountered error while converting JSON field to Arrow field: {e:?}");
309 false
310 }
311 }
312 }
313
314 fn to_arrow_field(&self) -> Result<Field> {
317 let field =
319 serde_json::to_value(self).map_err(|error| ArrowError::JsonError(error.to_string()))?;
320 field_from_json(&field)
321 }
322}
323
324pub fn record_batch_from_json(
326 schema: &Schema,
327 json_batch: ArrowJsonBatch,
328 json_dictionaries: Option<&HashMap<i64, ArrowJsonDictionaryBatch>>,
329) -> Result<RecordBatch> {
330 let mut columns = vec![];
331
332 for (field, json_col) in schema.fields().iter().zip(json_batch.columns) {
333 let col = array_from_json(field, json_col, json_dictionaries)?;
334 columns.push(col);
335 }
336
337 RecordBatch::try_new(Arc::new(schema.clone()), columns)
338}
339
340pub fn array_from_json(
342 field: &Field,
343 json_col: ArrowJsonColumn,
344 dictionaries: Option<&HashMap<i64, ArrowJsonDictionaryBatch>>,
345) -> Result<ArrayRef> {
346 match field.data_type() {
347 DataType::Null => Ok(Arc::new(NullArray::new(json_col.count))),
348 DataType::Boolean => {
349 let mut b = BooleanBuilder::with_capacity(json_col.count);
350 for (is_valid, value) in json_col
351 .validity
352 .as_ref()
353 .unwrap()
354 .iter()
355 .zip(json_col.data.unwrap())
356 {
357 match is_valid {
358 1 => b.append_value(value.as_bool().unwrap()),
359 _ => b.append_null(),
360 }
361 }
362 Ok(Arc::new(b.finish()))
363 }
364 DataType::Int8 => {
365 let mut b = Int8Builder::with_capacity(json_col.count);
366 for (is_valid, value) in json_col
367 .validity
368 .as_ref()
369 .unwrap()
370 .iter()
371 .zip(json_col.data.unwrap())
372 {
373 match is_valid {
374 1 => b.append_value(json_int("int8 value", &value)?),
375 _ => b.append_null(),
376 }
377 }
378 Ok(Arc::new(b.finish()))
379 }
380 DataType::Int16 => {
381 let mut b = Int16Builder::with_capacity(json_col.count);
382 for (is_valid, value) in json_col
383 .validity
384 .as_ref()
385 .unwrap()
386 .iter()
387 .zip(json_col.data.unwrap())
388 {
389 match is_valid {
390 1 => b.append_value(json_int("int16 value", &value)?),
391 _ => b.append_null(),
392 }
393 }
394 Ok(Arc::new(b.finish()))
395 }
396 DataType::Int32 | DataType::Date32 | DataType::Time32(_) => {
397 let mut b = Int32Builder::with_capacity(json_col.count);
398 for (is_valid, value) in json_col
399 .validity
400 .as_ref()
401 .unwrap()
402 .iter()
403 .zip(json_col.data.unwrap())
404 {
405 match is_valid {
406 1 => b.append_value(json_int("int32 value", &value)?),
407 _ => b.append_null(),
408 }
409 }
410 let array = Arc::new(b.finish()) as ArrayRef;
411 arrow::compute::cast(&array, field.data_type())
412 }
413 DataType::Interval(IntervalUnit::YearMonth) => {
414 let mut b = IntervalYearMonthBuilder::with_capacity(json_col.count);
415 for (is_valid, value) in json_col
416 .validity
417 .as_ref()
418 .unwrap()
419 .iter()
420 .zip(json_col.data.unwrap())
421 {
422 match is_valid {
423 1 => b.append_value(json_int("year-month interval", &value)?),
424 _ => b.append_null(),
425 }
426 }
427 Ok(Arc::new(b.finish()))
428 }
429 DataType::Int64
430 | DataType::Date64
431 | DataType::Time64(_)
432 | DataType::Timestamp(_, _)
433 | DataType::Duration(_) => {
434 let mut b = Int64Builder::with_capacity(json_col.count);
435 for (is_valid, value) in json_col
436 .validity
437 .as_ref()
438 .unwrap()
439 .iter()
440 .zip(json_col.data.unwrap())
441 {
442 match is_valid {
443 1 => b.append_value(match value {
444 Value::Number(n) => n.as_i64().unwrap(),
445 Value::String(s) => s.parse().expect("Unable to parse string as i64"),
446 _ => panic!("Unable to parse {value:?} as number"),
447 }),
448 _ => b.append_null(),
449 }
450 }
451 let array = Arc::new(b.finish()) as ArrayRef;
452 arrow::compute::cast(&array, field.data_type())
453 }
454 DataType::Interval(IntervalUnit::DayTime) => {
455 let mut b = IntervalDayTimeBuilder::with_capacity(json_col.count);
456 for (is_valid, value) in json_col
457 .validity
458 .as_ref()
459 .unwrap()
460 .iter()
461 .zip(json_col.data.unwrap())
462 {
463 match is_valid {
464 1 => b.append_value(match value {
465 Value::Object(ref map)
466 if map.contains_key("days") && map.contains_key("milliseconds") =>
467 {
468 match field.data_type() {
469 DataType::Interval(IntervalUnit::DayTime) => {
470 let days = map.get("days").unwrap();
471 let milliseconds = map.get("milliseconds").unwrap();
472 let days = json_int("interval days", days)?;
473 let millis = json_int("interval milliseconds", milliseconds)?;
474 IntervalDayTime::new(days, millis)
475 }
476 _ => panic!("Unable to parse {value:?} as interval daytime"),
477 }
478 }
479 _ => panic!("Unable to parse {value:?} as number"),
480 }),
481 _ => b.append_null(),
482 }
483 }
484 Ok(Arc::new(b.finish()))
485 }
486 DataType::UInt8 => {
487 let mut b = UInt8Builder::with_capacity(json_col.count);
488 for (is_valid, value) in json_col
489 .validity
490 .as_ref()
491 .unwrap()
492 .iter()
493 .zip(json_col.data.unwrap())
494 {
495 match is_valid {
496 1 => b.append_value(json_uint("uint8 value", &value)?),
497 _ => b.append_null(),
498 }
499 }
500 Ok(Arc::new(b.finish()))
501 }
502 DataType::UInt16 => {
503 let mut b = UInt16Builder::with_capacity(json_col.count);
504 for (is_valid, value) in json_col
505 .validity
506 .as_ref()
507 .unwrap()
508 .iter()
509 .zip(json_col.data.unwrap())
510 {
511 match is_valid {
512 1 => b.append_value(json_uint("uint16 value", &value)?),
513 _ => b.append_null(),
514 }
515 }
516 Ok(Arc::new(b.finish()))
517 }
518 DataType::UInt32 => {
519 let mut b = UInt32Builder::with_capacity(json_col.count);
520 for (is_valid, value) in json_col
521 .validity
522 .as_ref()
523 .unwrap()
524 .iter()
525 .zip(json_col.data.unwrap())
526 {
527 match is_valid {
528 1 => b.append_value(json_uint("uint32 value", &value)?),
529 _ => b.append_null(),
530 }
531 }
532 Ok(Arc::new(b.finish()))
533 }
534 DataType::UInt64 => {
535 let mut b = UInt64Builder::with_capacity(json_col.count);
536 for (is_valid, value) in json_col
537 .validity
538 .as_ref()
539 .unwrap()
540 .iter()
541 .zip(json_col.data.unwrap())
542 {
543 match is_valid {
544 1 => {
545 if value.is_string() {
546 b.append_value(
547 value
548 .as_str()
549 .unwrap()
550 .parse()
551 .expect("Unable to parse string as u64"),
552 )
553 } else if value.is_number() {
554 b.append_value(value.as_u64().expect("Unable to read number as u64"))
555 } else {
556 panic!("Unable to parse value {value:?} as u64")
557 }
558 }
559 _ => b.append_null(),
560 }
561 }
562 Ok(Arc::new(b.finish()))
563 }
564 DataType::Interval(IntervalUnit::MonthDayNano) => {
565 let mut b = IntervalMonthDayNanoBuilder::with_capacity(json_col.count);
566 for (is_valid, value) in json_col
567 .validity
568 .as_ref()
569 .unwrap()
570 .iter()
571 .zip(json_col.data.unwrap())
572 {
573 match is_valid {
574 1 => b.append_value(match value {
575 Value::Object(v) => {
576 let months = v.get("months").unwrap();
577 let days = v.get("days").unwrap();
578 let nanoseconds = v.get("nanoseconds").unwrap();
579 match (months, days, nanoseconds) {
580 (
581 Value::Number(months),
582 Value::Number(days),
583 Value::Number(nanoseconds),
584 ) => {
585 let months = json_int(
586 "month-day-nano interval months",
587 &Value::Number(months.clone()),
588 )?;
589 let days = json_int(
590 "month-day-nano interval days",
591 &Value::Number(days.clone()),
592 )?;
593 let nanoseconds = nanoseconds.as_i64().unwrap();
594 IntervalMonthDayNano::new(months, days, nanoseconds)
595 }
596 (_, _, _) => {
597 panic!("Unable to parse {v:?} as MonthDayNano")
598 }
599 }
600 }
601 _ => panic!("Unable to parse {value:?} as MonthDayNano"),
602 }),
603 _ => b.append_null(),
604 }
605 }
606 Ok(Arc::new(b.finish()))
607 }
608 DataType::Float32 => {
609 let mut b = Float32Builder::with_capacity(json_col.count);
610 for (is_valid, value) in json_col
611 .validity
612 .as_ref()
613 .unwrap()
614 .iter()
615 .zip(json_col.data.unwrap())
616 {
617 match is_valid {
618 1 => b.append_value(value.as_f64().unwrap() as f32),
619 _ => b.append_null(),
620 }
621 }
622 Ok(Arc::new(b.finish()))
623 }
624 DataType::Float64 => {
625 let mut b = Float64Builder::with_capacity(json_col.count);
626 for (is_valid, value) in json_col
627 .validity
628 .as_ref()
629 .unwrap()
630 .iter()
631 .zip(json_col.data.unwrap())
632 {
633 match is_valid {
634 1 => b.append_value(value.as_f64().unwrap()),
635 _ => b.append_null(),
636 }
637 }
638 Ok(Arc::new(b.finish()))
639 }
640 DataType::Binary => {
641 let mut b = BinaryBuilder::with_capacity(json_col.count, 1024);
642 for (is_valid, value) in json_col
643 .validity
644 .as_ref()
645 .unwrap()
646 .iter()
647 .zip(json_col.data.unwrap())
648 {
649 match is_valid {
650 1 => {
651 let v = decode(value.as_str().unwrap()).unwrap();
652 b.append_value(&v)
653 }
654 _ => b.append_null(),
655 }
656 }
657 Ok(Arc::new(b.finish()))
658 }
659 DataType::LargeBinary => {
660 let mut b = LargeBinaryBuilder::with_capacity(json_col.count, 1024);
661 for (is_valid, value) in json_col
662 .validity
663 .as_ref()
664 .unwrap()
665 .iter()
666 .zip(json_col.data.unwrap())
667 {
668 match is_valid {
669 1 => {
670 let v = decode(value.as_str().unwrap()).unwrap();
671 b.append_value(&v)
672 }
673 _ => b.append_null(),
674 }
675 }
676 Ok(Arc::new(b.finish()))
677 }
678 DataType::Utf8 => {
679 let mut b = StringBuilder::with_capacity(json_col.count, 1024);
680 for (is_valid, value) in json_col
681 .validity
682 .as_ref()
683 .unwrap()
684 .iter()
685 .zip(json_col.data.unwrap())
686 {
687 match is_valid {
688 1 => b.append_value(value.as_str().unwrap()),
689 _ => b.append_null(),
690 }
691 }
692 Ok(Arc::new(b.finish()))
693 }
694 DataType::LargeUtf8 => {
695 let mut b = LargeStringBuilder::with_capacity(json_col.count, 1024);
696 for (is_valid, value) in json_col
697 .validity
698 .as_ref()
699 .unwrap()
700 .iter()
701 .zip(json_col.data.unwrap())
702 {
703 match is_valid {
704 1 => b.append_value(value.as_str().unwrap()),
705 _ => b.append_null(),
706 }
707 }
708 Ok(Arc::new(b.finish()))
709 }
710 DataType::FixedSizeBinary(len) => {
711 let byte_width = usize::try_from(*len).map_err(|_| {
712 ArrowError::JsonError("FixedSizeBinary byte width must be non-negative".to_string())
713 })?;
714 let capacity = json_col.count.checked_mul(byte_width).ok_or_else(|| {
715 ArrowError::JsonError("FixedSizeBinary capacity overflow".to_string())
716 })?;
717 std::alloc::Layout::array::<u8>(capacity).map_err(|_| {
718 ArrowError::JsonError("FixedSizeBinary capacity overflow".to_string())
719 })?;
720 let mut b = FixedSizeBinaryBuilder::with_capacity(json_col.count, *len);
721 for (is_valid, value) in json_col
722 .validity
723 .as_ref()
724 .unwrap()
725 .iter()
726 .zip(json_col.data.unwrap())
727 {
728 match is_valid {
729 1 => {
730 let v = hex::decode(value.as_str().unwrap()).unwrap();
731 b.append_value(&v)?
732 }
733 _ => b.append_null(),
734 }
735 }
736 Ok(Arc::new(b.finish()))
737 }
738 DataType::List(child_field) => {
739 let null_buf = create_null_buf(&json_col);
740 let children = json_col.children.clone().unwrap();
741 let child_array = array_from_json(child_field, children[0].clone(), dictionaries)?;
742 let offsets: Vec<i32> = json_col
743 .offset
744 .unwrap()
745 .iter()
746 .map(|v| json_int("list offset", v))
747 .collect::<Result<_>>()?;
748 let list_data = ArrayData::builder(field.data_type().clone())
749 .len(json_col.count)
750 .offset(0)
751 .add_buffer(Buffer::from(offsets.to_byte_slice()))
752 .add_child_data(child_array.into_data())
753 .null_bit_buffer(Some(null_buf))
754 .build()
755 .unwrap();
756 Ok(Arc::new(ListArray::from(list_data)))
757 }
758 DataType::LargeList(child_field) => {
759 let null_buf = create_null_buf(&json_col);
760 let children = json_col.children.clone().unwrap();
761 let child_array = array_from_json(child_field, children[0].clone(), dictionaries)?;
762 let offsets: Vec<i64> = json_col
763 .offset
764 .unwrap()
765 .iter()
766 .map(|v| match v {
767 Value::Number(n) => n.as_i64().unwrap(),
768 Value::String(s) => s.parse::<i64>().unwrap(),
769 _ => panic!("64-bit offset must be either string or number"),
770 })
771 .collect();
772 let list_data = ArrayData::builder(field.data_type().clone())
773 .len(json_col.count)
774 .offset(0)
775 .add_buffer(Buffer::from(offsets.to_byte_slice()))
776 .add_child_data(child_array.into_data())
777 .null_bit_buffer(Some(null_buf))
778 .build()
779 .unwrap();
780 Ok(Arc::new(LargeListArray::from(list_data)))
781 }
782 DataType::ListView(child_field) => {
783 let null_buf = create_null_buf(&json_col);
784 let children = json_col.children.clone().unwrap();
785 let child_array = array_from_json(child_field, children[0].clone(), dictionaries)?;
786 let offsets: Vec<i32> = json_col
787 .offset
788 .unwrap()
789 .iter()
790 .map(|v| json_int("list view offset", v))
791 .collect::<Result<_>>()?;
792 let sizes: Vec<i32> = json_col
793 .size
794 .unwrap()
795 .iter()
796 .map(|v| json_int("list view size", v))
797 .collect::<Result<_>>()?;
798 let list_data = ArrayData::builder(field.data_type().clone())
799 .len(json_col.count)
800 .add_buffer(Buffer::from(offsets.to_byte_slice()))
801 .add_buffer(Buffer::from(sizes.to_byte_slice()))
802 .add_child_data(child_array.into_data())
803 .null_bit_buffer(Some(null_buf))
804 .build()
805 .unwrap();
806 Ok(Arc::new(ListViewArray::from(list_data)))
807 }
808 DataType::LargeListView(child_field) => {
809 let null_buf = create_null_buf(&json_col);
810 let children = json_col.children.clone().unwrap();
811 let child_array = array_from_json(child_field, children[0].clone(), dictionaries)?;
812 let offsets: Vec<i64> = json_col
813 .offset
814 .unwrap()
815 .iter()
816 .map(|v| match v {
817 Value::Number(n) => n.as_i64().unwrap(),
818 Value::String(s) => s.parse::<i64>().unwrap(),
819 _ => panic!("64-bit offset must be either string or number"),
820 })
821 .collect();
822 let sizes: Vec<i64> = json_col
823 .size
824 .unwrap()
825 .iter()
826 .map(|v| match v {
827 Value::Number(n) => n.as_i64().unwrap(),
828 Value::String(s) => s.parse::<i64>().unwrap(),
829 _ => panic!("64-bit size must be either string or number"),
830 })
831 .collect();
832 let list_data = ArrayData::builder(field.data_type().clone())
833 .len(json_col.count)
834 .add_buffer(Buffer::from(offsets.to_byte_slice()))
835 .add_buffer(Buffer::from(sizes.to_byte_slice()))
836 .add_child_data(child_array.into_data())
837 .null_bit_buffer(Some(null_buf))
838 .build()
839 .unwrap();
840 Ok(Arc::new(LargeListViewArray::from(list_data)))
841 }
842 DataType::FixedSizeList(child_field, _) => {
843 let children = json_col.children.clone().unwrap();
844 let child_array = array_from_json(child_field, children[0].clone(), dictionaries)?;
845 let null_buf = create_null_buf(&json_col);
846 let list_data = ArrayData::builder(field.data_type().clone())
847 .len(json_col.count)
848 .add_child_data(child_array.into_data())
849 .null_bit_buffer(Some(null_buf))
850 .build()
851 .unwrap();
852 Ok(Arc::new(FixedSizeListArray::from(list_data)))
853 }
854 DataType::Struct(fields) => {
855 let null_buf = create_null_buf(&json_col);
857 let mut array_data = ArrayData::builder(field.data_type().clone())
858 .len(json_col.count)
859 .null_bit_buffer(Some(null_buf));
860
861 for (field, col) in fields.iter().zip(json_col.children.unwrap()) {
862 let array = array_from_json(field, col, dictionaries)?;
863 array_data = array_data.add_child_data(array.into_data());
864 }
865
866 let array = StructArray::from(array_data.build().unwrap());
867 Ok(Arc::new(array))
868 }
869 DataType::Dictionary(key_type, value_type) => {
870 #[expect(deprecated)]
871 let dict_id = field.dict_id().ok_or_else(|| {
872 ArrowError::JsonError(format!("Unable to find dict_id for field {field}"))
873 })?;
874 let dictionary = dictionaries
876 .ok_or_else(|| {
877 ArrowError::JsonError(format!(
878 "Unable to find any dictionaries for field {field}"
879 ))
880 })?
881 .get(&dict_id);
882 match dictionary {
883 Some(dictionary) => dictionary_array_from_json(
884 field,
885 json_col,
886 key_type,
887 value_type,
888 dictionary,
889 dictionaries,
890 ),
891 None => Err(ArrowError::JsonError(format!(
892 "Unable to find dictionary for field {field}"
893 ))),
894 }
895 }
896 DataType::Decimal32(precision, scale) => {
897 let mut b = Decimal32Builder::with_capacity(json_col.count);
898 for (is_valid, value) in json_col
899 .validity
900 .as_ref()
901 .unwrap()
902 .iter()
903 .zip(json_col.data.unwrap())
904 {
905 match is_valid {
906 1 => b.append_value(value.as_str().unwrap().parse::<i32>().unwrap()),
907 _ => b.append_null(),
908 }
909 }
910 Ok(Arc::new(
911 b.finish().with_precision_and_scale(*precision, *scale)?,
912 ))
913 }
914 DataType::Decimal64(precision, scale) => {
915 let mut b = Decimal64Builder::with_capacity(json_col.count);
916 for (is_valid, value) in json_col
917 .validity
918 .as_ref()
919 .unwrap()
920 .iter()
921 .zip(json_col.data.unwrap())
922 {
923 match is_valid {
924 1 => b.append_value(value.as_str().unwrap().parse::<i64>().unwrap()),
925 _ => b.append_null(),
926 }
927 }
928 Ok(Arc::new(
929 b.finish().with_precision_and_scale(*precision, *scale)?,
930 ))
931 }
932 DataType::Decimal128(precision, scale) => {
933 let mut b = Decimal128Builder::with_capacity(json_col.count);
934 for (is_valid, value) in json_col
935 .validity
936 .as_ref()
937 .unwrap()
938 .iter()
939 .zip(json_col.data.unwrap())
940 {
941 match is_valid {
942 1 => b.append_value(value.as_str().unwrap().parse::<i128>().unwrap()),
943 _ => b.append_null(),
944 }
945 }
946 Ok(Arc::new(
947 b.finish().with_precision_and_scale(*precision, *scale)?,
948 ))
949 }
950 DataType::Decimal256(precision, scale) => {
951 let mut b = Decimal256Builder::with_capacity(json_col.count);
952 for (is_valid, value) in json_col
953 .validity
954 .as_ref()
955 .unwrap()
956 .iter()
957 .zip(json_col.data.unwrap())
958 {
959 match is_valid {
960 1 => {
961 let str = value.as_str().unwrap();
962 let integer = BigInt::parse_bytes(str.as_bytes(), 10).unwrap();
963 let integer_bytes = integer.to_signed_bytes_le();
964 let mut bytes = if integer.is_negative() {
968 [255_u8; 32]
969 } else {
970 [0_u8; 32]
971 };
972 if integer_bytes.len() > bytes.len() {
973 return Err(ArrowError::JsonError(
974 "Decimal256 value exceeds 256 bits".to_string(),
975 ));
976 }
977 bytes[0..integer_bytes.len()].copy_from_slice(integer_bytes.as_slice());
978 b.append_value(i256::from_le_bytes(bytes));
979 }
980 _ => b.append_null(),
981 }
982 }
983 Ok(Arc::new(
984 b.finish().with_precision_and_scale(*precision, *scale)?,
985 ))
986 }
987 DataType::Map(child_field, _) => {
988 let null_buf = create_null_buf(&json_col);
989 let children = json_col.children.clone().unwrap();
990 let child_array = array_from_json(child_field, children[0].clone(), dictionaries)?;
991 let offsets: Vec<i32> = json_col
992 .offset
993 .unwrap()
994 .iter()
995 .map(|v| json_int("map offset", v))
996 .collect::<Result<_>>()?;
997 let array_data = ArrayData::builder(field.data_type().clone())
998 .len(json_col.count)
999 .add_buffer(Buffer::from(offsets.to_byte_slice()))
1000 .add_child_data(child_array.into_data())
1001 .null_bit_buffer(Some(null_buf))
1002 .build()
1003 .unwrap();
1004
1005 let array = MapArray::from(array_data);
1006 Ok(Arc::new(array))
1007 }
1008 DataType::Union(fields, _) => {
1009 let Some(type_ids) = json_col.type_id else {
1010 return Err(ArrowError::JsonError(
1011 "Cannot find expected type_id in json column".to_string(),
1012 ));
1013 };
1014
1015 let offset: Option<ScalarBuffer<i32>> = json_col
1016 .offset
1017 .map(|offsets| {
1018 offsets
1019 .iter()
1020 .map(|v| json_int("union offset", v))
1021 .collect::<Result<_>>()
1022 })
1023 .transpose()?;
1024
1025 let mut children = Vec::with_capacity(fields.len());
1026 for ((_, field), col) in fields.iter().zip(json_col.children.unwrap()) {
1027 let array = array_from_json(field, col, dictionaries)?;
1028 children.push(array);
1029 }
1030
1031 let array =
1032 UnionArray::try_new(fields.clone(), type_ids.into(), offset, children).unwrap();
1033 Ok(Arc::new(array))
1034 }
1035 DataType::Utf8View => {
1036 let views = json_col.views.ok_or_else(|| {
1037 ArrowError::JsonError("Utf8View requires VIEWS field".to_string())
1038 })?;
1039 let variadic_buffers = json_col.variadic_data_buffers.unwrap_or_default();
1040 let validity = json_col.validity.as_ref();
1041
1042 let mut builder = StringViewBuilder::new();
1043 for (i, view) in views.iter().enumerate() {
1044 let is_valid = validity.map_or(1, |v| v[i]);
1045 if is_valid == 0 {
1046 builder.append_null();
1047 } else {
1048 let view_obj = view.as_object().unwrap();
1049 let size: usize = json_uint("view size", &view_obj["SIZE"])?;
1050 if let Some(inlined) = view_obj.get("INLINED") {
1052 builder.append_value(inlined.as_str().unwrap());
1053 } else {
1054 let buffer_index: usize =
1056 json_uint("view buffer index", &view_obj["BUFFER_INDEX"])?;
1057 let offset: usize = json_uint("view offset", &view_obj["OFFSET"])?;
1058 let end = offset.checked_add(size).ok_or_else(|| {
1059 ArrowError::JsonError("View offset overflow".to_string())
1060 })?;
1061 let buffer = variadic_buffers.get(buffer_index).ok_or_else(|| {
1062 ArrowError::JsonError("View buffer index out of bounds".to_string())
1063 })?;
1064 let buffer_data = hex::decode(buffer).unwrap();
1065 let data = buffer_data.get(offset..end).ok_or_else(|| {
1066 ArrowError::JsonError("View range out of bounds".to_string())
1067 })?;
1068 let s = std::str::from_utf8(data).unwrap();
1069 builder.append_value(s);
1070 }
1071 }
1072 }
1073 Ok(Arc::new(builder.finish()))
1074 }
1075 DataType::BinaryView => {
1076 let views = json_col.views.ok_or_else(|| {
1077 ArrowError::JsonError("BinaryView requires VIEWS field".to_string())
1078 })?;
1079 let variadic_buffers = json_col.variadic_data_buffers.unwrap_or_default();
1080 let validity = json_col.validity.as_ref();
1081
1082 let mut builder = BinaryViewBuilder::new();
1083 for (i, view) in views.iter().enumerate() {
1084 let is_valid = validity.map_or(1, |v| v[i]);
1085 if is_valid == 0 {
1086 builder.append_null();
1087 } else {
1088 let view_obj = view.as_object().unwrap();
1089 let size: usize = json_uint("view size", &view_obj["SIZE"])?;
1090 if let Some(inlined) = view_obj.get("INLINED") {
1092 let data = hex::decode(inlined.as_str().unwrap()).unwrap();
1093 builder.append_value(&data);
1094 } else {
1095 let buffer_index: usize =
1097 json_uint("view buffer index", &view_obj["BUFFER_INDEX"])?;
1098 let offset: usize = json_uint("view offset", &view_obj["OFFSET"])?;
1099 let end = offset.checked_add(size).ok_or_else(|| {
1100 ArrowError::JsonError("View offset overflow".to_string())
1101 })?;
1102 let buffer = variadic_buffers.get(buffer_index).ok_or_else(|| {
1103 ArrowError::JsonError("View buffer index out of bounds".to_string())
1104 })?;
1105 let buffer_data = hex::decode(buffer).unwrap();
1106 let data = buffer_data.get(offset..end).ok_or_else(|| {
1107 ArrowError::JsonError("View range out of bounds".to_string())
1108 })?;
1109 builder.append_value(data);
1110 }
1111 }
1112 }
1113 Ok(Arc::new(builder.finish()))
1114 }
1115 DataType::RunEndEncoded(run_ends_field, values_field) => {
1116 let children = json_col.children.clone().unwrap();
1117 if children.len() != 2 {
1118 return Err(ArrowError::JsonError(
1119 "RunEndEncoded requires exactly 2 children".to_string(),
1120 ));
1121 }
1122 let run_ends_array =
1123 array_from_json(run_ends_field, children[0].clone(), dictionaries)?;
1124 let values_array = array_from_json(values_field, children[1].clone(), dictionaries)?;
1125
1126 let run_array_data = ArrayData::builder(field.data_type().clone())
1127 .len(json_col.count)
1128 .add_child_data(run_ends_array.into_data())
1129 .add_child_data(values_array.into_data())
1130 .build()
1131 .unwrap();
1132
1133 Ok(make_array(run_array_data))
1134 }
1135 t => Err(ArrowError::JsonError(format!(
1136 "data type {t} not supported"
1137 ))),
1138 }
1139}
1140
1141pub fn dictionary_array_from_json(
1143 field: &Field,
1144 json_col: ArrowJsonColumn,
1145 dict_key: &DataType,
1146 dict_value: &DataType,
1147 dictionary: &ArrowJsonDictionaryBatch,
1148 dictionaries: Option<&HashMap<i64, ArrowJsonDictionaryBatch>>,
1149) -> Result<ArrayRef> {
1150 match dict_key {
1151 DataType::Int8
1152 | DataType::Int16
1153 | DataType::Int32
1154 | DataType::Int64
1155 | DataType::UInt8
1156 | DataType::UInt16
1157 | DataType::UInt32
1158 | DataType::UInt64 => {
1159 let null_buf = create_null_buf(&json_col);
1160
1161 #[expect(deprecated)]
1163 let key_field = Field::new_dict(
1164 "key",
1165 dict_key.clone(),
1166 field.is_nullable(),
1167 #[expect(deprecated)]
1168 field
1169 .dict_id()
1170 .expect("Dictionary fields must have a dict_id value"),
1171 field
1172 .dict_is_ordered()
1173 .expect("Dictionary fields must have a dict_is_ordered value"),
1174 );
1175 let keys = array_from_json(&key_field, json_col, None)?;
1176 let value_field = Field::new("value", dict_value.clone(), true);
1178 let values = array_from_json(
1179 &value_field,
1180 dictionary.data.columns[0].clone(),
1181 dictionaries,
1182 )?;
1183
1184 let dict_data = ArrayData::builder(field.data_type().clone())
1186 .len(keys.len())
1187 .add_buffer(keys.to_data().buffers()[0].clone())
1188 .null_bit_buffer(Some(null_buf))
1189 .add_child_data(values.into_data())
1190 .build()
1191 .unwrap();
1192
1193 let array = match dict_key {
1194 DataType::Int8 => Arc::new(Int8DictionaryArray::from(dict_data)) as ArrayRef,
1195 DataType::Int16 => Arc::new(Int16DictionaryArray::from(dict_data)),
1196 DataType::Int32 => Arc::new(Int32DictionaryArray::from(dict_data)),
1197 DataType::Int64 => Arc::new(Int64DictionaryArray::from(dict_data)),
1198 DataType::UInt8 => Arc::new(UInt8DictionaryArray::from(dict_data)),
1199 DataType::UInt16 => Arc::new(UInt16DictionaryArray::from(dict_data)),
1200 DataType::UInt32 => Arc::new(UInt32DictionaryArray::from(dict_data)),
1201 DataType::UInt64 => Arc::new(UInt64DictionaryArray::from(dict_data)),
1202 _ => unreachable!(),
1203 };
1204 Ok(array)
1205 }
1206 _ => Err(ArrowError::JsonError(format!(
1207 "Dictionary key type {dict_key:?} not supported"
1208 ))),
1209 }
1210}
1211
1212fn create_null_buf(json_col: &ArrowJsonColumn) -> Buffer {
1214 let num_bytes = bit_util::ceil(json_col.count, 8);
1215 let mut null_buf = MutableBuffer::new(num_bytes).with_bitset(num_bytes, false);
1216 json_col
1217 .validity
1218 .clone()
1219 .unwrap()
1220 .iter()
1221 .enumerate()
1222 .for_each(|(i, v)| {
1223 let null_slice = null_buf.as_slice_mut();
1224 if *v != 0 {
1225 bit_util::set_bit(null_slice, i);
1226 }
1227 });
1228 null_buf.into()
1229}
1230
1231impl ArrowJsonBatch {
1232 pub fn from_batch(batch: &RecordBatch) -> ArrowJsonBatch {
1244 let mut json_batch = ArrowJsonBatch {
1245 count: batch.num_rows(),
1246 columns: Vec::with_capacity(batch.num_columns()),
1247 };
1248
1249 for (col, field) in batch.columns().iter().zip(batch.schema().fields.iter()) {
1250 let json_col = match field.data_type() {
1251 DataType::Int8 => {
1252 let col = col.as_any().downcast_ref::<Int8Array>().unwrap();
1253
1254 let mut validity: Vec<u8> = Vec::with_capacity(col.len());
1255 let mut data: Vec<Value> = Vec::with_capacity(col.len());
1256
1257 for i in 0..col.len() {
1258 if col.is_null(i) {
1259 validity.push(1);
1260 data.push(0i8.into());
1261 } else {
1262 validity.push(0);
1263 data.push(col.value(i).into());
1264 }
1265 }
1266
1267 ArrowJsonColumn {
1268 name: field.name().clone(),
1269 count: col.len(),
1270 validity: Some(validity),
1271 data: Some(data),
1272 offset: None,
1273 type_id: None,
1274 size: None,
1275 views: None,
1276 variadic_data_buffers: None,
1277 children: None,
1278 }
1279 }
1280 _ => ArrowJsonColumn {
1281 name: field.name().clone(),
1282 count: col.len(),
1283 validity: None,
1284 data: None,
1285 offset: None,
1286 type_id: None,
1287 size: None,
1288 views: None,
1289 variadic_data_buffers: None,
1290 children: None,
1291 },
1292 };
1293
1294 json_batch.columns.push(json_col);
1295 }
1296
1297 json_batch
1298 }
1299}
1300
1301#[cfg(test)]
1302mod tests {
1303 use super::*;
1304
1305 #[test]
1306 fn test_decimal256_from_json() {
1307 let field = Field::new("c", DataType::Decimal256(76, 0), false);
1308 let col: ArrowJsonColumn = serde_json::from_str(
1309 r#"{
1310 "name": "c",
1311 "count": 5,
1312 "VALIDITY": [1, 1, 1, 1, 1],
1313 "DATA": [
1314 "0",
1315 "1",
1316 "-1",
1317 "123456789012345678901234567890",
1318 "-123456789012345678901234567890"
1319 ]
1320 }"#,
1321 )
1322 .unwrap();
1323 let arr = array_from_json(&field, col, None).unwrap();
1324 let arr = arr.as_any().downcast_ref::<Decimal256Array>().unwrap();
1325 assert_eq!(arr.value(0), i256::ZERO);
1326 assert_eq!(arr.value(1), i256::from_i128(1));
1327 assert_eq!(arr.value(2), i256::from_i128(-1));
1328 let big = i256::from_string("123456789012345678901234567890").unwrap();
1329 assert_eq!(arr.value(3), big);
1330 assert_eq!(arr.value(4), i256::ZERO - big);
1331 }
1332
1333 #[test]
1334 fn test_schema_equality() {
1335 let json = r#"
1336 {
1337 "fields": [
1338 {
1339 "name": "c1",
1340 "type": {"name": "int", "isSigned": true, "bitWidth": 32},
1341 "nullable": true,
1342 "children": []
1343 },
1344 {
1345 "name": "c2",
1346 "type": {"name": "floatingpoint", "precision": "DOUBLE"},
1347 "nullable": true,
1348 "children": []
1349 },
1350 {
1351 "name": "c3",
1352 "type": {"name": "utf8"},
1353 "nullable": true,
1354 "children": []
1355 },
1356 {
1357 "name": "c4",
1358 "type": {
1359 "name": "list"
1360 },
1361 "nullable": true,
1362 "children": [
1363 {
1364 "name": "custom_item",
1365 "type": {
1366 "name": "int",
1367 "isSigned": true,
1368 "bitWidth": 32
1369 },
1370 "nullable": false,
1371 "children": []
1372 }
1373 ]
1374 }
1375 ]
1376 }"#;
1377 let json_schema: ArrowJsonSchema = serde_json::from_str(json).unwrap();
1378 let schema = Schema::new(vec![
1379 Field::new("c1", DataType::Int32, true),
1380 Field::new("c2", DataType::Float64, true),
1381 Field::new("c3", DataType::Utf8, true),
1382 Field::new(
1383 "c4",
1384 DataType::List(Arc::new(Field::new("custom_item", DataType::Int32, false))),
1385 true,
1386 ),
1387 ]);
1388 assert!(json_schema.equals_schema(&schema));
1389 }
1390
1391 #[test]
1392 fn test_arrow_data_equality() {
1393 let secs_tz = Some("Europe/Budapest".into());
1394 let millis_tz = Some("America/New_York".into());
1395 let micros_tz = Some("UTC".into());
1396 let nanos_tz = Some("Africa/Johannesburg".into());
1397
1398 let schema = Schema::new(vec![
1399 Field::new("bools-with-metadata-map", DataType::Boolean, true)
1400 .with_metadata([("k", "v")]),
1401 Field::new("bools-with-metadata-vec", DataType::Boolean, true)
1402 .with_metadata([("k2", "v2")]),
1403 Field::new("bools", DataType::Boolean, true),
1404 Field::new("int8s", DataType::Int8, true),
1405 Field::new("int16s", DataType::Int16, true),
1406 Field::new("int32s", DataType::Int32, true),
1407 Field::new("int64s", DataType::Int64, true),
1408 Field::new("uint8s", DataType::UInt8, true),
1409 Field::new("uint16s", DataType::UInt16, true),
1410 Field::new("uint32s", DataType::UInt32, true),
1411 Field::new("uint64s", DataType::UInt64, true),
1412 Field::new("float32s", DataType::Float32, true),
1413 Field::new("float64s", DataType::Float64, true),
1414 Field::new("date_days", DataType::Date32, true),
1415 Field::new("date_millis", DataType::Date64, true),
1416 Field::new("time_secs", DataType::Time32(TimeUnit::Second), true),
1417 Field::new("time_millis", DataType::Time32(TimeUnit::Millisecond), true),
1418 Field::new("time_micros", DataType::Time64(TimeUnit::Microsecond), true),
1419 Field::new("time_nanos", DataType::Time64(TimeUnit::Nanosecond), true),
1420 Field::new("ts_secs", DataType::Timestamp(TimeUnit::Second, None), true),
1421 Field::new(
1422 "ts_millis",
1423 DataType::Timestamp(TimeUnit::Millisecond, None),
1424 true,
1425 ),
1426 Field::new(
1427 "ts_micros",
1428 DataType::Timestamp(TimeUnit::Microsecond, None),
1429 true,
1430 ),
1431 Field::new(
1432 "ts_nanos",
1433 DataType::Timestamp(TimeUnit::Nanosecond, None),
1434 true,
1435 ),
1436 Field::new(
1437 "ts_secs_tz",
1438 DataType::Timestamp(TimeUnit::Second, secs_tz.clone()),
1439 true,
1440 ),
1441 Field::new(
1442 "ts_millis_tz",
1443 DataType::Timestamp(TimeUnit::Millisecond, millis_tz.clone()),
1444 true,
1445 ),
1446 Field::new(
1447 "ts_micros_tz",
1448 DataType::Timestamp(TimeUnit::Microsecond, micros_tz.clone()),
1449 true,
1450 ),
1451 Field::new(
1452 "ts_nanos_tz",
1453 DataType::Timestamp(TimeUnit::Nanosecond, nanos_tz.clone()),
1454 true,
1455 ),
1456 Field::new("utf8s", DataType::Utf8, true),
1457 Field::new(
1458 "lists",
1459 DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))),
1460 true,
1461 ),
1462 Field::new(
1463 "structs",
1464 DataType::Struct(Fields::from(vec![
1465 Field::new("int32s", DataType::Int32, true),
1466 Field::new("utf8s", DataType::Utf8, true),
1467 ])),
1468 true,
1469 ),
1470 Field::new("utf8views", DataType::Utf8View, true),
1471 Field::new("binaryviews", DataType::BinaryView, true),
1472 Field::new(
1473 "listviews",
1474 DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, true))),
1475 true,
1476 ),
1477 Field::new(
1478 "largelistviews",
1479 DataType::LargeListView(Arc::new(Field::new_list_field(DataType::Int32, true))),
1480 true,
1481 ),
1482 Field::new(
1483 "runendencoded",
1484 DataType::RunEndEncoded(
1485 Arc::new(Field::new(
1486 Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
1487 DataType::Int16,
1488 false,
1489 )),
1490 Arc::new(Field::new(
1491 Field::REE_VALUES_FIELD_DEFAULT_NAME,
1492 DataType::Int32,
1493 true,
1494 )),
1495 ),
1496 true,
1497 ),
1498 ]);
1499
1500 let bools_with_metadata_map = BooleanArray::from(vec![Some(true), None, Some(false)]);
1501 let bools_with_metadata_vec = BooleanArray::from(vec![Some(true), None, Some(false)]);
1502 let bools = BooleanArray::from(vec![Some(true), None, Some(false)]);
1503 let int8s = Int8Array::from(vec![Some(1), None, Some(3)]);
1504 let int16s = Int16Array::from(vec![Some(1), None, Some(3)]);
1505 let int32s = Int32Array::from(vec![Some(1), None, Some(3)]);
1506 let int64s = Int64Array::from(vec![Some(1), None, Some(3)]);
1507 let uint8s = UInt8Array::from(vec![Some(1), None, Some(3)]);
1508 let uint16s = UInt16Array::from(vec![Some(1), None, Some(3)]);
1509 let uint32s = UInt32Array::from(vec![Some(1), None, Some(3)]);
1510 let uint64s = UInt64Array::from(vec![Some(1), None, Some(3)]);
1511 let float32s = Float32Array::from(vec![Some(1.0), None, Some(3.0)]);
1512 let float64s = Float64Array::from(vec![Some(1.0), None, Some(3.0)]);
1513 let date_days = Date32Array::from(vec![Some(1196848), None, None]);
1514 let date_millis = Date64Array::from(vec![
1515 Some(167903550396207),
1516 Some(29923997007884),
1517 Some(30612271819236),
1518 ]);
1519 let time_secs = Time32SecondArray::from(vec![Some(27974), Some(78592), Some(43207)]);
1520 let time_millis =
1521 Time32MillisecondArray::from(vec![Some(6613125), Some(74667230), Some(52260079)]);
1522 let time_micros = Time64MicrosecondArray::from(vec![Some(62522958593), None, None]);
1523 let time_nanos =
1524 Time64NanosecondArray::from(vec![Some(73380123595985), None, Some(16584393546415)]);
1525 let ts_secs = TimestampSecondArray::from(vec![None, Some(193438817552), None]);
1526 let ts_millis =
1527 TimestampMillisecondArray::from(vec![None, Some(38606916383008), Some(58113709376587)]);
1528 let ts_micros = TimestampMicrosecondArray::from(vec![None, None, None]);
1529 let ts_nanos = TimestampNanosecondArray::from(vec![None, None, Some(-6473623571954960143)]);
1530 let ts_secs_tz = TimestampSecondArray::from(vec![None, Some(193438817552), None])
1531 .with_timezone_opt(secs_tz);
1532 let ts_millis_tz =
1533 TimestampMillisecondArray::from(vec![None, Some(38606916383008), Some(58113709376587)])
1534 .with_timezone_opt(millis_tz);
1535 let ts_micros_tz =
1536 TimestampMicrosecondArray::from(vec![None, None, None]).with_timezone_opt(micros_tz);
1537 let ts_nanos_tz =
1538 TimestampNanosecondArray::from(vec![None, None, Some(-6473623571954960143)])
1539 .with_timezone_opt(nanos_tz);
1540 let utf8s = StringArray::from(vec![Some("aa"), None, Some("bbb")]);
1541
1542 let value_data = Int32Array::from(vec![None, Some(2), None, None]);
1543 let value_offsets = Buffer::from_slice_ref([0, 3, 4, 4]);
1544 let list_data_type = DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true)));
1545 let list_data = ArrayData::builder(list_data_type)
1546 .len(3)
1547 .add_buffer(value_offsets)
1548 .add_child_data(value_data.into_data())
1549 .null_bit_buffer(Some(Buffer::from([0b00000011])))
1550 .build()
1551 .unwrap();
1552 let lists = ListArray::from(list_data);
1553
1554 let structs_int32s = Int32Array::from(vec![None, Some(-2), None]);
1555 let structs_utf8s = StringArray::from(vec![None, None, Some("aaaaaa")]);
1556 let struct_data_type = DataType::Struct(Fields::from(vec![
1557 Field::new("int32s", DataType::Int32, true),
1558 Field::new("utf8s", DataType::Utf8, true),
1559 ]));
1560 let struct_data = ArrayData::builder(struct_data_type)
1561 .len(3)
1562 .add_child_data(structs_int32s.into_data())
1563 .add_child_data(structs_utf8s.into_data())
1564 .null_bit_buffer(Some(Buffer::from([0b00000011])))
1565 .build()
1566 .unwrap();
1567 let structs = StructArray::from(struct_data);
1568
1569 let utf8views =
1570 StringViewArray::from(vec![Some("hello"), None, Some("this is not inlined")]);
1571 let binaryviews = BinaryViewArray::from_iter(vec![
1572 Some(b"\xf3\x4d".as_slice()),
1573 Some(b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f".as_slice()),
1574 None,
1575 ]);
1576
1577 let listview_value_data = Int32Array::from(vec![Some(1), Some(2), Some(3), None, Some(5)]);
1578 let listview_offsets = Buffer::from_slice_ref([0i32, 2, 2]);
1579 let listview_sizes = Buffer::from_slice_ref([2i32, 0, 3]);
1580 let listview_data_type =
1581 DataType::ListView(Arc::new(Field::new_list_field(DataType::Int32, true)));
1582 let listview_data = ArrayData::builder(listview_data_type)
1583 .len(3)
1584 .add_buffer(listview_offsets)
1585 .add_buffer(listview_sizes)
1586 .add_child_data(listview_value_data.into_data())
1587 .null_bit_buffer(Some(Buffer::from([0b00000101])))
1588 .build()
1589 .unwrap();
1590 let listviews = ListViewArray::from(listview_data);
1591
1592 let largelistview_value_data = Int32Array::from(vec![Some(10), None, Some(30)]);
1593 let largelistview_offsets = Buffer::from_slice_ref([0i64, 2, 3]);
1594 let largelistview_sizes = Buffer::from_slice_ref([2i64, 1, 0]);
1595 let largelistview_data_type =
1596 DataType::LargeListView(Arc::new(Field::new_list_field(DataType::Int32, true)));
1597 let largelistview_data = ArrayData::builder(largelistview_data_type)
1598 .len(3)
1599 .add_buffer(largelistview_offsets)
1600 .add_buffer(largelistview_sizes)
1601 .add_child_data(largelistview_value_data.into_data())
1602 .null_bit_buffer(Some(Buffer::from([0b00000011])))
1603 .build()
1604 .unwrap();
1605 let largelistviews = LargeListViewArray::from(largelistview_data);
1606
1607 let ree_run_ends = Int16Array::from(vec![2, 3]);
1608 let ree_values = Int32Array::from(vec![Some(100), None]);
1609 let ree_data_type = DataType::RunEndEncoded(
1610 Arc::new(Field::new(
1611 Field::REE_RUN_ENDS_FIELD_DEFAULT_NAME,
1612 DataType::Int16,
1613 false,
1614 )),
1615 Arc::new(Field::new(
1616 Field::REE_VALUES_FIELD_DEFAULT_NAME,
1617 DataType::Int32,
1618 true,
1619 )),
1620 );
1621 let ree_data = ArrayData::builder(ree_data_type)
1622 .len(3)
1623 .add_child_data(ree_run_ends.into_data())
1624 .add_child_data(ree_values.into_data())
1625 .build()
1626 .unwrap();
1627 let runendencoded = RunArray::<Int16Type>::from(ree_data);
1628
1629 let record_batch = RecordBatch::try_new(
1630 Arc::new(schema.clone()),
1631 vec![
1632 Arc::new(bools_with_metadata_map),
1633 Arc::new(bools_with_metadata_vec),
1634 Arc::new(bools),
1635 Arc::new(int8s),
1636 Arc::new(int16s),
1637 Arc::new(int32s),
1638 Arc::new(int64s),
1639 Arc::new(uint8s),
1640 Arc::new(uint16s),
1641 Arc::new(uint32s),
1642 Arc::new(uint64s),
1643 Arc::new(float32s),
1644 Arc::new(float64s),
1645 Arc::new(date_days),
1646 Arc::new(date_millis),
1647 Arc::new(time_secs),
1648 Arc::new(time_millis),
1649 Arc::new(time_micros),
1650 Arc::new(time_nanos),
1651 Arc::new(ts_secs),
1652 Arc::new(ts_millis),
1653 Arc::new(ts_micros),
1654 Arc::new(ts_nanos),
1655 Arc::new(ts_secs_tz),
1656 Arc::new(ts_millis_tz),
1657 Arc::new(ts_micros_tz),
1658 Arc::new(ts_nanos_tz),
1659 Arc::new(utf8s),
1660 Arc::new(lists),
1661 Arc::new(structs),
1662 Arc::new(utf8views),
1663 Arc::new(binaryviews),
1664 Arc::new(listviews),
1665 Arc::new(largelistviews),
1666 Arc::new(runendencoded),
1667 ],
1668 )
1669 .unwrap();
1670 let json = std::fs::read_to_string("data/integration.json").unwrap();
1671 let arrow_json: ArrowJson = serde_json::from_str(&json).unwrap();
1672 assert!(arrow_json.schema.equals_schema(&schema));
1674 assert_eq!(arrow_json.get_record_batches().unwrap()[0], record_batch);
1676 }
1677}