1use arrow_schema::ArrowError;
8use parquet_variant::{ObjectFieldBuilder, Variant, VariantBuilderExt};
9use serde_json::{Number, Value};
10
11pub trait JsonToVariant {
54 fn append_json(&mut self, json: &str) -> Result<(), ArrowError>;
56}
57
58impl<T: VariantBuilderExt> JsonToVariant for T {
59 fn append_json(&mut self, json: &str) -> Result<(), ArrowError> {
60 let json: Value = serde_json::from_str(json)
61 .map_err(|e| ArrowError::InvalidArgumentError(format!("JSON format error: {e}")))?;
62
63 append_json(&json, self)?;
64 Ok(())
65 }
66}
67
68fn variant_from_number<'m, 'v>(n: &Number) -> Result<Variant<'m, 'v>, ArrowError> {
69 if let Some(i) = n.as_i64() {
70 if i as i8 as i64 == i {
72 Ok((i as i8).into())
73 } else if i as i16 as i64 == i {
74 Ok((i as i16).into())
75 } else if i as i32 as i64 == i {
76 Ok((i as i32).into())
77 } else {
78 Ok(i.into())
79 }
80 } else {
81 match n.as_f64() {
84 Some(f) => return Ok(f.into()),
85 None => Err(ArrowError::InvalidArgumentError(format!(
86 "Failed to parse {n} as number",
87 ))),
88 }?
89 }
90}
91
92pub fn append_json(json: &Value, builder: &mut impl VariantBuilderExt) -> Result<(), ArrowError> {
93 match json {
94 Value::Null => builder.append_value(Variant::Null),
95 Value::Bool(b) => builder.append_value(*b),
96 Value::Number(n) => {
97 builder.append_value(variant_from_number(n)?);
98 }
99 Value::String(s) => builder.append_value(s.as_str()),
100 Value::Array(arr) => {
101 let mut list_builder = builder.try_new_list()?;
102 for val in arr {
103 append_json(val, &mut list_builder)?;
104 }
105 list_builder.finish();
106 }
107 Value::Object(obj) => {
108 let mut obj_builder = builder.try_new_object()?;
109 for (key, value) in obj {
110 let mut field_builder = ObjectFieldBuilder::new(key, &mut obj_builder);
111 append_json(value, &mut field_builder)?;
112 }
113 obj_builder.finish();
114 }
115 }
116 Ok(())
117}
118
119#[cfg(test)]
120mod test {
121 use super::*;
122 use crate::VariantToJson;
123 use arrow_schema::ArrowError;
124 use parquet_variant::{
125 ShortString, Variant, VariantBuilder, VariantDecimal4, VariantDecimal8, VariantDecimal16,
126 };
127
128 struct JsonToVariantTest<'a> {
129 json: &'a str,
130 expected: Variant<'a, 'a>,
131 }
132
133 impl JsonToVariantTest<'_> {
134 fn run(self) -> Result<(), ArrowError> {
135 let mut variant_builder = VariantBuilder::new();
136 variant_builder.append_json(self.json)?;
137 let (metadata, value) = variant_builder.finish();
138 let variant = Variant::try_new(&metadata, &value)?;
139 assert_eq!(variant, self.expected);
140 Ok(())
141 }
142 }
143
144 #[test]
145 fn test_json_to_variant_null() -> Result<(), ArrowError> {
146 JsonToVariantTest {
147 json: "null",
148 expected: Variant::Null,
149 }
150 .run()
151 }
152
153 #[test]
154 fn test_json_to_variant_boolean_true() -> Result<(), ArrowError> {
155 JsonToVariantTest {
156 json: "true",
157 expected: Variant::BooleanTrue,
158 }
159 .run()
160 }
161
162 #[test]
163 fn test_json_to_variant_boolean_false() -> Result<(), ArrowError> {
164 JsonToVariantTest {
165 json: "false",
166 expected: Variant::BooleanFalse,
167 }
168 .run()
169 }
170
171 #[test]
172 fn test_json_to_variant_int8_positive() -> Result<(), ArrowError> {
173 JsonToVariantTest {
174 json: " 127 ",
175 expected: Variant::Int8(127),
176 }
177 .run()
178 }
179
180 #[test]
181 fn test_json_to_variant_int8_negative() -> Result<(), ArrowError> {
182 JsonToVariantTest {
183 json: " -128 ",
184 expected: Variant::Int8(-128),
185 }
186 .run()
187 }
188
189 #[test]
190 fn test_json_to_variant_int16() -> Result<(), ArrowError> {
191 JsonToVariantTest {
192 json: " 27134 ",
193 expected: Variant::Int16(27134),
194 }
195 .run()
196 }
197
198 #[test]
199 fn test_json_to_variant_int32() -> Result<(), ArrowError> {
200 JsonToVariantTest {
201 json: " -32767431 ",
202 expected: Variant::Int32(-32767431),
203 }
204 .run()
205 }
206
207 #[test]
208 fn test_json_to_variant_int64() -> Result<(), ArrowError> {
209 JsonToVariantTest {
210 json: "92842754201389",
211 expected: Variant::Int64(92842754201389),
212 }
213 .run()
214 }
215
216 #[ignore]
217 #[test]
218 fn test_json_to_variant_decimal4_basic() -> Result<(), ArrowError> {
219 JsonToVariantTest {
220 json: "1.23",
221 expected: Variant::from(VariantDecimal4::try_new(123, 2)?),
222 }
223 .run()
224 }
225
226 #[ignore]
227 #[test]
228 fn test_json_to_variant_decimal4_large_positive() -> Result<(), ArrowError> {
229 JsonToVariantTest {
230 json: "99999999.9",
231 expected: Variant::from(VariantDecimal4::try_new(999999999, 1)?),
232 }
233 .run()
234 }
235
236 #[ignore]
237 #[test]
238 fn test_json_to_variant_decimal4_large_negative() -> Result<(), ArrowError> {
239 JsonToVariantTest {
240 json: "-99999999.9",
241 expected: Variant::from(VariantDecimal4::try_new(-999999999, 1)?),
242 }
243 .run()
244 }
245
246 #[ignore]
247 #[test]
248 fn test_json_to_variant_decimal4_small_positive() -> Result<(), ArrowError> {
249 JsonToVariantTest {
250 json: "0.999999999",
251 expected: Variant::from(VariantDecimal4::try_new(999999999, 9)?),
252 }
253 .run()
254 }
255
256 #[ignore]
257 #[test]
258 fn test_json_to_variant_decimal4_tiny_positive() -> Result<(), ArrowError> {
259 JsonToVariantTest {
260 json: "0.000000001",
261 expected: Variant::from(VariantDecimal4::try_new(1, 9)?),
262 }
263 .run()
264 }
265
266 #[ignore]
267 #[test]
268 fn test_json_to_variant_decimal4_small_negative() -> Result<(), ArrowError> {
269 JsonToVariantTest {
270 json: "-0.999999999",
271 expected: Variant::from(VariantDecimal4::try_new(-999999999, 9)?),
272 }
273 .run()
274 }
275
276 #[ignore]
277 #[test]
278 fn test_json_to_variant_decimal8_positive() -> Result<(), ArrowError> {
279 JsonToVariantTest {
280 json: "999999999.0",
281 expected: Variant::from(VariantDecimal8::try_new(9999999990, 1)?),
282 }
283 .run()
284 }
285
286 #[ignore]
287 #[test]
288 fn test_json_to_variant_decimal8_negative() -> Result<(), ArrowError> {
289 JsonToVariantTest {
290 json: "-999999999.0",
291 expected: Variant::from(VariantDecimal8::try_new(-9999999990, 1)?),
292 }
293 .run()
294 }
295
296 #[ignore]
297 #[test]
298 fn test_json_to_variant_decimal8_high_precision() -> Result<(), ArrowError> {
299 JsonToVariantTest {
300 json: "0.999999999999999999",
301 expected: Variant::from(VariantDecimal8::try_new(999999999999999999, 18)?),
302 }
303 .run()
304 }
305
306 #[ignore]
307 #[test]
308 fn test_json_to_variant_decimal8_large_with_scale() -> Result<(), ArrowError> {
309 JsonToVariantTest {
310 json: "9999999999999999.99",
311 expected: Variant::from(VariantDecimal8::try_new(999999999999999999, 2)?),
312 }
313 .run()
314 }
315
316 #[ignore]
317 #[test]
318 fn test_json_to_variant_decimal8_large_negative_with_scale() -> Result<(), ArrowError> {
319 JsonToVariantTest {
320 json: "-9999999999999999.99",
321 expected: Variant::from(VariantDecimal8::try_new(-999999999999999999, 2)?),
322 }
323 .run()
324 }
325
326 #[ignore]
327 #[test]
328 fn test_json_to_variant_decimal16_large_integer() -> Result<(), ArrowError> {
329 JsonToVariantTest {
330 json: "9999999999999999999", expected: Variant::from(VariantDecimal16::try_new(9999999999999999999, 0)?),
332 }
333 .run()
334 }
335
336 #[ignore]
337 #[test]
338 fn test_json_to_variant_decimal16_high_precision() -> Result<(), ArrowError> {
339 JsonToVariantTest {
340 json: "0.9999999999999999999",
341 expected: Variant::from(VariantDecimal16::try_new(9999999999999999999, 19)?),
342 }
343 .run()
344 }
345
346 #[ignore]
347 #[test]
348 fn test_json_to_variant_decimal16_max_value() -> Result<(), ArrowError> {
349 JsonToVariantTest {
350 json: "79228162514264337593543950335", expected: Variant::from(VariantDecimal16::try_new(79228162514264337593543950335, 0)?),
352 }
353 .run()
354 }
355
356 #[ignore]
357 #[test]
358 fn test_json_to_variant_decimal16_max_scale() -> Result<(), ArrowError> {
359 JsonToVariantTest {
360 json: "7.9228162514264337593543950335", expected: Variant::from(VariantDecimal16::try_new(
363 79228162514264337593543950335,
364 28,
365 )?),
366 }
367 .run()
368 }
369
370 #[test]
371 fn test_json_to_variant_double_precision() -> Result<(), ArrowError> {
372 JsonToVariantTest {
373 json: "0.79228162514264337593543950335",
374 expected: Variant::Double(0.792_281_625_142_643_4_f64),
375 }
376 .run()
377 }
378
379 #[test]
380 fn test_json_to_variant_double_scientific_positive() -> Result<(), ArrowError> {
381 JsonToVariantTest {
382 json: "15e-1",
383 expected: Variant::Double(15e-1f64),
384 }
385 .run()
386 }
387
388 #[test]
389 fn test_json_to_variant_double_scientific_negative() -> Result<(), ArrowError> {
390 JsonToVariantTest {
391 json: "-15e-1",
392 expected: Variant::Double(-15e-1f64),
393 }
394 .run()
395 }
396
397 #[test]
398 fn test_json_to_variant_short_string() -> Result<(), ArrowError> {
399 JsonToVariantTest {
400 json: "\"harsh\"",
401 expected: Variant::ShortString(ShortString::try_new("harsh")?),
402 }
403 .run()
404 }
405
406 #[test]
407 fn test_json_to_variant_short_string_max_length() -> Result<(), ArrowError> {
408 JsonToVariantTest {
409 json: &format!("\"{}\"", "a".repeat(63)),
410 expected: Variant::ShortString(ShortString::try_new(&"a".repeat(63))?),
411 }
412 .run()
413 }
414
415 #[test]
416 fn test_json_to_variant_long_string() -> Result<(), ArrowError> {
417 JsonToVariantTest {
418 json: &format!("\"{}\"", "a".repeat(64)),
419 expected: Variant::String(&"a".repeat(64)),
420 }
421 .run()
422 }
423
424 #[test]
425 fn test_json_to_variant_very_long_string() -> Result<(), ArrowError> {
426 JsonToVariantTest {
427 json: &format!("\"{}\"", "b".repeat(100000)),
428 expected: Variant::String(&"b".repeat(100000)),
429 }
430 .run()
431 }
432
433 #[test]
434 fn test_json_to_variant_array_simple() -> Result<(), ArrowError> {
435 let mut variant_builder = VariantBuilder::new();
436 let mut list_builder = variant_builder.new_list();
437 list_builder.append_value(Variant::Int8(127));
438 list_builder.append_value(Variant::Int16(128));
439 list_builder.append_value(Variant::Int32(-32767431));
440 list_builder.finish();
441 let (metadata, value) = variant_builder.finish();
442 let variant = Variant::try_new(&metadata, &value)?;
443
444 JsonToVariantTest {
445 json: "[127, 128, -32767431]",
446 expected: variant,
447 }
448 .run()
449 }
450
451 #[test]
452 fn test_json_to_variant_array_with_object() -> Result<(), ArrowError> {
453 let mut variant_builder = VariantBuilder::new();
454 let mut list_builder = variant_builder.new_list();
455 let mut object_builder_inner = list_builder.new_object();
456 object_builder_inner.insert("age", Variant::Int8(32));
457 object_builder_inner.finish();
458 list_builder.append_value(Variant::Int16(128));
459 list_builder.append_value(Variant::BooleanFalse);
460 list_builder.finish();
461 let (metadata, value) = variant_builder.finish();
462 let variant = Variant::try_new(&metadata, &value)?;
463
464 JsonToVariantTest {
465 json: "[{\"age\": 32}, 128, false]",
466 expected: variant,
467 }
468 .run()
469 }
470
471 #[test]
472 fn test_json_to_variant_array_large_u16_offset() -> Result<(), ArrowError> {
473 let mut variant_builder = VariantBuilder::new();
475 let mut list_builder = variant_builder.new_list();
476 for _ in 0..128 {
477 list_builder.append_value(Variant::Int8(1));
478 }
479 list_builder.append_value(Variant::BooleanTrue);
480 list_builder.finish();
481 let (metadata, value) = variant_builder.finish();
482 let variant = Variant::try_new(&metadata, &value)?;
483
484 JsonToVariantTest {
485 json: &format!("[{} true]", "1, ".repeat(128)),
486 expected: variant,
487 }
488 .run()
489 }
490
491 #[test]
492 fn test_json_to_variant_array_nested_large() -> Result<(), ArrowError> {
493 let mut variant_builder = VariantBuilder::new();
495 let mut list_builder = variant_builder.new_list();
496 for _ in 0..256 {
497 let mut list_builder_inner = list_builder.new_list();
498 for _ in 0..255 {
499 list_builder_inner.append_value(Variant::Null);
500 }
501 list_builder_inner.finish();
502 }
503 list_builder.finish();
504 let (metadata, value) = variant_builder.finish();
505 let variant = Variant::try_new(&metadata, &value)?;
506 let intermediate = format!("[{}]", vec!["null"; 255].join(", "));
507 let json = format!("[{}]", vec![intermediate; 256].join(", "));
508 JsonToVariantTest {
509 json: json.as_str(),
510 expected: variant,
511 }
512 .run()
513 }
514
515 #[test]
516 fn test_json_to_variant_object_simple() -> Result<(), ArrowError> {
517 let mut variant_builder = VariantBuilder::new();
518 let mut object_builder = variant_builder.new_object();
519 object_builder.insert("a", Variant::Int8(3));
520 object_builder.insert("b", Variant::Int8(2));
521 object_builder.finish();
522 let (metadata, value) = variant_builder.finish();
523 let variant = Variant::try_new(&metadata, &value)?;
524 JsonToVariantTest {
525 json: "{\"b\": 2, \"a\": 1, \"a\": 3}",
526 expected: variant,
527 }
528 .run()
529 }
530
531 #[test]
532 fn test_json_to_variant_object_complex() -> Result<(), ArrowError> {
533 let mut variant_builder = VariantBuilder::new();
534 let mut object_builder = variant_builder.new_object();
535 let mut inner_list_builder = object_builder.new_list("booleans");
536 inner_list_builder.append_value(Variant::BooleanTrue);
537 inner_list_builder.append_value(Variant::BooleanFalse);
538 inner_list_builder.finish();
539 object_builder.insert("null", Variant::Null);
540 let mut inner_list_builder = object_builder.new_list("numbers");
541 inner_list_builder.append_value(Variant::Int8(4));
542 inner_list_builder.append_value(Variant::Double(-3e0));
543 inner_list_builder.append_value(Variant::Double(1001e-3));
544 inner_list_builder.finish();
545 object_builder.finish();
546 let (metadata, value) = variant_builder.finish();
547 let variant = Variant::try_new(&metadata, &value)?;
548 JsonToVariantTest {
549 json: "{\"numbers\": [4, -3e0, 1001e-3], \"null\": null, \"booleans\": [true, false]}",
550 expected: variant,
551 }
552 .run()
553 }
554
555 #[test]
556 fn test_json_to_variant_object_very_large() -> Result<(), ArrowError> {
557 let keys: Vec<String> = (0..=255).map(|n| format!("{n:03}")).collect();
560 let innermost_list: String = format!(
561 "[{}]",
562 (0..=127)
563 .map(|n| format!("{n}"))
564 .collect::<Vec<_>>()
565 .join(",")
566 );
567 let inner_keys: Vec<String> = (240..=495).map(|n| format!("{n}")).collect();
568 let inner_object = format!(
569 "{{{}:{}}}",
570 inner_keys
571 .iter()
572 .map(|k| format!("\"{k}\""))
573 .collect::<Vec<String>>()
574 .join(format!(":{innermost_list},").as_str()),
575 innermost_list
576 );
577 let json = format!(
578 "{{{}:{}}}",
579 keys.iter()
580 .map(|k| format!("\"{k}\""))
581 .collect::<Vec<String>>()
582 .join(format!(":{inner_object},").as_str()),
583 inner_object
584 );
585 let mut variant_builder = VariantBuilder::new();
587 variant_builder.append_json(&json)?;
588 let (metadata, value) = variant_builder.finish();
589 let v = Variant::try_new(&metadata, &value)?;
590 let output_string = v.to_json_string()?;
591 assert_eq!(output_string, json);
592 assert_eq!(metadata.len(), 2485);
594 assert_eq!(value.len(), 34147849);
599
600 let mut variant_builder = VariantBuilder::new();
601 let mut object_builder = variant_builder.new_object();
602 keys.iter().for_each(|key| {
603 let mut inner_object_builder = object_builder.new_object(key);
604 inner_keys.iter().for_each(|inner_key| {
605 let mut list_builder = inner_object_builder.new_list(inner_key);
606 for i in 0..=127 {
607 list_builder.append_value(Variant::Int8(i));
608 }
609 list_builder.finish();
610 });
611 inner_object_builder.finish();
612 });
613 object_builder.finish();
614 let (metadata, value) = variant_builder.finish();
615 let variant = Variant::try_new(&metadata, &value)?;
616
617 JsonToVariantTest {
618 json: &json,
619 expected: variant,
620 }
621 .run()
622 }
623
624 #[test]
625 fn test_json_to_variant_unicode() -> Result<(), ArrowError> {
626 let json = "{\"爱\":\"अ\",\"a\":1}";
627 let mut variant_builder = VariantBuilder::new();
628 variant_builder.append_json(json)?;
629 let (metadata, value) = variant_builder.finish();
630 let v = Variant::try_new(&metadata, &value)?;
631 let output_string = v.to_json_string()?;
632 assert_eq!(output_string, "{\"a\":1,\"爱\":\"अ\"}");
633 let mut variant_builder = VariantBuilder::new();
634 let mut object_builder = variant_builder.new_object();
635 object_builder.insert("a", Variant::Int8(1));
636 object_builder.insert("爱", Variant::ShortString(ShortString::try_new("अ")?));
637 object_builder.finish();
638 let (metadata, value) = variant_builder.finish();
639 let variant = Variant::try_new(&metadata, &value)?;
640
641 assert_eq!(
642 value,
643 &[
644 2u8, 2u8, 0u8, 1u8, 0u8, 2u8, 6u8, 12u8, 1u8, 13u8, 0xe0u8, 0xa4u8, 0x85u8
645 ]
646 );
647 assert_eq!(
648 metadata,
649 &[17u8, 2u8, 0u8, 1u8, 4u8, 97u8, 0xe7u8, 0x88u8, 0xb1u8]
650 );
651 JsonToVariantTest {
652 json,
653 expected: variant,
654 }
655 .run()
656 }
657}