Skip to main content

parquet_variant_json/
from_json.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Module for parsing JSON strings as Variant
6
7use arrow_schema::ArrowError;
8use parquet_variant::{ObjectFieldBuilder, Variant, VariantBuilderExt};
9use serde_json::{Number, Value};
10
11/// Converts a JSON string to Variant using a [`VariantBuilderExt`], such as
12/// [`VariantBuilder`].
13///
14/// The resulting `value` and `metadata` buffers can be
15/// extracted using `builder.finish()`
16///
17/// # Arguments
18/// * `json` - The JSON string to parse as Variant.
19///
20/// # Returns
21///
22/// * `Ok(())` if successful
23/// * `Err` with error details if the conversion fails
24///
25/// [`VariantBuilder`]: parquet_variant::VariantBuilder
26///
27/// ```rust
28/// # use parquet_variant::VariantBuilder;
29/// # use parquet_variant_json::{JsonToVariant, VariantToJson};
30///
31/// let mut variant_builder = VariantBuilder::new();
32/// let person_string = "{\"name\":\"Alice\", \"age\":30, ".to_string()
33/// + "\"email\":\"alice@example.com\", \"is_active\": true, \"score\": 95.7,"
34/// + "\"additional_info\": null}";
35/// variant_builder.append_json(&person_string)?;
36///
37/// let (metadata, value) = variant_builder.finish();
38///
39/// let variant = parquet_variant::Variant::try_new(&metadata, &value)?;
40///
41/// let json_result = variant.to_json_string()?;
42/// let json_value = variant.to_json_value()?;
43///
44/// let mut buffer = Vec::new();
45/// variant.to_json(&mut buffer)?;
46/// let buffer_result = String::from_utf8(buffer)?;
47/// assert_eq!(json_result, "{\"additional_info\":null,\"age\":30,".to_string() +
48/// "\"email\":\"alice@example.com\",\"is_active\":true,\"name\":\"Alice\",\"score\":95.7}");
49/// assert_eq!(json_result, buffer_result);
50/// assert_eq!(json_result, serde_json::to_string(&json_value)?);
51/// # Ok::<(), Box<dyn std::error::Error>>(())
52/// ```
53pub trait JsonToVariant {
54    /// Create a Variant from a JSON string
55    fn append_json(&mut self, json: &str) -> Result<(), ArrowError>;
56}
57
58impl<T: VariantBuilderExt> JsonToVariant for T {
59    fn append_json(&mut self, json: &str) -> Result<(), ArrowError> {
60        let json: Value = serde_json::from_str(json)
61            .map_err(|e| ArrowError::InvalidArgumentError(format!("JSON format error: {e}")))?;
62
63        append_json(&json, self)?;
64        Ok(())
65    }
66}
67
68fn variant_from_number<'m, 'v>(n: &Number) -> Result<Variant<'m, 'v>, ArrowError> {
69    if let Some(i) = n.as_i64() {
70        // Find minimum Integer width to fit
71        if i as i8 as i64 == i {
72            Ok((i as i8).into())
73        } else if i as i16 as i64 == i {
74            Ok((i as i16).into())
75        } else if i as i32 as i64 == i {
76            Ok((i as i32).into())
77        } else {
78            Ok(i.into())
79        }
80    } else {
81        // Todo: Try decimal once we implement custom JSON parsing where we have access to strings
82        // Try double - currently json_to_variant does not produce decimal
83        match n.as_f64() {
84            Some(f) => return Ok(f.into()),
85            None => Err(ArrowError::InvalidArgumentError(format!(
86                "Failed to parse {n} as number",
87            ))),
88        }?
89    }
90}
91
92pub fn append_json(json: &Value, builder: &mut impl VariantBuilderExt) -> Result<(), ArrowError> {
93    match json {
94        Value::Null => builder.append_value(Variant::Null),
95        Value::Bool(b) => builder.append_value(*b),
96        Value::Number(n) => {
97            builder.append_value(variant_from_number(n)?);
98        }
99        Value::String(s) => builder.append_value(s.as_str()),
100        Value::Array(arr) => {
101            let mut list_builder = builder.try_new_list()?;
102            for val in arr {
103                append_json(val, &mut list_builder)?;
104            }
105            list_builder.finish();
106        }
107        Value::Object(obj) => {
108            let mut obj_builder = builder.try_new_object()?;
109            for (key, value) in obj {
110                let mut field_builder = ObjectFieldBuilder::new(key, &mut obj_builder);
111                append_json(value, &mut field_builder)?;
112            }
113            obj_builder.finish();
114        }
115    }
116    Ok(())
117}
118
119#[cfg(test)]
120mod test {
121    use super::*;
122    use crate::VariantToJson;
123    use arrow_schema::ArrowError;
124    use parquet_variant::{
125        ShortString, Variant, VariantBuilder, VariantDecimal4, VariantDecimal8, VariantDecimal16,
126    };
127
128    struct JsonToVariantTest<'a> {
129        json: &'a str,
130        expected: Variant<'a, 'a>,
131    }
132
133    impl JsonToVariantTest<'_> {
134        fn run(self) -> Result<(), ArrowError> {
135            let mut variant_builder = VariantBuilder::new();
136            variant_builder.append_json(self.json)?;
137            let (metadata, value) = variant_builder.finish();
138            let variant = Variant::try_new(&metadata, &value)?;
139            assert_eq!(variant, self.expected);
140            Ok(())
141        }
142    }
143
144    #[test]
145    fn test_json_to_variant_null() -> Result<(), ArrowError> {
146        JsonToVariantTest {
147            json: "null",
148            expected: Variant::Null,
149        }
150        .run()
151    }
152
153    #[test]
154    fn test_json_to_variant_boolean_true() -> Result<(), ArrowError> {
155        JsonToVariantTest {
156            json: "true",
157            expected: Variant::BooleanTrue,
158        }
159        .run()
160    }
161
162    #[test]
163    fn test_json_to_variant_boolean_false() -> Result<(), ArrowError> {
164        JsonToVariantTest {
165            json: "false",
166            expected: Variant::BooleanFalse,
167        }
168        .run()
169    }
170
171    #[test]
172    fn test_json_to_variant_int8_positive() -> Result<(), ArrowError> {
173        JsonToVariantTest {
174            json: "  127 ",
175            expected: Variant::Int8(127),
176        }
177        .run()
178    }
179
180    #[test]
181    fn test_json_to_variant_int8_negative() -> Result<(), ArrowError> {
182        JsonToVariantTest {
183            json: "  -128 ",
184            expected: Variant::Int8(-128),
185        }
186        .run()
187    }
188
189    #[test]
190    fn test_json_to_variant_int16() -> Result<(), ArrowError> {
191        JsonToVariantTest {
192            json: "  27134  ",
193            expected: Variant::Int16(27134),
194        }
195        .run()
196    }
197
198    #[test]
199    fn test_json_to_variant_int32() -> Result<(), ArrowError> {
200        JsonToVariantTest {
201            json: " -32767431  ",
202            expected: Variant::Int32(-32767431),
203        }
204        .run()
205    }
206
207    #[test]
208    fn test_json_to_variant_int64() -> Result<(), ArrowError> {
209        JsonToVariantTest {
210            json: "92842754201389",
211            expected: Variant::Int64(92842754201389),
212        }
213        .run()
214    }
215
216    #[ignore]
217    #[test]
218    fn test_json_to_variant_decimal4_basic() -> Result<(), ArrowError> {
219        JsonToVariantTest {
220            json: "1.23",
221            expected: Variant::from(VariantDecimal4::try_new(123, 2)?),
222        }
223        .run()
224    }
225
226    #[ignore]
227    #[test]
228    fn test_json_to_variant_decimal4_large_positive() -> Result<(), ArrowError> {
229        JsonToVariantTest {
230            json: "99999999.9",
231            expected: Variant::from(VariantDecimal4::try_new(999999999, 1)?),
232        }
233        .run()
234    }
235
236    #[ignore]
237    #[test]
238    fn test_json_to_variant_decimal4_large_negative() -> Result<(), ArrowError> {
239        JsonToVariantTest {
240            json: "-99999999.9",
241            expected: Variant::from(VariantDecimal4::try_new(-999999999, 1)?),
242        }
243        .run()
244    }
245
246    #[ignore]
247    #[test]
248    fn test_json_to_variant_decimal4_small_positive() -> Result<(), ArrowError> {
249        JsonToVariantTest {
250            json: "0.999999999",
251            expected: Variant::from(VariantDecimal4::try_new(999999999, 9)?),
252        }
253        .run()
254    }
255
256    #[ignore]
257    #[test]
258    fn test_json_to_variant_decimal4_tiny_positive() -> Result<(), ArrowError> {
259        JsonToVariantTest {
260            json: "0.000000001",
261            expected: Variant::from(VariantDecimal4::try_new(1, 9)?),
262        }
263        .run()
264    }
265
266    #[ignore]
267    #[test]
268    fn test_json_to_variant_decimal4_small_negative() -> Result<(), ArrowError> {
269        JsonToVariantTest {
270            json: "-0.999999999",
271            expected: Variant::from(VariantDecimal4::try_new(-999999999, 9)?),
272        }
273        .run()
274    }
275
276    #[ignore]
277    #[test]
278    fn test_json_to_variant_decimal8_positive() -> Result<(), ArrowError> {
279        JsonToVariantTest {
280            json: "999999999.0",
281            expected: Variant::from(VariantDecimal8::try_new(9999999990, 1)?),
282        }
283        .run()
284    }
285
286    #[ignore]
287    #[test]
288    fn test_json_to_variant_decimal8_negative() -> Result<(), ArrowError> {
289        JsonToVariantTest {
290            json: "-999999999.0",
291            expected: Variant::from(VariantDecimal8::try_new(-9999999990, 1)?),
292        }
293        .run()
294    }
295
296    #[ignore]
297    #[test]
298    fn test_json_to_variant_decimal8_high_precision() -> Result<(), ArrowError> {
299        JsonToVariantTest {
300            json: "0.999999999999999999",
301            expected: Variant::from(VariantDecimal8::try_new(999999999999999999, 18)?),
302        }
303        .run()
304    }
305
306    #[ignore]
307    #[test]
308    fn test_json_to_variant_decimal8_large_with_scale() -> Result<(), ArrowError> {
309        JsonToVariantTest {
310            json: "9999999999999999.99",
311            expected: Variant::from(VariantDecimal8::try_new(999999999999999999, 2)?),
312        }
313        .run()
314    }
315
316    #[ignore]
317    #[test]
318    fn test_json_to_variant_decimal8_large_negative_with_scale() -> Result<(), ArrowError> {
319        JsonToVariantTest {
320            json: "-9999999999999999.99",
321            expected: Variant::from(VariantDecimal8::try_new(-999999999999999999, 2)?),
322        }
323        .run()
324    }
325
326    #[ignore]
327    #[test]
328    fn test_json_to_variant_decimal16_large_integer() -> Result<(), ArrowError> {
329        JsonToVariantTest {
330            json: "9999999999999999999", // integer larger than i64
331            expected: Variant::from(VariantDecimal16::try_new(9999999999999999999, 0)?),
332        }
333        .run()
334    }
335
336    #[ignore]
337    #[test]
338    fn test_json_to_variant_decimal16_high_precision() -> Result<(), ArrowError> {
339        JsonToVariantTest {
340            json: "0.9999999999999999999",
341            expected: Variant::from(VariantDecimal16::try_new(9999999999999999999, 19)?),
342        }
343        .run()
344    }
345
346    #[ignore]
347    #[test]
348    fn test_json_to_variant_decimal16_max_value() -> Result<(), ArrowError> {
349        JsonToVariantTest {
350            json: "79228162514264337593543950335", // 2 ^ 96 - 1
351            expected: Variant::from(VariantDecimal16::try_new(79228162514264337593543950335, 0)?),
352        }
353        .run()
354    }
355
356    #[ignore]
357    #[test]
358    fn test_json_to_variant_decimal16_max_scale() -> Result<(), ArrowError> {
359        JsonToVariantTest {
360            json: "7.9228162514264337593543950335", // using scale higher than this falls into double
361            // since the max scale is 28.
362            expected: Variant::from(VariantDecimal16::try_new(
363                79228162514264337593543950335,
364                28,
365            )?),
366        }
367        .run()
368    }
369
370    #[test]
371    fn test_json_to_variant_double_precision() -> Result<(), ArrowError> {
372        JsonToVariantTest {
373            json: "0.79228162514264337593543950335",
374            expected: Variant::Double(0.792_281_625_142_643_4_f64),
375        }
376        .run()
377    }
378
379    #[test]
380    fn test_json_to_variant_double_scientific_positive() -> Result<(), ArrowError> {
381        JsonToVariantTest {
382            json: "15e-1",
383            expected: Variant::Double(15e-1f64),
384        }
385        .run()
386    }
387
388    #[test]
389    fn test_json_to_variant_double_scientific_negative() -> Result<(), ArrowError> {
390        JsonToVariantTest {
391            json: "-15e-1",
392            expected: Variant::Double(-15e-1f64),
393        }
394        .run()
395    }
396
397    #[test]
398    fn test_json_to_variant_short_string() -> Result<(), ArrowError> {
399        JsonToVariantTest {
400            json: "\"harsh\"",
401            expected: Variant::ShortString(ShortString::try_new("harsh")?),
402        }
403        .run()
404    }
405
406    #[test]
407    fn test_json_to_variant_short_string_max_length() -> Result<(), ArrowError> {
408        JsonToVariantTest {
409            json: &format!("\"{}\"", "a".repeat(63)),
410            expected: Variant::ShortString(ShortString::try_new(&"a".repeat(63))?),
411        }
412        .run()
413    }
414
415    #[test]
416    fn test_json_to_variant_long_string() -> Result<(), ArrowError> {
417        JsonToVariantTest {
418            json: &format!("\"{}\"", "a".repeat(64)),
419            expected: Variant::String(&"a".repeat(64)),
420        }
421        .run()
422    }
423
424    #[test]
425    fn test_json_to_variant_very_long_string() -> Result<(), ArrowError> {
426        JsonToVariantTest {
427            json: &format!("\"{}\"", "b".repeat(100000)),
428            expected: Variant::String(&"b".repeat(100000)),
429        }
430        .run()
431    }
432
433    #[test]
434    fn test_json_to_variant_array_simple() -> Result<(), ArrowError> {
435        let mut variant_builder = VariantBuilder::new();
436        let mut list_builder = variant_builder.new_list();
437        list_builder.append_value(Variant::Int8(127));
438        list_builder.append_value(Variant::Int16(128));
439        list_builder.append_value(Variant::Int32(-32767431));
440        list_builder.finish();
441        let (metadata, value) = variant_builder.finish();
442        let variant = Variant::try_new(&metadata, &value)?;
443
444        JsonToVariantTest {
445            json: "[127, 128, -32767431]",
446            expected: variant,
447        }
448        .run()
449    }
450
451    #[test]
452    fn test_json_to_variant_array_with_object() -> Result<(), ArrowError> {
453        let mut variant_builder = VariantBuilder::new();
454        let mut list_builder = variant_builder.new_list();
455        let mut object_builder_inner = list_builder.new_object();
456        object_builder_inner.insert("age", Variant::Int8(32));
457        object_builder_inner.finish();
458        list_builder.append_value(Variant::Int16(128));
459        list_builder.append_value(Variant::BooleanFalse);
460        list_builder.finish();
461        let (metadata, value) = variant_builder.finish();
462        let variant = Variant::try_new(&metadata, &value)?;
463
464        JsonToVariantTest {
465            json: "[{\"age\": 32}, 128, false]",
466            expected: variant,
467        }
468        .run()
469    }
470
471    #[test]
472    fn test_json_to_variant_array_large_u16_offset() -> Result<(), ArrowError> {
473        // u16 offset - 128 i8's + 1 "true" = 257 bytes
474        let mut variant_builder = VariantBuilder::new();
475        let mut list_builder = variant_builder.new_list();
476        for _ in 0..128 {
477            list_builder.append_value(Variant::Int8(1));
478        }
479        list_builder.append_value(Variant::BooleanTrue);
480        list_builder.finish();
481        let (metadata, value) = variant_builder.finish();
482        let variant = Variant::try_new(&metadata, &value)?;
483
484        JsonToVariantTest {
485            json: &format!("[{} true]", "1, ".repeat(128)),
486            expected: variant,
487        }
488        .run()
489    }
490
491    #[test]
492    fn test_json_to_variant_array_nested_large() -> Result<(), ArrowError> {
493        // verify u24, and large_size
494        let mut variant_builder = VariantBuilder::new();
495        let mut list_builder = variant_builder.new_list();
496        for _ in 0..256 {
497            let mut list_builder_inner = list_builder.new_list();
498            for _ in 0..255 {
499                list_builder_inner.append_value(Variant::Null);
500            }
501            list_builder_inner.finish();
502        }
503        list_builder.finish();
504        let (metadata, value) = variant_builder.finish();
505        let variant = Variant::try_new(&metadata, &value)?;
506        let intermediate = format!("[{}]", vec!["null"; 255].join(", "));
507        let json = format!("[{}]", vec![intermediate; 256].join(", "));
508        JsonToVariantTest {
509            json: json.as_str(),
510            expected: variant,
511        }
512        .run()
513    }
514
515    #[test]
516    fn test_json_to_variant_object_simple() -> Result<(), ArrowError> {
517        let mut variant_builder = VariantBuilder::new();
518        let mut object_builder = variant_builder.new_object();
519        object_builder.insert("a", Variant::Int8(3));
520        object_builder.insert("b", Variant::Int8(2));
521        object_builder.finish();
522        let (metadata, value) = variant_builder.finish();
523        let variant = Variant::try_new(&metadata, &value)?;
524        JsonToVariantTest {
525            json: "{\"b\": 2, \"a\": 1, \"a\": 3}",
526            expected: variant,
527        }
528        .run()
529    }
530
531    #[test]
532    fn test_json_to_variant_object_complex() -> Result<(), ArrowError> {
533        let mut variant_builder = VariantBuilder::new();
534        let mut object_builder = variant_builder.new_object();
535        let mut inner_list_builder = object_builder.new_list("booleans");
536        inner_list_builder.append_value(Variant::BooleanTrue);
537        inner_list_builder.append_value(Variant::BooleanFalse);
538        inner_list_builder.finish();
539        object_builder.insert("null", Variant::Null);
540        let mut inner_list_builder = object_builder.new_list("numbers");
541        inner_list_builder.append_value(Variant::Int8(4));
542        inner_list_builder.append_value(Variant::Double(-3e0));
543        inner_list_builder.append_value(Variant::Double(1001e-3));
544        inner_list_builder.finish();
545        object_builder.finish();
546        let (metadata, value) = variant_builder.finish();
547        let variant = Variant::try_new(&metadata, &value)?;
548        JsonToVariantTest {
549            json: "{\"numbers\": [4, -3e0, 1001e-3], \"null\": null, \"booleans\": [true, false]}",
550            expected: variant,
551        }
552        .run()
553    }
554
555    #[test]
556    fn test_json_to_variant_object_very_large() -> Result<(), ArrowError> {
557        // 256 elements (keys: 000-255) - each element is an object of 256 elements (240-495) - each
558        // element a list of numbers from 0-127
559        let keys: Vec<String> = (0..=255).map(|n| format!("{n:03}")).collect();
560        let innermost_list: String = format!(
561            "[{}]",
562            (0..=127)
563                .map(|n| format!("{n}"))
564                .collect::<Vec<_>>()
565                .join(",")
566        );
567        let inner_keys: Vec<String> = (240..=495).map(|n| format!("{n}")).collect();
568        let inner_object = format!(
569            "{{{}:{}}}",
570            inner_keys
571                .iter()
572                .map(|k| format!("\"{k}\""))
573                .collect::<Vec<String>>()
574                .join(format!(":{innermost_list},").as_str()),
575            innermost_list
576        );
577        let json = format!(
578            "{{{}:{}}}",
579            keys.iter()
580                .map(|k| format!("\"{k}\""))
581                .collect::<Vec<String>>()
582                .join(format!(":{inner_object},").as_str()),
583            inner_object
584        );
585        // Manually verify raw JSON value size
586        let mut variant_builder = VariantBuilder::new();
587        variant_builder.append_json(&json)?;
588        let (metadata, value) = variant_builder.finish();
589        let v = Variant::try_new(&metadata, &value)?;
590        let output_string = v.to_json_string()?;
591        assert_eq!(output_string, json);
592        // Verify metadata size = 1 + 2 + 2 * 497 + 3 * 496
593        assert_eq!(metadata.len(), 2485);
594        // Verify value size.
595        // Size of innermost_list: 1 + 1 + 2*(128 + 1) + 2*128 = 516
596        // Size of inner object: 1 + 4 + 2*256 + 3*(256 + 1) + 256 * 516 = 133384
597        // Size of json: 1 + 4 + 2*256 + 4*(256 + 1) + 256 * 133384 = 34147849
598        assert_eq!(value.len(), 34147849);
599
600        let mut variant_builder = VariantBuilder::new();
601        let mut object_builder = variant_builder.new_object();
602        keys.iter().for_each(|key| {
603            let mut inner_object_builder = object_builder.new_object(key);
604            inner_keys.iter().for_each(|inner_key| {
605                let mut list_builder = inner_object_builder.new_list(inner_key);
606                for i in 0..=127 {
607                    list_builder.append_value(Variant::Int8(i));
608                }
609                list_builder.finish();
610            });
611            inner_object_builder.finish();
612        });
613        object_builder.finish();
614        let (metadata, value) = variant_builder.finish();
615        let variant = Variant::try_new(&metadata, &value)?;
616
617        JsonToVariantTest {
618            json: &json,
619            expected: variant,
620        }
621        .run()
622    }
623
624    #[test]
625    fn test_json_to_variant_unicode() -> Result<(), ArrowError> {
626        let json = "{\"爱\":\"अ\",\"a\":1}";
627        let mut variant_builder = VariantBuilder::new();
628        variant_builder.append_json(json)?;
629        let (metadata, value) = variant_builder.finish();
630        let v = Variant::try_new(&metadata, &value)?;
631        let output_string = v.to_json_string()?;
632        assert_eq!(output_string, "{\"a\":1,\"爱\":\"अ\"}");
633        let mut variant_builder = VariantBuilder::new();
634        let mut object_builder = variant_builder.new_object();
635        object_builder.insert("a", Variant::Int8(1));
636        object_builder.insert("爱", Variant::ShortString(ShortString::try_new("अ")?));
637        object_builder.finish();
638        let (metadata, value) = variant_builder.finish();
639        let variant = Variant::try_new(&metadata, &value)?;
640
641        assert_eq!(
642            value,
643            &[
644                2u8, 2u8, 0u8, 1u8, 0u8, 2u8, 6u8, 12u8, 1u8, 13u8, 0xe0u8, 0xa4u8, 0x85u8
645            ]
646        );
647        assert_eq!(
648            metadata,
649            &[17u8, 2u8, 0u8, 1u8, 4u8, 97u8, 0xe7u8, 0x88u8, 0xb1u8]
650        );
651        JsonToVariantTest {
652            json,
653            expected: variant,
654        }
655        .run()
656    }
657}