parquet_variant_json/
to_json.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Module for converting Variant data to JSON format
19use arrow_schema::ArrowError;
20use base64::{engine::general_purpose, Engine as _};
21use serde_json::Value;
22use std::io::Write;
23
24use parquet_variant::{Variant, VariantList, VariantObject};
25
26// Format string constants to avoid duplication and reduce errors
27const DATE_FORMAT: &str = "%Y-%m-%d";
28const TIMESTAMP_NTZ_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.6f";
29
30// Helper functions for consistent formatting
31fn format_date_string(date: &chrono::NaiveDate) -> String {
32    date.format(DATE_FORMAT).to_string()
33}
34
35fn format_timestamp_ntz_string(ts: &chrono::NaiveDateTime) -> String {
36    ts.format(TIMESTAMP_NTZ_FORMAT).to_string()
37}
38
39fn format_binary_base64(bytes: &[u8]) -> String {
40    general_purpose::STANDARD.encode(bytes)
41}
42
43///
44/// This function writes JSON directly to any type that implements [`Write`],
45/// making it efficient for streaming or when you want to control the output destination.
46///
47/// See [`variant_to_json_string`] for a convenience function that returns a
48/// JSON string.
49///
50/// # Arguments
51///
52/// * `writer` - Writer to output JSON to
53/// * `variant` - The Variant value to convert
54///
55/// # Returns
56///
57/// * `Ok(())` if successful
58/// * `Err` with error details if conversion fails
59///
60/// # Examples
61///
62///
63/// ```rust
64/// # use parquet_variant::{Variant};
65/// # use parquet_variant_json::variant_to_json;
66/// # use arrow_schema::ArrowError;
67/// let variant = Variant::from("Hello, World!");
68/// let mut buffer = Vec::new();
69/// variant_to_json(&mut buffer, &variant)?;
70/// assert_eq!(String::from_utf8(buffer).unwrap(), "\"Hello, World!\"");
71/// # Ok::<(), ArrowError>(())
72/// ```
73///
74/// # Example: Create a [`Variant::Object`] and convert to JSON
75/// ```rust
76/// # use parquet_variant::{Variant, VariantBuilder};
77/// # use parquet_variant_json::variant_to_json;
78/// # use arrow_schema::ArrowError;
79/// let mut builder = VariantBuilder::new();
80/// // Create an object builder that will write fields to the object
81/// let mut object_builder = builder.new_object();
82/// object_builder.insert("first_name", "Jiaying");
83/// object_builder.insert("last_name", "Li");
84/// object_builder.finish();
85/// // Finish the builder to get the metadata and value
86/// let (metadata, value) = builder.finish();
87/// // Create the Variant and convert to JSON
88/// let variant = Variant::try_new(&metadata, &value)?;
89/// let mut writer = Vec::new();
90/// variant_to_json(&mut writer, &variant,)?;
91/// assert_eq!(br#"{"first_name":"Jiaying","last_name":"Li"}"#, writer.as_slice());
92/// # Ok::<(), ArrowError>(())
93/// ```
94pub fn variant_to_json(json_buffer: &mut impl Write, variant: &Variant) -> Result<(), ArrowError> {
95    match variant {
96        Variant::Null => write!(json_buffer, "null")?,
97        Variant::BooleanTrue => write!(json_buffer, "true")?,
98        Variant::BooleanFalse => write!(json_buffer, "false")?,
99        Variant::Int8(i) => write!(json_buffer, "{i}")?,
100        Variant::Int16(i) => write!(json_buffer, "{i}")?,
101        Variant::Int32(i) => write!(json_buffer, "{i}")?,
102        Variant::Int64(i) => write!(json_buffer, "{i}")?,
103        Variant::Float(f) => write!(json_buffer, "{f}")?,
104        Variant::Double(f) => write!(json_buffer, "{f}")?,
105        Variant::Decimal4(decimal) => write!(json_buffer, "{decimal}")?,
106        Variant::Decimal8(decimal) => write!(json_buffer, "{decimal}")?,
107        Variant::Decimal16(decimal) => write!(json_buffer, "{decimal}")?,
108        Variant::Date(date) => write!(json_buffer, "\"{}\"", format_date_string(date))?,
109        Variant::TimestampMicros(ts) => write!(json_buffer, "\"{}\"", ts.to_rfc3339())?,
110        Variant::TimestampNtzMicros(ts) => {
111            write!(json_buffer, "\"{}\"", format_timestamp_ntz_string(ts))?
112        }
113        Variant::Binary(bytes) => {
114            // Encode binary as base64 string
115            let base64_str = format_binary_base64(bytes);
116            let json_str = serde_json::to_string(&base64_str).map_err(|e| {
117                ArrowError::InvalidArgumentError(format!("JSON encoding error: {e}"))
118            })?;
119            write!(json_buffer, "{json_str}")?
120        }
121        Variant::String(s) => {
122            // Use serde_json to properly escape the string
123            let json_str = serde_json::to_string(s).map_err(|e| {
124                ArrowError::InvalidArgumentError(format!("JSON encoding error: {e}"))
125            })?;
126            write!(json_buffer, "{json_str}")?
127        }
128        Variant::ShortString(s) => {
129            // Use serde_json to properly escape the string
130            let json_str = serde_json::to_string(s.as_str()).map_err(|e| {
131                ArrowError::InvalidArgumentError(format!("JSON encoding error: {e}"))
132            })?;
133            write!(json_buffer, "{json_str}")?
134        }
135        Variant::Object(obj) => {
136            convert_object_to_json(json_buffer, obj)?;
137        }
138        Variant::List(arr) => {
139            convert_array_to_json(json_buffer, arr)?;
140        }
141    }
142    Ok(())
143}
144
145/// Convert object fields to JSON
146fn convert_object_to_json(buffer: &mut impl Write, obj: &VariantObject) -> Result<(), ArrowError> {
147    write!(buffer, "{{")?;
148
149    // Get all fields from the object
150    let mut first = true;
151
152    for (key, value) in obj.iter() {
153        if !first {
154            write!(buffer, ",")?;
155        }
156        first = false;
157
158        // Write the key (properly escaped)
159        let json_key = serde_json::to_string(key).map_err(|e| {
160            ArrowError::InvalidArgumentError(format!("JSON key encoding error: {e}"))
161        })?;
162        write!(buffer, "{json_key}:")?;
163
164        // Recursively convert the value
165        variant_to_json(buffer, &value)?;
166    }
167
168    write!(buffer, "}}")?;
169    Ok(())
170}
171
172/// Convert array elements to JSON
173fn convert_array_to_json(buffer: &mut impl Write, arr: &VariantList) -> Result<(), ArrowError> {
174    write!(buffer, "[")?;
175
176    let mut first = true;
177    for element in arr.iter() {
178        if !first {
179            write!(buffer, ",")?;
180        }
181        first = false;
182
183        variant_to_json(buffer, &element)?;
184    }
185
186    write!(buffer, "]")?;
187    Ok(())
188}
189
190/// Convert [`Variant`] to JSON [`String`]
191///
192/// This is a convenience function that converts a Variant to a JSON string.
193/// This is the same as calling [`variant_to_json`] with a [`Vec`].
194/// It's the simplest way to get a JSON representation when you just need a String result.
195///
196/// # Arguments
197///
198/// * `variant` - The Variant value to convert
199///
200/// # Returns
201///
202/// * `Ok(String)` containing the JSON representation
203/// * `Err` with error details if conversion fails
204///
205/// # Examples
206///
207/// ```rust
208/// # use parquet_variant::{Variant};
209/// # use parquet_variant_json::variant_to_json_string;
210/// # use arrow_schema::ArrowError;
211/// let variant = Variant::Int32(42);
212/// let json = variant_to_json_string(&variant)?;
213/// assert_eq!(json, "42");
214/// # Ok::<(), ArrowError>(())
215/// ```
216///
217/// # Example: Create a [`Variant::Object`] and convert to JSON
218///
219/// This example shows how to create an object with two fields and convert it to JSON:
220/// ```json
221/// {
222///   "first_name": "Jiaying",
223///   "last_name": "Li"
224/// }
225/// ```
226///
227/// ```rust
228/// # use parquet_variant::{Variant, VariantBuilder};
229/// # use parquet_variant_json::variant_to_json_string;
230/// # use arrow_schema::ArrowError;
231/// let mut builder = VariantBuilder::new();
232/// // Create an object builder that will write fields to the object
233/// let mut object_builder = builder.new_object();
234/// object_builder.insert("first_name", "Jiaying");
235/// object_builder.insert("last_name", "Li");
236/// object_builder.finish();
237/// // Finish the builder to get the metadata and value
238/// let (metadata, value) = builder.finish();
239/// // Create the Variant and convert to JSON
240/// let variant = Variant::try_new(&metadata, &value)?;
241/// let json = variant_to_json_string(&variant)?;
242/// assert_eq!(r#"{"first_name":"Jiaying","last_name":"Li"}"#, json);
243/// # Ok::<(), ArrowError>(())
244/// ```
245pub fn variant_to_json_string(variant: &Variant) -> Result<String, ArrowError> {
246    let mut buffer = Vec::new();
247    variant_to_json(&mut buffer, variant)?;
248    String::from_utf8(buffer)
249        .map_err(|e| ArrowError::InvalidArgumentError(format!("UTF-8 conversion error: {e}")))
250}
251
252/// Convert [`Variant`] to [`serde_json::Value`]
253///
254/// This function converts a Variant to a [`serde_json::Value`], which is useful
255/// when you need to work with the JSON data programmatically or integrate with
256/// other serde-based JSON processing.
257///
258/// # Arguments
259///
260/// * `variant` - The Variant value to convert
261///
262/// # Returns
263///
264/// * `Ok(Value)` containing the JSON value
265/// * `Err` with error details if conversion fails
266///
267/// # Examples
268///
269/// ```rust
270/// # use parquet_variant::{Variant};
271/// # use parquet_variant_json::variant_to_json_value;
272/// # use serde_json::Value;
273/// # use arrow_schema::ArrowError;
274/// let variant = Variant::from("hello");
275/// let json_value = variant_to_json_value(&variant)?;
276/// assert_eq!(json_value, Value::String("hello".to_string()));
277/// # Ok::<(), ArrowError>(())
278/// ```
279pub fn variant_to_json_value(variant: &Variant) -> Result<Value, ArrowError> {
280    match variant {
281        Variant::Null => Ok(Value::Null),
282        Variant::BooleanTrue => Ok(Value::Bool(true)),
283        Variant::BooleanFalse => Ok(Value::Bool(false)),
284        Variant::Int8(i) => Ok(Value::Number((*i).into())),
285        Variant::Int16(i) => Ok(Value::Number((*i).into())),
286        Variant::Int32(i) => Ok(Value::Number((*i).into())),
287        Variant::Int64(i) => Ok(Value::Number((*i).into())),
288        Variant::Float(f) => serde_json::Number::from_f64((*f).into())
289            .map(Value::Number)
290            .ok_or_else(|| ArrowError::InvalidArgumentError("Invalid float value".to_string())),
291        Variant::Double(f) => serde_json::Number::from_f64(*f)
292            .map(Value::Number)
293            .ok_or_else(|| ArrowError::InvalidArgumentError("Invalid double value".to_string())),
294        Variant::Decimal4(decimal4) => {
295            let scale = decimal4.scale();
296            let integer = decimal4.integer();
297
298            let integer = if scale == 0 {
299                integer
300            } else {
301                let divisor = 10_i32.pow(scale as u32);
302                if integer % divisor != 0 {
303                    // fall back to floating point
304                    return Ok(Value::from(integer as f64 / divisor as f64));
305                }
306                integer / divisor
307            };
308            Ok(Value::from(integer))
309        }
310        Variant::Decimal8(decimal8) => {
311            let scale = decimal8.scale();
312            let integer = decimal8.integer();
313
314            let integer = if scale == 0 {
315                integer
316            } else {
317                let divisor = 10_i64.pow(scale as u32);
318                if integer % divisor != 0 {
319                    // fall back to floating point
320                    return Ok(Value::from(integer as f64 / divisor as f64));
321                }
322                integer / divisor
323            };
324            Ok(Value::from(integer))
325        }
326        Variant::Decimal16(decimal16) => {
327            let scale = decimal16.scale();
328            let integer = decimal16.integer();
329
330            let integer = if scale == 0 {
331                integer
332            } else {
333                let divisor = 10_i128.pow(scale as u32);
334                if integer % divisor != 0 {
335                    // fall back to floating point
336                    return Ok(Value::from(integer as f64 / divisor as f64));
337                }
338                integer / divisor
339            };
340            // i128 has higher precision than any 64-bit type. Try a lossless narrowing cast to
341            // i64 or u64 first, falling back to a lossy narrowing cast to f64 if necessary.
342            let value = i64::try_from(integer)
343                .map(Value::from)
344                .or_else(|_| u64::try_from(integer).map(Value::from))
345                .unwrap_or_else(|_| Value::from(integer as f64));
346            Ok(value)
347        }
348        Variant::Date(date) => Ok(Value::String(format_date_string(date))),
349        Variant::TimestampMicros(ts) => Ok(Value::String(ts.to_rfc3339())),
350        Variant::TimestampNtzMicros(ts) => Ok(Value::String(format_timestamp_ntz_string(ts))),
351        Variant::Binary(bytes) => Ok(Value::String(format_binary_base64(bytes))),
352        Variant::String(s) => Ok(Value::String(s.to_string())),
353        Variant::ShortString(s) => Ok(Value::String(s.to_string())),
354        Variant::Object(obj) => {
355            let map = obj
356                .iter()
357                .map(|(k, v)| variant_to_json_value(&v).map(|json_val| (k.to_string(), json_val)))
358                .collect::<Result<_, _>>()?;
359            Ok(Value::Object(map))
360        }
361        Variant::List(arr) => {
362            let vec = arr
363                .iter()
364                .map(|element| variant_to_json_value(&element))
365                .collect::<Result<_, _>>()?;
366            Ok(Value::Array(vec))
367        }
368    }
369}
370
371#[cfg(test)]
372mod tests {
373    use super::*;
374    use chrono::{DateTime, NaiveDate, Utc};
375    use parquet_variant::{VariantDecimal16, VariantDecimal4, VariantDecimal8};
376
377    #[test]
378    fn test_decimal_edge_cases() -> Result<(), ArrowError> {
379        // Test negative decimal
380        let negative_variant = Variant::from(VariantDecimal4::try_new(-12345, 3)?);
381        let negative_json = variant_to_json_string(&negative_variant)?;
382        assert_eq!(negative_json, "-12.345");
383
384        // Test large scale decimal
385        let large_scale_variant = Variant::from(VariantDecimal8::try_new(123456789, 6)?);
386        let large_scale_json = variant_to_json_string(&large_scale_variant)?;
387        assert_eq!(large_scale_json, "123.456789");
388
389        Ok(())
390    }
391
392    #[test]
393    fn test_decimal16_to_json() -> Result<(), ArrowError> {
394        let variant = Variant::from(VariantDecimal16::try_new(123456789012345, 4)?);
395        let json = variant_to_json_string(&variant)?;
396        assert_eq!(json, "12345678901.2345");
397
398        let json_value = variant_to_json_value(&variant)?;
399        assert!(matches!(json_value, Value::Number(_)));
400
401        // Test very large number
402        let large_variant = Variant::from(VariantDecimal16::try_new(999999999999999999, 2)?);
403        let large_json = variant_to_json_string(&large_variant)?;
404        // Due to f64 precision limits, very large numbers may lose precision
405        assert!(
406            large_json.starts_with("9999999999999999")
407                || large_json.starts_with("10000000000000000")
408        );
409        Ok(())
410    }
411
412    #[test]
413    fn test_date_to_json() -> Result<(), ArrowError> {
414        let date = NaiveDate::from_ymd_opt(2023, 12, 25).unwrap();
415        let variant = Variant::Date(date);
416        let json = variant_to_json_string(&variant)?;
417        assert_eq!(json, "\"2023-12-25\"");
418
419        let json_value = variant_to_json_value(&variant)?;
420        assert_eq!(json_value, Value::String("2023-12-25".to_string()));
421
422        // Test leap year date
423        let leap_date = NaiveDate::from_ymd_opt(2024, 2, 29).unwrap();
424        let leap_variant = Variant::Date(leap_date);
425        let leap_json = variant_to_json_string(&leap_variant)?;
426        assert_eq!(leap_json, "\"2024-02-29\"");
427        Ok(())
428    }
429
430    #[test]
431    fn test_timestamp_micros_to_json() -> Result<(), ArrowError> {
432        let timestamp = DateTime::parse_from_rfc3339("2023-12-25T10:30:45Z")
433            .unwrap()
434            .with_timezone(&Utc);
435        let variant = Variant::TimestampMicros(timestamp);
436        let json = variant_to_json_string(&variant)?;
437        assert!(json.contains("2023-12-25T10:30:45"));
438        assert!(json.starts_with('"') && json.ends_with('"'));
439
440        let json_value = variant_to_json_value(&variant)?;
441        assert!(matches!(json_value, Value::String(_)));
442        Ok(())
443    }
444
445    #[test]
446    fn test_timestamp_ntz_micros_to_json() -> Result<(), ArrowError> {
447        let naive_timestamp = DateTime::from_timestamp(1703505045, 123456)
448            .unwrap()
449            .naive_utc();
450        let variant = Variant::TimestampNtzMicros(naive_timestamp);
451        let json = variant_to_json_string(&variant)?;
452        assert!(json.contains("2023-12-25"));
453        assert!(json.starts_with('"') && json.ends_with('"'));
454
455        let json_value = variant_to_json_value(&variant)?;
456        assert!(matches!(json_value, Value::String(_)));
457        Ok(())
458    }
459
460    #[test]
461    fn test_binary_to_json() -> Result<(), ArrowError> {
462        let binary_data = b"Hello, World!";
463        let variant = Variant::Binary(binary_data);
464        let json = variant_to_json_string(&variant)?;
465
466        // Should be base64 encoded and quoted
467        assert!(json.starts_with('"') && json.ends_with('"'));
468        assert!(json.len() > 2); // Should have content
469
470        let json_value = variant_to_json_value(&variant)?;
471        assert!(matches!(json_value, Value::String(_)));
472
473        // Test empty binary
474        let empty_variant = Variant::Binary(b"");
475        let empty_json = variant_to_json_string(&empty_variant)?;
476        assert_eq!(empty_json, "\"\"");
477
478        // Test binary with special bytes
479        let special_variant = Variant::Binary(&[0, 255, 128, 64]);
480        let special_json = variant_to_json_string(&special_variant)?;
481        assert!(special_json.starts_with('"') && special_json.ends_with('"'));
482        Ok(())
483    }
484
485    #[test]
486    fn test_string_to_json() -> Result<(), ArrowError> {
487        let variant = Variant::from("hello world");
488        let json = variant_to_json_string(&variant)?;
489        assert_eq!(json, "\"hello world\"");
490
491        let json_value = variant_to_json_value(&variant)?;
492        assert_eq!(json_value, Value::String("hello world".to_string()));
493        Ok(())
494    }
495
496    #[test]
497    fn test_short_string_to_json() -> Result<(), ArrowError> {
498        use parquet_variant::ShortString;
499        let short_string = ShortString::try_new("short")?;
500        let variant = Variant::ShortString(short_string);
501        let json = variant_to_json_string(&variant)?;
502        assert_eq!(json, "\"short\"");
503
504        let json_value = variant_to_json_value(&variant)?;
505        assert_eq!(json_value, Value::String("short".to_string()));
506        Ok(())
507    }
508
509    #[test]
510    fn test_string_escaping() -> Result<(), ArrowError> {
511        let variant = Variant::from("hello\nworld\t\"quoted\"");
512        let json = variant_to_json_string(&variant)?;
513        assert_eq!(json, "\"hello\\nworld\\t\\\"quoted\\\"\"");
514
515        let json_value = variant_to_json_value(&variant)?;
516        assert_eq!(
517            json_value,
518            Value::String("hello\nworld\t\"quoted\"".to_string())
519        );
520        Ok(())
521    }
522
523    #[test]
524    fn test_json_buffer_writing() -> Result<(), ArrowError> {
525        let variant = Variant::Int8(123);
526        let mut buffer = Vec::new();
527        variant_to_json(&mut buffer, &variant)?;
528
529        let result = String::from_utf8(buffer)
530            .map_err(|e| ArrowError::InvalidArgumentError(e.to_string()))?;
531        assert_eq!(result, "123");
532        Ok(())
533    }
534
535    /// Reusable test structure for JSON conversion testing
536    struct JsonTest {
537        variant: Variant<'static, 'static>,
538        expected_json: &'static str,
539        expected_value: Value,
540    }
541
542    impl JsonTest {
543        fn run(self) {
544            let json_string = variant_to_json_string(&self.variant)
545                .expect("variant_to_json_string should succeed");
546            assert_eq!(
547                json_string, self.expected_json,
548                "JSON string mismatch for variant: {:?}",
549                self.variant
550            );
551
552            let json_value =
553                variant_to_json_value(&self.variant).expect("variant_to_json_value should succeed");
554
555            // For floating point numbers, we need special comparison due to JSON number representation
556            match (&json_value, &self.expected_value) {
557                (Value::Number(actual), Value::Number(expected)) => {
558                    let actual_f64 = actual.as_f64().unwrap_or(0.0);
559                    let expected_f64 = expected.as_f64().unwrap_or(0.0);
560                    assert!(
561                        (actual_f64 - expected_f64).abs() < f64::EPSILON,
562                        "JSON value mismatch for variant: {:?}, got {}, expected {}",
563                        self.variant,
564                        actual_f64,
565                        expected_f64
566                    );
567                }
568                _ => {
569                    assert_eq!(
570                        json_value, self.expected_value,
571                        "JSON value mismatch for variant: {:?}",
572                        self.variant
573                    );
574                }
575            }
576
577            // Verify roundtrip: JSON string should parse to same value
578            let parsed: Value =
579                serde_json::from_str(&json_string).expect("Generated JSON should be valid");
580            // Same floating point handling for roundtrip
581            match (&parsed, &self.expected_value) {
582                (Value::Number(actual), Value::Number(expected)) => {
583                    let actual_f64 = actual.as_f64().unwrap_or(0.0);
584                    let expected_f64 = expected.as_f64().unwrap_or(0.0);
585                    assert!(
586                        (actual_f64 - expected_f64).abs() < f64::EPSILON,
587                        "Parsed JSON mismatch for variant: {:?}, got {}, expected {}",
588                        self.variant,
589                        actual_f64,
590                        expected_f64
591                    );
592                }
593                _ => {
594                    assert_eq!(
595                        parsed, self.expected_value,
596                        "Parsed JSON mismatch for variant: {:?}",
597                        self.variant
598                    );
599                }
600            }
601        }
602    }
603
604    #[test]
605    fn test_primitive_json_conversion() {
606        use parquet_variant::ShortString;
607
608        // Null
609        JsonTest {
610            variant: Variant::Null,
611            expected_json: "null",
612            expected_value: Value::Null,
613        }
614        .run();
615
616        // Booleans
617        JsonTest {
618            variant: Variant::BooleanTrue,
619            expected_json: "true",
620            expected_value: Value::Bool(true),
621        }
622        .run();
623
624        JsonTest {
625            variant: Variant::BooleanFalse,
626            expected_json: "false",
627            expected_value: Value::Bool(false),
628        }
629        .run();
630
631        // Integers - positive and negative edge cases
632        JsonTest {
633            variant: Variant::Int8(42),
634            expected_json: "42",
635            expected_value: Value::Number(42.into()),
636        }
637        .run();
638
639        JsonTest {
640            variant: Variant::Int8(-128),
641            expected_json: "-128",
642            expected_value: Value::Number((-128).into()),
643        }
644        .run();
645
646        JsonTest {
647            variant: Variant::Int16(32767),
648            expected_json: "32767",
649            expected_value: Value::Number(32767.into()),
650        }
651        .run();
652
653        JsonTest {
654            variant: Variant::Int16(-32768),
655            expected_json: "-32768",
656            expected_value: Value::Number((-32768).into()),
657        }
658        .run();
659
660        JsonTest {
661            variant: Variant::Int32(2147483647),
662            expected_json: "2147483647",
663            expected_value: Value::Number(2147483647.into()),
664        }
665        .run();
666
667        JsonTest {
668            variant: Variant::Int32(-2147483648),
669            expected_json: "-2147483648",
670            expected_value: Value::Number((-2147483648).into()),
671        }
672        .run();
673
674        JsonTest {
675            variant: Variant::Int64(9223372036854775807),
676            expected_json: "9223372036854775807",
677            expected_value: Value::Number(9223372036854775807i64.into()),
678        }
679        .run();
680
681        JsonTest {
682            variant: Variant::Int64(-9223372036854775808),
683            expected_json: "-9223372036854775808",
684            expected_value: Value::Number((-9223372036854775808i64).into()),
685        }
686        .run();
687
688        // Floats
689        JsonTest {
690            variant: Variant::Float(3.5),
691            expected_json: "3.5",
692            expected_value: serde_json::Number::from_f64(3.5)
693                .map(Value::Number)
694                .unwrap(),
695        }
696        .run();
697
698        JsonTest {
699            variant: Variant::Float(0.0),
700            expected_json: "0",
701            expected_value: Value::Number(0.into()), // Use integer 0 to match JSON parsing
702        }
703        .run();
704
705        JsonTest {
706            variant: Variant::Float(-1.5),
707            expected_json: "-1.5",
708            expected_value: serde_json::Number::from_f64(-1.5)
709                .map(Value::Number)
710                .unwrap(),
711        }
712        .run();
713
714        JsonTest {
715            variant: Variant::Double(std::f64::consts::E),
716            expected_json: "2.718281828459045",
717            expected_value: serde_json::Number::from_f64(std::f64::consts::E)
718                .map(Value::Number)
719                .unwrap(),
720        }
721        .run();
722
723        // Decimals
724        JsonTest {
725            variant: Variant::from(VariantDecimal4::try_new(12345, 2).unwrap()),
726            expected_json: "123.45",
727            expected_value: serde_json::Number::from_f64(123.45)
728                .map(Value::Number)
729                .unwrap(),
730        }
731        .run();
732
733        JsonTest {
734            variant: Variant::from(VariantDecimal4::try_new(42, 0).unwrap()),
735            expected_json: "42",
736            expected_value: serde_json::Number::from_f64(42.0)
737                .map(Value::Number)
738                .unwrap(),
739        }
740        .run();
741
742        JsonTest {
743            variant: Variant::from(VariantDecimal8::try_new(1234567890, 3).unwrap()),
744            expected_json: "1234567.89",
745            expected_value: serde_json::Number::from_f64(1234567.89)
746                .map(Value::Number)
747                .unwrap(),
748        }
749        .run();
750
751        JsonTest {
752            variant: Variant::from(VariantDecimal16::try_new(123456789012345, 4).unwrap()),
753            expected_json: "12345678901.2345",
754            expected_value: serde_json::Number::from_f64(12345678901.2345)
755                .map(Value::Number)
756                .unwrap(),
757        }
758        .run();
759
760        // Strings
761        JsonTest {
762            variant: Variant::from("hello world"),
763            expected_json: "\"hello world\"",
764            expected_value: Value::String("hello world".to_string()),
765        }
766        .run();
767
768        JsonTest {
769            variant: Variant::from(""),
770            expected_json: "\"\"",
771            expected_value: Value::String("".to_string()),
772        }
773        .run();
774
775        JsonTest {
776            variant: Variant::ShortString(ShortString::try_new("test").unwrap()),
777            expected_json: "\"test\"",
778            expected_value: Value::String("test".to_string()),
779        }
780        .run();
781
782        // Date and timestamps
783        JsonTest {
784            variant: Variant::Date(NaiveDate::from_ymd_opt(2023, 12, 25).unwrap()),
785            expected_json: "\"2023-12-25\"",
786            expected_value: Value::String("2023-12-25".to_string()),
787        }
788        .run();
789
790        // Binary data (base64 encoded)
791        JsonTest {
792            variant: Variant::Binary(b"test"),
793            expected_json: "\"dGVzdA==\"", // base64 encoded "test"
794            expected_value: Value::String("dGVzdA==".to_string()),
795        }
796        .run();
797
798        JsonTest {
799            variant: Variant::Binary(b""),
800            expected_json: "\"\"", // empty base64
801            expected_value: Value::String("".to_string()),
802        }
803        .run();
804
805        JsonTest {
806            variant: Variant::Binary(b"binary data"),
807            expected_json: "\"YmluYXJ5IGRhdGE=\"", // base64 encoded "binary data"
808            expected_value: Value::String("YmluYXJ5IGRhdGE=".to_string()),
809        }
810        .run();
811    }
812
813    #[test]
814    fn test_string_escaping_comprehensive() {
815        // Test comprehensive string escaping scenarios
816        JsonTest {
817            variant: Variant::from("line1\nline2\ttab\"quote\"\\backslash"),
818            expected_json: "\"line1\\nline2\\ttab\\\"quote\\\"\\\\backslash\"",
819            expected_value: Value::String("line1\nline2\ttab\"quote\"\\backslash".to_string()),
820        }
821        .run();
822
823        JsonTest {
824            variant: Variant::from("Hello 世界 🌍"),
825            expected_json: "\"Hello 世界 🌍\"",
826            expected_value: Value::String("Hello 世界 🌍".to_string()),
827        }
828        .run();
829    }
830
831    #[test]
832    fn test_buffer_writing_variants() -> Result<(), ArrowError> {
833        use crate::variant_to_json;
834
835        let variant = Variant::from("test buffer writing");
836
837        // Test writing to a Vec<u8>
838        let mut buffer = Vec::new();
839        variant_to_json(&mut buffer, &variant)?;
840        let result = String::from_utf8(buffer)
841            .map_err(|e| ArrowError::InvalidArgumentError(e.to_string()))?;
842        assert_eq!(result, "\"test buffer writing\"");
843
844        // Test writing to vec![]
845        let mut buffer = vec![];
846        variant_to_json(&mut buffer, &variant)?;
847        let result = String::from_utf8(buffer)
848            .map_err(|e| ArrowError::InvalidArgumentError(e.to_string()))?;
849        assert_eq!(result, "\"test buffer writing\"");
850
851        Ok(())
852    }
853
854    #[test]
855    fn test_simple_object_to_json() -> Result<(), ArrowError> {
856        use parquet_variant::VariantBuilder;
857
858        // Create a simple object with various field types
859        let mut builder = VariantBuilder::new();
860
861        builder
862            .new_object()
863            .with_field("name", "Alice")
864            .with_field("age", 30i32)
865            .with_field("active", true)
866            .with_field("score", 95.5f64)
867            .finish()
868            .unwrap();
869
870        let (metadata, value) = builder.finish();
871        let variant = Variant::try_new(&metadata, &value)?;
872        let json = variant_to_json_string(&variant)?;
873
874        // Parse the JSON to verify structure - handle JSON parsing errors manually
875        let parsed: Value = serde_json::from_str(&json).unwrap();
876        let obj = parsed.as_object().expect("expected JSON object");
877        assert_eq!(obj.get("name"), Some(&Value::String("Alice".to_string())));
878        assert_eq!(obj.get("age"), Some(&Value::Number(30.into())));
879        assert_eq!(obj.get("active"), Some(&Value::Bool(true)));
880        assert!(matches!(obj.get("score"), Some(Value::Number(_))));
881        assert_eq!(obj.len(), 4);
882
883        // Test variant_to_json_value as well
884        let json_value = variant_to_json_value(&variant)?;
885        assert!(matches!(json_value, Value::Object(_)));
886
887        Ok(())
888    }
889
890    #[test]
891    fn test_empty_object_to_json() -> Result<(), ArrowError> {
892        use parquet_variant::VariantBuilder;
893
894        let mut builder = VariantBuilder::new();
895
896        {
897            let obj = builder.new_object();
898            obj.finish().unwrap();
899        }
900
901        let (metadata, value) = builder.finish();
902        let variant = Variant::try_new(&metadata, &value)?;
903        let json = variant_to_json_string(&variant)?;
904        assert_eq!(json, "{}");
905
906        let json_value = variant_to_json_value(&variant)?;
907        assert_eq!(json_value, Value::Object(serde_json::Map::new()));
908
909        Ok(())
910    }
911
912    #[test]
913    fn test_object_with_special_characters_to_json() -> Result<(), ArrowError> {
914        use parquet_variant::VariantBuilder;
915
916        let mut builder = VariantBuilder::new();
917
918        builder
919            .new_object()
920            .with_field("message", "Hello \"World\"\nWith\tTabs")
921            .with_field("path", "C:\\Users\\Alice\\Documents")
922            .with_field("unicode", "😀 Smiley")
923            .finish()
924            .unwrap();
925
926        let (metadata, value) = builder.finish();
927        let variant = Variant::try_new(&metadata, &value)?;
928        let json = variant_to_json_string(&variant)?;
929
930        // Verify that special characters are properly escaped
931        assert!(json.contains("Hello \\\"World\\\"\\nWith\\tTabs"));
932        assert!(json.contains("C:\\\\Users\\\\Alice\\\\Documents"));
933        assert!(json.contains("😀 Smiley"));
934
935        // Verify that the JSON can be parsed back
936        let parsed: Value = serde_json::from_str(&json).unwrap();
937        assert!(matches!(parsed, Value::Object(_)));
938
939        Ok(())
940    }
941
942    #[test]
943    fn test_simple_list_to_json() -> Result<(), ArrowError> {
944        use parquet_variant::VariantBuilder;
945
946        let mut builder = VariantBuilder::new();
947
948        builder
949            .new_list()
950            .with_value(1i32)
951            .with_value(2i32)
952            .with_value(3i32)
953            .with_value(4i32)
954            .with_value(5i32)
955            .finish();
956
957        let (metadata, value) = builder.finish();
958        let variant = Variant::try_new(&metadata, &value)?;
959        let json = variant_to_json_string(&variant)?;
960        assert_eq!(json, "[1,2,3,4,5]");
961
962        let json_value = variant_to_json_value(&variant)?;
963        let arr = json_value.as_array().expect("expected JSON array");
964        assert_eq!(arr.len(), 5);
965        assert_eq!(arr[0], Value::Number(1.into()));
966        assert_eq!(arr[4], Value::Number(5.into()));
967
968        Ok(())
969    }
970
971    #[test]
972    fn test_empty_list_to_json() -> Result<(), ArrowError> {
973        use parquet_variant::VariantBuilder;
974
975        let mut builder = VariantBuilder::new();
976
977        {
978            let list = builder.new_list();
979            list.finish();
980        }
981
982        let (metadata, value) = builder.finish();
983        let variant = Variant::try_new(&metadata, &value)?;
984        let json = variant_to_json_string(&variant)?;
985        assert_eq!(json, "[]");
986
987        let json_value = variant_to_json_value(&variant)?;
988        assert_eq!(json_value, Value::Array(vec![]));
989
990        Ok(())
991    }
992
993    #[test]
994    fn test_mixed_type_list_to_json() -> Result<(), ArrowError> {
995        use parquet_variant::VariantBuilder;
996
997        let mut builder = VariantBuilder::new();
998
999        builder
1000            .new_list()
1001            .with_value("hello")
1002            .with_value(42i32)
1003            .with_value(true)
1004            .with_value(()) // null
1005            .with_value(std::f64::consts::PI)
1006            .finish();
1007
1008        let (metadata, value) = builder.finish();
1009        let variant = Variant::try_new(&metadata, &value)?;
1010        let json = variant_to_json_string(&variant)?;
1011
1012        let parsed: Value = serde_json::from_str(&json).unwrap();
1013        let arr = parsed.as_array().expect("expected JSON array");
1014        assert_eq!(arr.len(), 5);
1015        assert_eq!(arr[0], Value::String("hello".to_string()));
1016        assert_eq!(arr[1], Value::Number(42.into()));
1017        assert_eq!(arr[2], Value::Bool(true));
1018        assert_eq!(arr[3], Value::Null);
1019        assert!(matches!(arr[4], Value::Number(_)));
1020
1021        Ok(())
1022    }
1023
1024    #[test]
1025    fn test_object_field_ordering_in_json() -> Result<(), ArrowError> {
1026        use parquet_variant::VariantBuilder;
1027
1028        let mut builder = VariantBuilder::new();
1029
1030        {
1031            let mut obj = builder.new_object();
1032            // Add fields in non-alphabetical order
1033            obj.insert("zebra", "last");
1034            obj.insert("alpha", "first");
1035            obj.insert("beta", "second");
1036            obj.finish().unwrap();
1037        }
1038
1039        let (metadata, value) = builder.finish();
1040        let variant = Variant::try_new(&metadata, &value)?;
1041        let json = variant_to_json_string(&variant)?;
1042
1043        // Parse and verify all fields are present
1044        let parsed: Value = serde_json::from_str(&json).unwrap();
1045        let obj = parsed.as_object().expect("expected JSON object");
1046        assert_eq!(obj.len(), 3);
1047        assert_eq!(obj.get("alpha"), Some(&Value::String("first".to_string())));
1048        assert_eq!(obj.get("beta"), Some(&Value::String("second".to_string())));
1049        assert_eq!(obj.get("zebra"), Some(&Value::String("last".to_string())));
1050
1051        Ok(())
1052    }
1053
1054    #[test]
1055    fn test_list_with_various_primitive_types_to_json() -> Result<(), ArrowError> {
1056        use parquet_variant::VariantBuilder;
1057
1058        let mut builder = VariantBuilder::new();
1059
1060        builder
1061            .new_list()
1062            .with_value("string_value")
1063            .with_value(42i32)
1064            .with_value(true)
1065            .with_value(std::f64::consts::PI)
1066            .with_value(false)
1067            .with_value(()) // null
1068            .with_value(100i64)
1069            .finish();
1070
1071        let (metadata, value) = builder.finish();
1072        let variant = Variant::try_new(&metadata, &value)?;
1073        let json = variant_to_json_string(&variant)?;
1074
1075        let parsed: Value = serde_json::from_str(&json).unwrap();
1076        let arr = parsed.as_array().expect("expected JSON array");
1077        assert_eq!(arr.len(), 7);
1078        assert_eq!(arr[0], Value::String("string_value".to_string()));
1079        assert_eq!(arr[1], Value::Number(42.into()));
1080        assert_eq!(arr[2], Value::Bool(true));
1081        assert!(matches!(arr[3], Value::Number(_))); // float
1082        assert_eq!(arr[4], Value::Bool(false));
1083        assert_eq!(arr[5], Value::Null);
1084        assert_eq!(arr[6], Value::Number(100.into()));
1085
1086        Ok(())
1087    }
1088
1089    #[test]
1090    fn test_object_with_various_primitive_types_to_json() -> Result<(), ArrowError> {
1091        use parquet_variant::VariantBuilder;
1092
1093        let mut builder = VariantBuilder::new();
1094
1095        {
1096            let mut obj = builder.new_object();
1097            obj.insert("string_field", "test_string");
1098            obj.insert("int_field", 123i32);
1099            obj.insert("bool_field", true);
1100            obj.insert("float_field", 2.71f64);
1101            obj.insert("null_field", ());
1102            obj.insert("long_field", 999i64);
1103            obj.finish().unwrap();
1104        }
1105
1106        let (metadata, value) = builder.finish();
1107        let variant = Variant::try_new(&metadata, &value)?;
1108        let json = variant_to_json_string(&variant)?;
1109
1110        let parsed: Value = serde_json::from_str(&json).unwrap();
1111        let obj = parsed.as_object().expect("expected JSON object");
1112        assert_eq!(obj.len(), 6);
1113        assert_eq!(
1114            obj.get("string_field"),
1115            Some(&Value::String("test_string".to_string()))
1116        );
1117        assert_eq!(obj.get("int_field"), Some(&Value::Number(123.into())));
1118        assert_eq!(obj.get("bool_field"), Some(&Value::Bool(true)));
1119        assert!(matches!(obj.get("float_field"), Some(Value::Number(_))));
1120        assert_eq!(obj.get("null_field"), Some(&Value::Null));
1121        assert_eq!(obj.get("long_field"), Some(&Value::Number(999.into())));
1122
1123        Ok(())
1124    }
1125
1126    #[test]
1127    fn test_decimal_precision_behavior() -> Result<(), ArrowError> {
1128        // Test case that demonstrates f64 precision limits
1129        // This is a 63-bit precision decimal8 value that f64 cannot represent exactly
1130        let high_precision_decimal8 = Variant::from(VariantDecimal8::try_new(
1131            9007199254740993, // 2^53 + 1, exceeds f64 precision
1132            6,
1133        )?);
1134
1135        let json_string = variant_to_json_string(&high_precision_decimal8)?;
1136        let json_value = variant_to_json_value(&high_precision_decimal8)?;
1137
1138        // Due to f64 precision limits, we expect precision loss for values > 2^53
1139        // Both functions should produce consistent results (even if not exact)
1140        let parsed: Value = serde_json::from_str(&json_string).unwrap();
1141        assert_eq!(parsed, json_value);
1142
1143        // Test a case that can be exactly represented (integer result)
1144        let exact_decimal = Variant::from(VariantDecimal8::try_new(
1145            1234567890000, // Should result in 1234567.89 (trailing zeros trimmed)
1146            6,
1147        )?);
1148
1149        let json_string_exact = variant_to_json_string(&exact_decimal)?;
1150        assert_eq!(json_string_exact, "1234567.89");
1151
1152        // Test integer case (should be exact)
1153        let integer_decimal = Variant::from(VariantDecimal8::try_new(
1154            42000000, // Should result in 42 (integer)
1155            6,
1156        )?);
1157
1158        let json_string_integer = variant_to_json_string(&integer_decimal)?;
1159        assert_eq!(json_string_integer, "42");
1160
1161        Ok(())
1162    }
1163
1164    #[test]
1165    fn test_float_nan_inf_handling() -> Result<(), ArrowError> {
1166        // Test NaN handling - should return an error since JSON doesn't support NaN
1167        let nan_variant = Variant::Float(f32::NAN);
1168        let nan_result = variant_to_json_value(&nan_variant);
1169        assert!(nan_result.is_err());
1170        assert!(nan_result
1171            .unwrap_err()
1172            .to_string()
1173            .contains("Invalid float value"));
1174
1175        // Test positive infinity - should return an error since JSON doesn't support Infinity
1176        let pos_inf_variant = Variant::Float(f32::INFINITY);
1177        let pos_inf_result = variant_to_json_value(&pos_inf_variant);
1178        assert!(pos_inf_result.is_err());
1179        assert!(pos_inf_result
1180            .unwrap_err()
1181            .to_string()
1182            .contains("Invalid float value"));
1183
1184        // Test negative infinity - should return an error since JSON doesn't support -Infinity
1185        let neg_inf_variant = Variant::Float(f32::NEG_INFINITY);
1186        let neg_inf_result = variant_to_json_value(&neg_inf_variant);
1187        assert!(neg_inf_result.is_err());
1188        assert!(neg_inf_result
1189            .unwrap_err()
1190            .to_string()
1191            .contains("Invalid float value"));
1192
1193        // Test the same for Double variants
1194        let nan_double_variant = Variant::Double(f64::NAN);
1195        let nan_double_result = variant_to_json_value(&nan_double_variant);
1196        assert!(nan_double_result.is_err());
1197        assert!(nan_double_result
1198            .unwrap_err()
1199            .to_string()
1200            .contains("Invalid double value"));
1201
1202        let pos_inf_double_variant = Variant::Double(f64::INFINITY);
1203        let pos_inf_double_result = variant_to_json_value(&pos_inf_double_variant);
1204        assert!(pos_inf_double_result.is_err());
1205        assert!(pos_inf_double_result
1206            .unwrap_err()
1207            .to_string()
1208            .contains("Invalid double value"));
1209
1210        let neg_inf_double_variant = Variant::Double(f64::NEG_INFINITY);
1211        let neg_inf_double_result = variant_to_json_value(&neg_inf_double_variant);
1212        assert!(neg_inf_double_result.is_err());
1213        assert!(neg_inf_double_result
1214            .unwrap_err()
1215            .to_string()
1216            .contains("Invalid double value"));
1217
1218        // Test normal float values still work
1219        let normal_float = Variant::Float(std::f32::consts::PI);
1220        let normal_result = variant_to_json_value(&normal_float)?;
1221        assert!(matches!(normal_result, Value::Number(_)));
1222
1223        let normal_double = Variant::Double(std::f64::consts::E);
1224        let normal_double_result = variant_to_json_value(&normal_double)?;
1225        assert!(matches!(normal_double_result, Value::Number(_)));
1226
1227        Ok(())
1228    }
1229}
parquet_variant_json/to_json.rs

parquet_variant_json/
to_json.rs
