Skip to main content

arrow_json/reader/
primitive_array.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use std::marker::PhantomData;
6use std::sync::Arc;
7
8use arrow_array::builder::PrimitiveBuilder;
9use arrow_array::{ArrayRef, ArrowPrimitiveType};
10use arrow_cast::parse::Parser;
11use arrow_schema::{ArrowError, DataType};
12use half::f16;
13use num_traits::NumCast;
14
15use crate::reader::tape::{Tape, TapeElement};
16use crate::reader::{ArrayDecoder, DecoderContext};
17
18/// A trait for JSON-specific primitive parsing logic
19///
20/// According to the specification unquoted fields should be parsed as a double-precision
21/// floating point numbers, including scientific representation such as `2e3`
22///
23/// In practice, it is common to serialize numbers outside the range of an `f64` and expect
24/// them to round-trip correctly. As such when parsing integers we first parse as the integer
25/// and fallback to parsing as a floating point if this fails
26trait ParseJsonNumber: Sized {
27    fn parse(s: &[u8]) -> Option<Self>;
28}
29
30macro_rules! primitive_parse {
31    ($($t:ty),+) => {
32        $(impl ParseJsonNumber for $t {
33            fn parse(s: &[u8]) -> Option<Self> {
34                match lexical_core::parse::<Self>(s) {
35                    Ok(f) => Some(f),
36                    Err(_) => lexical_core::parse::<f64>(s).ok().and_then(NumCast::from),
37                }
38            }
39        })+
40    };
41}
42
43primitive_parse!(i8, i16, i32, i64, u8, u16, u32, u64);
44
45impl ParseJsonNumber for f16 {
46    fn parse(s: &[u8]) -> Option<Self> {
47        lexical_core::parse::<f32>(s).ok().map(f16::from_f32)
48    }
49}
50
51impl ParseJsonNumber for f32 {
52    fn parse(s: &[u8]) -> Option<Self> {
53        lexical_core::parse::<Self>(s).ok()
54    }
55}
56
57impl ParseJsonNumber for f64 {
58    fn parse(s: &[u8]) -> Option<Self> {
59        lexical_core::parse::<Self>(s).ok()
60    }
61}
62
63pub struct PrimitiveArrayDecoder<P: ArrowPrimitiveType> {
64    data_type: DataType,
65    ignore_type_conflicts: bool,
66    // Invariant and Send
67    phantom: PhantomData<fn(P) -> P>,
68}
69
70impl<P: ArrowPrimitiveType> PrimitiveArrayDecoder<P> {
71    pub fn new(ctx: &DecoderContext, data_type: &DataType) -> Self {
72        Self {
73            data_type: data_type.clone(),
74            ignore_type_conflicts: ctx.ignore_type_conflicts(),
75            phantom: Default::default(),
76        }
77    }
78}
79
80impl<P> ArrayDecoder for PrimitiveArrayDecoder<P>
81where
82    P: ArrowPrimitiveType + Parser,
83    P::Native: ParseJsonNumber + NumCast,
84{
85    fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef, ArrowError> {
86        let mut builder =
87            PrimitiveBuilder::<P>::with_capacity(pos.len()).with_data_type(self.data_type.clone());
88        let d = &self.data_type;
89
90        for p in pos {
91            let value = match tape.get(*p) {
92                TapeElement::Null => {
93                    builder.append_null();
94                    continue;
95                }
96                TapeElement::String(idx) => {
97                    let s = tape.get_string(idx);
98                    P::parse(s).ok_or_else(|| {
99                        ArrowError::JsonError(format!("failed to parse \"{s}\" as {d}"))
100                    })
101                }
102                TapeElement::Number(idx) => {
103                    let s = tape.get_string(idx);
104                    ParseJsonNumber::parse(s.as_bytes())
105                        .ok_or_else(|| ArrowError::JsonError(format!("failed to parse {s} as {d}")))
106                }
107                TapeElement::F32(v) => {
108                    let v = f32::from_bits(v);
109                    NumCast::from(v)
110                        .ok_or_else(|| ArrowError::JsonError(format!("failed to parse {v} as {d}")))
111                }
112                TapeElement::I32(v) => NumCast::from(v)
113                    .ok_or_else(|| ArrowError::JsonError(format!("failed to parse {v} as {d}"))),
114                TapeElement::F64(high) => match tape.get(p + 1) {
115                    TapeElement::F32(low) => {
116                        let v = f64::from_bits(((high as u64) << 32) | low as u64);
117                        NumCast::from(v).ok_or_else(|| {
118                            ArrowError::JsonError(format!("failed to parse {v} as {d}"))
119                        })
120                    }
121                    _ => unreachable!(),
122                },
123                TapeElement::I64(high) => match tape.get(p + 1) {
124                    TapeElement::I32(low) => {
125                        let v = ((high as i64) << 32) | (low as u32) as i64;
126                        NumCast::from(v).ok_or_else(|| {
127                            ArrowError::JsonError(format!("failed to parse {v} as {d}"))
128                        })
129                    }
130                    _ => unreachable!(),
131                },
132                _ => Err(tape.error(*p, "primitive")),
133            };
134
135            match value {
136                Ok(value) => builder.append_value(value),
137                Err(_) if self.ignore_type_conflicts => builder.append_null(),
138                Err(e) => return Err(e),
139            }
140        }
141
142        Ok(Arc::new(builder.finish()))
143    }
144}