arrow_json/reader/
binary_array.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::io::Write;
19use std::marker::PhantomData;
20use std::sync::Arc;
21
22use arrow_array::builder::{BinaryViewBuilder, FixedSizeBinaryBuilder, GenericBinaryBuilder};
23use arrow_array::{ArrayRef, GenericStringArray, OffsetSizeTrait};
24use arrow_schema::ArrowError;
25
26use crate::reader::ArrayDecoder;
27use crate::reader::tape::{Tape, TapeElement};
28
29#[inline]
30fn decode_hex_digit(byte: u8) -> Option<u8> {
31    match byte {
32        b'0'..=b'9' => Some(byte - b'0'),
33        b'a'..=b'f' => Some(byte - b'a' + 10),
34        b'A'..=b'F' => Some(byte - b'A' + 10),
35        _ => None,
36    }
37}
38
39fn invalid_hex_error_at(index: usize, byte: u8) -> ArrowError {
40    ArrowError::JsonError(format!(
41        "invalid hex encoding in binary data: invalid digit 0x{byte:02x} at position {index}"
42    ))
43}
44
45fn decode_hex_to_writer<W: Write>(hex_string: &str, writer: &mut W) -> Result<(), ArrowError> {
46    let bytes = hex_string.as_bytes();
47    let (iter, remainder) = bytes.as_chunks::<2>();
48    let mut buffer = [0u8; 64];
49    let mut buffered = 0;
50
51    for (pair_index, pair) in iter.iter().enumerate() {
52        let base = pair_index * 2;
53        let high = decode_hex_digit(pair[0]).ok_or_else(|| invalid_hex_error_at(base, pair[0]))?;
54        let low =
55            decode_hex_digit(pair[1]).ok_or_else(|| invalid_hex_error_at(base + 1, pair[1]))?;
56        buffer[buffered] = (high << 4) | low;
57        buffered += 1;
58
59        if buffered == buffer.len() {
60            writer
61                .write_all(&buffer)
62                .map_err(|e| ArrowError::JsonError(format!("failed to write binary data: {e}")))?;
63            buffered = 0;
64        }
65    }
66
67    if !remainder.is_empty() {
68        let index = (bytes.len() / 2) * 2;
69        let low = decode_hex_digit(remainder[0])
70            .ok_or_else(|| invalid_hex_error_at(index, remainder[0]))?;
71        buffer[buffered] = low;
72        buffered += 1;
73    }
74
75    if buffered > 0 {
76        writer
77            .write_all(&buffer[..buffered])
78            .map_err(|e| ArrowError::JsonError(format!("failed to write binary data: {e}")))?;
79    }
80
81    Ok(())
82}
83
84#[derive(Default)]
85pub struct BinaryArrayDecoder<O: OffsetSizeTrait> {
86    phantom: PhantomData<O>,
87}
88
89impl<O: OffsetSizeTrait> ArrayDecoder for BinaryArrayDecoder<O> {
90    fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef, ArrowError> {
91        let data_capacity = estimate_data_capacity(tape, pos)?;
92
93        if O::from_usize(data_capacity).is_none() {
94            return Err(ArrowError::JsonError(format!(
95                "offset overflow decoding {}",
96                GenericStringArray::<O>::DATA_TYPE
97            )));
98        }
99
100        let mut builder = GenericBinaryBuilder::<O>::with_capacity(pos.len(), data_capacity);
101
102        for p in pos {
103            match tape.get(*p) {
104                TapeElement::String(idx) => {
105                    let string = tape.get_string(idx);
106                    // Decode directly into the builder for performance. If decoding fails,
107                    // the error is terminal and the builder is discarded by the caller.
108                    decode_hex_to_writer(string, &mut builder)?;
109                    builder.append_value(b"");
110                }
111                TapeElement::Null => builder.append_null(),
112                _ => unreachable!(),
113            }
114        }
115
116        Ok(Arc::new(builder.finish()))
117    }
118}
119
120#[derive(Default)]
121pub struct FixedSizeBinaryArrayDecoder {
122    len: i32,
123}
124
125impl FixedSizeBinaryArrayDecoder {
126    pub fn new(len: i32) -> Self {
127        Self { len }
128    }
129}
130
131impl ArrayDecoder for FixedSizeBinaryArrayDecoder {
132    fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef, ArrowError> {
133        let mut builder = FixedSizeBinaryBuilder::with_capacity(pos.len(), self.len);
134        // Preallocate for the decoded byte width (FixedSizeBinary len), not the hex string length.
135        let capacity: usize = self.len.try_into().map_err(|_| {
136            ArrowError::InvalidArgumentError(format!("Cannot convert size '{}' to usize", self.len))
137        })?;
138        let mut scratch = Vec::with_capacity(capacity);
139
140        for p in pos {
141            match tape.get(*p) {
142                TapeElement::String(idx) => {
143                    let string = tape.get_string(idx);
144                    scratch.clear();
145                    scratch.reserve(string.len().div_ceil(2));
146                    decode_hex_to_writer(string, &mut scratch)?;
147                    builder.append_value(&scratch)?;
148                }
149                TapeElement::Null => builder.append_null(),
150                _ => unreachable!(),
151            }
152        }
153
154        Ok(Arc::new(builder.finish()))
155    }
156}
157
158#[derive(Default)]
159pub struct BinaryViewDecoder {}
160
161impl ArrayDecoder for BinaryViewDecoder {
162    fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef, ArrowError> {
163        let data_capacity = estimate_data_capacity(tape, pos)?;
164        let mut builder = BinaryViewBuilder::with_capacity(data_capacity);
165        let mut scratch = Vec::new();
166
167        for p in pos {
168            match tape.get(*p) {
169                TapeElement::String(idx) => {
170                    let string = tape.get_string(idx);
171                    scratch.clear();
172                    scratch.reserve(string.len().div_ceil(2));
173                    decode_hex_to_writer(string, &mut scratch)?;
174                    builder.append_value(&scratch);
175                }
176                TapeElement::Null => builder.append_null(),
177                _ => unreachable!(),
178            }
179        }
180
181        Ok(Arc::new(builder.finish()))
182    }
183}
184
185fn estimate_data_capacity(tape: &Tape<'_>, pos: &[u32]) -> Result<usize, ArrowError> {
186    let mut data_capacity = 0;
187    for p in pos {
188        match tape.get(*p) {
189            TapeElement::String(idx) => {
190                let string_len = tape.get_string(idx).len();
191                // two hex characters represent one byte
192                let decoded_len = string_len.div_ceil(2);
193                data_capacity += decoded_len;
194            }
195            TapeElement::Null => {}
196            _ => {
197                return Err(tape.error(*p, "binary data encoded as string"));
198            }
199        }
200    }
201    Ok(data_capacity)
202}
203
204#[cfg(test)]
205mod tests {
206    use super::*;
207    use crate::ReaderBuilder;
208    use arrow_schema::{DataType, Field};
209    use std::io::Cursor;
210
211    #[test]
212    fn test_decode_hex_to_writer_empty() {
213        let mut out = Vec::new();
214        decode_hex_to_writer("", &mut out).unwrap();
215        assert!(out.is_empty());
216    }
217
218    #[test]
219    fn test_decode_hex_to_writer_odd_length() {
220        let mut out = Vec::new();
221        decode_hex_to_writer("0f0", &mut out).unwrap();
222        assert_eq!(out, vec![0x0f, 0x00]);
223
224        out.clear();
225        decode_hex_to_writer("a", &mut out).unwrap();
226        assert_eq!(out, vec![0x0a]);
227    }
228
229    #[test]
230    fn test_decode_hex_to_writer_invalid() {
231        let mut out = Vec::new();
232        let err = decode_hex_to_writer("0f0g", &mut out).unwrap_err();
233        match err {
234            ArrowError::JsonError(msg) => {
235                assert!(msg.contains("invalid hex encoding in binary data"));
236                assert!(msg.contains("position 3"));
237            }
238            _ => panic!("expected JsonError"),
239        }
240    }
241
242    #[test]
243    fn test_binary_reader_invalid_hex_is_terminal() {
244        let field = Field::new("item", DataType::Binary, false);
245        let data = b"\"0f0g\"\n\"0f00\"\n";
246        let mut reader = ReaderBuilder::new_with_field(field)
247            .build(Cursor::new(data))
248            .unwrap();
249
250        let err = reader.next().unwrap().unwrap_err().to_string();
251        assert!(err.contains("invalid hex encoding in binary data"));
252
253        match reader.next() {
254            None => {}
255            Some(Err(_)) => {}
256            Some(Ok(_)) => panic!("expected terminal error after invalid hex"),
257        }
258    }
259}
arrow_json/reader/binary_array.rs

arrow_json/reader/
binary_array.rs
