Skip to main content

arrow_json/reader/
binary_array.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use std::io::Write;
6use std::marker::PhantomData;
7use std::sync::Arc;
8
9use arrow_array::builder::{BinaryViewBuilder, FixedSizeBinaryBuilder, GenericBinaryBuilder};
10use arrow_array::{ArrayRef, GenericStringArray, OffsetSizeTrait};
11use arrow_schema::ArrowError;
12
13use crate::reader::ArrayDecoder;
14use crate::reader::tape::{Tape, TapeElement};
15
16#[inline]
17fn decode_hex_digit(byte: u8) -> Option<u8> {
18    match byte {
19        b'0'..=b'9' => Some(byte - b'0'),
20        b'a'..=b'f' => Some(byte - b'a' + 10),
21        b'A'..=b'F' => Some(byte - b'A' + 10),
22        _ => None,
23    }
24}
25
26fn invalid_hex_error_at(index: usize, byte: u8) -> ArrowError {
27    ArrowError::JsonError(format!(
28        "invalid hex encoding in binary data: invalid digit 0x{byte:02x} at position {index}"
29    ))
30}
31
32fn decode_hex_to_writer<W: Write>(hex_string: &str, writer: &mut W) -> Result<(), ArrowError> {
33    let bytes = hex_string.as_bytes();
34    let (iter, remainder) = bytes.as_chunks::<2>();
35    let mut buffer = [0u8; 64];
36    let mut buffered = 0;
37
38    for (pair_index, pair) in iter.iter().enumerate() {
39        let base = pair_index * 2;
40        let high = decode_hex_digit(pair[0]).ok_or_else(|| invalid_hex_error_at(base, pair[0]))?;
41        let low =
42            decode_hex_digit(pair[1]).ok_or_else(|| invalid_hex_error_at(base + 1, pair[1]))?;
43        buffer[buffered] = (high << 4) | low;
44        buffered += 1;
45
46        if buffered == buffer.len() {
47            writer
48                .write_all(&buffer)
49                .map_err(|e| ArrowError::JsonError(format!("failed to write binary data: {e}")))?;
50            buffered = 0;
51        }
52    }
53
54    if !remainder.is_empty() {
55        let index = (bytes.len() / 2) * 2;
56        let low = decode_hex_digit(remainder[0])
57            .ok_or_else(|| invalid_hex_error_at(index, remainder[0]))?;
58        buffer[buffered] = low;
59        buffered += 1;
60    }
61
62    if buffered > 0 {
63        writer
64            .write_all(&buffer[..buffered])
65            .map_err(|e| ArrowError::JsonError(format!("failed to write binary data: {e}")))?;
66    }
67
68    Ok(())
69}
70
71#[derive(Default)]
72pub struct BinaryArrayDecoder<O: OffsetSizeTrait> {
73    phantom: PhantomData<O>,
74}
75
76impl<O: OffsetSizeTrait> ArrayDecoder for BinaryArrayDecoder<O> {
77    fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef, ArrowError> {
78        let data_capacity = estimate_data_capacity(tape, pos)?;
79
80        if O::from_usize(data_capacity).is_none() {
81            return Err(ArrowError::JsonError(format!(
82                "offset overflow decoding {}",
83                GenericStringArray::<O>::DATA_TYPE
84            )));
85        }
86
87        let mut builder = GenericBinaryBuilder::<O>::with_capacity(pos.len(), data_capacity);
88
89        for p in pos {
90            match tape.get(*p) {
91                TapeElement::String(idx) => {
92                    let string = tape.get_string(idx);
93                    // Decode directly into the builder for performance. If decoding fails,
94                    // the error is terminal and the builder is discarded by the caller.
95                    decode_hex_to_writer(string, &mut builder)?;
96                    builder.append_value(b"");
97                }
98                TapeElement::Null => builder.append_null(),
99                _ => unreachable!(),
100            }
101        }
102
103        Ok(Arc::new(builder.finish()))
104    }
105}
106
107#[derive(Default)]
108pub struct FixedSizeBinaryArrayDecoder {
109    len: i32,
110}
111
112impl FixedSizeBinaryArrayDecoder {
113    pub fn new(len: i32) -> Self {
114        Self { len }
115    }
116}
117
118impl ArrayDecoder for FixedSizeBinaryArrayDecoder {
119    fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef, ArrowError> {
120        let mut builder = FixedSizeBinaryBuilder::with_capacity(pos.len(), self.len);
121        // Preallocate for the decoded byte width (FixedSizeBinary len), not the hex string length.
122        let capacity: usize = self.len.try_into().map_err(|_| {
123            ArrowError::InvalidArgumentError(format!("Cannot convert size '{}' to usize", self.len))
124        })?;
125        let mut scratch = Vec::with_capacity(capacity);
126
127        for p in pos {
128            match tape.get(*p) {
129                TapeElement::String(idx) => {
130                    let string = tape.get_string(idx);
131                    scratch.clear();
132                    scratch.reserve(string.len().div_ceil(2));
133                    decode_hex_to_writer(string, &mut scratch)?;
134                    builder.append_value(&scratch)?;
135                }
136                TapeElement::Null => builder.append_null(),
137                _ => unreachable!(),
138            }
139        }
140
141        Ok(Arc::new(builder.finish()))
142    }
143}
144
145#[derive(Default)]
146pub struct BinaryViewDecoder {}
147
148impl ArrayDecoder for BinaryViewDecoder {
149    fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef, ArrowError> {
150        let data_capacity = estimate_data_capacity(tape, pos)?;
151        let mut builder = BinaryViewBuilder::with_capacity(data_capacity);
152        let mut scratch = Vec::new();
153
154        for p in pos {
155            match tape.get(*p) {
156                TapeElement::String(idx) => {
157                    let string = tape.get_string(idx);
158                    scratch.clear();
159                    scratch.reserve(string.len().div_ceil(2));
160                    decode_hex_to_writer(string, &mut scratch)?;
161                    builder.append_value(&scratch);
162                }
163                TapeElement::Null => builder.append_null(),
164                _ => unreachable!(),
165            }
166        }
167
168        Ok(Arc::new(builder.finish()))
169    }
170}
171
172fn estimate_data_capacity(tape: &Tape<'_>, pos: &[u32]) -> Result<usize, ArrowError> {
173    let mut data_capacity = 0;
174    for p in pos {
175        match tape.get(*p) {
176            TapeElement::String(idx) => {
177                let string_len = tape.get_string(idx).len();
178                // two hex characters represent one byte
179                let decoded_len = string_len.div_ceil(2);
180                data_capacity += decoded_len;
181            }
182            TapeElement::Null => {}
183            _ => {
184                return Err(tape.error(*p, "binary data encoded as string"));
185            }
186        }
187    }
188    Ok(data_capacity)
189}
190
191#[cfg(test)]
192mod tests {
193    use super::*;
194    use crate::ReaderBuilder;
195    use arrow_schema::{DataType, Field};
196    use std::io::Cursor;
197
198    #[test]
199    fn test_decode_hex_to_writer_empty() {
200        let mut out = Vec::new();
201        decode_hex_to_writer("", &mut out).unwrap();
202        assert!(out.is_empty());
203    }
204
205    #[test]
206    fn test_decode_hex_to_writer_odd_length() {
207        let mut out = Vec::new();
208        decode_hex_to_writer("0f0", &mut out).unwrap();
209        assert_eq!(out, vec![0x0f, 0x00]);
210
211        out.clear();
212        decode_hex_to_writer("a", &mut out).unwrap();
213        assert_eq!(out, vec![0x0a]);
214    }
215
216    #[test]
217    fn test_decode_hex_to_writer_invalid() {
218        let mut out = Vec::new();
219        let err = decode_hex_to_writer("0f0g", &mut out).unwrap_err();
220        match err {
221            ArrowError::JsonError(msg) => {
222                assert!(msg.contains("invalid hex encoding in binary data"));
223                assert!(msg.contains("position 3"));
224            }
225            _ => panic!("expected JsonError"),
226        }
227    }
228
229    #[test]
230    fn test_binary_reader_invalid_hex_is_terminal() {
231        let field = Field::new("item", DataType::Binary, false);
232        let data = b"\"0f0g\"\n\"0f00\"\n";
233        let mut reader = ReaderBuilder::new_with_field(field)
234            .build(Cursor::new(data))
235            .unwrap();
236
237        let err = reader.next().unwrap().unwrap_err().to_string();
238        assert!(err.contains("invalid hex encoding in binary data"));
239
240        match reader.next() {
241            None => {}
242            Some(Err(_)) => {}
243            Some(Ok(_)) => panic!("expected terminal error after invalid hex"),
244        }
245    }
246}