arrow_json/reader/
binary_array.rs1use std::io::Write;
6use std::marker::PhantomData;
7use std::sync::Arc;
8
9use arrow_array::builder::{BinaryViewBuilder, FixedSizeBinaryBuilder, GenericBinaryBuilder};
10use arrow_array::{ArrayRef, GenericStringArray, OffsetSizeTrait};
11use arrow_schema::ArrowError;
12
13use crate::reader::ArrayDecoder;
14use crate::reader::tape::{Tape, TapeElement};
15
16#[inline]
17fn decode_hex_digit(byte: u8) -> Option<u8> {
18 match byte {
19 b'0'..=b'9' => Some(byte - b'0'),
20 b'a'..=b'f' => Some(byte - b'a' + 10),
21 b'A'..=b'F' => Some(byte - b'A' + 10),
22 _ => None,
23 }
24}
25
26fn invalid_hex_error_at(index: usize, byte: u8) -> ArrowError {
27 ArrowError::JsonError(format!(
28 "invalid hex encoding in binary data: invalid digit 0x{byte:02x} at position {index}"
29 ))
30}
31
32fn decode_hex_to_writer<W: Write>(hex_string: &str, writer: &mut W) -> Result<(), ArrowError> {
33 let bytes = hex_string.as_bytes();
34 let (iter, remainder) = bytes.as_chunks::<2>();
35 let mut buffer = [0u8; 64];
36 let mut buffered = 0;
37
38 for (pair_index, pair) in iter.iter().enumerate() {
39 let base = pair_index * 2;
40 let high = decode_hex_digit(pair[0]).ok_or_else(|| invalid_hex_error_at(base, pair[0]))?;
41 let low =
42 decode_hex_digit(pair[1]).ok_or_else(|| invalid_hex_error_at(base + 1, pair[1]))?;
43 buffer[buffered] = (high << 4) | low;
44 buffered += 1;
45
46 if buffered == buffer.len() {
47 writer
48 .write_all(&buffer)
49 .map_err(|e| ArrowError::JsonError(format!("failed to write binary data: {e}")))?;
50 buffered = 0;
51 }
52 }
53
54 if !remainder.is_empty() {
55 let index = (bytes.len() / 2) * 2;
56 let low = decode_hex_digit(remainder[0])
57 .ok_or_else(|| invalid_hex_error_at(index, remainder[0]))?;
58 buffer[buffered] = low;
59 buffered += 1;
60 }
61
62 if buffered > 0 {
63 writer
64 .write_all(&buffer[..buffered])
65 .map_err(|e| ArrowError::JsonError(format!("failed to write binary data: {e}")))?;
66 }
67
68 Ok(())
69}
70
71#[derive(Default)]
72pub struct BinaryArrayDecoder<O: OffsetSizeTrait> {
73 phantom: PhantomData<O>,
74}
75
76impl<O: OffsetSizeTrait> ArrayDecoder for BinaryArrayDecoder<O> {
77 fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef, ArrowError> {
78 let data_capacity = estimate_data_capacity(tape, pos)?;
79
80 if O::from_usize(data_capacity).is_none() {
81 return Err(ArrowError::JsonError(format!(
82 "offset overflow decoding {}",
83 GenericStringArray::<O>::DATA_TYPE
84 )));
85 }
86
87 let mut builder = GenericBinaryBuilder::<O>::with_capacity(pos.len(), data_capacity);
88
89 for p in pos {
90 match tape.get(*p) {
91 TapeElement::String(idx) => {
92 let string = tape.get_string(idx);
93 decode_hex_to_writer(string, &mut builder)?;
96 builder.append_value(b"");
97 }
98 TapeElement::Null => builder.append_null(),
99 _ => unreachable!(),
100 }
101 }
102
103 Ok(Arc::new(builder.finish()))
104 }
105}
106
107#[derive(Default)]
108pub struct FixedSizeBinaryArrayDecoder {
109 len: i32,
110}
111
112impl FixedSizeBinaryArrayDecoder {
113 pub fn new(len: i32) -> Self {
114 Self { len }
115 }
116}
117
118impl ArrayDecoder for FixedSizeBinaryArrayDecoder {
119 fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef, ArrowError> {
120 let mut builder = FixedSizeBinaryBuilder::with_capacity(pos.len(), self.len);
121 let capacity: usize = self.len.try_into().map_err(|_| {
123 ArrowError::InvalidArgumentError(format!("Cannot convert size '{}' to usize", self.len))
124 })?;
125 let mut scratch = Vec::with_capacity(capacity);
126
127 for p in pos {
128 match tape.get(*p) {
129 TapeElement::String(idx) => {
130 let string = tape.get_string(idx);
131 scratch.clear();
132 scratch.reserve(string.len().div_ceil(2));
133 decode_hex_to_writer(string, &mut scratch)?;
134 builder.append_value(&scratch)?;
135 }
136 TapeElement::Null => builder.append_null(),
137 _ => unreachable!(),
138 }
139 }
140
141 Ok(Arc::new(builder.finish()))
142 }
143}
144
145#[derive(Default)]
146pub struct BinaryViewDecoder {}
147
148impl ArrayDecoder for BinaryViewDecoder {
149 fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef, ArrowError> {
150 let data_capacity = estimate_data_capacity(tape, pos)?;
151 let mut builder = BinaryViewBuilder::with_capacity(data_capacity);
152 let mut scratch = Vec::new();
153
154 for p in pos {
155 match tape.get(*p) {
156 TapeElement::String(idx) => {
157 let string = tape.get_string(idx);
158 scratch.clear();
159 scratch.reserve(string.len().div_ceil(2));
160 decode_hex_to_writer(string, &mut scratch)?;
161 builder.append_value(&scratch);
162 }
163 TapeElement::Null => builder.append_null(),
164 _ => unreachable!(),
165 }
166 }
167
168 Ok(Arc::new(builder.finish()))
169 }
170}
171
172fn estimate_data_capacity(tape: &Tape<'_>, pos: &[u32]) -> Result<usize, ArrowError> {
173 let mut data_capacity = 0;
174 for p in pos {
175 match tape.get(*p) {
176 TapeElement::String(idx) => {
177 let string_len = tape.get_string(idx).len();
178 let decoded_len = string_len.div_ceil(2);
180 data_capacity += decoded_len;
181 }
182 TapeElement::Null => {}
183 _ => {
184 return Err(tape.error(*p, "binary data encoded as string"));
185 }
186 }
187 }
188 Ok(data_capacity)
189}
190
191#[cfg(test)]
192mod tests {
193 use super::*;
194 use crate::ReaderBuilder;
195 use arrow_schema::{DataType, Field};
196 use std::io::Cursor;
197
198 #[test]
199 fn test_decode_hex_to_writer_empty() {
200 let mut out = Vec::new();
201 decode_hex_to_writer("", &mut out).unwrap();
202 assert!(out.is_empty());
203 }
204
205 #[test]
206 fn test_decode_hex_to_writer_odd_length() {
207 let mut out = Vec::new();
208 decode_hex_to_writer("0f0", &mut out).unwrap();
209 assert_eq!(out, vec![0x0f, 0x00]);
210
211 out.clear();
212 decode_hex_to_writer("a", &mut out).unwrap();
213 assert_eq!(out, vec![0x0a]);
214 }
215
216 #[test]
217 fn test_decode_hex_to_writer_invalid() {
218 let mut out = Vec::new();
219 let err = decode_hex_to_writer("0f0g", &mut out).unwrap_err();
220 match err {
221 ArrowError::JsonError(msg) => {
222 assert!(msg.contains("invalid hex encoding in binary data"));
223 assert!(msg.contains("position 3"));
224 }
225 _ => panic!("expected JsonError"),
226 }
227 }
228
229 #[test]
230 fn test_binary_reader_invalid_hex_is_terminal() {
231 let field = Field::new("item", DataType::Binary, false);
232 let data = b"\"0f0g\"\n\"0f00\"\n";
233 let mut reader = ReaderBuilder::new_with_field(field)
234 .build(Cursor::new(data))
235 .unwrap();
236
237 let err = reader.next().unwrap().unwrap_err().to_string();
238 assert!(err.contains("invalid hex encoding in binary data"));
239
240 match reader.next() {
241 None => {}
242 Some(Err(_)) => {}
243 Some(Ok(_)) => panic!("expected terminal error after invalid hex"),
244 }
245 }
246}