1use arrow_schema::ArrowError;
6use csv_core::{ReadRecordResult, Reader};
7
8const AVERAGE_FIELD_SIZE: usize = 8;
10
11const MIN_CAPACITY: usize = 1024;
13
14#[derive(Debug)]
16pub struct RecordDecoder {
17 delimiter: Reader,
18
19 num_columns: usize,
21
22 line_number: usize,
24
25 offsets: Vec<usize>,
27
28 offsets_len: usize,
32
33 current_field: usize,
35
36 num_rows: usize,
38
39 data: Vec<u8>,
41
42 data_len: usize,
46
47 truncated_rows: bool,
52
53 truncated_row_count: usize,
59}
60
61impl RecordDecoder {
62 pub fn new(delimiter: Reader, num_columns: usize, truncated_rows: bool) -> Self {
63 Self {
64 delimiter,
65 num_columns,
66 line_number: 1,
67 offsets: vec![],
68 offsets_len: 1, current_field: 0,
70 data_len: 0,
71 data: vec![],
72 num_rows: 0,
73 truncated_rows,
74 truncated_row_count: 0,
75 }
76 }
77
78 pub fn decode(&mut self, input: &[u8], to_read: usize) -> Result<(usize, usize), ArrowError> {
82 if to_read == 0 {
83 return Ok((0, 0));
84 }
85
86 self.offsets
88 .resize(self.offsets_len + to_read * self.num_columns, 0);
89
90 let mut input_offset = 0;
92
93 let mut read = 0;
95
96 loop {
97 let remaining_rows = to_read - read;
99 let capacity = remaining_rows * self.num_columns * AVERAGE_FIELD_SIZE;
100 let estimated_data = capacity.max(MIN_CAPACITY);
101 self.data.resize(self.data_len + estimated_data, 0);
102
103 loop {
105 let (result, bytes_read, bytes_written, end_positions) =
106 self.delimiter.read_record(
107 &input[input_offset..],
108 &mut self.data[self.data_len..],
109 &mut self.offsets[self.offsets_len..],
110 );
111
112 self.current_field += end_positions;
113 self.offsets_len += end_positions;
114 input_offset += bytes_read;
115 self.data_len += bytes_written;
116
117 match result {
118 ReadRecordResult::End | ReadRecordResult::InputEmpty => {
119 return Ok((read, input_offset));
121 }
122 ReadRecordResult::OutputFull => break,
124 ReadRecordResult::OutputEndsFull => {
125 return Err(ArrowError::CsvError(format!(
126 "incorrect number of fields for line {}, expected {} got more than {}",
127 self.line_number, self.num_columns, self.current_field
128 )));
129 }
130 ReadRecordResult::Record => {
131 if self.current_field != self.num_columns {
132 if self.truncated_rows && self.current_field < self.num_columns {
133 let fill_count = self.num_columns - self.current_field;
135 let fill_value = self.offsets[self.offsets_len - 1];
136 self.offsets[self.offsets_len..self.offsets_len + fill_count]
137 .fill(fill_value);
138 self.offsets_len += fill_count;
139 self.truncated_row_count += 1;
140 } else {
141 return Err(ArrowError::CsvError(format!(
142 "incorrect number of fields for line {}, expected {} got {}",
143 self.line_number, self.num_columns, self.current_field
144 )));
145 }
146 }
147 read += 1;
148 self.current_field = 0;
149 self.line_number += 1;
150 self.num_rows += 1;
151
152 if read == to_read {
153 return Ok((read, input_offset));
155 }
156
157 if input.len() == input_offset {
158 return Ok((read, input_offset));
162 }
163 }
164 }
165 }
166 }
167 }
168
169 pub fn len(&self) -> usize {
171 self.num_rows
172 }
173
174 pub fn is_empty(&self) -> bool {
176 self.num_rows == 0
177 }
178
179 pub fn truncated_row_count(&self) -> usize {
183 self.truncated_row_count
184 }
185
186 pub fn clear(&mut self) {
188 self.offsets_len = 1;
190 self.data_len = 0;
191 self.num_rows = 0;
192 self.truncated_row_count = 0;
195 }
196
197 pub fn flush(&mut self) -> Result<StringRecords<'_>, ArrowError> {
199 if self.current_field != 0 {
200 return Err(ArrowError::CsvError(
201 "Cannot flush part way through record".to_string(),
202 ));
203 }
204
205 let mut row_offset: usize = 0;
208 self.offsets[1..self.offsets_len]
209 .chunks_exact_mut(self.num_columns)
210 .try_for_each(|row| -> Result<(), ArrowError> {
211 let offset = row_offset;
212 row.iter_mut().try_for_each(|x| -> Result<(), ArrowError> {
213 *x = x.checked_add(offset).ok_or_else(|| {
214 ArrowError::CsvError(
215 "CSV record offsets overflowed usize while flushing".to_string(),
216 )
217 })?;
218 row_offset = *x;
219 Ok(())
220 })
221 })?;
222
223 let data = std::str::from_utf8(&self.data[..self.data_len]).map_err(|e| {
225 let valid_up_to = e.valid_up_to();
226
227 let idx = self.offsets[..self.offsets_len]
229 .iter()
230 .rposition(|x| *x <= valid_up_to)
231 .unwrap();
232
233 let field = idx % self.num_columns + 1;
234 let line_offset = self.line_number - self.num_rows;
235 let line = line_offset + idx / self.num_columns;
236
237 ArrowError::CsvError(format!(
238 "Encountered invalid UTF-8 data for line {line} and field {field}"
239 ))
240 })?;
241
242 let offsets = &self.offsets[..self.offsets_len];
243 let num_rows = self.num_rows;
244
245 self.offsets_len = 1;
249 self.data_len = 0;
250 self.num_rows = 0;
251
252 Ok(StringRecords {
253 num_rows,
254 num_columns: self.num_columns,
255 offsets,
256 data,
257 })
258 }
259}
260
261#[derive(Debug)]
263pub struct StringRecords<'a> {
264 num_columns: usize,
265 num_rows: usize,
266 offsets: &'a [usize],
267 data: &'a str,
268}
269
270impl<'a> StringRecords<'a> {
271 fn get(&self, index: usize) -> StringRecord<'a> {
272 let field_idx = index * self.num_columns;
273 StringRecord {
274 data: self.data,
275 offsets: &self.offsets[field_idx..field_idx + self.num_columns + 1],
276 }
277 }
278
279 pub fn len(&self) -> usize {
280 self.num_rows
281 }
282
283 pub fn iter(&self) -> impl Iterator<Item = StringRecord<'a>> + '_ {
284 (0..self.num_rows).map(|x| self.get(x))
285 }
286}
287
288#[derive(Debug, Clone, Copy)]
290pub struct StringRecord<'a> {
291 data: &'a str,
292 offsets: &'a [usize],
293}
294
295impl<'a> StringRecord<'a> {
296 pub fn get(&self, index: usize) -> &'a str {
297 let end = self.offsets[index + 1];
298 let start = self.offsets[index];
299
300 unsafe { self.data.get_unchecked(start..end) }
303 }
304}
305
306impl std::fmt::Display for StringRecord<'_> {
307 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
308 let num_fields = self.offsets.len() - 1;
309 write!(f, "[")?;
310 for i in 0..num_fields {
311 if i > 0 {
312 write!(f, ",")?;
313 }
314 write!(f, "{}", self.get(i))?;
315 }
316 write!(f, "]")?;
317 Ok(())
318 }
319}
320
321#[cfg(test)]
322mod tests {
323 use crate::reader::records::RecordDecoder;
324 use csv_core::Reader;
325 use std::io::{BufRead, BufReader, Cursor};
326
327 #[test]
328 fn test_basic() {
329 let csv = [
330 "foo,bar,baz",
331 "a,b,c",
332 "12,3,5",
333 "\"asda\"\"asas\",\"sdffsnsd\", as",
334 ]
335 .join("\n");
336
337 let mut expected = vec![
338 vec!["foo", "bar", "baz"],
339 vec!["a", "b", "c"],
340 vec!["12", "3", "5"],
341 vec!["asda\"asas", "sdffsnsd", " as"],
342 ]
343 .into_iter();
344
345 let mut reader = BufReader::with_capacity(3, Cursor::new(csv.as_bytes()));
346 let mut decoder = RecordDecoder::new(Reader::new(), 3, false);
347
348 loop {
349 let to_read = 3;
350 let mut read = 0;
351 loop {
352 let buf = reader.fill_buf().unwrap();
353 let (records, bytes) = decoder.decode(buf, to_read - read).unwrap();
354
355 reader.consume(bytes);
356 read += records;
357
358 if read == to_read || bytes == 0 {
359 break;
360 }
361 }
362 if read == 0 {
363 break;
364 }
365
366 let b = decoder.flush().unwrap();
367 b.iter().zip(&mut expected).for_each(|(record, expected)| {
368 let actual = (0..3)
369 .map(|field_idx| record.get(field_idx))
370 .collect::<Vec<_>>();
371 assert_eq!(actual, expected)
372 });
373 }
374 assert!(expected.next().is_none());
375 }
376
377 #[test]
378 fn test_invalid_fields() {
379 let csv = "a,b\nb,c\na\n";
380 let mut decoder = RecordDecoder::new(Reader::new(), 2, false);
381 let err = decoder.decode(csv.as_bytes(), 4).unwrap_err().to_string();
382
383 let expected = "Csv error: incorrect number of fields for line 3, expected 2 got 1";
384
385 assert_eq!(err, expected);
386
387 let mut decoder = RecordDecoder::new(Reader::new(), 2, false);
389 let (skipped, bytes) = decoder.decode(csv.as_bytes(), 1).unwrap();
390 assert_eq!(skipped, 1);
391 decoder.clear();
392
393 let remaining = &csv.as_bytes()[bytes..];
394 let err = decoder.decode(remaining, 3).unwrap_err().to_string();
395 assert_eq!(err, expected);
396 }
397
398 #[test]
399 fn test_skip_insufficient_rows() {
400 let csv = "a\nv\n";
401 let mut decoder = RecordDecoder::new(Reader::new(), 1, false);
402 let (read, bytes) = decoder.decode(csv.as_bytes(), 3).unwrap();
403 assert_eq!(read, 2);
404 assert_eq!(bytes, csv.len());
405 }
406
407 #[test]
408 fn test_truncated_rows() {
409 let csv = "a,b\nv\n,1\n,2\n,3\n";
410 let mut decoder = RecordDecoder::new(Reader::new(), 2, true);
411 let (read, bytes) = decoder.decode(csv.as_bytes(), 5).unwrap();
412 assert_eq!(read, 5);
413 assert_eq!(bytes, csv.len());
414 assert_eq!(decoder.truncated_row_count(), 1);
416 }
417
418 #[test]
419 fn test_truncated_row_count_not_reset_by_flush() {
420 let csv = "a\nb,2\nc\n";
421 let mut decoder = RecordDecoder::new(Reader::new(), 2, true);
422
423 let (read, _) = decoder.decode(csv.as_bytes(), 2).unwrap();
424 assert_eq!(read, 2);
425 assert_eq!(decoder.truncated_row_count(), 1);
426 decoder.flush().unwrap();
427 assert_eq!(decoder.truncated_row_count(), 1);
428
429 let (read, _) = decoder.decode(&csv.as_bytes()[6..], 1).unwrap();
430 assert_eq!(read, 1);
431 assert_eq!(decoder.truncated_row_count(), 2);
432 decoder.flush().unwrap();
433 assert_eq!(decoder.truncated_row_count(), 2);
434 }
435
436 #[test]
437 fn test_truncated_row_count_reset_by_clear() {
438 let csv = "a\nb,2\n";
439 let mut decoder = RecordDecoder::new(Reader::new(), 2, true);
440
441 let (read, _) = decoder.decode(csv.as_bytes(), 2).unwrap();
442 assert_eq!(read, 2);
443 assert_eq!(decoder.truncated_row_count(), 1);
444
445 decoder.clear();
447 assert_eq!(decoder.truncated_row_count(), 0);
448 }
449
450 #[test]
457 fn test_flush_offset_overflow_returns_csv_error() {
458 let mut decoder = RecordDecoder::new(Reader::new(), 1, false);
459 decoder.offsets = vec![0, usize::MAX, 1];
460 decoder.offsets_len = 3;
461 decoder.num_rows = 2;
462 let err = decoder.flush().unwrap_err();
463 assert_eq!(
464 err.to_string(),
465 "Csv error: CSV record offsets overflowed usize while flushing"
466 );
467 }
468}