Skip to main content

arrow_csv/reader/
records.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use arrow_schema::ArrowError;
6use csv_core::{ReadRecordResult, Reader};
7
8/// The estimated length of a field in bytes
9const AVERAGE_FIELD_SIZE: usize = 8;
10
11/// The minimum amount of data in a single read
12const MIN_CAPACITY: usize = 1024;
13
14/// [`RecordDecoder`] provides a push-based interface to decoder [`StringRecords`]
15#[derive(Debug)]
16pub struct RecordDecoder {
17    delimiter: Reader,
18
19    /// The expected number of fields per row
20    num_columns: usize,
21
22    /// The current line number
23    line_number: usize,
24
25    /// Offsets delimiting field start positions
26    offsets: Vec<usize>,
27
28    /// The current offset into `self.offsets`
29    ///
30    /// We track this independently of Vec to avoid re-zeroing memory
31    offsets_len: usize,
32
33    /// The number of fields read for the current record
34    current_field: usize,
35
36    /// The number of rows buffered
37    num_rows: usize,
38
39    /// Decoded field data
40    data: Vec<u8>,
41
42    /// Offsets into data
43    ///
44    /// We track this independently of Vec to avoid re-zeroing memory
45    data_len: usize,
46
47    /// Whether rows with less than expected columns are considered valid
48    ///
49    /// Default value is false
50    /// When enabled fills in missing columns with null
51    truncated_rows: bool,
52
53    /// The number of rows padded because they had fewer fields than expected
54    ///
55    /// Only incremented when `truncated_rows` is enabled. Cumulative over the
56    /// lifetime of this decoder, i.e. not reset by [`Self::flush`], but reset by
57    /// [`Self::clear`] along with the buffered rows it counted
58    truncated_row_count: usize,
59}
60
61impl RecordDecoder {
62    pub fn new(delimiter: Reader, num_columns: usize, truncated_rows: bool) -> Self {
63        Self {
64            delimiter,
65            num_columns,
66            line_number: 1,
67            offsets: vec![],
68            offsets_len: 1, // The first offset is always 0
69            current_field: 0,
70            data_len: 0,
71            data: vec![],
72            num_rows: 0,
73            truncated_rows,
74            truncated_row_count: 0,
75        }
76    }
77
78    /// Decodes records from `input` returning the number of records and bytes read
79    ///
80    /// Note: this expects to be called with an empty `input` to signal EOF
81    pub fn decode(&mut self, input: &[u8], to_read: usize) -> Result<(usize, usize), ArrowError> {
82        if to_read == 0 {
83            return Ok((0, 0));
84        }
85
86        // Reserve sufficient capacity in offsets
87        self.offsets
88            .resize(self.offsets_len + to_read * self.num_columns, 0);
89
90        // The current offset into `input`
91        let mut input_offset = 0;
92
93        // The number of rows decoded in this pass
94        let mut read = 0;
95
96        loop {
97            // Reserve necessary space in output data based on best estimate
98            let remaining_rows = to_read - read;
99            let capacity = remaining_rows * self.num_columns * AVERAGE_FIELD_SIZE;
100            let estimated_data = capacity.max(MIN_CAPACITY);
101            self.data.resize(self.data_len + estimated_data, 0);
102
103            // Try to read a record
104            loop {
105                let (result, bytes_read, bytes_written, end_positions) =
106                    self.delimiter.read_record(
107                        &input[input_offset..],
108                        &mut self.data[self.data_len..],
109                        &mut self.offsets[self.offsets_len..],
110                    );
111
112                self.current_field += end_positions;
113                self.offsets_len += end_positions;
114                input_offset += bytes_read;
115                self.data_len += bytes_written;
116
117                match result {
118                    ReadRecordResult::End | ReadRecordResult::InputEmpty => {
119                        // Reached end of input
120                        return Ok((read, input_offset));
121                    }
122                    // Need to allocate more capacity
123                    ReadRecordResult::OutputFull => break,
124                    ReadRecordResult::OutputEndsFull => {
125                        return Err(ArrowError::CsvError(format!(
126                            "incorrect number of fields for line {}, expected {} got more than {}",
127                            self.line_number, self.num_columns, self.current_field
128                        )));
129                    }
130                    ReadRecordResult::Record => {
131                        if self.current_field != self.num_columns {
132                            if self.truncated_rows && self.current_field < self.num_columns {
133                                // If the number of fields is less than expected, pad with nulls
134                                let fill_count = self.num_columns - self.current_field;
135                                let fill_value = self.offsets[self.offsets_len - 1];
136                                self.offsets[self.offsets_len..self.offsets_len + fill_count]
137                                    .fill(fill_value);
138                                self.offsets_len += fill_count;
139                                self.truncated_row_count += 1;
140                            } else {
141                                return Err(ArrowError::CsvError(format!(
142                                    "incorrect number of fields for line {}, expected {} got {}",
143                                    self.line_number, self.num_columns, self.current_field
144                                )));
145                            }
146                        }
147                        read += 1;
148                        self.current_field = 0;
149                        self.line_number += 1;
150                        self.num_rows += 1;
151
152                        if read == to_read {
153                            // Read sufficient rows
154                            return Ok((read, input_offset));
155                        }
156
157                        if input.len() == input_offset {
158                            // Input exhausted, need to read more
159                            // Without this read_record will interpret the empty input
160                            // byte array as indicating the end of the file
161                            return Ok((read, input_offset));
162                        }
163                    }
164                }
165            }
166        }
167    }
168
169    /// Returns the current number of buffered records
170    pub fn len(&self) -> usize {
171        self.num_rows
172    }
173
174    /// Returns true if the decoder is empty
175    pub fn is_empty(&self) -> bool {
176        self.num_rows == 0
177    }
178
179    /// Returns the number of rows padded because they had fewer fields than expected
180    ///
181    /// Cumulative across [`Self::flush`] calls, reset by [`Self::clear`]
182    pub fn truncated_row_count(&self) -> usize {
183        self.truncated_row_count
184    }
185
186    /// Clears the current contents of the decoder
187    pub fn clear(&mut self) {
188        // This does not reset current_field to allow clearing part way through a record
189        self.offsets_len = 1;
190        self.data_len = 0;
191        self.num_rows = 0;
192        // The rows counted so far are being discarded along with the buffered data,
193        // so they must not be reported as padded
194        self.truncated_row_count = 0;
195    }
196
197    /// Flushes the current contents of the reader
198    pub fn flush(&mut self) -> Result<StringRecords<'_>, ArrowError> {
199        if self.current_field != 0 {
200            return Err(ArrowError::CsvError(
201                "Cannot flush part way through record".to_string(),
202            ));
203        }
204
205        // csv_core::Reader writes end offsets relative to the start of the row
206        // Therefore scan through and offset these based on the cumulative row offsets
207        let mut row_offset: usize = 0;
208        self.offsets[1..self.offsets_len]
209            .chunks_exact_mut(self.num_columns)
210            .try_for_each(|row| -> Result<(), ArrowError> {
211                let offset = row_offset;
212                row.iter_mut().try_for_each(|x| -> Result<(), ArrowError> {
213                    *x = x.checked_add(offset).ok_or_else(|| {
214                        ArrowError::CsvError(
215                            "CSV record offsets overflowed usize while flushing".to_string(),
216                        )
217                    })?;
218                    row_offset = *x;
219                    Ok(())
220                })
221            })?;
222
223        // Need to truncate data t1o the actual amount of data read
224        let data = std::str::from_utf8(&self.data[..self.data_len]).map_err(|e| {
225            let valid_up_to = e.valid_up_to();
226
227            // We can't use binary search because of empty fields
228            let idx = self.offsets[..self.offsets_len]
229                .iter()
230                .rposition(|x| *x <= valid_up_to)
231                .unwrap();
232
233            let field = idx % self.num_columns + 1;
234            let line_offset = self.line_number - self.num_rows;
235            let line = line_offset + idx / self.num_columns;
236
237            ArrowError::CsvError(format!(
238                "Encountered invalid UTF-8 data for line {line} and field {field}"
239            ))
240        })?;
241
242        let offsets = &self.offsets[..self.offsets_len];
243        let num_rows = self.num_rows;
244
245        // Reset state
246        // `truncated_row_count` is deliberately left alone so that it accumulates
247        // across the batches produced by a single decoder
248        self.offsets_len = 1;
249        self.data_len = 0;
250        self.num_rows = 0;
251
252        Ok(StringRecords {
253            num_rows,
254            num_columns: self.num_columns,
255            offsets,
256            data,
257        })
258    }
259}
260
261/// A collection of parsed, UTF-8 CSV records
262#[derive(Debug)]
263pub struct StringRecords<'a> {
264    num_columns: usize,
265    num_rows: usize,
266    offsets: &'a [usize],
267    data: &'a str,
268}
269
270impl<'a> StringRecords<'a> {
271    fn get(&self, index: usize) -> StringRecord<'a> {
272        let field_idx = index * self.num_columns;
273        StringRecord {
274            data: self.data,
275            offsets: &self.offsets[field_idx..field_idx + self.num_columns + 1],
276        }
277    }
278
279    pub fn len(&self) -> usize {
280        self.num_rows
281    }
282
283    pub fn iter(&self) -> impl Iterator<Item = StringRecord<'a>> + '_ {
284        (0..self.num_rows).map(|x| self.get(x))
285    }
286}
287
288/// A single parsed, UTF-8 CSV record
289#[derive(Debug, Clone, Copy)]
290pub struct StringRecord<'a> {
291    data: &'a str,
292    offsets: &'a [usize],
293}
294
295impl<'a> StringRecord<'a> {
296    pub fn get(&self, index: usize) -> &'a str {
297        let end = self.offsets[index + 1];
298        let start = self.offsets[index];
299
300        // SAFETY:
301        // Parsing produces offsets at valid byte boundaries
302        unsafe { self.data.get_unchecked(start..end) }
303    }
304}
305
306impl std::fmt::Display for StringRecord<'_> {
307    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
308        let num_fields = self.offsets.len() - 1;
309        write!(f, "[")?;
310        for i in 0..num_fields {
311            if i > 0 {
312                write!(f, ",")?;
313            }
314            write!(f, "{}", self.get(i))?;
315        }
316        write!(f, "]")?;
317        Ok(())
318    }
319}
320
321#[cfg(test)]
322mod tests {
323    use crate::reader::records::RecordDecoder;
324    use csv_core::Reader;
325    use std::io::{BufRead, BufReader, Cursor};
326
327    #[test]
328    fn test_basic() {
329        let csv = [
330            "foo,bar,baz",
331            "a,b,c",
332            "12,3,5",
333            "\"asda\"\"asas\",\"sdffsnsd\", as",
334        ]
335        .join("\n");
336
337        let mut expected = vec![
338            vec!["foo", "bar", "baz"],
339            vec!["a", "b", "c"],
340            vec!["12", "3", "5"],
341            vec!["asda\"asas", "sdffsnsd", " as"],
342        ]
343        .into_iter();
344
345        let mut reader = BufReader::with_capacity(3, Cursor::new(csv.as_bytes()));
346        let mut decoder = RecordDecoder::new(Reader::new(), 3, false);
347
348        loop {
349            let to_read = 3;
350            let mut read = 0;
351            loop {
352                let buf = reader.fill_buf().unwrap();
353                let (records, bytes) = decoder.decode(buf, to_read - read).unwrap();
354
355                reader.consume(bytes);
356                read += records;
357
358                if read == to_read || bytes == 0 {
359                    break;
360                }
361            }
362            if read == 0 {
363                break;
364            }
365
366            let b = decoder.flush().unwrap();
367            b.iter().zip(&mut expected).for_each(|(record, expected)| {
368                let actual = (0..3)
369                    .map(|field_idx| record.get(field_idx))
370                    .collect::<Vec<_>>();
371                assert_eq!(actual, expected)
372            });
373        }
374        assert!(expected.next().is_none());
375    }
376
377    #[test]
378    fn test_invalid_fields() {
379        let csv = "a,b\nb,c\na\n";
380        let mut decoder = RecordDecoder::new(Reader::new(), 2, false);
381        let err = decoder.decode(csv.as_bytes(), 4).unwrap_err().to_string();
382
383        let expected = "Csv error: incorrect number of fields for line 3, expected 2 got 1";
384
385        assert_eq!(err, expected);
386
387        // Test with initial skip
388        let mut decoder = RecordDecoder::new(Reader::new(), 2, false);
389        let (skipped, bytes) = decoder.decode(csv.as_bytes(), 1).unwrap();
390        assert_eq!(skipped, 1);
391        decoder.clear();
392
393        let remaining = &csv.as_bytes()[bytes..];
394        let err = decoder.decode(remaining, 3).unwrap_err().to_string();
395        assert_eq!(err, expected);
396    }
397
398    #[test]
399    fn test_skip_insufficient_rows() {
400        let csv = "a\nv\n";
401        let mut decoder = RecordDecoder::new(Reader::new(), 1, false);
402        let (read, bytes) = decoder.decode(csv.as_bytes(), 3).unwrap();
403        assert_eq!(read, 2);
404        assert_eq!(bytes, csv.len());
405    }
406
407    #[test]
408    fn test_truncated_rows() {
409        let csv = "a,b\nv\n,1\n,2\n,3\n";
410        let mut decoder = RecordDecoder::new(Reader::new(), 2, true);
411        let (read, bytes) = decoder.decode(csv.as_bytes(), 5).unwrap();
412        assert_eq!(read, 5);
413        assert_eq!(bytes, csv.len());
414        // Only "v" is short, the rows starting with a delimiter have both fields
415        assert_eq!(decoder.truncated_row_count(), 1);
416    }
417
418    #[test]
419    fn test_truncated_row_count_not_reset_by_flush() {
420        let csv = "a\nb,2\nc\n";
421        let mut decoder = RecordDecoder::new(Reader::new(), 2, true);
422
423        let (read, _) = decoder.decode(csv.as_bytes(), 2).unwrap();
424        assert_eq!(read, 2);
425        assert_eq!(decoder.truncated_row_count(), 1);
426        decoder.flush().unwrap();
427        assert_eq!(decoder.truncated_row_count(), 1);
428
429        let (read, _) = decoder.decode(&csv.as_bytes()[6..], 1).unwrap();
430        assert_eq!(read, 1);
431        assert_eq!(decoder.truncated_row_count(), 2);
432        decoder.flush().unwrap();
433        assert_eq!(decoder.truncated_row_count(), 2);
434    }
435
436    #[test]
437    fn test_truncated_row_count_reset_by_clear() {
438        let csv = "a\nb,2\n";
439        let mut decoder = RecordDecoder::new(Reader::new(), 2, true);
440
441        let (read, _) = decoder.decode(csv.as_bytes(), 2).unwrap();
442        assert_eq!(read, 2);
443        assert_eq!(decoder.truncated_row_count(), 1);
444
445        // The rows are discarded, so the padding done to them is discarded too
446        decoder.clear();
447        assert_eq!(decoder.truncated_row_count(), 0);
448    }
449
450    /// Regression test for an overflow path found by the `arrow-csv`
451    /// cargo-fuzz harness being prototyped for #5332. Stages the
452    /// `RecordDecoder` state directly so that rebasing the second row's
453    /// end offset overflows `usize`. With the previous `*x += offset` this
454    /// panicked with `attempt to add with overflow`; the patched code
455    /// surfaces the condition as `ArrowError::CsvError`.
456    #[test]
457    fn test_flush_offset_overflow_returns_csv_error() {
458        let mut decoder = RecordDecoder::new(Reader::new(), 1, false);
459        decoder.offsets = vec![0, usize::MAX, 1];
460        decoder.offsets_len = 3;
461        decoder.num_rows = 2;
462        let err = decoder.flush().unwrap_err();
463        assert_eq!(
464            err.to_string(),
465            "Csv error: CSV record offsets overflowed usize while flushing"
466        );
467    }
468}