Skip to main content

arrow_json/reader/
value_iter.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use std::io::BufRead;
6
7use arrow_schema::ArrowError;
8use serde_json::Value;
9
10/// JSON file reader that produces a serde_json::Value iterator from a Read trait
11///
12/// # Example
13///
14/// ```
15/// use std::fs::File;
16/// use std::io::BufReader;
17/// use arrow_json::reader::ValueIter;
18///
19/// let mut reader =
20///     BufReader::new(File::open("test/data/mixed_arrays.json").unwrap());
21/// let mut value_reader = ValueIter::new(&mut reader, None);
22/// for value in value_reader {
23///     println!("JSON value: {}", value.unwrap());
24/// }
25/// ```
26#[derive(Debug)]
27pub struct ValueIter<R: BufRead> {
28    reader: R,
29    max_read_records: Option<usize>,
30    record_count: usize,
31    // reuse line buffer to avoid allocation on each record
32    line_buf: String,
33}
34
35impl<R: BufRead> ValueIter<R> {
36    /// Creates a new `ValueIter`
37    pub fn new(reader: R, max_read_records: Option<usize>) -> Self {
38        Self {
39            reader,
40            max_read_records,
41            record_count: 0,
42            line_buf: String::new(),
43        }
44    }
45
46    /// Returns the number of records this iterator has consumed
47    pub fn record_count(&self) -> usize {
48        self.record_count
49    }
50}
51
52impl<R: BufRead> Iterator for ValueIter<R> {
53    type Item = Result<Value, ArrowError>;
54
55    fn next(&mut self) -> Option<Self::Item> {
56        if let Some(max) = self.max_read_records
57            && self.record_count >= max
58        {
59            return None;
60        }
61
62        loop {
63            self.line_buf.clear();
64            match self.reader.read_line(&mut self.line_buf) {
65                Ok(0) => {
66                    // read_line returns 0 when stream reached EOF
67                    return None;
68                }
69                Err(e) => {
70                    return Some(Err(ArrowError::JsonError(format!(
71                        "Failed to read JSON record: {e}"
72                    ))));
73                }
74                _ => {
75                    let trimmed_s = self.line_buf.trim();
76                    if trimmed_s.is_empty() {
77                        // ignore empty lines
78                        continue;
79                    }
80
81                    self.record_count += 1;
82                    return Some(
83                        serde_json::from_str(trimmed_s)
84                            .map_err(|e| ArrowError::JsonError(format!("Not valid JSON: {e}"))),
85                    );
86                }
87            }
88        }
89    }
90}