arrow_json/reader/value_iter.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use std::io::BufRead;
6
7use arrow_schema::ArrowError;
8use serde_json::Value;
9
10/// JSON file reader that produces a serde_json::Value iterator from a Read trait
11///
12/// # Example
13///
14/// ```
15/// use std::fs::File;
16/// use std::io::BufReader;
17/// use arrow_json::reader::ValueIter;
18///
19/// let mut reader =
20/// BufReader::new(File::open("test/data/mixed_arrays.json").unwrap());
21/// let mut value_reader = ValueIter::new(&mut reader, None);
22/// for value in value_reader {
23/// println!("JSON value: {}", value.unwrap());
24/// }
25/// ```
26#[derive(Debug)]
27pub struct ValueIter<R: BufRead> {
28 reader: R,
29 max_read_records: Option<usize>,
30 record_count: usize,
31 // reuse line buffer to avoid allocation on each record
32 line_buf: String,
33}
34
35impl<R: BufRead> ValueIter<R> {
36 /// Creates a new `ValueIter`
37 pub fn new(reader: R, max_read_records: Option<usize>) -> Self {
38 Self {
39 reader,
40 max_read_records,
41 record_count: 0,
42 line_buf: String::new(),
43 }
44 }
45
46 /// Returns the number of records this iterator has consumed
47 pub fn record_count(&self) -> usize {
48 self.record_count
49 }
50}
51
52impl<R: BufRead> Iterator for ValueIter<R> {
53 type Item = Result<Value, ArrowError>;
54
55 fn next(&mut self) -> Option<Self::Item> {
56 if let Some(max) = self.max_read_records
57 && self.record_count >= max
58 {
59 return None;
60 }
61
62 loop {
63 self.line_buf.clear();
64 match self.reader.read_line(&mut self.line_buf) {
65 Ok(0) => {
66 // read_line returns 0 when stream reached EOF
67 return None;
68 }
69 Err(e) => {
70 return Some(Err(ArrowError::JsonError(format!(
71 "Failed to read JSON record: {e}"
72 ))));
73 }
74 _ => {
75 let trimmed_s = self.line_buf.trim();
76 if trimmed_s.is_empty() {
77 // ignore empty lines
78 continue;
79 }
80
81 self.record_count += 1;
82 return Some(
83 serde_json::from_str(trimmed_s)
84 .map_err(|e| ArrowError::JsonError(format!("Not valid JSON: {e}"))),
85 );
86 }
87 }
88 }
89 }
90}