Expand description
CSV Reading: Reader and ReaderBuilder
§Basic Usage
This CSV reader allows CSV files to be read into the Arrow memory model. Records are loaded in batches and are then converted from row-based data to columnar data.
Example:
let schema = Schema::new(vec![
Field::new("city", DataType::Utf8, false),
Field::new("lat", DataType::Float64, false),
Field::new("lng", DataType::Float64, false),
]);
let file = File::open("test/data/uk_cities.csv").unwrap();
let mut csv = ReaderBuilder::new(Arc::new(schema)).build(file).unwrap();
let batch = csv.next().unwrap().unwrap();§Timestamps
Timestamps are parsed with [arrow_cast::parse::string_to_datetime]. A
timestamp without an explicit offset that falls in a daylight savings gap or
overlap of the target timezone is resolved as described there, rather than
returning an error.
§Example: Numeric calculations on CSV
This code finds the maximum value in column 0 of a CSV file containing
c1,c2,c3,c4
1,1.1,"hong kong",true
3,323.12,"XiAn",false
10,131323.12,"cheng du",false// Open the example file
let file = File::open("test/data/example.csv").unwrap();
let csv_schema = Schema::new(vec![
Field::new("c1", DataType::Int16, true),
Field::new("c2", DataType::Float32, true),
Field::new("c3", DataType::Utf8, true),
Field::new("c4", DataType::Boolean, true),
]);
let mut reader = ReaderBuilder::new(Arc::new(csv_schema))
.with_header(true)
.build(file)
.unwrap();
// find the maximum value in column 0 across all batches
let mut max_c0 = 0;
while let Some(r) = reader.next() {
let r = r.unwrap(); // handle error
// get the max value in column(0) for this batch
let col = r.column(0).as_primitive::<Int16Type>();
let batch_max = col.iter().max().flatten().unwrap_or_default();
max_c0 = max_c0.max(batch_max);
}
assert_eq!(max_c0, 10);§Async Usage
The lower-level Decoder can be integrated with various forms of async data streams,
and is designed to be agnostic to the various different kinds of async IO primitives found
within the Rust ecosystem.
For example, see below for how it can be used with an arbitrary Stream of Bytes
fn decode_stream<S: Stream<Item = Bytes> + Unpin>(
mut decoder: Decoder,
mut input: S,
) -> impl Stream<Item = Result<RecordBatch, ArrowError>> {
let mut buffered = Bytes::new();
futures::stream::poll_fn(move |cx| {
loop {
if buffered.is_empty() {
if let Some(b) = ready!(input.poll_next_unpin(cx)) {
buffered = b;
}
// Note: don't break on `None` as the decoder needs
// to be called with an empty array to delimit the
// final record
}
let decoded = match decoder.decode(buffered.as_ref()) {
Ok(0) => break,
Ok(decoded) => decoded,
Err(e) => return Poll::Ready(Some(Err(e))),
};
buffered.advance(decoded);
}
Poll::Ready(decoder.flush().transpose())
})
}
In a similar vein, it can also be used with tokio-based IO primitives
fn decode_stream<R: AsyncBufRead + Unpin>(
mut decoder: Decoder,
mut reader: R,
) -> impl Stream<Item = Result<RecordBatch, ArrowError>> {
futures::stream::poll_fn(move |cx| {
loop {
let b = match ready!(Pin::new(&mut reader).poll_fill_buf(cx)) {
Ok(b) => b,
Err(e) => return Poll::Ready(Some(Err(e.into()))),
};
let decoded = match decoder.decode(b) {
// Note: the decoder needs to be called with an empty
// array to delimit the final record
Ok(0) => break,
Ok(decoded) => decoded,
Err(e) => return Poll::Ready(Some(Err(e))),
};
Pin::new(&mut reader).consume(decoded);
}
Poll::Ready(decoder.flush().transpose())
})
}Re-exports§
use arrow_array::builder::NullBuilder;🔒use arrow_array::builder::PrimitiveBuilder;🔒use arrow_array::builder::StringDictionaryBuilder;🔒use arrow_cast::parse::Parser;🔒use arrow_cast::parse::parse_decimal;🔒use arrow_cast::parse::string_to_datetime;🔒use chrono::TimeZone;🔒use chrono::Utc;🔒use csv::StringRecord;🔒use regex::Regex;🔒use regex::RegexSet;🔒use std::fmt;🔒use std::fmt::Debug;🔒use std::fmt::Debug;🔒use std::fs::File;🔒use std::io::BufRead;🔒use std::io::BufReader as StdBufReader;🔒use std::io::Read;🔒use std::sync::Arc;🔒use std::sync::LazyLock;🔒use crate::map_csv_error;🔒use crate::reader::records::RecordDecoder;🔒use crate::reader::records::StringRecords;🔒use arrow_array::timezone::Tz;🔒use arrow_array::types::*;🔒use arrow_array::*;🔒use arrow_schema::*;🔒
Modules§
- records 🔒
Structs§
- BufReader
- CSV file reader implementation. See
Readerfor usage - Decoder
- A push-based interface for decoding CSV data from an arbitrary byte stream
- Format
- The format specification for the CSV file
- Inferred
Data 🔒Type - Null
Regex 🔒 - A wrapper over
Option<Regex>to check if the value isNULL. - Reader
Builder - Builder for CSV
Readers
Statics§
- REGEX_
SET 🔒 - Order should match
InferredDataType
Functions§
- build_
boolean_ 🔒array - build_
decimal_ 🔒array - build_
dictionary_ 🔒array - build_
primitive_ 🔒array - build_
timestamp_ 🔒array - build_
timestamp_ 🔒array_ impl - infer_
schema_ from_ files - Infer schema from a list of CSV files by reading through first n records
with
max_read_recordscontrolling the maximum number of records to read. - parse 🔒
- Parses a slice of
StringRecordsinto a [RecordBatch] - parse_
bool 🔒 - validate_
header 🔒
Type Aliases§
- Bounds 🔒
- Reader
- CSV file reader using
std::io::BufReader