Skip to main content

parquet/file/
reader.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! File reader API and methods to access file metadata, row group
6//! readers to read individual column chunks, or access record
7//! iterator.
8
9use bytes::{Buf, Bytes};
10use std::fs::File;
11use std::io::{BufReader, Seek, SeekFrom};
12use std::{io::Read, sync::Arc};
13
14use crate::bloom_filter::Sbbf;
15use crate::column::page::PageIterator;
16use crate::column::{page::PageReader, reader::ColumnReader};
17use crate::errors::{ParquetError, Result};
18use crate::file::metadata::*;
19pub use crate::file::serialized_reader::{SerializedFileReader, SerializedPageReader};
20use crate::record::reader::RowIter;
21use crate::schema::types::Type as SchemaType;
22
23use crate::basic::Type;
24
25use crate::column::reader::ColumnReaderImpl;
26
27/// Length should return the total number of bytes in the input source.
28/// It's mainly used to read the metadata, which is at the end of the source.
29#[expect(clippy::len_without_is_empty)]
30pub trait Length {
31    /// Returns the amount of bytes of the inner source.
32    fn len(&self) -> u64;
33}
34
35/// Generates [`Read`]ers to read chunks of a Parquet data source.
36///
37/// The Parquet reader uses [`ChunkReader`] to access Parquet data, allowing
38/// multiple decoders to read concurrently from different locations in the same
39/// file.
40///
41/// The trait functions both as a reader and a factory for readers.
42/// * random access via [`Self::get_bytes`]
43/// * sequential access via the reader returned via factory method [`Self::get_read`]
44///
45/// # Provided Implementations
46/// * [`File`] for reading from local file system
47/// * [`Bytes`] for reading from an in-memory buffer
48///
49/// User provided implementations can implement more sophisticated behaviors
50/// such as on-demand buffering or scan sharing.
51pub trait ChunkReader: Length + Send + Sync {
52    /// The concrete type of reader returned by this trait
53    type T: Read;
54
55    /// Get a [`Read`] instance starting at the provided file offset
56    ///
57    /// Returned readers follow the model of [`File::try_clone`] where mutations
58    /// of one reader affect all readers. Thus subsequent or concurrent calls to
59    /// [`Self::get_read`] or [`Self::get_bytes`] may cause side-effects on
60    /// previously returned readers. Callers of `get_read` should take care
61    /// to avoid race conditions.
62    fn get_read(&self, start: u64) -> Result<Self::T>;
63
64    /// Get a range of data in memory as [`Bytes`]
65    ///
66    /// Similarly to [`Self::get_read`], this method may have side-effects on
67    /// previously returned readers.
68    fn get_bytes(&self, start: u64, length: usize) -> Result<Bytes>;
69}
70
71impl Length for File {
72    fn len(&self) -> u64 {
73        self.metadata().map(|m| m.len()).unwrap_or(0u64)
74    }
75}
76
77impl ChunkReader for File {
78    type T = BufReader<File>;
79
80    fn get_read(&self, start: u64) -> Result<Self::T> {
81        let mut reader = self.try_clone()?;
82        reader.seek(SeekFrom::Start(start))?;
83        Ok(BufReader::new(reader))
84    }
85
86    fn get_bytes(&self, start: u64, length: usize) -> Result<Bytes> {
87        let mut buffer = Vec::with_capacity(length);
88        let mut reader = self.try_clone()?;
89        reader.seek(SeekFrom::Start(start))?;
90        let read = reader.take(length as _).read_to_end(&mut buffer)?;
91
92        if read != length {
93            return Err(eof_err!(
94                "Expected to read {} bytes, read only {}",
95                length,
96                read
97            ));
98        }
99        Ok(buffer.into())
100    }
101}
102
103impl Length for Bytes {
104    fn len(&self) -> u64 {
105        self.len() as u64
106    }
107}
108
109impl ChunkReader for Bytes {
110    type T = bytes::buf::Reader<Bytes>;
111
112    fn get_read(&self, start: u64) -> Result<Self::T> {
113        let start = start as usize;
114        if start > self.len() {
115            return Err(eof_err!(
116                "Expected to read at offset {start}, while file has length {}",
117                self.len()
118            ));
119        }
120        Ok(self.slice(start..).reader())
121    }
122
123    fn get_bytes(&self, start: u64, length: usize) -> Result<Bytes> {
124        let start = start as usize;
125        if start > self.len() || start + length > self.len() {
126            return Err(eof_err!(
127                "Expected to read {} bytes at offset {}, while file has length {}",
128                length,
129                start,
130                self.len()
131            ));
132        }
133        Ok(self.slice(start..start + length))
134    }
135}
136
137// ----------------------------------------------------------------------
138// APIs for file & row group readers
139
140/// Parquet file reader API. With this, user can get metadata information about the
141/// Parquet file, can get reader for each row group, and access record iterator.
142pub trait FileReader: Send + Sync {
143    /// Get metadata information about this file.
144    fn metadata(&self) -> &ParquetMetaData;
145
146    /// Get the total number of row groups for this file.
147    fn num_row_groups(&self) -> usize;
148
149    /// Get the `i`th row group reader. Note this doesn't do bound check.
150    fn get_row_group(&self, i: usize) -> Result<Box<dyn RowGroupReader + '_>>;
151
152    /// Get an iterator over the row in this file, see [`RowIter`] for caveats.
153    ///
154    /// Iterator will automatically load the next row group to advance.
155    ///
156    /// Projected schema can be a subset of or equal to the file schema, when it is None,
157    /// full file schema is assumed.
158    fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter<'_>>;
159}
160
161/// Parquet row group reader API. With this, user can get metadata information about the
162/// row group, as well as readers for each individual column chunk.
163pub trait RowGroupReader: Send + Sync {
164    /// Get metadata information about this row group.
165    fn metadata(&self) -> &RowGroupMetaData;
166
167    /// Get the total number of column chunks in this row group.
168    fn num_columns(&self) -> usize;
169
170    /// Get page reader for the `i`th column chunk.
171    fn get_column_page_reader(&self, i: usize) -> Result<Box<dyn PageReader>>;
172
173    /// Get value reader for the `i`th column chunk.
174    fn get_column_reader(&self, i: usize) -> Result<ColumnReader> {
175        let schema_descr = self.metadata().schema_descr();
176        let col_descr = schema_descr.column(i);
177        let col_page_reader = self.get_column_page_reader(i)?;
178        let col_reader = match col_descr.physical_type() {
179            Type::BOOLEAN => {
180                ColumnReader::BoolColumnReader(ColumnReaderImpl::new(col_descr, col_page_reader))
181            }
182            Type::INT32 => {
183                ColumnReader::Int32ColumnReader(ColumnReaderImpl::new(col_descr, col_page_reader))
184            }
185            Type::INT64 => {
186                ColumnReader::Int64ColumnReader(ColumnReaderImpl::new(col_descr, col_page_reader))
187            }
188            Type::INT96 => {
189                ColumnReader::Int96ColumnReader(ColumnReaderImpl::new(col_descr, col_page_reader))
190            }
191            Type::FLOAT => {
192                ColumnReader::FloatColumnReader(ColumnReaderImpl::new(col_descr, col_page_reader))
193            }
194            Type::DOUBLE => {
195                ColumnReader::DoubleColumnReader(ColumnReaderImpl::new(col_descr, col_page_reader))
196            }
197            Type::BYTE_ARRAY => ColumnReader::ByteArrayColumnReader(ColumnReaderImpl::new(
198                col_descr,
199                col_page_reader,
200            )),
201            Type::FIXED_LEN_BYTE_ARRAY => ColumnReader::FixedLenByteArrayColumnReader(
202                ColumnReaderImpl::new(col_descr, col_page_reader),
203            ),
204        };
205        Ok(col_reader)
206    }
207
208    /// Get bloom filter for the `i`th column chunk, if present and the reader was configured
209    /// to read bloom filters.
210    fn get_column_bloom_filter(&self, i: usize) -> Option<&Sbbf>;
211
212    /// Get an iterator over the row in this file, see [`RowIter`] for caveats.
213    ///
214    /// Projected schema can be a subset of or equal to the file schema, when it is None,
215    /// full file schema is assumed.
216    fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter<'_>>;
217}
218
219// ----------------------------------------------------------------------
220// Iterator
221
222/// Implementation of page iterator for parquet file.
223pub struct FilePageIterator {
224    column_index: usize,
225    row_group_indices: Box<dyn Iterator<Item = usize> + Send>,
226    file_reader: Arc<dyn FileReader>,
227}
228
229impl FilePageIterator {
230    /// Creates a page iterator for all row groups in file.
231    pub fn new(column_index: usize, file_reader: Arc<dyn FileReader>) -> Result<Self> {
232        let num_row_groups = file_reader.metadata().num_row_groups();
233
234        let row_group_indices = Box::new(0..num_row_groups);
235
236        Self::with_row_groups(column_index, row_group_indices, file_reader)
237    }
238
239    /// Create page iterator from parquet file reader with only some row groups.
240    pub fn with_row_groups(
241        column_index: usize,
242        row_group_indices: Box<dyn Iterator<Item = usize> + Send>,
243        file_reader: Arc<dyn FileReader>,
244    ) -> Result<Self> {
245        // Check that column_index is valid
246        let num_columns = file_reader
247            .metadata()
248            .file_metadata()
249            .schema_descr()
250            .num_columns();
251
252        if column_index >= num_columns {
253            return Err(ParquetError::IndexOutOfBound(column_index, num_columns));
254        }
255
256        // We don't check iterators here because iterator may be infinite
257        Ok(Self {
258            column_index,
259            row_group_indices,
260            file_reader,
261        })
262    }
263}
264
265impl Iterator for FilePageIterator {
266    type Item = Result<Box<dyn PageReader>>;
267
268    fn next(&mut self) -> Option<Result<Box<dyn PageReader>>> {
269        self.row_group_indices.next().map(|row_group_index| {
270            self.file_reader
271                .get_row_group(row_group_index)?
272                .get_column_page_reader(self.column_index)
273        })
274    }
275}
276
277impl PageIterator for FilePageIterator {}
278
279#[cfg(test)]
280mod tests {
281    use super::*;
282
283    #[test]
284    fn test_bytes_chunk_reader_get_read_out_of_bounds() {
285        let data = Bytes::from(vec![0, 1, 2, 3]);
286        let err = data.get_read(5).unwrap_err();
287        assert_eq!(
288            err.to_string(),
289            "EOF: Expected to read at offset 5, while file has length 4"
290        );
291    }
292
293    #[test]
294    fn test_bytes_chunk_reader_get_bytes_out_of_bounds() {
295        let data = Bytes::from(vec![0, 1, 2, 3]);
296        let err = data.get_bytes(5, 1).unwrap_err();
297        assert_eq!(
298            err.to_string(),
299            "EOF: Expected to read 1 bytes at offset 5, while file has length 4"
300        );
301
302        let err = data.get_bytes(2, 3).unwrap_err();
303        assert_eq!(
304            err.to_string(),
305            "EOF: Expected to read 3 bytes at offset 2, while file has length 4"
306        );
307    }
308}