1use bytes::{Buf, Bytes};
10use std::fs::File;
11use std::io::{BufReader, Seek, SeekFrom};
12use std::{io::Read, sync::Arc};
13
14use crate::bloom_filter::Sbbf;
15use crate::column::page::PageIterator;
16use crate::column::{page::PageReader, reader::ColumnReader};
17use crate::errors::{ParquetError, Result};
18use crate::file::metadata::*;
19pub use crate::file::serialized_reader::{SerializedFileReader, SerializedPageReader};
20use crate::record::reader::RowIter;
21use crate::schema::types::Type as SchemaType;
22
23use crate::basic::Type;
24
25use crate::column::reader::ColumnReaderImpl;
26
27#[expect(clippy::len_without_is_empty)]
30pub trait Length {
31 fn len(&self) -> u64;
33}
34
35pub trait ChunkReader: Length + Send + Sync {
52 type T: Read;
54
55 fn get_read(&self, start: u64) -> Result<Self::T>;
63
64 fn get_bytes(&self, start: u64, length: usize) -> Result<Bytes>;
69}
70
71impl Length for File {
72 fn len(&self) -> u64 {
73 self.metadata().map(|m| m.len()).unwrap_or(0u64)
74 }
75}
76
77impl ChunkReader for File {
78 type T = BufReader<File>;
79
80 fn get_read(&self, start: u64) -> Result<Self::T> {
81 let mut reader = self.try_clone()?;
82 reader.seek(SeekFrom::Start(start))?;
83 Ok(BufReader::new(reader))
84 }
85
86 fn get_bytes(&self, start: u64, length: usize) -> Result<Bytes> {
87 let mut buffer = Vec::with_capacity(length);
88 let mut reader = self.try_clone()?;
89 reader.seek(SeekFrom::Start(start))?;
90 let read = reader.take(length as _).read_to_end(&mut buffer)?;
91
92 if read != length {
93 return Err(eof_err!(
94 "Expected to read {} bytes, read only {}",
95 length,
96 read
97 ));
98 }
99 Ok(buffer.into())
100 }
101}
102
103impl Length for Bytes {
104 fn len(&self) -> u64 {
105 self.len() as u64
106 }
107}
108
109impl ChunkReader for Bytes {
110 type T = bytes::buf::Reader<Bytes>;
111
112 fn get_read(&self, start: u64) -> Result<Self::T> {
113 let start = start as usize;
114 if start > self.len() {
115 return Err(eof_err!(
116 "Expected to read at offset {start}, while file has length {}",
117 self.len()
118 ));
119 }
120 Ok(self.slice(start..).reader())
121 }
122
123 fn get_bytes(&self, start: u64, length: usize) -> Result<Bytes> {
124 let start = start as usize;
125 if start > self.len() || start + length > self.len() {
126 return Err(eof_err!(
127 "Expected to read {} bytes at offset {}, while file has length {}",
128 length,
129 start,
130 self.len()
131 ));
132 }
133 Ok(self.slice(start..start + length))
134 }
135}
136
137pub trait FileReader: Send + Sync {
143 fn metadata(&self) -> &ParquetMetaData;
145
146 fn num_row_groups(&self) -> usize;
148
149 fn get_row_group(&self, i: usize) -> Result<Box<dyn RowGroupReader + '_>>;
151
152 fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter<'_>>;
159}
160
161pub trait RowGroupReader: Send + Sync {
164 fn metadata(&self) -> &RowGroupMetaData;
166
167 fn num_columns(&self) -> usize;
169
170 fn get_column_page_reader(&self, i: usize) -> Result<Box<dyn PageReader>>;
172
173 fn get_column_reader(&self, i: usize) -> Result<ColumnReader> {
175 let schema_descr = self.metadata().schema_descr();
176 let col_descr = schema_descr.column(i);
177 let col_page_reader = self.get_column_page_reader(i)?;
178 let col_reader = match col_descr.physical_type() {
179 Type::BOOLEAN => {
180 ColumnReader::BoolColumnReader(ColumnReaderImpl::new(col_descr, col_page_reader))
181 }
182 Type::INT32 => {
183 ColumnReader::Int32ColumnReader(ColumnReaderImpl::new(col_descr, col_page_reader))
184 }
185 Type::INT64 => {
186 ColumnReader::Int64ColumnReader(ColumnReaderImpl::new(col_descr, col_page_reader))
187 }
188 Type::INT96 => {
189 ColumnReader::Int96ColumnReader(ColumnReaderImpl::new(col_descr, col_page_reader))
190 }
191 Type::FLOAT => {
192 ColumnReader::FloatColumnReader(ColumnReaderImpl::new(col_descr, col_page_reader))
193 }
194 Type::DOUBLE => {
195 ColumnReader::DoubleColumnReader(ColumnReaderImpl::new(col_descr, col_page_reader))
196 }
197 Type::BYTE_ARRAY => ColumnReader::ByteArrayColumnReader(ColumnReaderImpl::new(
198 col_descr,
199 col_page_reader,
200 )),
201 Type::FIXED_LEN_BYTE_ARRAY => ColumnReader::FixedLenByteArrayColumnReader(
202 ColumnReaderImpl::new(col_descr, col_page_reader),
203 ),
204 };
205 Ok(col_reader)
206 }
207
208 fn get_column_bloom_filter(&self, i: usize) -> Option<&Sbbf>;
211
212 fn get_row_iter(&self, projection: Option<SchemaType>) -> Result<RowIter<'_>>;
217}
218
219pub struct FilePageIterator {
224 column_index: usize,
225 row_group_indices: Box<dyn Iterator<Item = usize> + Send>,
226 file_reader: Arc<dyn FileReader>,
227}
228
229impl FilePageIterator {
230 pub fn new(column_index: usize, file_reader: Arc<dyn FileReader>) -> Result<Self> {
232 let num_row_groups = file_reader.metadata().num_row_groups();
233
234 let row_group_indices = Box::new(0..num_row_groups);
235
236 Self::with_row_groups(column_index, row_group_indices, file_reader)
237 }
238
239 pub fn with_row_groups(
241 column_index: usize,
242 row_group_indices: Box<dyn Iterator<Item = usize> + Send>,
243 file_reader: Arc<dyn FileReader>,
244 ) -> Result<Self> {
245 let num_columns = file_reader
247 .metadata()
248 .file_metadata()
249 .schema_descr()
250 .num_columns();
251
252 if column_index >= num_columns {
253 return Err(ParquetError::IndexOutOfBound(column_index, num_columns));
254 }
255
256 Ok(Self {
258 column_index,
259 row_group_indices,
260 file_reader,
261 })
262 }
263}
264
265impl Iterator for FilePageIterator {
266 type Item = Result<Box<dyn PageReader>>;
267
268 fn next(&mut self) -> Option<Result<Box<dyn PageReader>>> {
269 self.row_group_indices.next().map(|row_group_index| {
270 self.file_reader
271 .get_row_group(row_group_index)?
272 .get_column_page_reader(self.column_index)
273 })
274 }
275}
276
277impl PageIterator for FilePageIterator {}
278
279#[cfg(test)]
280mod tests {
281 use super::*;
282
283 #[test]
284 fn test_bytes_chunk_reader_get_read_out_of_bounds() {
285 let data = Bytes::from(vec![0, 1, 2, 3]);
286 let err = data.get_read(5).unwrap_err();
287 assert_eq!(
288 err.to_string(),
289 "EOF: Expected to read at offset 5, while file has length 4"
290 );
291 }
292
293 #[test]
294 fn test_bytes_chunk_reader_get_bytes_out_of_bounds() {
295 let data = Bytes::from(vec![0, 1, 2, 3]);
296 let err = data.get_bytes(5, 1).unwrap_err();
297 assert_eq!(
298 err.to_string(),
299 "EOF: Expected to read 1 bytes at offset 5, while file has length 4"
300 );
301
302 let err = data.get_bytes(2, 3).unwrap_err();
303 assert_eq!(
304 err.to_string(),
305 "EOF: Expected to read 3 bytes at offset 2, while file has length 4"
306 );
307 }
308}