parquet/arrow/async_reader/mod.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! `async` API for reading Parquet files as [`RecordBatch`]es
6//!
7//! See the [crate-level documentation](crate) for more details.
8//!
9//! See example on [`ParquetRecordBatchStreamBuilder::new`]
10
11use std::fmt::Formatter;
12use std::io::SeekFrom;
13use std::ops::Range;
14use std::pin::Pin;
15use std::sync::Arc;
16use std::task::{Context, Poll};
17
18use bytes::Bytes;
19use futures::future::{BoxFuture, FutureExt};
20use futures::stream::Stream;
21use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt};
22
23use arrow_array::{ArrayRef, RecordBatch};
24use arrow_schema::{Schema, SchemaRef};
25
26use crate::arrow::arrow_reader::{
27 ArrowReaderBuilder, ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReader,
28};
29
30use crate::bloom_filter::Sbbf;
31use crate::errors::{ParquetError, Result};
32use crate::file::metadata::{ParquetMetaData, ParquetMetaDataReader};
33
34mod metadata;
35pub use metadata::*;
36
37mod spawn;
38pub use spawn::SpawnedReader;
39
40/// Re-exported so [`ParquetRecordBatchStreamBuilder::with_row_group_selections`]
41/// can be used without importing from another module.
42pub use crate::arrow::arrow_reader::RowGroupSelection;
43
44#[cfg(feature = "object_store")]
45mod store;
46
47use crate::DecodeResult;
48use crate::arrow::push_decoder::{ParquetPushDecoder, ParquetPushDecoderBuilder, PushDecoderInput};
49#[cfg(feature = "object_store")]
50pub use store::*;
51
52/// The asynchronous interface used by [`ParquetRecordBatchStream`] to read parquet files
53///
54/// Notes:
55///
56/// 1. There is a default implementation for types that implement [`AsyncRead`]
57/// and [`AsyncSeek`], for example [`tokio::fs::File`].
58///
59/// 2. Implementations for remote storage, such as the `object_store` crate,
60/// can implement this interface directly, typically by pairing a store
61/// handle with an object path and delegating [`Self::get_bytes`] and
62/// [`Self::get_byte_ranges`] to ranged reads. [`SpawnedReader`] can wrap
63/// such a reader to perform its I/O on a dedicated runtime, and
64/// [`ParquetMetaDataReader::with_arrow_reader_options`] simplifies
65/// implementing [`Self::get_metadata`].
66///
67/// # Example: implementing `AsyncFileReader` for the `object_store` crate
68///
69/// ```no_run
70/// # use std::ops::Range;
71/// # use std::sync::Arc;
72/// use bytes::Bytes;
73/// use futures::future::BoxFuture;
74/// use futures::{FutureExt, TryFutureExt};
75/// use object_store::path::Path;
76/// use object_store::{GetOptions, GetRange, ObjectStore, ObjectStoreExt};
77/// use parquet::arrow::arrow_reader::ArrowReaderOptions;
78/// use parquet::arrow::async_reader::{AsyncFileReader, MetadataSuffixFetch};
79/// use parquet::errors::{ParquetError, Result};
80/// use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataReader};
81///
82/// fn to_parquet_err(e: object_store::Error) -> ParquetError {
83/// ParquetError::External(Box::new(e))
84/// }
85///
86/// #[derive(Clone)]
87/// struct ObjectStoreReader {
88/// store: Arc<dyn ObjectStore>,
89/// path: Path,
90/// }
91///
92/// impl AsyncFileReader for ObjectStoreReader {
93/// fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes>> {
94/// self.store
95/// .get_range(&self.path, range)
96/// .map_err(to_parquet_err)
97/// .boxed()
98/// }
99///
100/// fn get_byte_ranges(&mut self, ranges: Vec<Range<u64>>) -> BoxFuture<'_, Result<Vec<Bytes>>> {
101/// async move {
102/// self.store
103/// .get_ranges(&self.path, &ranges)
104/// .await
105/// .map_err(to_parquet_err)
106/// }
107/// .boxed()
108/// }
109///
110/// fn get_metadata<'a>(
111/// &'a mut self,
112/// options: Option<&'a ArrowReaderOptions>,
113/// ) -> BoxFuture<'a, Result<Arc<ParquetMetaData>>> {
114/// async move {
115/// let metadata = ParquetMetaDataReader::new()
116/// .with_arrow_reader_options(options)
117/// .load_via_suffix_and_finish(self)
118/// .await?;
119/// Ok(Arc::new(metadata))
120/// }
121/// .boxed()
122/// }
123/// }
124///
125/// /// Supports fetching the parquet footer without knowing the file size,
126/// /// via suffix range requests
127/// impl MetadataSuffixFetch for &mut ObjectStoreReader {
128/// fn fetch_suffix(&mut self, suffix: usize) -> BoxFuture<'_, Result<Bytes>> {
129/// let options = GetOptions {
130/// range: Some(GetRange::Suffix(suffix as u64)),
131/// ..Default::default()
132/// };
133/// async move {
134/// let resp = self
135/// .store
136/// .get_opts(&self.path, options)
137/// .await
138/// .map_err(to_parquet_err)?;
139/// resp.bytes().await.map_err(to_parquet_err)
140/// }
141/// .boxed()
142/// }
143/// }
144/// ```
145///
146/// [`ParquetMetaDataReader::with_arrow_reader_options`]: crate::file::metadata::ParquetMetaDataReader::with_arrow_reader_options
147///
148/// [`tokio::fs::File`]: https://docs.rs/tokio/latest/tokio/fs/struct.File.html
149pub trait AsyncFileReader: Send {
150 /// Retrieve the bytes in `range`
151 fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes>>;
152
153 /// Retrieve multiple byte ranges. The default implementation will call `get_bytes` sequentially
154 fn get_byte_ranges(&mut self, ranges: Vec<Range<u64>>) -> BoxFuture<'_, Result<Vec<Bytes>>> {
155 async move {
156 let mut result = Vec::with_capacity(ranges.len());
157
158 for range in ranges {
159 let data = self.get_bytes(range).await?;
160 result.push(data);
161 }
162
163 Ok(result)
164 }
165 .boxed()
166 }
167
168 /// Return a future which results in the [`ParquetMetaData`] for this Parquet file.
169 ///
170 /// This is an asynchronous operation as it may involve reading the file
171 /// footer and potentially other metadata from disk or a remote source.
172 ///
173 /// Reading data from Parquet requires the metadata to understand the
174 /// schema, row groups, and location of pages within the file. This metadata
175 /// is stored primarily in the footer of the Parquet file, and can be read using
176 /// [`ParquetMetaDataReader`].
177 ///
178 /// However, implementations can significantly speed up reading Parquet by
179 /// supplying cached metadata or pre-fetched metadata via this API.
180 ///
181 /// # Parameters
182 /// * `options`: Optional [`ArrowReaderOptions`] that may contain decryption
183 /// and other options that affect how the metadata is read.
184 fn get_metadata<'a>(
185 &'a mut self,
186 options: Option<&'a ArrowReaderOptions>,
187 ) -> BoxFuture<'a, Result<Arc<ParquetMetaData>>>;
188}
189
190/// This allows Box<dyn AsyncFileReader + '_> to be used as an AsyncFileReader,
191impl AsyncFileReader for Box<dyn AsyncFileReader + '_> {
192 fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes>> {
193 self.as_mut().get_bytes(range)
194 }
195
196 fn get_byte_ranges(&mut self, ranges: Vec<Range<u64>>) -> BoxFuture<'_, Result<Vec<Bytes>>> {
197 self.as_mut().get_byte_ranges(ranges)
198 }
199
200 fn get_metadata<'a>(
201 &'a mut self,
202 options: Option<&'a ArrowReaderOptions>,
203 ) -> BoxFuture<'a, Result<Arc<ParquetMetaData>>> {
204 self.as_mut().get_metadata(options)
205 }
206}
207
208impl<T: AsyncFileReader + MetadataFetch + AsyncRead + AsyncSeek + Unpin> MetadataSuffixFetch for T {
209 fn fetch_suffix(&mut self, suffix: usize) -> BoxFuture<'_, Result<Bytes>> {
210 async move {
211 self.seek(SeekFrom::End(-(suffix as i64))).await?;
212 let mut buf = Vec::with_capacity(suffix);
213 self.take(suffix as _).read_to_end(&mut buf).await?;
214 Ok(buf.into())
215 }
216 .boxed()
217 }
218}
219
220impl<T: AsyncRead + AsyncSeek + Unpin + Send> AsyncFileReader for T {
221 fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes>> {
222 async move {
223 self.seek(SeekFrom::Start(range.start)).await?;
224
225 let to_read = range.end - range.start;
226 let mut buffer = Vec::with_capacity(to_read.try_into()?);
227 let read = self.take(to_read).read_to_end(&mut buffer).await?;
228 if read as u64 != to_read {
229 return Err(eof_err!("expected to read {} bytes, got {}", to_read, read));
230 }
231
232 Ok(buffer.into())
233 }
234 .boxed()
235 }
236
237 fn get_metadata<'a>(
238 &'a mut self,
239 options: Option<&'a ArrowReaderOptions>,
240 ) -> BoxFuture<'a, Result<Arc<ParquetMetaData>>> {
241 async move {
242 let metadata_reader = ParquetMetaDataReader::new().with_arrow_reader_options(options);
243 let parquet_metadata = metadata_reader.load_via_suffix_and_finish(self).await?;
244 Ok(Arc::new(parquet_metadata))
245 }
246 .boxed()
247 }
248}
249
250impl ArrowReaderMetadata {
251 /// Returns a new [`ArrowReaderMetadata`] for this builder
252 ///
253 /// See [`ParquetRecordBatchStreamBuilder::new_with_metadata`] for how this can be used
254 pub async fn load_async<T: AsyncFileReader>(
255 input: &mut T,
256 options: ArrowReaderOptions,
257 ) -> Result<Self> {
258 let metadata = input.get_metadata(Some(&options)).await?;
259 Self::try_new(metadata, options)
260 }
261}
262
263#[doc(hidden)]
264/// Newtype (wrapper) used within [`ArrowReaderBuilder`] to distinguish sync readers from async
265///
266/// Allows sharing the same builder for different readers while keeping the same
267/// ParquetRecordBatchStreamBuilder API
268pub struct AsyncReader<T>(T);
269
270/// A builder for reading parquet files from an `async` source as [`ParquetRecordBatchStream`]
271///
272/// This can be used to decode a Parquet file in streaming fashion (without
273/// downloading the whole file at once) from a remote source, such as an object store.
274///
275/// This builder handles reading the parquet file metadata, allowing consumers
276/// to use this information to select what specific columns, row groups, etc.
277/// they wish to be read by the resulting stream.
278///
279/// See examples on [`ParquetRecordBatchStreamBuilder::new`], including how to
280/// issue multiple I/O requests in parallel using multiple streams.
281///
282/// # See also:
283/// * [`ParquetPushDecoderBuilder`] for lower level control over buffering and
284/// decoding.
285/// * [`ParquetRecordBatchStream::next_row_group`] for I/O prefetching
286///
287///
288/// See [`ArrowReaderBuilder`] for additional member functions
289pub type ParquetRecordBatchStreamBuilder<T> = ArrowReaderBuilder<AsyncReader<T>>;
290
291impl<T: AsyncFileReader + Send + 'static> ParquetRecordBatchStreamBuilder<T> {
292 /// Create a new [`ParquetRecordBatchStreamBuilder`] for reading from the
293 /// specified source.
294 ///
295 /// # Examples:
296 /// * [Basic example reading from an async source](#example)
297 /// * [Configuring options and reading metadata](#example-configuring-options-and-reading-metadata)
298 /// * [Reading Row Groups in Parallel](#example-reading-row-groups-in-parallel)
299 ///
300 /// # Example
301 /// ```
302 /// # #[tokio::main(flavor="current_thread")]
303 /// # async fn main() {
304 /// #
305 /// # use arrow_array::RecordBatch;
306 /// # use arrow::util::pretty::pretty_format_batches;
307 /// # use futures::TryStreamExt;
308 /// #
309 /// # use parquet::arrow::{ParquetRecordBatchStreamBuilder, ProjectionMask};
310 /// #
311 /// # fn assert_batches_eq(batches: &[RecordBatch], expected_lines: &[&str]) {
312 /// # let formatted = pretty_format_batches(batches).unwrap().to_string();
313 /// # let actual_lines: Vec<_> = formatted.trim().lines().collect();
314 /// # assert_eq!(
315 /// # &actual_lines, expected_lines,
316 /// # "\n\nexpected:\n\n{:#?}\nactual:\n\n{:#?}\n\n",
317 /// # expected_lines, actual_lines
318 /// # );
319 /// # }
320 /// #
321 /// # let testdata = arrow::util::test_util::parquet_test_data();
322 /// # let path = format!("{}/alltypes_plain.parquet", testdata);
323 /// // Use tokio::fs::File to read data using an async I/O. This can be replaced with
324 /// // another async I/O reader such as a reader from an object store.
325 /// let file = tokio::fs::File::open(path).await.unwrap();
326 ///
327 /// // Configure options for reading from the async source
328 /// let builder = ParquetRecordBatchStreamBuilder::new(file)
329 /// .await
330 /// .unwrap();
331 /// // Building the stream opens the parquet file (reads metadata, etc) and returns
332 /// // a stream that can be used to incrementally read the data in batches
333 /// let stream = builder.build().unwrap();
334 /// // In this example, we collect the stream into a Vec<RecordBatch>
335 /// // but real applications would likely process the batches as they are read
336 /// let results = stream.try_collect::<Vec<_>>().await.unwrap();
337 /// // Demonstrate the results are as expected
338 /// assert_batches_eq(
339 /// &results,
340 /// &[
341 /// "+----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+---------------------+",
342 /// "| id | bool_col | tinyint_col | smallint_col | int_col | bigint_col | float_col | double_col | date_string_col | string_col | timestamp_col |",
343 /// "+----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+---------------------+",
344 /// "| 4 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30332f30312f3039 | 30 | 2009-03-01T00:00:00 |",
345 /// "| 5 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30332f30312f3039 | 31 | 2009-03-01T00:01:00 |",
346 /// "| 6 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30342f30312f3039 | 30 | 2009-04-01T00:00:00 |",
347 /// "| 7 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30342f30312f3039 | 31 | 2009-04-01T00:01:00 |",
348 /// "| 2 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30322f30312f3039 | 30 | 2009-02-01T00:00:00 |",
349 /// "| 3 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30322f30312f3039 | 31 | 2009-02-01T00:01:00 |",
350 /// "| 0 | true | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 30312f30312f3039 | 30 | 2009-01-01T00:00:00 |",
351 /// "| 1 | false | 1 | 1 | 1 | 10 | 1.1 | 10.1 | 30312f30312f3039 | 31 | 2009-01-01T00:01:00 |",
352 /// "+----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+---------------------+",
353 /// ],
354 /// );
355 /// # }
356 /// ```
357 ///
358 /// # Example Configuring Options and Reading Metadata
359 ///
360 /// There are many options that control the behavior of the reader, such as
361 /// `with_batch_size`, `with_projection`, `with_filter`, etc...
362 ///
363 /// ```
364 /// # #[tokio::main(flavor="current_thread")]
365 /// # async fn main() {
366 /// #
367 /// # use arrow_array::RecordBatch;
368 /// # use arrow::util::pretty::pretty_format_batches;
369 /// # use futures::TryStreamExt;
370 /// #
371 /// # use parquet::arrow::{ParquetRecordBatchStreamBuilder, ProjectionMask};
372 /// #
373 /// # fn assert_batches_eq(batches: &[RecordBatch], expected_lines: &[&str]) {
374 /// # let formatted = pretty_format_batches(batches).unwrap().to_string();
375 /// # let actual_lines: Vec<_> = formatted.trim().lines().collect();
376 /// # assert_eq!(
377 /// # &actual_lines, expected_lines,
378 /// # "\n\nexpected:\n\n{:#?}\nactual:\n\n{:#?}\n\n",
379 /// # expected_lines, actual_lines
380 /// # );
381 /// # }
382 /// #
383 /// # let testdata = arrow::util::test_util::parquet_test_data();
384 /// # let path = format!("{}/alltypes_plain.parquet", testdata);
385 /// // As before, use tokio::fs::File to read data using an async I/O.
386 /// let file = tokio::fs::File::open(path).await.unwrap();
387 ///
388 /// // Configure options for reading from the async source, in this case we set the batch size
389 /// // to 3 which produces 3 rows at a time.
390 /// let builder = ParquetRecordBatchStreamBuilder::new(file)
391 /// .await
392 /// .unwrap()
393 /// .with_batch_size(3);
394 ///
395 /// // We can also read the metadata to inspect the schema and other metadata
396 /// // before actually reading the data
397 /// let file_metadata = builder.metadata().file_metadata();
398 /// // Specify that we only want to read the 1st, 2nd, and 6th columns
399 /// let mask = ProjectionMask::roots(file_metadata.schema_descr(), [1, 2, 6]);
400 ///
401 /// let stream = builder.with_projection(mask).build().unwrap();
402 /// let results = stream.try_collect::<Vec<_>>().await.unwrap();
403 /// // Print out the results
404 /// assert_batches_eq(
405 /// &results,
406 /// &[
407 /// "+----------+-------------+-----------+",
408 /// "| bool_col | tinyint_col | float_col |",
409 /// "+----------+-------------+-----------+",
410 /// "| true | 0 | 0.0 |",
411 /// "| false | 1 | 1.1 |",
412 /// "| true | 0 | 0.0 |",
413 /// "| false | 1 | 1.1 |",
414 /// "| true | 0 | 0.0 |",
415 /// "| false | 1 | 1.1 |",
416 /// "| true | 0 | 0.0 |",
417 /// "| false | 1 | 1.1 |",
418 /// "+----------+-------------+-----------+",
419 /// ],
420 /// );
421 ///
422 /// // The results has 8 rows, so since we set the batch size to 3, we expect
423 /// // 3 batches, two with 3 rows each and the last batch with 2 rows.
424 /// assert_eq!(results.len(), 3);
425 /// # }
426 /// ```
427 ///
428 /// # Example reading Row Groups in Parallel
429 ///
430 /// Each [`ParquetRecordBatchStream`] is independent and can be used to read
431 /// from the same underlying source in parallel. Use
432 /// [`ParquetRecordBatchStream::next_row_group`] with a single stream to
433 /// begin prefetching the next Row Group. To read a file in parallel, create
434 /// a stream for each subset of the file. For example, you can read each
435 /// row group in parallel by creating a stream for each row group using the
436 /// [`ParquetRecordBatchStreamBuilder::with_row_groups`] API as shown below
437 ///
438 /// ```
439 /// # use std::sync::Arc;
440 /// # use arrow_array::{ArrayRef, Int32Array, RecordBatch};
441 /// # use arrow::util::pretty::pretty_format_batches;
442 /// # use futures::{StreamExt, TryStreamExt};
443 /// # use tempfile::NamedTempFile;
444 /// # use parquet::arrow::{ArrowWriter, ParquetRecordBatchStreamBuilder, ProjectionMask};
445 /// # use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions};
446 /// # use parquet::file::metadata::ParquetMetaDataReader;
447 /// # use parquet::file::properties::{WriterProperties};
448 /// # // write to a temporary file with 10 RowGroups and read back with async API
449 /// # fn write_file() -> parquet::errors::Result<NamedTempFile> {
450 /// # let mut file = NamedTempFile::new().unwrap();
451 /// # let small_batch = RecordBatch::try_from_iter([
452 /// # ("id", Arc::new(Int32Array::from(vec![0, 1, 2, 3, 4])) as ArrayRef),
453 /// # ]).unwrap();
454 /// # let props = WriterProperties::builder()
455 /// # .set_max_row_group_row_count(Some(5))
456 /// # .set_write_batch_size(5)
457 /// # .build();
458 /// # let mut writer = ArrowWriter::try_new(&mut file, small_batch.schema(), Some(props))?;
459 /// # for i in 0..10 {
460 /// # writer.write(&small_batch)?
461 /// # };
462 /// # writer.close()?;
463 /// # Ok(file)
464 /// # }
465 /// # #[tokio::main(flavor="current_thread")]
466 /// # async fn main() -> parquet::errors::Result<()> {
467 /// # let t = write_file()?;
468 /// # let path = t.path();
469 /// // This example uses a tokio::fs::File as the async source, but it
470 /// // could be any async source such as an object store reader)
471 /// let mut file = tokio::fs::File::open(path).await?;
472 /// // To read Row Groups in parallel, create a separate stream builder for each Row Group.
473 /// // First get the metadata to find the row group information
474 /// let file_size = file.metadata().await?.len();
475 /// let metadata = ParquetMetaDataReader::new().load_and_finish(&mut file, file_size).await?;
476 /// assert_eq!(metadata.num_row_groups(), 10); // file has 10 row groups with 5 rows each
477 /// // Create a stream reader for each row group
478 /// let reader_metadata = ArrowReaderMetadata::try_new(
479 /// Arc::new(metadata),
480 /// ArrowReaderOptions::new()
481 /// )?;
482 /// let mut streams = vec![];
483 /// for row_group_index in 0..10 {
484 /// // Each stream needs its own source instance to issue
485 /// // parallel IO requests, so clone the file for each stream
486 /// let this_file = file.try_clone().await?;
487 /// let stream = ParquetRecordBatchStreamBuilder::new_with_metadata(
488 /// this_file,
489 /// reader_metadata.clone()
490 /// )
491 /// .with_row_groups(vec![row_group_index]) // read only this row group
492 /// .build()?;
493 /// streams.push(stream);
494 /// }
495 /// // Each reader can now be polled independently and in parallel, for
496 /// // example using StreamExt::buffered to read from 3 at a time
497 /// let results = futures::stream::iter(streams)
498 /// .map(|stream| async move { stream })
499 /// .buffered(3)
500 /// .flatten()
501 /// .try_collect::<Vec<_>>().await?;
502 /// // read all 50 rows (10 row groups x 5 rows per group)
503 /// assert_eq!(50, results.iter().map(|s| s.num_rows()).sum::<usize>());
504 /// # Ok(())
505 /// # }
506 /// ```
507 pub async fn new(input: T) -> Result<Self> {
508 Self::new_with_options(input, Default::default()).await
509 }
510
511 /// Create a new [`ParquetRecordBatchStreamBuilder`] with the provided async source
512 /// and [`ArrowReaderOptions`].
513 pub async fn new_with_options(mut input: T, options: ArrowReaderOptions) -> Result<Self> {
514 let metadata = ArrowReaderMetadata::load_async(&mut input, options).await?;
515 Ok(Self::new_with_metadata(input, metadata))
516 }
517
518 /// Create a [`ParquetRecordBatchStreamBuilder`] from the provided [`ArrowReaderMetadata`]
519 ///
520 /// This allows loading metadata once and using it to create multiple builders with
521 /// potentially different settings, that can be read in parallel.
522 ///
523 /// # Example of reading from multiple streams in parallel
524 ///
525 /// ```
526 /// # use std::fs::metadata;
527 /// # use std::sync::Arc;
528 /// # use bytes::Bytes;
529 /// # use arrow_array::{Int32Array, RecordBatch};
530 /// # use arrow_schema::{DataType, Field, Schema};
531 /// # use parquet::arrow::arrow_reader::ArrowReaderMetadata;
532 /// # use parquet::arrow::{ArrowWriter, ParquetRecordBatchStreamBuilder};
533 /// # use tempfile::tempfile;
534 /// # use futures::StreamExt;
535 /// # #[tokio::main(flavor="current_thread")]
536 /// # async fn main() {
537 /// #
538 /// # let mut file = tempfile().unwrap();
539 /// # let schema = Arc::new(Schema::new(vec![Field::new("i32", DataType::Int32, false)]));
540 /// # let mut writer = ArrowWriter::try_new(&mut file, schema.clone(), None).unwrap();
541 /// # let batch = RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]).unwrap();
542 /// # writer.write(&batch).unwrap();
543 /// # writer.close().unwrap();
544 /// // open file with parquet data
545 /// let mut file = tokio::fs::File::from_std(file);
546 /// // load metadata once
547 /// let meta = ArrowReaderMetadata::load_async(&mut file, Default::default()).await.unwrap();
548 /// // create two readers, a and b, from the same underlying file
549 /// // without reading the metadata again
550 /// let mut a = ParquetRecordBatchStreamBuilder::new_with_metadata(
551 /// file.try_clone().await.unwrap(),
552 /// meta.clone()
553 /// ).build().unwrap();
554 /// let mut b = ParquetRecordBatchStreamBuilder::new_with_metadata(file, meta).build().unwrap();
555 ///
556 /// // Can read batches from both readers in parallel
557 /// assert_eq!(
558 /// a.next().await.unwrap().unwrap(),
559 /// b.next().await.unwrap().unwrap(),
560 /// );
561 /// # }
562 /// ```
563 pub fn new_with_metadata(input: T, metadata: ArrowReaderMetadata) -> Self {
564 Self::new_builder(AsyncReader(input), metadata)
565 }
566
567 /// Read bloom filter for a column in a row group
568 ///
569 /// Returns `None` if the column does not have a bloom filter
570 ///
571 /// We should call this function after other forms pruning, such as projection and predicate pushdown.
572 pub async fn get_row_group_column_bloom_filter(
573 &mut self,
574 row_group_idx: usize,
575 column_idx: usize,
576 ) -> Result<Option<Sbbf>> {
577 let metadata = self.metadata.row_group(row_group_idx);
578 let column_metadata = metadata.column(column_idx);
579
580 Sbbf::read_from_column_chunk_async(column_metadata, &mut self.input.0).await
581 }
582
583 /// Select row groups and rows using row-group-local coordinates.
584 ///
585 /// Entries are decoded in the supplied order, omitted row groups are
586 /// skipped, and a `None` selection reads the whole row group. This is
587 /// mutually exclusive with [`ArrowReaderBuilder::with_row_groups`] and
588 /// [`ArrowReaderBuilder::with_row_selection`]; combining them returns an
589 /// error from [`Self::build`].
590 ///
591 /// See [`ParquetPushDecoderBuilder::with_row_group_selections`] for the
592 /// full semantics and a worked example. This builder supports the same API
593 /// because the async stream is implemented using the push decoder; the
594 /// synchronous reader does not support row-group-local selections.
595 ///
596 /// [`ParquetPushDecoderBuilder::with_row_group_selections`]: crate::arrow::push_decoder::ParquetPushDecoderBuilder::with_row_group_selections
597 pub fn with_row_group_selections(
598 mut self,
599 row_group_selections: Vec<RowGroupSelection>,
600 ) -> Self {
601 self.row_group_plan
602 .set_row_group_selections(row_group_selections);
603 self
604 }
605
606 /// Read and decode the dictionary page for a column in a row group, if any.
607 ///
608 /// Returns `Ok(None)` if the column chunk has no dictionary page, or if
609 /// its physical type is not `BYTE_ARRAY` (the only physical type
610 /// currently supported).
611 ///
612 /// The returned array contains raw `Binary` values, even for columns
613 /// annotated as strings. Callers can compare byte slices directly or
614 /// convert values to UTF-8 explicitly.
615 ///
616 /// This can be used to inspect dictionary values when selecting or pruning
617 /// row groups before passing the selected indices to
618 /// [`ParquetRecordBatchStreamBuilder::with_row_groups`].
619 ///
620 /// Note this does not verify that the *entire* column chunk is
621 /// dictionary-encoded -- callers that need that guarantee (e.g. to treat
622 /// the dictionary as an exhaustive set of the column's values) should
623 /// check
624 /// [`crate::file::metadata::ColumnChunkMetaData::page_encoding_stats_mask`].
625 pub async fn get_column_chunk_dictionary(
626 &mut self,
627 row_group_idx: usize,
628 column_idx: usize,
629 ) -> Result<Option<ArrayRef>> {
630 ParquetMetaDataReader::read_column_dictionary_async(
631 &mut self.input.0,
632 &self.metadata,
633 row_group_idx,
634 column_idx,
635 )
636 .await
637 }
638
639 /// Build a new [`ParquetRecordBatchStream`]
640 ///
641 /// See examples on [`ParquetRecordBatchStreamBuilder::new`]
642 pub fn build(self) -> Result<ParquetRecordBatchStream<T>> {
643 let Self {
644 input,
645 metadata,
646 schema,
647 fields,
648 batch_size,
649 row_group_plan,
650 projection,
651 filter,
652 row_selection_policy: selection_strategy,
653 limit,
654 offset,
655 metrics,
656 max_predicate_cache_size,
657 } = self;
658
659 // Ensure schema of ParquetRecordBatchStream respects projection, and does
660 // not store metadata (same as for ParquetRecordBatchReader and emitted RecordBatches)
661 let projection_len = projection.mask.as_ref().map_or(usize::MAX, |m| m.len());
662 let projected_fields = schema
663 .fields
664 .filter_leaves(|idx, _| idx < projection_len && projection.leaf_included(idx));
665 let projected_schema = Arc::new(Schema::new(projected_fields));
666
667 let decoder = ParquetPushDecoderBuilder {
668 input: PushDecoderInput::default(),
669 metadata,
670 schema,
671 fields,
672 projection,
673 filter,
674 row_group_plan,
675 row_selection_policy: selection_strategy,
676 batch_size,
677 limit,
678 offset,
679 metrics,
680 max_predicate_cache_size,
681 }
682 .build()?;
683
684 let request_state = RequestState::None { input: input.0 };
685
686 Ok(ParquetRecordBatchStream {
687 schema: projected_schema,
688 decoder,
689 request_state,
690 })
691 }
692}
693
694/// State machine that tracks outstanding requests to fetch data
695///
696/// The parameter `T` is the input, typically an `AsyncFileReader`
697enum RequestState<T> {
698 /// No outstanding requests
699 None {
700 input: T,
701 },
702 /// There is an outstanding request for data
703 Outstanding {
704 /// Ranges that have been requested
705 ranges: Vec<Range<u64>>,
706 /// Future that will resolve (input, requested_ranges)
707 ///
708 /// Note the future owns the reader while the request is outstanding
709 /// and returns it upon completion
710 future: BoxFuture<'static, Result<(T, Vec<Bytes>)>>,
711 },
712 Done,
713}
714
715impl<T> RequestState<T>
716where
717 T: AsyncFileReader + Unpin + Send + 'static,
718{
719 /// Issue a request to fetch `ranges`, returning the Outstanding state
720 fn begin_request(mut input: T, ranges: Vec<Range<u64>>) -> Self {
721 let ranges_captured = ranges.clone();
722
723 // Note this must move the input *into* the future
724 // because the get_byte_ranges future has a lifetime
725 // (aka can have references internally) and thus must
726 // own the input while the request is outstanding.
727 let future = async move {
728 let data = input.get_byte_ranges(ranges_captured).await?;
729 Ok((input, data))
730 }
731 .boxed();
732 RequestState::Outstanding { ranges, future }
733 }
734}
735
736impl<T> std::fmt::Debug for RequestState<T> {
737 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
738 match self {
739 RequestState::None { input: _ } => f
740 .debug_struct("RequestState::None")
741 .field("input", &"...")
742 .finish(),
743 RequestState::Outstanding { ranges, .. } => f
744 .debug_struct("RequestState::Outstanding")
745 .field("ranges", &ranges)
746 .finish(),
747 RequestState::Done => {
748 write!(f, "RequestState::Done")
749 }
750 }
751 }
752}
753
754/// An asynchronous [`Stream`]of [`RecordBatch`] constructed using [`ParquetRecordBatchStreamBuilder`] to read parquet files.
755///
756/// `ParquetRecordBatchStream` also provides [`ParquetRecordBatchStream::next_row_group`] for fetching row groups,
757/// allowing users to decode record batches separately from I/O.
758///
759/// # I/O Buffering
760///
761/// `ParquetRecordBatchStream` buffers *all* data pages selected after predicates
762/// (projection + filtering, etc) and decodes the rows from those buffered pages.
763///
764/// For example, if all rows and columns are selected, the entire row group is
765/// buffered in memory during decode. This minimizes the number of IO operations
766/// required, which is especially important for object stores, where IO operations
767/// have latencies in the hundreds of milliseconds
768///
769/// See [`ParquetPushDecoderBuilder`] for an API with lower level control over
770/// buffering.
771///
772/// [`Stream`]: https://docs.rs/futures/latest/futures/stream/trait.Stream.html
773pub struct ParquetRecordBatchStream<T> {
774 /// Output schema of the stream
775 schema: SchemaRef,
776 /// Input and Outstanding IO request, if any
777 request_state: RequestState<T>,
778 /// Decoding state machine (no IO)
779 decoder: ParquetPushDecoder,
780}
781
782impl<T> std::fmt::Debug for ParquetRecordBatchStream<T> {
783 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
784 f.debug_struct("ParquetRecordBatchStream")
785 .field("request_state", &self.request_state)
786 .finish()
787 }
788}
789
790impl<T> ParquetRecordBatchStream<T> {
791 /// Returns the projected [`SchemaRef`] for reading the parquet file.
792 ///
793 /// Note that the schema metadata will be stripped here. See
794 /// [`ParquetRecordBatchStreamBuilder::schema`] if the metadata is desired.
795 pub fn schema(&self) -> &SchemaRef {
796 &self.schema
797 }
798}
799
800impl<T> ParquetRecordBatchStream<T>
801where
802 T: AsyncFileReader + Unpin + Send + 'static,
803{
804 /// Fetches the next row group from the stream.
805 ///
806 /// Users can continue to call this function to get row groups and decode them concurrently.
807 ///
808 /// ## Notes
809 ///
810 /// ParquetRecordBatchStream should be used either as a `Stream` or with `next_row_group`; they should not be used simultaneously.
811 ///
812 /// ## Returns
813 ///
814 /// - `Ok(None)` if the stream has ended.
815 /// - `Err(error)` if the stream has errored. All subsequent calls will return `Ok(None)`.
816 /// - `Ok(Some(reader))` which holds all the data for the row group.
817 pub async fn next_row_group(&mut self) -> Result<Option<ParquetRecordBatchReader>> {
818 loop {
819 // Take ownership of request state to process, leaving self in a
820 // valid state
821 let request_state = std::mem::replace(&mut self.request_state, RequestState::Done);
822 match request_state {
823 // No outstanding requests, proceed to setup next row group
824 RequestState::None { input } => {
825 match self.decoder.try_next_reader()? {
826 DecodeResult::NeedsData(ranges) => {
827 self.request_state = RequestState::begin_request(input, ranges);
828 // Will loop again: the input might be ready immediately.
829 }
830 DecodeResult::Data(reader) => {
831 self.request_state = RequestState::None { input };
832 return Ok(Some(reader));
833 }
834 DecodeResult::Finished => return Ok(None),
835 }
836 }
837 RequestState::Outstanding { ranges, future } => {
838 let (input, data) = future.await?;
839 // Push the requested data to the decoder and try again
840 self.decoder.push_ranges(ranges, data)?;
841 self.request_state = RequestState::None { input };
842 // Will try and decode on the next iteration.
843 }
844 RequestState::Done => {
845 self.request_state = RequestState::Done;
846 return Ok(None);
847 }
848 }
849 }
850 }
851}
852
853impl<T> Stream for ParquetRecordBatchStream<T>
854where
855 T: AsyncFileReader + Unpin + Send + 'static,
856{
857 type Item = Result<RecordBatch>;
858 fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
859 match self.poll_next_inner(cx) {
860 Ok(res) => {
861 // Successfully decoded a batch, or reached end of stream.
862 // convert Option<RecordBatch> to Option<Result<RecordBatch>>
863 res.map(|res| Ok(res).transpose())
864 }
865 Err(e) => {
866 self.request_state = RequestState::Done;
867 Poll::Ready(Some(Err(e)))
868 }
869 }
870 }
871}
872
873impl<T> ParquetRecordBatchStream<T>
874where
875 T: AsyncFileReader + Unpin + Send + 'static,
876{
877 /// Inner state machine
878 ///
879 /// Note this is separate from poll_next so we can use ? operator to check for errors
880 /// as it returns `Result<Poll<Option<RecordBatch>>>`
881 fn poll_next_inner(&mut self, cx: &mut Context<'_>) -> Result<Poll<Option<RecordBatch>>> {
882 loop {
883 let request_state = std::mem::replace(&mut self.request_state, RequestState::Done);
884 match request_state {
885 RequestState::None { input } => {
886 // No outstanding requests, proceed to decode the next batch
887 match self.decoder.try_decode()? {
888 DecodeResult::NeedsData(ranges) => {
889 self.request_state = RequestState::begin_request(input, ranges);
890 // Will loop again: the input might be ready immediately.
891 }
892 DecodeResult::Data(batch) => {
893 self.request_state = RequestState::None { input };
894 return Ok(Poll::Ready(Some(batch)));
895 }
896 DecodeResult::Finished => {
897 self.request_state = RequestState::Done;
898 return Ok(Poll::Ready(None));
899 }
900 }
901 }
902 RequestState::Outstanding { ranges, mut future } => match future.poll_unpin(cx) {
903 // Data was ready, push it to the decoder and continue
904 Poll::Ready(result) => {
905 let (input, data) = result?;
906 // Push the requested data to the decoder
907 self.decoder.push_ranges(ranges, data)?;
908 self.request_state = RequestState::None { input };
909 // The next iteration will try to decode the next batch.
910 }
911 Poll::Pending => {
912 self.request_state = RequestState::Outstanding { ranges, future };
913 return Ok(Poll::Pending);
914 }
915 },
916 RequestState::Done => {
917 // Stream is done (error or end), return None
918 self.request_state = RequestState::Done;
919 return Ok(Poll::Ready(None));
920 }
921 }
922 }
923 }
924}
925
926#[cfg(test)]
927mod tests {
928 use super::*;
929 use crate::arrow::ArrowWriter;
930 use crate::arrow::ProjectionMask;
931 use crate::arrow::arrow_reader::tests::test_row_numbers_with_multiple_row_groups_helper;
932 use crate::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions};
933 use crate::arrow::schema::virtual_type::RowNumber;
934 use crate::basic::Encoding;
935 use crate::file::metadata::PageIndexPolicy;
936 use crate::file::metadata::ParquetMetaDataReader;
937 use crate::file::metadata::page_index::PageIndex;
938 use crate::file::properties::WriterProperties;
939 use arrow_array::cast::AsArray;
940 use arrow_array::{Array, ArrayRef, BinaryArray, StringArray};
941 use arrow_schema::{DataType, Field, Schema};
942 use futures::TryStreamExt;
943 use std::sync::{Arc, Mutex};
944
945 #[derive(Clone)]
946 struct TestReader {
947 data: Bytes,
948 metadata: Option<Arc<ParquetMetaData>>,
949 requests: Arc<Mutex<Vec<Range<usize>>>>,
950 }
951
952 impl TestReader {
953 fn new(data: Bytes) -> Self {
954 Self {
955 data,
956 metadata: Default::default(),
957 requests: Default::default(),
958 }
959 }
960 }
961
962 impl AsyncFileReader for TestReader {
963 fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes>> {
964 let range = range.clone();
965 self.requests
966 .lock()
967 .unwrap()
968 .push(range.start as usize..range.end as usize);
969 futures::future::ready(Ok(self
970 .data
971 .slice(range.start as usize..range.end as usize)))
972 .boxed()
973 }
974
975 fn get_metadata<'a>(
976 &'a mut self,
977 options: Option<&'a ArrowReaderOptions>,
978 ) -> BoxFuture<'a, Result<Arc<ParquetMetaData>>> {
979 let metadata_reader = ParquetMetaDataReader::new().with_arrow_reader_options(options);
980 self.metadata = Some(Arc::new(
981 metadata_reader.parse_and_finish(&self.data).unwrap(),
982 ));
983 futures::future::ready(Ok(self.metadata.clone().unwrap().clone())).boxed()
984 }
985 }
986
987 #[tokio::test]
988 async fn test_get_column_chunk_dictionary() {
989 let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)]));
990 let values: Vec<&str> = ["alpha", "beta", "gamma"]
991 .iter()
992 .copied()
993 .cycle()
994 .take(30)
995 .collect();
996 let array: ArrayRef = Arc::new(StringArray::from(values));
997 let batch = RecordBatch::try_new(schema.clone(), vec![array]).unwrap();
998
999 let props = WriterProperties::builder()
1000 .set_dictionary_enabled(true)
1001 .build();
1002 let mut buf = Vec::new();
1003 {
1004 let mut writer = ArrowWriter::try_new(&mut buf, schema, Some(props)).unwrap();
1005 writer.write(&batch).unwrap();
1006 writer.close().unwrap();
1007 }
1008 let data = Bytes::from(buf);
1009
1010 let direct_metadata = ParquetMetaDataReader::new()
1011 .parse_and_finish(&data)
1012 .unwrap();
1013 let mut direct_reader = TestReader::new(data.clone());
1014 let direct = ParquetMetaDataReader::read_column_dictionary_async(
1015 &mut direct_reader,
1016 &direct_metadata,
1017 0,
1018 0,
1019 )
1020 .await
1021 .unwrap()
1022 .unwrap();
1023 let direct = direct.as_any().downcast_ref::<BinaryArray>().unwrap();
1024 assert_eq!(direct.value(0), b"alpha");
1025
1026 let async_reader = TestReader::new(data);
1027 let mut builder = ParquetRecordBatchStreamBuilder::new(async_reader)
1028 .await
1029 .unwrap();
1030
1031 let dictionary = builder
1032 .get_column_chunk_dictionary(0, 0)
1033 .await
1034 .unwrap()
1035 .unwrap();
1036 let dictionary = dictionary.as_any().downcast_ref::<BinaryArray>().unwrap();
1037 let dictionary_values: Vec<&[u8]> = dictionary.iter().map(|v| v.unwrap()).collect();
1038 assert_eq!(
1039 dictionary_values,
1040 vec![b"alpha".as_slice(), b"beta", b"gamma"]
1041 );
1042 }
1043
1044 // This test demonstrates row group pruning using dictionary pages,
1045 // verifying that data pages of skipped row groups are not read
1046 #[tokio::test]
1047 async fn test_dictionary_selects_row_groups_without_reading_skipped_data() {
1048 // Write two row groups, each dictionary-encoded and containing a single
1049 // distinct string repeated 30 times: row group 0 is all "skip", row
1050 // group 1 is all "target".
1051 let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)]));
1052 let row_group_values = ["skip", "target"];
1053 let props = WriterProperties::builder()
1054 .set_dictionary_enabled(true)
1055 .build();
1056 let mut buf = Vec::new();
1057 {
1058 let mut writer = ArrowWriter::try_new(&mut buf, schema.clone(), Some(props)).unwrap();
1059 for value in row_group_values {
1060 let array: ArrayRef = Arc::new(StringArray::from(vec![value; 30]));
1061 let batch = RecordBatch::try_new(schema.clone(), vec![array]).unwrap();
1062 writer.write(&batch).unwrap();
1063 writer.flush().unwrap();
1064 }
1065 writer.close().unwrap();
1066 }
1067
1068 let async_reader = TestReader::new(Bytes::from(buf));
1069 // `requests` records the byte ranges fetched from the underlying reader,
1070 // which we later use to check that we read only the needed page
1071 let requests = async_reader.requests.clone();
1072 let mut builder = ParquetRecordBatchStreamBuilder::new(async_reader)
1073 .await
1074 .unwrap();
1075 let metadata = builder.metadata().clone();
1076 assert_eq!(metadata.num_row_groups(), 2);
1077
1078 // For each row group, fetch and decode just the dictionary page
1079 // to decide whether to read the whole row group
1080 let mut selected_row_groups = Vec::new();
1081 let mut dictionary_ranges = Vec::new();
1082 for row_group_idx in 0..metadata.num_row_groups() {
1083 let column = metadata.row_group(row_group_idx).column(0);
1084 let encoding_mask = column.page_encoding_stats_mask().unwrap();
1085 assert!(
1086 encoding_mask.is_only(Encoding::PLAIN_DICTIONARY)
1087 || encoding_mask.is_only(Encoding::RLE_DICTIONARY)
1088 );
1089
1090 let dictionary_start = column.dictionary_page_offset().unwrap() as usize;
1091 let data_start = column.data_page_offset() as usize;
1092 dictionary_ranges.push(dictionary_start..data_start);
1093
1094 let dictionary = builder
1095 .get_column_chunk_dictionary(row_group_idx, 0)
1096 .await
1097 .unwrap()
1098 .unwrap();
1099 let dictionary = dictionary.as_binary::<i32>();
1100 if dictionary
1101 .iter()
1102 .any(|value| value == Some(b"target".as_slice()))
1103 {
1104 selected_row_groups.push(row_group_idx);
1105 }
1106 }
1107 assert_eq!(selected_row_groups, vec![1]);
1108
1109 // Read the filtered row group and verify the values
1110 let batches: Vec<_> = builder
1111 .with_row_groups(selected_row_groups)
1112 .build()
1113 .unwrap()
1114 .try_collect()
1115 .await
1116 .unwrap();
1117 let values: Vec<_> = batches
1118 .iter()
1119 .flat_map(|batch| batch.column(0).as_string::<i32>().iter())
1120 .collect();
1121 assert_eq!(values, vec![Some("target"); 30]);
1122
1123 // Finally, verify none of the requests overlapped the data pages
1124 // of the skipped row group
1125 let skipped_column = metadata.row_group(0).column(0);
1126 let (skipped_start, skipped_len) = skipped_column.byte_range();
1127 let skipped_data_range =
1128 skipped_column.data_page_offset() as usize..(skipped_start + skipped_len) as usize;
1129 let requests = requests.lock().unwrap();
1130 for dictionary_range in dictionary_ranges {
1131 assert!(requests.contains(&dictionary_range));
1132 }
1133 assert!(requests.iter().all(|request| {
1134 request.end <= skipped_data_range.start || request.start >= skipped_data_range.end
1135 }));
1136 }
1137
1138 #[tokio::test]
1139 async fn test_batch_size_overallocate() {
1140 let testdata = arrow::util::test_util::parquet_test_data();
1141 // `alltypes_plain.parquet` only have 8 rows
1142 let path = format!("{testdata}/alltypes_plain.parquet");
1143 let data = Bytes::from(std::fs::read(path).unwrap());
1144
1145 let async_reader = TestReader::new(data.clone());
1146
1147 let builder = ParquetRecordBatchStreamBuilder::new(async_reader)
1148 .await
1149 .unwrap();
1150
1151 let file_rows = builder.metadata().file_metadata().num_rows() as usize;
1152
1153 let builder = builder
1154 .with_projection(ProjectionMask::all())
1155 .with_batch_size(1024);
1156
1157 // even though the batch size is set to 1024, it should adjust to the max
1158 // number of rows in the file (8)
1159 assert_ne!(1024, file_rows);
1160 assert_eq!(builder.batch_size, file_rows);
1161
1162 let _stream = builder.build().unwrap();
1163 }
1164
1165 #[tokio::test]
1166 async fn empty_offset_index_doesnt_panic_in_read_row_group() {
1167 use tokio::fs::File;
1168 let testdata = arrow::util::test_util::parquet_test_data();
1169 let path = format!("{testdata}/alltypes_plain.parquet");
1170 let mut file = File::open(&path).await.unwrap();
1171 let file_size = file.metadata().await.unwrap().len();
1172 let mut metadata = ParquetMetaDataReader::new()
1173 .with_page_index_policy(PageIndexPolicy::Required)
1174 .load_and_finish(&mut file, file_size)
1175 .await
1176 .unwrap();
1177
1178 let page_index = PageIndex::try_new(None, Some(vec![])).unwrap();
1179 metadata.set_page_index(Some(Arc::new(page_index)));
1180 let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required);
1181 let arrow_reader_metadata = ArrowReaderMetadata::try_new(metadata.into(), options).unwrap();
1182 let reader =
1183 ParquetRecordBatchStreamBuilder::new_with_metadata(file, arrow_reader_metadata)
1184 .build()
1185 .unwrap();
1186
1187 let result = reader.try_collect::<Vec<_>>().await.unwrap();
1188 assert_eq!(result.len(), 1);
1189 }
1190
1191 #[test]
1192 fn test_row_numbers_with_multiple_row_groups() {
1193 test_row_numbers_with_multiple_row_groups_helper(
1194 false,
1195 |path, selection, _row_filter, batch_size| {
1196 let runtime = tokio::runtime::Builder::new_current_thread()
1197 .enable_all()
1198 .build()
1199 .expect("Could not create runtime");
1200 runtime.block_on(async move {
1201 let file = tokio::fs::File::open(path).await.unwrap();
1202 let row_number_field = Arc::new(
1203 Field::new("row_number", DataType::Int64, false)
1204 .with_extension_type(RowNumber),
1205 );
1206 let options = ArrowReaderOptions::new()
1207 .with_virtual_columns(vec![row_number_field])
1208 .unwrap();
1209 let reader = ParquetRecordBatchStreamBuilder::new_with_options(file, options)
1210 .await
1211 .unwrap()
1212 .with_row_selection(selection)
1213 .with_batch_size(batch_size)
1214 .build()
1215 .expect("Could not create reader");
1216 reader.try_collect::<Vec<_>>().await.unwrap()
1217 })
1218 },
1219 );
1220 }
1221
1222 #[test]
1223 fn test_row_numbers_with_multiple_row_groups_and_filter() {
1224 test_row_numbers_with_multiple_row_groups_helper(
1225 true,
1226 |path, selection, row_filter, batch_size| {
1227 let runtime = tokio::runtime::Builder::new_current_thread()
1228 .enable_all()
1229 .build()
1230 .expect("Could not create runtime");
1231 runtime.block_on(async move {
1232 let file = tokio::fs::File::open(path).await.unwrap();
1233 let row_number_field = Arc::new(
1234 Field::new("row_number", DataType::Int64, false)
1235 .with_extension_type(RowNumber),
1236 );
1237 let options = ArrowReaderOptions::new()
1238 .with_virtual_columns(vec![row_number_field])
1239 .unwrap();
1240 let reader = ParquetRecordBatchStreamBuilder::new_with_options(file, options)
1241 .await
1242 .unwrap()
1243 .with_row_selection(selection)
1244 .with_row_filter(row_filter.expect("No row filter"))
1245 .with_batch_size(batch_size)
1246 .build()
1247 .expect("Could not create reader");
1248 reader.try_collect::<Vec<_>>().await.unwrap()
1249 })
1250 },
1251 );
1252 }
1253}