Skip to main content

parquet/arrow/async_reader/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! `async` API for reading Parquet files as [`RecordBatch`]es
6//!
7//! See the [crate-level documentation](crate) for more details.
8//!
9//! See example on [`ParquetRecordBatchStreamBuilder::new`]
10
11use std::fmt::Formatter;
12use std::io::SeekFrom;
13use std::ops::Range;
14use std::pin::Pin;
15use std::sync::Arc;
16use std::task::{Context, Poll};
17
18use bytes::Bytes;
19use futures::future::{BoxFuture, FutureExt};
20use futures::stream::Stream;
21use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt};
22
23use arrow_array::{ArrayRef, RecordBatch};
24use arrow_schema::{Schema, SchemaRef};
25
26use crate::arrow::arrow_reader::{
27    ArrowReaderBuilder, ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReader,
28};
29
30use crate::bloom_filter::Sbbf;
31use crate::errors::{ParquetError, Result};
32use crate::file::metadata::{ParquetMetaData, ParquetMetaDataReader};
33
34mod metadata;
35pub use metadata::*;
36
37mod spawn;
38pub use spawn::SpawnedReader;
39
40/// Re-exported so [`ParquetRecordBatchStreamBuilder::with_row_group_selections`]
41/// can be used without importing from another module.
42pub use crate::arrow::arrow_reader::RowGroupSelection;
43
44#[cfg(feature = "object_store")]
45mod store;
46
47use crate::DecodeResult;
48use crate::arrow::push_decoder::{ParquetPushDecoder, ParquetPushDecoderBuilder, PushDecoderInput};
49#[cfg(feature = "object_store")]
50pub use store::*;
51
52/// The asynchronous interface used by [`ParquetRecordBatchStream`] to read parquet files
53///
54/// Notes:
55///
56/// 1. There is a default implementation for types that implement [`AsyncRead`]
57///    and [`AsyncSeek`], for example [`tokio::fs::File`].
58///
59/// 2. Implementations for remote storage, such as the `object_store` crate,
60///    can implement this interface directly, typically by pairing a store
61///    handle with an object path and delegating [`Self::get_bytes`] and
62///    [`Self::get_byte_ranges`] to ranged reads. [`SpawnedReader`] can wrap
63///    such a reader to perform its I/O on a dedicated runtime, and
64///    [`ParquetMetaDataReader::with_arrow_reader_options`] simplifies
65///    implementing [`Self::get_metadata`].
66///
67/// # Example: implementing `AsyncFileReader` for the `object_store` crate
68///
69/// ```no_run
70/// # use std::ops::Range;
71/// # use std::sync::Arc;
72/// use bytes::Bytes;
73/// use futures::future::BoxFuture;
74/// use futures::{FutureExt, TryFutureExt};
75/// use object_store::path::Path;
76/// use object_store::{GetOptions, GetRange, ObjectStore, ObjectStoreExt};
77/// use parquet::arrow::arrow_reader::ArrowReaderOptions;
78/// use parquet::arrow::async_reader::{AsyncFileReader, MetadataSuffixFetch};
79/// use parquet::errors::{ParquetError, Result};
80/// use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataReader};
81///
82/// fn to_parquet_err(e: object_store::Error) -> ParquetError {
83///     ParquetError::External(Box::new(e))
84/// }
85///
86/// #[derive(Clone)]
87/// struct ObjectStoreReader {
88///     store: Arc<dyn ObjectStore>,
89///     path: Path,
90/// }
91///
92/// impl AsyncFileReader for ObjectStoreReader {
93///     fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes>> {
94///         self.store
95///             .get_range(&self.path, range)
96///             .map_err(to_parquet_err)
97///             .boxed()
98///     }
99///
100///     fn get_byte_ranges(&mut self, ranges: Vec<Range<u64>>) -> BoxFuture<'_, Result<Vec<Bytes>>> {
101///         async move {
102///             self.store
103///                 .get_ranges(&self.path, &ranges)
104///                 .await
105///                 .map_err(to_parquet_err)
106///         }
107///         .boxed()
108///     }
109///
110///     fn get_metadata<'a>(
111///         &'a mut self,
112///         options: Option<&'a ArrowReaderOptions>,
113///     ) -> BoxFuture<'a, Result<Arc<ParquetMetaData>>> {
114///         async move {
115///             let metadata = ParquetMetaDataReader::new()
116///                 .with_arrow_reader_options(options)
117///                 .load_via_suffix_and_finish(self)
118///                 .await?;
119///             Ok(Arc::new(metadata))
120///         }
121///         .boxed()
122///     }
123/// }
124///
125/// /// Supports fetching the parquet footer without knowing the file size,
126/// /// via suffix range requests
127/// impl MetadataSuffixFetch for &mut ObjectStoreReader {
128///     fn fetch_suffix(&mut self, suffix: usize) -> BoxFuture<'_, Result<Bytes>> {
129///         let options = GetOptions {
130///             range: Some(GetRange::Suffix(suffix as u64)),
131///             ..Default::default()
132///         };
133///         async move {
134///             let resp = self
135///                 .store
136///                 .get_opts(&self.path, options)
137///                 .await
138///                 .map_err(to_parquet_err)?;
139///             resp.bytes().await.map_err(to_parquet_err)
140///         }
141///         .boxed()
142///     }
143/// }
144/// ```
145///
146/// [`ParquetMetaDataReader::with_arrow_reader_options`]: crate::file::metadata::ParquetMetaDataReader::with_arrow_reader_options
147///
148/// [`tokio::fs::File`]: https://docs.rs/tokio/latest/tokio/fs/struct.File.html
149pub trait AsyncFileReader: Send {
150    /// Retrieve the bytes in `range`
151    fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes>>;
152
153    /// Retrieve multiple byte ranges. The default implementation will call `get_bytes` sequentially
154    fn get_byte_ranges(&mut self, ranges: Vec<Range<u64>>) -> BoxFuture<'_, Result<Vec<Bytes>>> {
155        async move {
156            let mut result = Vec::with_capacity(ranges.len());
157
158            for range in ranges {
159                let data = self.get_bytes(range).await?;
160                result.push(data);
161            }
162
163            Ok(result)
164        }
165        .boxed()
166    }
167
168    /// Return a future which results in the [`ParquetMetaData`] for this Parquet file.
169    ///
170    /// This is an asynchronous operation as it may involve reading the file
171    /// footer and potentially other metadata from disk or a remote source.
172    ///
173    /// Reading data from Parquet requires the metadata to understand the
174    /// schema, row groups, and location of pages within the file. This metadata
175    /// is stored primarily in the footer of the Parquet file, and can be read using
176    /// [`ParquetMetaDataReader`].
177    ///
178    /// However, implementations can significantly speed up reading Parquet by
179    /// supplying cached metadata or pre-fetched metadata via this API.
180    ///
181    /// # Parameters
182    /// * `options`: Optional [`ArrowReaderOptions`] that may contain decryption
183    ///   and other options that affect how the metadata is read.
184    fn get_metadata<'a>(
185        &'a mut self,
186        options: Option<&'a ArrowReaderOptions>,
187    ) -> BoxFuture<'a, Result<Arc<ParquetMetaData>>>;
188}
189
190/// This allows Box<dyn AsyncFileReader + '_> to be used as an AsyncFileReader,
191impl AsyncFileReader for Box<dyn AsyncFileReader + '_> {
192    fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes>> {
193        self.as_mut().get_bytes(range)
194    }
195
196    fn get_byte_ranges(&mut self, ranges: Vec<Range<u64>>) -> BoxFuture<'_, Result<Vec<Bytes>>> {
197        self.as_mut().get_byte_ranges(ranges)
198    }
199
200    fn get_metadata<'a>(
201        &'a mut self,
202        options: Option<&'a ArrowReaderOptions>,
203    ) -> BoxFuture<'a, Result<Arc<ParquetMetaData>>> {
204        self.as_mut().get_metadata(options)
205    }
206}
207
208impl<T: AsyncFileReader + MetadataFetch + AsyncRead + AsyncSeek + Unpin> MetadataSuffixFetch for T {
209    fn fetch_suffix(&mut self, suffix: usize) -> BoxFuture<'_, Result<Bytes>> {
210        async move {
211            self.seek(SeekFrom::End(-(suffix as i64))).await?;
212            let mut buf = Vec::with_capacity(suffix);
213            self.take(suffix as _).read_to_end(&mut buf).await?;
214            Ok(buf.into())
215        }
216        .boxed()
217    }
218}
219
220impl<T: AsyncRead + AsyncSeek + Unpin + Send> AsyncFileReader for T {
221    fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes>> {
222        async move {
223            self.seek(SeekFrom::Start(range.start)).await?;
224
225            let to_read = range.end - range.start;
226            let mut buffer = Vec::with_capacity(to_read.try_into()?);
227            let read = self.take(to_read).read_to_end(&mut buffer).await?;
228            if read as u64 != to_read {
229                return Err(eof_err!("expected to read {} bytes, got {}", to_read, read));
230            }
231
232            Ok(buffer.into())
233        }
234        .boxed()
235    }
236
237    fn get_metadata<'a>(
238        &'a mut self,
239        options: Option<&'a ArrowReaderOptions>,
240    ) -> BoxFuture<'a, Result<Arc<ParquetMetaData>>> {
241        async move {
242            let metadata_reader = ParquetMetaDataReader::new().with_arrow_reader_options(options);
243            let parquet_metadata = metadata_reader.load_via_suffix_and_finish(self).await?;
244            Ok(Arc::new(parquet_metadata))
245        }
246        .boxed()
247    }
248}
249
250impl ArrowReaderMetadata {
251    /// Returns a new [`ArrowReaderMetadata`] for this builder
252    ///
253    /// See [`ParquetRecordBatchStreamBuilder::new_with_metadata`] for how this can be used
254    pub async fn load_async<T: AsyncFileReader>(
255        input: &mut T,
256        options: ArrowReaderOptions,
257    ) -> Result<Self> {
258        let metadata = input.get_metadata(Some(&options)).await?;
259        Self::try_new(metadata, options)
260    }
261}
262
263#[doc(hidden)]
264/// Newtype (wrapper) used within [`ArrowReaderBuilder`] to distinguish sync readers from async
265///
266/// Allows sharing the same builder for different readers while keeping the same
267/// ParquetRecordBatchStreamBuilder API
268pub struct AsyncReader<T>(T);
269
270/// A builder for reading parquet files from an `async` source as  [`ParquetRecordBatchStream`]
271///
272/// This can be used to decode a Parquet file in streaming fashion (without
273/// downloading the whole file at once) from a remote source, such as an object store.
274///
275/// This builder handles reading the parquet file metadata, allowing consumers
276/// to use this information to select what specific columns, row groups, etc.
277/// they wish to be read by the resulting stream.
278///
279/// See examples on [`ParquetRecordBatchStreamBuilder::new`], including how to
280/// issue multiple I/O requests in parallel using multiple streams.
281///
282/// # See also:
283/// * [`ParquetPushDecoderBuilder`] for lower level control over buffering and
284///   decoding.
285/// * [`ParquetRecordBatchStream::next_row_group`] for I/O prefetching
286///
287///
288/// See [`ArrowReaderBuilder`] for additional member functions
289pub type ParquetRecordBatchStreamBuilder<T> = ArrowReaderBuilder<AsyncReader<T>>;
290
291impl<T: AsyncFileReader + Send + 'static> ParquetRecordBatchStreamBuilder<T> {
292    /// Create a new [`ParquetRecordBatchStreamBuilder`] for reading from the
293    /// specified source.
294    ///
295    /// # Examples:
296    /// * [Basic example reading from an async source](#example)
297    /// * [Configuring options and reading metadata](#example-configuring-options-and-reading-metadata)
298    /// * [Reading Row Groups in Parallel](#example-reading-row-groups-in-parallel)
299    ///
300    /// # Example
301    /// ```
302    /// # #[tokio::main(flavor="current_thread")]
303    /// # async fn main() {
304    /// #
305    /// # use arrow_array::RecordBatch;
306    /// # use arrow::util::pretty::pretty_format_batches;
307    /// # use futures::TryStreamExt;
308    /// #
309    /// # use parquet::arrow::{ParquetRecordBatchStreamBuilder, ProjectionMask};
310    /// #
311    /// # fn assert_batches_eq(batches: &[RecordBatch], expected_lines: &[&str]) {
312    /// #     let formatted = pretty_format_batches(batches).unwrap().to_string();
313    /// #     let actual_lines: Vec<_> = formatted.trim().lines().collect();
314    /// #     assert_eq!(
315    /// #          &actual_lines, expected_lines,
316    /// #          "\n\nexpected:\n\n{:#?}\nactual:\n\n{:#?}\n\n",
317    /// #          expected_lines, actual_lines
318    /// #      );
319    /// #  }
320    /// #
321    /// # let testdata = arrow::util::test_util::parquet_test_data();
322    /// # let path = format!("{}/alltypes_plain.parquet", testdata);
323    /// // Use tokio::fs::File to read data using an async I/O. This can be replaced with
324    /// // another async I/O reader such as a reader from an object store.
325    /// let file = tokio::fs::File::open(path).await.unwrap();
326    ///
327    /// // Configure options for reading from the async source
328    /// let builder = ParquetRecordBatchStreamBuilder::new(file)
329    ///     .await
330    ///     .unwrap();
331    /// // Building the stream opens the parquet file (reads metadata, etc) and returns
332    /// // a stream that can be used to incrementally read the data in batches
333    /// let stream = builder.build().unwrap();
334    /// // In this example, we collect the stream into a Vec<RecordBatch>
335    /// // but real applications would likely process the batches as they are read
336    /// let results = stream.try_collect::<Vec<_>>().await.unwrap();
337    /// // Demonstrate the results are as expected
338    /// assert_batches_eq(
339    ///     &results,
340    ///     &[
341    ///       "+----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+---------------------+",
342    ///       "| id | bool_col | tinyint_col | smallint_col | int_col | bigint_col | float_col | double_col | date_string_col  | string_col | timestamp_col       |",
343    ///       "+----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+---------------------+",
344    ///       "| 4  | true     | 0           | 0            | 0       | 0          | 0.0       | 0.0        | 30332f30312f3039 | 30         | 2009-03-01T00:00:00 |",
345    ///       "| 5  | false    | 1           | 1            | 1       | 10         | 1.1       | 10.1       | 30332f30312f3039 | 31         | 2009-03-01T00:01:00 |",
346    ///       "| 6  | true     | 0           | 0            | 0       | 0          | 0.0       | 0.0        | 30342f30312f3039 | 30         | 2009-04-01T00:00:00 |",
347    ///       "| 7  | false    | 1           | 1            | 1       | 10         | 1.1       | 10.1       | 30342f30312f3039 | 31         | 2009-04-01T00:01:00 |",
348    ///       "| 2  | true     | 0           | 0            | 0       | 0          | 0.0       | 0.0        | 30322f30312f3039 | 30         | 2009-02-01T00:00:00 |",
349    ///       "| 3  | false    | 1           | 1            | 1       | 10         | 1.1       | 10.1       | 30322f30312f3039 | 31         | 2009-02-01T00:01:00 |",
350    ///       "| 0  | true     | 0           | 0            | 0       | 0          | 0.0       | 0.0        | 30312f30312f3039 | 30         | 2009-01-01T00:00:00 |",
351    ///       "| 1  | false    | 1           | 1            | 1       | 10         | 1.1       | 10.1       | 30312f30312f3039 | 31         | 2009-01-01T00:01:00 |",
352    ///       "+----+----------+-------------+--------------+---------+------------+-----------+------------+------------------+------------+---------------------+",
353    ///      ],
354    ///  );
355    /// # }
356    /// ```
357    ///
358    /// # Example Configuring Options and Reading Metadata
359    ///
360    /// There are many options that control the behavior of the reader, such as
361    /// `with_batch_size`, `with_projection`, `with_filter`, etc...
362    ///
363    /// ```
364    /// # #[tokio::main(flavor="current_thread")]
365    /// # async fn main() {
366    /// #
367    /// # use arrow_array::RecordBatch;
368    /// # use arrow::util::pretty::pretty_format_batches;
369    /// # use futures::TryStreamExt;
370    /// #
371    /// # use parquet::arrow::{ParquetRecordBatchStreamBuilder, ProjectionMask};
372    /// #
373    /// # fn assert_batches_eq(batches: &[RecordBatch], expected_lines: &[&str]) {
374    /// #     let formatted = pretty_format_batches(batches).unwrap().to_string();
375    /// #     let actual_lines: Vec<_> = formatted.trim().lines().collect();
376    /// #     assert_eq!(
377    /// #          &actual_lines, expected_lines,
378    /// #          "\n\nexpected:\n\n{:#?}\nactual:\n\n{:#?}\n\n",
379    /// #          expected_lines, actual_lines
380    /// #      );
381    /// #  }
382    /// #
383    /// # let testdata = arrow::util::test_util::parquet_test_data();
384    /// # let path = format!("{}/alltypes_plain.parquet", testdata);
385    /// // As before, use tokio::fs::File to read data using an async I/O.
386    /// let file = tokio::fs::File::open(path).await.unwrap();
387    ///
388    /// // Configure options for reading from the async source, in this case we set the batch size
389    /// // to 3 which produces 3 rows at a time.
390    /// let builder = ParquetRecordBatchStreamBuilder::new(file)
391    ///     .await
392    ///     .unwrap()
393    ///     .with_batch_size(3);
394    ///
395    /// // We can also read the metadata to inspect the schema and other metadata
396    /// // before actually reading the data
397    /// let file_metadata = builder.metadata().file_metadata();
398    /// // Specify that we only want to read the 1st, 2nd, and 6th columns
399    /// let mask = ProjectionMask::roots(file_metadata.schema_descr(), [1, 2, 6]);
400    ///
401    /// let stream = builder.with_projection(mask).build().unwrap();
402    /// let results = stream.try_collect::<Vec<_>>().await.unwrap();
403    /// // Print out the results
404    /// assert_batches_eq(
405    ///     &results,
406    ///     &[
407    ///         "+----------+-------------+-----------+",
408    ///         "| bool_col | tinyint_col | float_col |",
409    ///         "+----------+-------------+-----------+",
410    ///         "| true     | 0           | 0.0       |",
411    ///         "| false    | 1           | 1.1       |",
412    ///         "| true     | 0           | 0.0       |",
413    ///         "| false    | 1           | 1.1       |",
414    ///         "| true     | 0           | 0.0       |",
415    ///         "| false    | 1           | 1.1       |",
416    ///         "| true     | 0           | 0.0       |",
417    ///         "| false    | 1           | 1.1       |",
418    ///         "+----------+-------------+-----------+",
419    ///      ],
420    ///  );
421    ///
422    /// // The results has 8 rows, so since we set the batch size to 3, we expect
423    /// // 3 batches, two with 3 rows each and the last batch with 2 rows.
424    /// assert_eq!(results.len(), 3);
425    /// # }
426    /// ```
427    ///
428    /// # Example reading Row Groups in Parallel
429    ///
430    /// Each [`ParquetRecordBatchStream`] is independent and can be used to read
431    /// from the same underlying source in parallel. Use
432    /// [`ParquetRecordBatchStream::next_row_group`] with a single stream to
433    /// begin prefetching the next Row Group. To read a file in parallel, create
434    /// a stream for each subset of the file. For example, you can read each
435    /// row group in parallel by creating a stream for each row group using the
436    /// [`ParquetRecordBatchStreamBuilder::with_row_groups`] API as shown below
437    ///
438    /// ```
439    /// # use std::sync::Arc;
440    /// # use arrow_array::{ArrayRef, Int32Array, RecordBatch};
441    /// # use arrow::util::pretty::pretty_format_batches;
442    /// # use futures::{StreamExt, TryStreamExt};
443    /// # use tempfile::NamedTempFile;
444    /// # use parquet::arrow::{ArrowWriter, ParquetRecordBatchStreamBuilder, ProjectionMask};
445    /// # use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions};
446    /// # use parquet::file::metadata::ParquetMetaDataReader;
447    /// # use parquet::file::properties::{WriterProperties};
448    /// # // write to a temporary file with 10 RowGroups and read back with async API
449    /// # fn write_file() -> parquet::errors::Result<NamedTempFile> {
450    /// #   let mut file = NamedTempFile::new().unwrap();
451    /// #   let small_batch = RecordBatch::try_from_iter([
452    /// #      ("id", Arc::new(Int32Array::from(vec![0, 1, 2, 3, 4])) as ArrayRef),
453    /// #   ]).unwrap();
454    /// #   let props = WriterProperties::builder()
455    /// #     .set_max_row_group_row_count(Some(5))
456    /// #     .set_write_batch_size(5)
457    /// #     .build();
458    /// #   let mut writer = ArrowWriter::try_new(&mut file, small_batch.schema(), Some(props))?;
459    /// #   for i in 0..10 {
460    /// #     writer.write(&small_batch)?
461    /// #   };
462    /// #   writer.close()?;
463    /// #   Ok(file)
464    /// # }
465    /// # #[tokio::main(flavor="current_thread")]
466    /// # async fn main() -> parquet::errors::Result<()> {
467    /// # let t = write_file()?;
468    /// # let path = t.path();
469    /// // This example uses a tokio::fs::File as the async source, but it
470    /// // could be any async source such as an object store reader)
471    /// let mut file = tokio::fs::File::open(path).await?;
472    /// // To read Row Groups in parallel, create a separate stream builder for each Row Group.
473    /// // First get the metadata to find the row group information
474    /// let file_size = file.metadata().await?.len();
475    /// let metadata = ParquetMetaDataReader::new().load_and_finish(&mut file, file_size).await?;
476    /// assert_eq!(metadata.num_row_groups(), 10); // file has 10 row groups with 5 rows each
477    /// // Create a stream reader for each row group
478    /// let reader_metadata = ArrowReaderMetadata::try_new(
479    ///   Arc::new(metadata),
480    ///   ArrowReaderOptions::new()
481    /// )?;
482    /// let mut streams = vec![];
483    ///  for row_group_index in 0..10 {
484    ///   // Each stream needs its own source instance to issue
485    ///   // parallel IO requests, so clone the file for each stream
486    ///   let this_file = file.try_clone().await?;
487    ///   let stream = ParquetRecordBatchStreamBuilder::new_with_metadata(
488    ///        this_file,
489    ///        reader_metadata.clone()
490    ///      )
491    ///      .with_row_groups(vec![row_group_index]) // read only this row group
492    ///      .build()?;
493    ///     streams.push(stream);
494    /// }
495    /// // Each reader can now be polled independently and in parallel, for
496    /// // example using StreamExt::buffered to read from 3 at a time
497    /// let results = futures::stream::iter(streams)
498    ///  .map(|stream| async move { stream })
499    ///  .buffered(3)
500    ///  .flatten()
501    ///  .try_collect::<Vec<_>>().await?;
502    /// // read all 50 rows (10 row groups x 5 rows per group)
503    /// assert_eq!(50, results.iter().map(|s| s.num_rows()).sum::<usize>());
504    /// # Ok(())
505    /// # }
506    /// ```
507    pub async fn new(input: T) -> Result<Self> {
508        Self::new_with_options(input, Default::default()).await
509    }
510
511    /// Create a new [`ParquetRecordBatchStreamBuilder`] with the provided async source
512    /// and [`ArrowReaderOptions`].
513    pub async fn new_with_options(mut input: T, options: ArrowReaderOptions) -> Result<Self> {
514        let metadata = ArrowReaderMetadata::load_async(&mut input, options).await?;
515        Ok(Self::new_with_metadata(input, metadata))
516    }
517
518    /// Create a [`ParquetRecordBatchStreamBuilder`] from the provided [`ArrowReaderMetadata`]
519    ///
520    /// This allows loading metadata once and using it to create multiple builders with
521    /// potentially different settings, that can be read in parallel.
522    ///
523    /// # Example of reading from multiple streams in parallel
524    ///
525    /// ```
526    /// # use std::fs::metadata;
527    /// # use std::sync::Arc;
528    /// # use bytes::Bytes;
529    /// # use arrow_array::{Int32Array, RecordBatch};
530    /// # use arrow_schema::{DataType, Field, Schema};
531    /// # use parquet::arrow::arrow_reader::ArrowReaderMetadata;
532    /// # use parquet::arrow::{ArrowWriter, ParquetRecordBatchStreamBuilder};
533    /// # use tempfile::tempfile;
534    /// # use futures::StreamExt;
535    /// # #[tokio::main(flavor="current_thread")]
536    /// # async fn main() {
537    /// #
538    /// # let mut file = tempfile().unwrap();
539    /// # let schema = Arc::new(Schema::new(vec![Field::new("i32", DataType::Int32, false)]));
540    /// # let mut writer = ArrowWriter::try_new(&mut file, schema.clone(), None).unwrap();
541    /// # let batch = RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]).unwrap();
542    /// # writer.write(&batch).unwrap();
543    /// # writer.close().unwrap();
544    /// // open file with parquet data
545    /// let mut file = tokio::fs::File::from_std(file);
546    /// // load metadata once
547    /// let meta = ArrowReaderMetadata::load_async(&mut file, Default::default()).await.unwrap();
548    /// // create two readers, a and b, from the same underlying file
549    /// // without reading the metadata again
550    /// let mut a = ParquetRecordBatchStreamBuilder::new_with_metadata(
551    ///     file.try_clone().await.unwrap(),
552    ///     meta.clone()
553    /// ).build().unwrap();
554    /// let mut b = ParquetRecordBatchStreamBuilder::new_with_metadata(file, meta).build().unwrap();
555    ///
556    /// // Can read batches from both readers in parallel
557    /// assert_eq!(
558    ///   a.next().await.unwrap().unwrap(),
559    ///   b.next().await.unwrap().unwrap(),
560    /// );
561    /// # }
562    /// ```
563    pub fn new_with_metadata(input: T, metadata: ArrowReaderMetadata) -> Self {
564        Self::new_builder(AsyncReader(input), metadata)
565    }
566
567    /// Read bloom filter for a column in a row group
568    ///
569    /// Returns `None` if the column does not have a bloom filter
570    ///
571    /// We should call this function after other forms pruning, such as projection and predicate pushdown.
572    pub async fn get_row_group_column_bloom_filter(
573        &mut self,
574        row_group_idx: usize,
575        column_idx: usize,
576    ) -> Result<Option<Sbbf>> {
577        let metadata = self.metadata.row_group(row_group_idx);
578        let column_metadata = metadata.column(column_idx);
579
580        Sbbf::read_from_column_chunk_async(column_metadata, &mut self.input.0).await
581    }
582
583    /// Select row groups and rows using row-group-local coordinates.
584    ///
585    /// Entries are decoded in the supplied order, omitted row groups are
586    /// skipped, and a `None` selection reads the whole row group. This is
587    /// mutually exclusive with [`ArrowReaderBuilder::with_row_groups`] and
588    /// [`ArrowReaderBuilder::with_row_selection`]; combining them returns an
589    /// error from [`Self::build`].
590    ///
591    /// See [`ParquetPushDecoderBuilder::with_row_group_selections`] for the
592    /// full semantics and a worked example. This builder supports the same API
593    /// because the async stream is implemented using the push decoder; the
594    /// synchronous reader does not support row-group-local selections.
595    ///
596    /// [`ParquetPushDecoderBuilder::with_row_group_selections`]: crate::arrow::push_decoder::ParquetPushDecoderBuilder::with_row_group_selections
597    pub fn with_row_group_selections(
598        mut self,
599        row_group_selections: Vec<RowGroupSelection>,
600    ) -> Self {
601        self.row_group_plan
602            .set_row_group_selections(row_group_selections);
603        self
604    }
605
606    /// Read and decode the dictionary page for a column in a row group, if any.
607    ///
608    /// Returns `Ok(None)` if the column chunk has no dictionary page, or if
609    /// its physical type is not `BYTE_ARRAY` (the only physical type
610    /// currently supported).
611    ///
612    /// The returned array contains raw `Binary` values, even for columns
613    /// annotated as strings. Callers can compare byte slices directly or
614    /// convert values to UTF-8 explicitly.
615    ///
616    /// This can be used to inspect dictionary values when selecting or pruning
617    /// row groups before passing the selected indices to
618    /// [`ParquetRecordBatchStreamBuilder::with_row_groups`].
619    ///
620    /// Note this does not verify that the *entire* column chunk is
621    /// dictionary-encoded -- callers that need that guarantee (e.g. to treat
622    /// the dictionary as an exhaustive set of the column's values) should
623    /// check
624    /// [`crate::file::metadata::ColumnChunkMetaData::page_encoding_stats_mask`].
625    pub async fn get_column_chunk_dictionary(
626        &mut self,
627        row_group_idx: usize,
628        column_idx: usize,
629    ) -> Result<Option<ArrayRef>> {
630        ParquetMetaDataReader::read_column_dictionary_async(
631            &mut self.input.0,
632            &self.metadata,
633            row_group_idx,
634            column_idx,
635        )
636        .await
637    }
638
639    /// Build a new [`ParquetRecordBatchStream`]
640    ///
641    /// See examples on [`ParquetRecordBatchStreamBuilder::new`]
642    pub fn build(self) -> Result<ParquetRecordBatchStream<T>> {
643        let Self {
644            input,
645            metadata,
646            schema,
647            fields,
648            batch_size,
649            row_group_plan,
650            projection,
651            filter,
652            row_selection_policy: selection_strategy,
653            limit,
654            offset,
655            metrics,
656            max_predicate_cache_size,
657        } = self;
658
659        // Ensure schema of ParquetRecordBatchStream respects projection, and does
660        // not store metadata (same as for ParquetRecordBatchReader and emitted RecordBatches)
661        let projection_len = projection.mask.as_ref().map_or(usize::MAX, |m| m.len());
662        let projected_fields = schema
663            .fields
664            .filter_leaves(|idx, _| idx < projection_len && projection.leaf_included(idx));
665        let projected_schema = Arc::new(Schema::new(projected_fields));
666
667        let decoder = ParquetPushDecoderBuilder {
668            input: PushDecoderInput::default(),
669            metadata,
670            schema,
671            fields,
672            projection,
673            filter,
674            row_group_plan,
675            row_selection_policy: selection_strategy,
676            batch_size,
677            limit,
678            offset,
679            metrics,
680            max_predicate_cache_size,
681        }
682        .build()?;
683
684        let request_state = RequestState::None { input: input.0 };
685
686        Ok(ParquetRecordBatchStream {
687            schema: projected_schema,
688            decoder,
689            request_state,
690        })
691    }
692}
693
694/// State machine that tracks outstanding requests to fetch data
695///
696/// The parameter `T` is the input, typically an `AsyncFileReader`
697enum RequestState<T> {
698    /// No outstanding requests
699    None {
700        input: T,
701    },
702    /// There is an outstanding request for data
703    Outstanding {
704        /// Ranges that have been requested
705        ranges: Vec<Range<u64>>,
706        /// Future that will resolve (input, requested_ranges)
707        ///
708        /// Note the future owns the reader while the request is outstanding
709        /// and returns it upon completion
710        future: BoxFuture<'static, Result<(T, Vec<Bytes>)>>,
711    },
712    Done,
713}
714
715impl<T> RequestState<T>
716where
717    T: AsyncFileReader + Unpin + Send + 'static,
718{
719    /// Issue a request to fetch `ranges`, returning the Outstanding state
720    fn begin_request(mut input: T, ranges: Vec<Range<u64>>) -> Self {
721        let ranges_captured = ranges.clone();
722
723        // Note this must move the input *into* the future
724        // because the get_byte_ranges future has a lifetime
725        // (aka can have references internally) and thus must
726        // own the input while the request is outstanding.
727        let future = async move {
728            let data = input.get_byte_ranges(ranges_captured).await?;
729            Ok((input, data))
730        }
731        .boxed();
732        RequestState::Outstanding { ranges, future }
733    }
734}
735
736impl<T> std::fmt::Debug for RequestState<T> {
737    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
738        match self {
739            RequestState::None { input: _ } => f
740                .debug_struct("RequestState::None")
741                .field("input", &"...")
742                .finish(),
743            RequestState::Outstanding { ranges, .. } => f
744                .debug_struct("RequestState::Outstanding")
745                .field("ranges", &ranges)
746                .finish(),
747            RequestState::Done => {
748                write!(f, "RequestState::Done")
749            }
750        }
751    }
752}
753
754/// An asynchronous [`Stream`]of [`RecordBatch`] constructed using [`ParquetRecordBatchStreamBuilder`] to read parquet files.
755///
756/// `ParquetRecordBatchStream` also provides [`ParquetRecordBatchStream::next_row_group`] for fetching row groups,
757/// allowing users to decode record batches separately from I/O.
758///
759/// # I/O Buffering
760///
761/// `ParquetRecordBatchStream` buffers *all* data pages selected after predicates
762/// (projection + filtering, etc) and decodes the rows from those buffered pages.
763///
764/// For example, if all rows and columns are selected, the entire row group is
765/// buffered in memory during decode. This minimizes the number of IO operations
766/// required, which is especially important for object stores, where IO operations
767/// have latencies in the hundreds of milliseconds
768///
769/// See [`ParquetPushDecoderBuilder`] for an API with lower level control over
770/// buffering.
771///
772/// [`Stream`]: https://docs.rs/futures/latest/futures/stream/trait.Stream.html
773pub struct ParquetRecordBatchStream<T> {
774    /// Output schema of the stream
775    schema: SchemaRef,
776    /// Input and Outstanding IO request, if any
777    request_state: RequestState<T>,
778    /// Decoding state machine (no IO)
779    decoder: ParquetPushDecoder,
780}
781
782impl<T> std::fmt::Debug for ParquetRecordBatchStream<T> {
783    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
784        f.debug_struct("ParquetRecordBatchStream")
785            .field("request_state", &self.request_state)
786            .finish()
787    }
788}
789
790impl<T> ParquetRecordBatchStream<T> {
791    /// Returns the projected [`SchemaRef`] for reading the parquet file.
792    ///
793    /// Note that the schema metadata will be stripped here. See
794    /// [`ParquetRecordBatchStreamBuilder::schema`] if the metadata is desired.
795    pub fn schema(&self) -> &SchemaRef {
796        &self.schema
797    }
798}
799
800impl<T> ParquetRecordBatchStream<T>
801where
802    T: AsyncFileReader + Unpin + Send + 'static,
803{
804    /// Fetches the next row group from the stream.
805    ///
806    /// Users can continue to call this function to get row groups and decode them concurrently.
807    ///
808    /// ## Notes
809    ///
810    /// ParquetRecordBatchStream should be used either as a `Stream` or with `next_row_group`; they should not be used simultaneously.
811    ///
812    /// ## Returns
813    ///
814    /// - `Ok(None)` if the stream has ended.
815    /// - `Err(error)` if the stream has errored. All subsequent calls will return `Ok(None)`.
816    /// - `Ok(Some(reader))` which holds all the data for the row group.
817    pub async fn next_row_group(&mut self) -> Result<Option<ParquetRecordBatchReader>> {
818        loop {
819            // Take ownership of request state to process, leaving self in a
820            // valid state
821            let request_state = std::mem::replace(&mut self.request_state, RequestState::Done);
822            match request_state {
823                // No outstanding requests, proceed to setup next row group
824                RequestState::None { input } => {
825                    match self.decoder.try_next_reader()? {
826                        DecodeResult::NeedsData(ranges) => {
827                            self.request_state = RequestState::begin_request(input, ranges);
828                            // Will loop again: the input might be ready immediately.
829                        }
830                        DecodeResult::Data(reader) => {
831                            self.request_state = RequestState::None { input };
832                            return Ok(Some(reader));
833                        }
834                        DecodeResult::Finished => return Ok(None),
835                    }
836                }
837                RequestState::Outstanding { ranges, future } => {
838                    let (input, data) = future.await?;
839                    // Push the requested data to the decoder and try again
840                    self.decoder.push_ranges(ranges, data)?;
841                    self.request_state = RequestState::None { input };
842                    // Will try and decode on the next iteration.
843                }
844                RequestState::Done => {
845                    self.request_state = RequestState::Done;
846                    return Ok(None);
847                }
848            }
849        }
850    }
851}
852
853impl<T> Stream for ParquetRecordBatchStream<T>
854where
855    T: AsyncFileReader + Unpin + Send + 'static,
856{
857    type Item = Result<RecordBatch>;
858    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
859        match self.poll_next_inner(cx) {
860            Ok(res) => {
861                // Successfully decoded a batch, or reached end of stream.
862                // convert Option<RecordBatch> to Option<Result<RecordBatch>>
863                res.map(|res| Ok(res).transpose())
864            }
865            Err(e) => {
866                self.request_state = RequestState::Done;
867                Poll::Ready(Some(Err(e)))
868            }
869        }
870    }
871}
872
873impl<T> ParquetRecordBatchStream<T>
874where
875    T: AsyncFileReader + Unpin + Send + 'static,
876{
877    /// Inner state machine
878    ///
879    /// Note this is separate from poll_next so we can use ? operator to check for errors
880    /// as it returns `Result<Poll<Option<RecordBatch>>>`
881    fn poll_next_inner(&mut self, cx: &mut Context<'_>) -> Result<Poll<Option<RecordBatch>>> {
882        loop {
883            let request_state = std::mem::replace(&mut self.request_state, RequestState::Done);
884            match request_state {
885                RequestState::None { input } => {
886                    // No outstanding requests, proceed to decode the next batch
887                    match self.decoder.try_decode()? {
888                        DecodeResult::NeedsData(ranges) => {
889                            self.request_state = RequestState::begin_request(input, ranges);
890                            // Will loop again: the input might be ready immediately.
891                        }
892                        DecodeResult::Data(batch) => {
893                            self.request_state = RequestState::None { input };
894                            return Ok(Poll::Ready(Some(batch)));
895                        }
896                        DecodeResult::Finished => {
897                            self.request_state = RequestState::Done;
898                            return Ok(Poll::Ready(None));
899                        }
900                    }
901                }
902                RequestState::Outstanding { ranges, mut future } => match future.poll_unpin(cx) {
903                    // Data was ready, push it to the decoder and continue
904                    Poll::Ready(result) => {
905                        let (input, data) = result?;
906                        // Push the requested data to the decoder
907                        self.decoder.push_ranges(ranges, data)?;
908                        self.request_state = RequestState::None { input };
909                        // The next iteration will try to decode the next batch.
910                    }
911                    Poll::Pending => {
912                        self.request_state = RequestState::Outstanding { ranges, future };
913                        return Ok(Poll::Pending);
914                    }
915                },
916                RequestState::Done => {
917                    // Stream is done (error or end), return None
918                    self.request_state = RequestState::Done;
919                    return Ok(Poll::Ready(None));
920                }
921            }
922        }
923    }
924}
925
926#[cfg(test)]
927mod tests {
928    use super::*;
929    use crate::arrow::ArrowWriter;
930    use crate::arrow::ProjectionMask;
931    use crate::arrow::arrow_reader::tests::test_row_numbers_with_multiple_row_groups_helper;
932    use crate::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions};
933    use crate::arrow::schema::virtual_type::RowNumber;
934    use crate::basic::Encoding;
935    use crate::file::metadata::PageIndexPolicy;
936    use crate::file::metadata::ParquetMetaDataReader;
937    use crate::file::metadata::page_index::PageIndex;
938    use crate::file::properties::WriterProperties;
939    use arrow_array::cast::AsArray;
940    use arrow_array::{Array, ArrayRef, BinaryArray, StringArray};
941    use arrow_schema::{DataType, Field, Schema};
942    use futures::TryStreamExt;
943    use std::sync::{Arc, Mutex};
944
945    #[derive(Clone)]
946    struct TestReader {
947        data: Bytes,
948        metadata: Option<Arc<ParquetMetaData>>,
949        requests: Arc<Mutex<Vec<Range<usize>>>>,
950    }
951
952    impl TestReader {
953        fn new(data: Bytes) -> Self {
954            Self {
955                data,
956                metadata: Default::default(),
957                requests: Default::default(),
958            }
959        }
960    }
961
962    impl AsyncFileReader for TestReader {
963        fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes>> {
964            let range = range.clone();
965            self.requests
966                .lock()
967                .unwrap()
968                .push(range.start as usize..range.end as usize);
969            futures::future::ready(Ok(self
970                .data
971                .slice(range.start as usize..range.end as usize)))
972            .boxed()
973        }
974
975        fn get_metadata<'a>(
976            &'a mut self,
977            options: Option<&'a ArrowReaderOptions>,
978        ) -> BoxFuture<'a, Result<Arc<ParquetMetaData>>> {
979            let metadata_reader = ParquetMetaDataReader::new().with_arrow_reader_options(options);
980            self.metadata = Some(Arc::new(
981                metadata_reader.parse_and_finish(&self.data).unwrap(),
982            ));
983            futures::future::ready(Ok(self.metadata.clone().unwrap().clone())).boxed()
984        }
985    }
986
987    #[tokio::test]
988    async fn test_get_column_chunk_dictionary() {
989        let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)]));
990        let values: Vec<&str> = ["alpha", "beta", "gamma"]
991            .iter()
992            .copied()
993            .cycle()
994            .take(30)
995            .collect();
996        let array: ArrayRef = Arc::new(StringArray::from(values));
997        let batch = RecordBatch::try_new(schema.clone(), vec![array]).unwrap();
998
999        let props = WriterProperties::builder()
1000            .set_dictionary_enabled(true)
1001            .build();
1002        let mut buf = Vec::new();
1003        {
1004            let mut writer = ArrowWriter::try_new(&mut buf, schema, Some(props)).unwrap();
1005            writer.write(&batch).unwrap();
1006            writer.close().unwrap();
1007        }
1008        let data = Bytes::from(buf);
1009
1010        let direct_metadata = ParquetMetaDataReader::new()
1011            .parse_and_finish(&data)
1012            .unwrap();
1013        let mut direct_reader = TestReader::new(data.clone());
1014        let direct = ParquetMetaDataReader::read_column_dictionary_async(
1015            &mut direct_reader,
1016            &direct_metadata,
1017            0,
1018            0,
1019        )
1020        .await
1021        .unwrap()
1022        .unwrap();
1023        let direct = direct.as_any().downcast_ref::<BinaryArray>().unwrap();
1024        assert_eq!(direct.value(0), b"alpha");
1025
1026        let async_reader = TestReader::new(data);
1027        let mut builder = ParquetRecordBatchStreamBuilder::new(async_reader)
1028            .await
1029            .unwrap();
1030
1031        let dictionary = builder
1032            .get_column_chunk_dictionary(0, 0)
1033            .await
1034            .unwrap()
1035            .unwrap();
1036        let dictionary = dictionary.as_any().downcast_ref::<BinaryArray>().unwrap();
1037        let dictionary_values: Vec<&[u8]> = dictionary.iter().map(|v| v.unwrap()).collect();
1038        assert_eq!(
1039            dictionary_values,
1040            vec![b"alpha".as_slice(), b"beta", b"gamma"]
1041        );
1042    }
1043
1044    // This test demonstrates row group pruning using dictionary pages,
1045    // verifying that data pages of skipped row groups are not read
1046    #[tokio::test]
1047    async fn test_dictionary_selects_row_groups_without_reading_skipped_data() {
1048        // Write two row groups, each dictionary-encoded and containing a single
1049        // distinct string repeated 30 times: row group 0 is all "skip", row
1050        // group 1 is all "target".
1051        let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)]));
1052        let row_group_values = ["skip", "target"];
1053        let props = WriterProperties::builder()
1054            .set_dictionary_enabled(true)
1055            .build();
1056        let mut buf = Vec::new();
1057        {
1058            let mut writer = ArrowWriter::try_new(&mut buf, schema.clone(), Some(props)).unwrap();
1059            for value in row_group_values {
1060                let array: ArrayRef = Arc::new(StringArray::from(vec![value; 30]));
1061                let batch = RecordBatch::try_new(schema.clone(), vec![array]).unwrap();
1062                writer.write(&batch).unwrap();
1063                writer.flush().unwrap();
1064            }
1065            writer.close().unwrap();
1066        }
1067
1068        let async_reader = TestReader::new(Bytes::from(buf));
1069        // `requests` records the byte ranges fetched from the underlying reader,
1070        // which we later use to check that we read only the needed page
1071        let requests = async_reader.requests.clone();
1072        let mut builder = ParquetRecordBatchStreamBuilder::new(async_reader)
1073            .await
1074            .unwrap();
1075        let metadata = builder.metadata().clone();
1076        assert_eq!(metadata.num_row_groups(), 2);
1077
1078        // For each row group, fetch and decode just the dictionary page
1079        // to decide whether to read the whole row group
1080        let mut selected_row_groups = Vec::new();
1081        let mut dictionary_ranges = Vec::new();
1082        for row_group_idx in 0..metadata.num_row_groups() {
1083            let column = metadata.row_group(row_group_idx).column(0);
1084            let encoding_mask = column.page_encoding_stats_mask().unwrap();
1085            assert!(
1086                encoding_mask.is_only(Encoding::PLAIN_DICTIONARY)
1087                    || encoding_mask.is_only(Encoding::RLE_DICTIONARY)
1088            );
1089
1090            let dictionary_start = column.dictionary_page_offset().unwrap() as usize;
1091            let data_start = column.data_page_offset() as usize;
1092            dictionary_ranges.push(dictionary_start..data_start);
1093
1094            let dictionary = builder
1095                .get_column_chunk_dictionary(row_group_idx, 0)
1096                .await
1097                .unwrap()
1098                .unwrap();
1099            let dictionary = dictionary.as_binary::<i32>();
1100            if dictionary
1101                .iter()
1102                .any(|value| value == Some(b"target".as_slice()))
1103            {
1104                selected_row_groups.push(row_group_idx);
1105            }
1106        }
1107        assert_eq!(selected_row_groups, vec![1]);
1108
1109        // Read the filtered row group and verify the values
1110        let batches: Vec<_> = builder
1111            .with_row_groups(selected_row_groups)
1112            .build()
1113            .unwrap()
1114            .try_collect()
1115            .await
1116            .unwrap();
1117        let values: Vec<_> = batches
1118            .iter()
1119            .flat_map(|batch| batch.column(0).as_string::<i32>().iter())
1120            .collect();
1121        assert_eq!(values, vec![Some("target"); 30]);
1122
1123        // Finally, verify none of the requests overlapped the data pages
1124        // of the skipped row group
1125        let skipped_column = metadata.row_group(0).column(0);
1126        let (skipped_start, skipped_len) = skipped_column.byte_range();
1127        let skipped_data_range =
1128            skipped_column.data_page_offset() as usize..(skipped_start + skipped_len) as usize;
1129        let requests = requests.lock().unwrap();
1130        for dictionary_range in dictionary_ranges {
1131            assert!(requests.contains(&dictionary_range));
1132        }
1133        assert!(requests.iter().all(|request| {
1134            request.end <= skipped_data_range.start || request.start >= skipped_data_range.end
1135        }));
1136    }
1137
1138    #[tokio::test]
1139    async fn test_batch_size_overallocate() {
1140        let testdata = arrow::util::test_util::parquet_test_data();
1141        // `alltypes_plain.parquet` only have 8 rows
1142        let path = format!("{testdata}/alltypes_plain.parquet");
1143        let data = Bytes::from(std::fs::read(path).unwrap());
1144
1145        let async_reader = TestReader::new(data.clone());
1146
1147        let builder = ParquetRecordBatchStreamBuilder::new(async_reader)
1148            .await
1149            .unwrap();
1150
1151        let file_rows = builder.metadata().file_metadata().num_rows() as usize;
1152
1153        let builder = builder
1154            .with_projection(ProjectionMask::all())
1155            .with_batch_size(1024);
1156
1157        // even though the batch size is set to 1024, it should adjust to the max
1158        // number of rows in the file (8)
1159        assert_ne!(1024, file_rows);
1160        assert_eq!(builder.batch_size, file_rows);
1161
1162        let _stream = builder.build().unwrap();
1163    }
1164
1165    #[tokio::test]
1166    async fn empty_offset_index_doesnt_panic_in_read_row_group() {
1167        use tokio::fs::File;
1168        let testdata = arrow::util::test_util::parquet_test_data();
1169        let path = format!("{testdata}/alltypes_plain.parquet");
1170        let mut file = File::open(&path).await.unwrap();
1171        let file_size = file.metadata().await.unwrap().len();
1172        let mut metadata = ParquetMetaDataReader::new()
1173            .with_page_index_policy(PageIndexPolicy::Required)
1174            .load_and_finish(&mut file, file_size)
1175            .await
1176            .unwrap();
1177
1178        let page_index = PageIndex::try_new(None, Some(vec![])).unwrap();
1179        metadata.set_page_index(Some(Arc::new(page_index)));
1180        let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required);
1181        let arrow_reader_metadata = ArrowReaderMetadata::try_new(metadata.into(), options).unwrap();
1182        let reader =
1183            ParquetRecordBatchStreamBuilder::new_with_metadata(file, arrow_reader_metadata)
1184                .build()
1185                .unwrap();
1186
1187        let result = reader.try_collect::<Vec<_>>().await.unwrap();
1188        assert_eq!(result.len(), 1);
1189    }
1190
1191    #[test]
1192    fn test_row_numbers_with_multiple_row_groups() {
1193        test_row_numbers_with_multiple_row_groups_helper(
1194            false,
1195            |path, selection, _row_filter, batch_size| {
1196                let runtime = tokio::runtime::Builder::new_current_thread()
1197                    .enable_all()
1198                    .build()
1199                    .expect("Could not create runtime");
1200                runtime.block_on(async move {
1201                    let file = tokio::fs::File::open(path).await.unwrap();
1202                    let row_number_field = Arc::new(
1203                        Field::new("row_number", DataType::Int64, false)
1204                            .with_extension_type(RowNumber),
1205                    );
1206                    let options = ArrowReaderOptions::new()
1207                        .with_virtual_columns(vec![row_number_field])
1208                        .unwrap();
1209                    let reader = ParquetRecordBatchStreamBuilder::new_with_options(file, options)
1210                        .await
1211                        .unwrap()
1212                        .with_row_selection(selection)
1213                        .with_batch_size(batch_size)
1214                        .build()
1215                        .expect("Could not create reader");
1216                    reader.try_collect::<Vec<_>>().await.unwrap()
1217                })
1218            },
1219        );
1220    }
1221
1222    #[test]
1223    fn test_row_numbers_with_multiple_row_groups_and_filter() {
1224        test_row_numbers_with_multiple_row_groups_helper(
1225            true,
1226            |path, selection, row_filter, batch_size| {
1227                let runtime = tokio::runtime::Builder::new_current_thread()
1228                    .enable_all()
1229                    .build()
1230                    .expect("Could not create runtime");
1231                runtime.block_on(async move {
1232                    let file = tokio::fs::File::open(path).await.unwrap();
1233                    let row_number_field = Arc::new(
1234                        Field::new("row_number", DataType::Int64, false)
1235                            .with_extension_type(RowNumber),
1236                    );
1237                    let options = ArrowReaderOptions::new()
1238                        .with_virtual_columns(vec![row_number_field])
1239                        .unwrap();
1240                    let reader = ParquetRecordBatchStreamBuilder::new_with_options(file, options)
1241                        .await
1242                        .unwrap()
1243                        .with_row_selection(selection)
1244                        .with_row_filter(row_filter.expect("No row filter"))
1245                        .with_batch_size(batch_size)
1246                        .build()
1247                        .expect("Could not create reader");
1248                    reader.try_collect::<Vec<_>>().await.unwrap()
1249                })
1250            },
1251        );
1252    }
1253}