Skip to main content

parquet/arrow/arrow_reader/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Contains reader which reads parquet data into arrow [`RecordBatch`]
6
7use arrow_array::cast::AsArray;
8use arrow_array::{BooleanArray, RecordBatch, RecordBatchReader};
9use arrow_buffer::{BooleanBuffer, BooleanBufferBuilder};
10use arrow_schema::{ArrowError, DataType as ArrowType, FieldRef, Schema, SchemaRef};
11use arrow_select::filter::filter_record_batch;
12pub use filter::{ArrowPredicate, ArrowPredicateFn, RowFilter};
13use selection::MaskCursor;
14pub use selection::{
15    MaskRunIter, RowSelection, RowSelectionCursor, RowSelectionPolicy, RowSelector,
16};
17use std::fmt::{Debug, Formatter};
18use std::sync::Arc;
19
20pub use crate::arrow::array_reader::RowGroups;
21use crate::arrow::array_reader::{ArrayReader, ArrayReaderBuilder};
22use crate::arrow::schema::{
23    ParquetField, parquet_to_arrow_schema_and_fields, virtual_type::is_virtual_column,
24};
25use crate::arrow::{FieldLevels, ProjectionMask, parquet_to_arrow_field_levels_with_virtual};
26use crate::bloom_filter::Sbbf;
27use crate::column::page::{PageIterator, PageReader};
28#[cfg(feature = "encryption")]
29use crate::encryption::decrypt::FileDecryptionProperties;
30use crate::errors::{ParquetError, Result};
31use crate::file::metadata::{
32    ColumnChunkMask, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions,
33    ParquetMetaDataReader, ParquetStatisticsPolicy, RowGroupMetaData,
34};
35use crate::file::reader::{ChunkReader, SerializedPageReader};
36use crate::schema::types::SchemaDescriptor;
37
38use crate::arrow::arrow_reader::metrics::ArrowReaderMetrics;
39// Exposed so integration tests and benchmarks can temporarily override the threshold.
40pub use read_plan::{PredicateOptions, ReadPlan, ReadPlanBuilder};
41
42mod filter;
43pub mod metrics;
44mod read_plan;
45pub(crate) mod selection;
46pub mod statistics;
47
48/// Default batch size for reading parquet files
49pub const DEFAULT_BATCH_SIZE: usize = 1024;
50
51/// A row group and its optional row-group-local [`RowSelection`].
52///
53/// A row-group-local selection is relative to the rows in this row group. For
54/// example, an offset of 100 refers to the row at offset 100 within the row
55/// group, not within the Parquet file.
56///
57/// A `None` selection reads the entire row group. Omitting a row group skips
58/// it. Entries are decoded in the supplied order.
59#[derive(Debug, Clone, PartialEq, Eq)]
60pub struct RowGroupSelection {
61    pub(crate) row_group_index: usize,
62    pub(crate) selection: Option<RowSelection>,
63}
64
65impl RowGroupSelection {
66    /// Creates a row-group-local selection.
67    pub fn new(row_group_index: usize, selection: Option<RowSelection>) -> Self {
68        Self {
69            row_group_index,
70            selection,
71        }
72    }
73
74    /// The index of the row group this selection applies to.
75    pub fn row_group_index(&self) -> usize {
76        self.row_group_index
77    }
78
79    /// The row-group-local selection, or `None` if the entire row group is
80    /// read.
81    pub fn selection(&self) -> Option<&RowSelection> {
82        self.selection.as_ref()
83    }
84}
85
86/// Row-selection configuration shared by the Arrow reader builders.
87#[derive(Debug)]
88pub(crate) enum RowGroupPlan {
89    /// First select `row_groups`, if provided, and then apply `selection`
90    /// across the concatenated rows from those row groups.
91    ///
92    /// This is formed by [`ArrowReaderBuilder::with_row_groups`] and
93    /// [`ArrowReaderBuilder::with_row_selection`].
94    Global {
95        row_groups: Option<Vec<usize>>,
96        selection: Option<RowSelection>,
97    },
98    /// Apply each row-group-local selection independently, in the order
99    /// supplied.
100    ///
101    /// This is formed by `with_row_group_selections` on the push decoder and
102    /// async stream builders.
103    PerRowGroup(Vec<RowGroupSelection>),
104    /// Mutually exclusive global and per-row-group configuration was supplied.
105    /// This is reported as an error when the reader is built.
106    Conflicting,
107}
108
109impl RowGroupPlan {
110    fn set_row_groups(&mut self, new_row_groups: Vec<usize>) {
111        match self {
112            Self::Global { row_groups, .. } => *row_groups = Some(new_row_groups),
113            Self::PerRowGroup(_) => *self = Self::Conflicting,
114            Self::Conflicting => {}
115        }
116    }
117
118    fn set_row_selection(&mut self, new_selection: RowSelection) {
119        match self {
120            Self::Global { selection, .. } => *selection = Some(new_selection),
121            Self::PerRowGroup(_) => *self = Self::Conflicting,
122            Self::Conflicting => {}
123        }
124    }
125
126    pub(crate) fn set_row_group_selections(
127        &mut self,
128        row_group_selections: Vec<RowGroupSelection>,
129    ) {
130        match self {
131            Self::Global {
132                row_groups: None,
133                selection: None,
134            }
135            | Self::PerRowGroup(_) => {
136                *self = Self::PerRowGroup(row_group_selections);
137            }
138            Self::Global { .. } => *self = Self::Conflicting,
139            Self::Conflicting => {}
140        }
141    }
142
143    pub(crate) fn conflict_error() -> ParquetError {
144        ParquetError::General(
145            "with_row_group_selections cannot be combined with with_row_groups or with_row_selection"
146                .to_string(),
147        )
148    }
149
150    fn into_global(self) -> Result<(Option<Vec<usize>>, Option<RowSelection>)> {
151        match self {
152            Self::Global {
153                row_groups,
154                selection,
155            } => Ok((row_groups, selection)),
156            Self::PerRowGroup(_) => Err(ParquetError::General(
157                "Row-group-local selections are not supported by the synchronous reader"
158                    .to_string(),
159            )),
160            Self::Conflicting => Err(Self::conflict_error()),
161        }
162    }
163}
164
165/// Builder for constructing Parquet readers that decode into [Apache Arrow]
166/// arrays.
167///
168/// Most users should use one of the following specializations:
169///
170/// * synchronous API: [`ParquetRecordBatchReaderBuilder`]
171/// * `async` API: [`ParquetRecordBatchStreamBuilder`]
172/// * decoder API: [`ParquetPushDecoderBuilder`]
173///
174/// # Features
175/// * Projection pushdown: [`Self::with_projection`]
176/// * Cached metadata: [`ArrowReaderMetadata::load`]
177/// * Offset skipping: [`Self::with_offset`] and [`Self::with_limit`]
178/// * Row group filtering: [`Self::with_row_groups`]
179/// * Range filtering: [`Self::with_row_selection`]
180/// * Row level filtering: [`Self::with_row_filter`]
181///
182/// # Implementing Predicate Pushdown
183///
184/// [`Self::with_row_filter`] permits filter evaluation *during* the decoding
185/// process, which is efficient and allows the most low level optimizations.
186///
187/// However, most Parquet based systems will apply filters at many steps prior
188/// to decoding such as pruning files, row groups and data pages. This crate
189/// provides the low level APIs needed to implement such filtering, but does not
190/// include any logic to actually evaluate predicates. For example:
191///
192/// * [`Self::with_row_groups`] for Row Group pruning
193/// * [`Self::with_row_selection`] for data page pruning
194/// * [`StatisticsConverter`] to convert Parquet statistics to Arrow arrays
195///
196/// The rationale for this design is that implementing predicate pushdown is a
197/// complex topic and varies significantly from system to system. For example
198///
199/// 1. Predicates supported (do you support predicates like prefix matching, user defined functions, etc)
200/// 2. Evaluating predicates on multiple files (with potentially different but compatible schemas)
201/// 3. Evaluating predicates using information from an external metadata catalog (e.g. Apache Iceberg or similar)
202/// 4. Interleaving fetching metadata, evaluating predicates, and decoding files
203///
204/// You can read more about this design in the [Querying Parquet with
205/// Millisecond Latency] Arrow blog post.
206///
207/// [`ParquetRecordBatchStreamBuilder`]: crate::arrow::async_reader::ParquetRecordBatchStreamBuilder
208/// [`ParquetPushDecoderBuilder`]: crate::arrow::push_decoder::ParquetPushDecoderBuilder
209/// [Apache Arrow]: https://arrow.apache.org/
210/// [`StatisticsConverter`]: statistics::StatisticsConverter
211/// [Querying Parquet with Millisecond Latency]: https://arrow.apache.org/blog/2022/12/26/querying-parquet-with-millisecond-latency/
212pub struct ArrowReaderBuilder<T> {
213    /// The "input" to read parquet data from.
214    ///
215    /// Note in the case of the [`ParquetPushDecoderBuilder`] there is no
216    /// underlying reader; the input is instead [`PushDecoderInput`], the buffer that
217    /// caller-pushed bytes accumulate in.
218    ///
219    /// [`ParquetPushDecoderBuilder`]: crate::arrow::push_decoder::ParquetPushDecoderBuilder
220    /// [`PushDecoderInput`]: crate::arrow::push_decoder::PushDecoderInput
221    pub(crate) input: T,
222
223    pub(crate) metadata: Arc<ParquetMetaData>,
224
225    pub(crate) schema: SchemaRef,
226
227    pub(crate) fields: Option<Arc<ParquetField>>,
228
229    pub(crate) batch_size: usize,
230
231    pub(crate) row_group_plan: RowGroupPlan,
232
233    pub(crate) projection: ProjectionMask,
234
235    pub(crate) filter: Option<RowFilter>,
236
237    pub(crate) row_selection_policy: RowSelectionPolicy,
238
239    pub(crate) limit: Option<usize>,
240
241    pub(crate) offset: Option<usize>,
242
243    pub(crate) metrics: ArrowReaderMetrics,
244
245    pub(crate) max_predicate_cache_size: usize,
246}
247
248impl<T: Debug> Debug for ArrowReaderBuilder<T> {
249    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
250        f.debug_struct("ArrowReaderBuilder<T>")
251            .field("input", &self.input)
252            .field("metadata", &self.metadata)
253            .field("schema", &self.schema)
254            .field("fields", &self.fields)
255            .field("batch_size", &self.batch_size)
256            .field("row_group_plan", &self.row_group_plan)
257            .field("projection", &self.projection)
258            .field("filter", &self.filter)
259            .field("row_selection_policy", &self.row_selection_policy)
260            .field("limit", &self.limit)
261            .field("offset", &self.offset)
262            .field("metrics", &self.metrics)
263            .finish()
264    }
265}
266
267impl<T> ArrowReaderBuilder<T> {
268    pub(crate) fn new_builder(input: T, metadata: ArrowReaderMetadata) -> Self {
269        Self {
270            input,
271            metadata: metadata.metadata,
272            schema: metadata.schema,
273            fields: metadata.fields,
274            batch_size: DEFAULT_BATCH_SIZE,
275            row_group_plan: RowGroupPlan::Global {
276                row_groups: None,
277                selection: None,
278            },
279            projection: ProjectionMask::all(),
280            filter: None,
281            row_selection_policy: RowSelectionPolicy::default(),
282            limit: None,
283            offset: None,
284            metrics: ArrowReaderMetrics::Disabled,
285            max_predicate_cache_size: 100 * 1024 * 1024, // 100MB default cache size
286        }
287    }
288
289    /// Returns a reference to the [`ParquetMetaData`] for this parquet file
290    pub fn metadata(&self) -> &Arc<ParquetMetaData> {
291        &self.metadata
292    }
293
294    /// Returns the parquet [`SchemaDescriptor`] for this parquet file
295    pub fn parquet_schema(&self) -> &SchemaDescriptor {
296        self.metadata.file_metadata().schema_descr()
297    }
298
299    /// Returns the arrow [`SchemaRef`] for this parquet file
300    pub fn schema(&self) -> &SchemaRef {
301        &self.schema
302    }
303
304    /// Set the size of [`RecordBatch`] to produce. Defaults to [`DEFAULT_BATCH_SIZE`].
305    ///
306    /// This may be used as a hint for internal allocations, but does not
307    /// guarantee exact internal buffer capacities.
308    ///
309    /// If `batch_size` is more than the file row count, use the file row count.
310    pub fn with_batch_size(self, batch_size: usize) -> Self {
311        // Try to avoid allocate large buffer
312        let batch_size = batch_size.min(self.metadata.file_metadata().num_rows() as usize);
313        Self { batch_size, ..self }
314    }
315
316    /// Only read data from the provided row group indexes
317    ///
318    /// This is also called row group filtering
319    ///
320    /// On [`ParquetPushDecoderBuilder`] and [`ParquetRecordBatchStreamBuilder`],
321    /// which additionally offer `with_row_group_selections`, this cannot be
322    /// combined with that method; attempting to do so returns an error from
323    /// `build`.
324    ///
325    /// [`ParquetPushDecoderBuilder`]: crate::arrow::push_decoder::ParquetPushDecoderBuilder
326    /// [`ParquetRecordBatchStreamBuilder`]: crate::arrow::async_reader::ParquetRecordBatchStreamBuilder
327    pub fn with_row_groups(mut self, row_groups: Vec<usize>) -> Self {
328        self.row_group_plan.set_row_groups(row_groups);
329        self
330    }
331
332    /// Only read data from the provided column indexes
333    pub fn with_projection(self, mask: ProjectionMask) -> Self {
334        Self {
335            projection: mask,
336            ..self
337        }
338    }
339
340    /// Configure how row selections should be materialised during execution
341    ///
342    /// See [`RowSelectionPolicy`] for more details
343    pub fn with_row_selection_policy(self, policy: RowSelectionPolicy) -> Self {
344        Self {
345            row_selection_policy: policy,
346            ..self
347        }
348    }
349
350    /// Provide a [`RowSelection`] to filter out rows, and avoid fetching their
351    /// data into memory.
352    ///
353    /// This feature is used to restrict which rows are decoded within row
354    /// groups, skipping ranges of rows that are not needed. Such selections
355    /// could be determined by evaluating predicates against the parquet page
356    /// [`Index`] or some other external information available to a query
357    /// engine.
358    ///
359    /// # Notes
360    ///
361    /// Row group filtering (see [`Self::with_row_groups`]) is applied prior to
362    /// applying the row selection, and therefore rows from skipped row groups
363    /// should not be included in the [`RowSelection`] (see example below)
364    ///
365    /// On [`ParquetPushDecoderBuilder`] and [`ParquetRecordBatchStreamBuilder`],
366    /// which additionally offer `with_row_group_selections`, this cannot be
367    /// combined with that method; attempting to do so returns an error from
368    /// `build`.
369    ///
370    /// [`ParquetPushDecoderBuilder`]: crate::arrow::push_decoder::ParquetPushDecoderBuilder
371    /// [`ParquetRecordBatchStreamBuilder`]: crate::arrow::async_reader::ParquetRecordBatchStreamBuilder
372    ///
373    /// It is recommended to enable writing the page index if using this
374    /// functionality, to allow more efficient skipping over data pages. See
375    /// [`ArrowReaderOptions::with_page_index_policy`].
376    ///
377    /// # Example
378    ///
379    /// Given a parquet file with 4 row groups, and a row group filter of `[0,
380    /// 2, 3]`, in order to scan rows 50-100 in row group 2 and rows 200-300 in
381    /// row group 3:
382    ///
383    /// ```text
384    ///   Row Group 0, 1000 rows (selected)
385    ///   Row Group 1, 1000 rows (skipped)
386    ///   Row Group 2, 1000 rows (selected, but want to only scan rows 50-100)
387    ///   Row Group 3, 1000 rows (selected, but want to only scan rows 200-300)
388    /// ```
389    ///
390    /// You could pass the following [`RowSelection`]:
391    ///
392    /// ```text
393    ///  Select 1000    (scan all rows in row group 0)
394    ///  Skip 50        (skip the first 50 rows in row group 2)
395    ///  Select 50      (scan rows 50-100 in row group 2)
396    ///  Skip 900       (skip the remaining rows in row group 2)
397    ///  Skip 200       (skip the first 200 rows in row group 3)
398    ///  Select 100     (scan rows 200-300 in row group 3)
399    ///  Skip 700       (skip the remaining rows in row group 3)
400    /// ```
401    /// Note there is no entry for the (entirely) skipped row group 1.
402    ///
403    /// Note you can represent the same selection with fewer entries. Instead of
404    ///
405    /// ```text
406    ///  Skip 900       (skip the remaining rows in row group 2)
407    ///  Skip 200       (skip the first 200 rows in row group 3)
408    /// ```
409    ///
410    /// you could use
411    ///
412    /// ```text
413    /// Skip 1100      (skip the remaining 900 rows in row group 2 and the first 200 rows in row group 3)
414    /// ```
415    ///
416    /// [`Index`]: crate::file::page_index::column_index::ColumnIndexMetaData
417    pub fn with_row_selection(mut self, selection: RowSelection) -> Self {
418        self.row_group_plan.set_row_selection(selection);
419        self
420    }
421
422    /// Provide a [`RowFilter`] to skip decoding rows
423    ///
424    /// Row filters are applied after row group selection and row selection
425    ///
426    /// It is recommended to enable reading the page index if using this functionality, to allow
427    /// more efficient skipping over data pages. See [`ArrowReaderOptions::with_page_index_policy`].
428    ///
429    /// See the [blog post on late materialization] for a more technical explanation.
430    ///
431    /// [blog post on late materialization]: https://arrow.apache.org/blog/2025/12/11/parquet-late-materialization-deep-dive
432    ///
433    /// # Example
434    /// ```rust
435    /// # use std::fs::File;
436    /// # use arrow_array::Int32Array;
437    /// # use parquet::arrow::ProjectionMask;
438    /// # use parquet::arrow::arrow_reader::{ArrowPredicateFn, ParquetRecordBatchReaderBuilder, RowFilter};
439    /// # fn main() -> Result<(), parquet::errors::ParquetError> {
440    /// # let testdata = arrow::util::test_util::parquet_test_data();
441    /// # let path = format!("{testdata}/alltypes_plain.parquet");
442    /// # let file = File::open(&path)?;
443    /// let builder = ParquetRecordBatchReaderBuilder::try_new(file)?;
444    /// let schema_desc = builder.metadata().file_metadata().schema_descr_ptr();
445    /// // Create predicate that evaluates `int_col != 1`.
446    /// // `int_col` column has index 4 (zero based) in the schema
447    /// let projection = ProjectionMask::leaves(&schema_desc, [4]);
448    /// // Only the projection columns are passed to the predicate so
449    /// // int_col is column 0 in the predicate
450    /// let predicate = ArrowPredicateFn::new(projection, |batch| {
451    ///     let int_col = batch.column(0);
452    ///     arrow::compute::kernels::cmp::neq(int_col, &Int32Array::new_scalar(1))
453    /// });
454    /// let row_filter = RowFilter::new(vec![Box::new(predicate)]);
455    /// // The filter will be invoked during the reading process
456    /// let reader = builder.with_row_filter(row_filter).build()?;
457    /// # for b in reader { let _ = b?; }
458    /// # Ok(())
459    /// # }
460    /// ```
461    pub fn with_row_filter(self, filter: RowFilter) -> Self {
462        Self {
463            filter: Some(filter),
464            ..self
465        }
466    }
467
468    /// Provide a limit to the number of rows to be read
469    ///
470    /// The limit will be applied after any [`Self::with_row_selection`] and [`Self::with_row_filter`]
471    /// allowing it to limit the final set of rows decoded after any pushed down predicates
472    ///
473    /// It is recommended to enable reading the page index if using this functionality, to allow
474    /// more efficient skipping over data pages. See [`ArrowReaderOptions::with_page_index_policy`]
475    pub fn with_limit(self, limit: usize) -> Self {
476        Self {
477            limit: Some(limit),
478            ..self
479        }
480    }
481
482    /// Provide an offset to skip over the given number of rows
483    ///
484    /// The offset will be applied after any [`Self::with_row_selection`] and [`Self::with_row_filter`]
485    /// allowing it to skip rows after any pushed down predicates
486    ///
487    /// It is recommended to enable reading the page index if using this functionality, to allow
488    /// more efficient skipping over data pages. See [`ArrowReaderOptions::with_page_index_policy`]
489    pub fn with_offset(self, offset: usize) -> Self {
490        Self {
491            offset: Some(offset),
492            ..self
493        }
494    }
495
496    /// Specify metrics collection during reading
497    ///
498    /// To access the metrics, create an [`ArrowReaderMetrics`] and pass a
499    /// clone of the provided metrics to the builder.
500    ///
501    /// For example:
502    ///
503    /// ```rust
504    /// # use std::sync::Arc;
505    /// # use bytes::Bytes;
506    /// # use arrow_array::{Int32Array, RecordBatch};
507    /// # use arrow_schema::{DataType, Field, Schema};
508    /// # use parquet::arrow::arrow_reader::{ParquetRecordBatchReader, ParquetRecordBatchReaderBuilder};
509    /// use parquet::arrow::arrow_reader::metrics::ArrowReaderMetrics;
510    /// # use parquet::arrow::ArrowWriter;
511    /// # let mut file: Vec<u8> = Vec::with_capacity(1024);
512    /// # let schema = Arc::new(Schema::new(vec![Field::new("i32", DataType::Int32, false)]));
513    /// # let mut writer = ArrowWriter::try_new(&mut file, schema.clone(), None).unwrap();
514    /// # let batch = RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]).unwrap();
515    /// # writer.write(&batch).unwrap();
516    /// # writer.close().unwrap();
517    /// # let file = Bytes::from(file);
518    /// // Create metrics object to pass into the reader
519    /// let metrics = ArrowReaderMetrics::enabled();
520    /// let reader = ParquetRecordBatchReaderBuilder::try_new(file).unwrap()
521    ///   // Configure the builder to use the metrics by passing a clone
522    ///   .with_metrics(metrics.clone())
523    ///   // Build the reader
524    ///   .build().unwrap();
525    /// // .. read data from the reader ..
526    ///
527    /// // check the metrics
528    /// assert!(metrics.records_read_from_inner().is_some());
529    /// ```
530    pub fn with_metrics(self, metrics: ArrowReaderMetrics) -> Self {
531        Self { metrics, ..self }
532    }
533
534    /// Set the maximum size (per row group) of the predicate cache in bytes for
535    /// the async decoder.
536    ///
537    /// Defaults to 100MB (across all columns). Set to `usize::MAX` to use
538    /// unlimited cache size.
539    ///
540    /// This cache is used to store decoded arrays that are used in
541    /// predicate evaluation ([`Self::with_row_filter`]).
542    ///
543    /// This cache is only used for the "async" decoder, [`ParquetRecordBatchStream`]. See
544    /// [this ticket] for more details and alternatives.
545    ///
546    /// [`ParquetRecordBatchStream`]: https://docs.rs/parquet/latest/parquet/arrow/async_reader/struct.ParquetRecordBatchStream.html
547    /// [this ticket]: https://github.com/apache/arrow-rs/issues/8000
548    pub fn with_max_predicate_cache_size(self, max_predicate_cache_size: usize) -> Self {
549        Self {
550            max_predicate_cache_size,
551            ..self
552        }
553    }
554}
555
556/// Options that control how [`ParquetMetaData`] is read when constructing
557/// an Arrow reader.
558///
559/// To use these options, pass them to one of the following methods:
560/// * [`ParquetRecordBatchReaderBuilder::try_new_with_options`]
561/// * [`ParquetRecordBatchStreamBuilder::new_with_options`]
562///
563/// For fine-grained control over metadata loading, use
564/// [`ArrowReaderMetadata::load`] to load metadata with these options,
565///
566/// See [`ArrowReaderBuilder`] for how to configure how the column data
567/// is then read from the file, including projection and filter pushdown
568///
569/// [`ParquetRecordBatchStreamBuilder::new_with_options`]: crate::arrow::async_reader::ParquetRecordBatchStreamBuilder::new_with_options
570#[derive(Debug, Clone, Default)]
571pub struct ArrowReaderOptions {
572    /// Should the reader strip any user defined metadata from the Arrow schema
573    skip_arrow_metadata: bool,
574    /// If provided, used as the schema hint when determining the Arrow schema,
575    /// otherwise the schema hint is read from the [ARROW_SCHEMA_META_KEY]
576    ///
577    /// [ARROW_SCHEMA_META_KEY]: crate::arrow::ARROW_SCHEMA_META_KEY
578    supplied_schema: Option<SchemaRef>,
579
580    column_index: PageIndexPolicy,
581    offset_index: PageIndexPolicy,
582    column_index_mask: ColumnChunkMask,
583    offset_index_mask: ColumnChunkMask,
584
585    /// Options to control reading of Parquet metadata
586    metadata_options: ParquetMetaDataOptions,
587    /// If encryption is enabled, the file decryption properties can be provided
588    #[cfg(feature = "encryption")]
589    pub(crate) file_decryption_properties: Option<Arc<FileDecryptionProperties>>,
590
591    virtual_columns: Vec<FieldRef>,
592}
593
594impl ArrowReaderOptions {
595    /// Create a new [`ArrowReaderOptions`] with the default settings
596    pub fn new() -> Self {
597        Self::default()
598    }
599
600    /// Skip decoding the embedded arrow metadata (defaults to `false`)
601    ///
602    /// Parquet files generated by some writers may contain embedded arrow
603    /// schema and metadata.
604    /// This may not be correct or compatible with your system,
605    /// for example, see [ARROW-16184](https://issues.apache.org/jira/browse/ARROW-16184)
606    pub fn with_skip_arrow_metadata(self, skip_arrow_metadata: bool) -> Self {
607        Self {
608            skip_arrow_metadata,
609            ..self
610        }
611    }
612
613    /// Provide a schema hint to use when reading the Parquet file.
614    ///
615    /// If provided, this schema takes precedence over any arrow schema embedded
616    /// in the metadata (see the [`arrow`] documentation for more details).
617    ///
618    /// If the provided schema is not compatible with the data stored in the
619    /// parquet file schema, an error will be returned when constructing the
620    /// builder.
621    ///
622    /// This option is only required if you want to explicitly control the
623    /// conversion of Parquet types to Arrow types, such as casting a column to
624    /// a different type. For example, if you wanted to read an Int64 in
625    /// a Parquet file to a [`TimestampMicrosecondArray`] in the Arrow schema.
626    ///
627    /// [`arrow`]: crate::arrow
628    /// [`TimestampMicrosecondArray`]: arrow_array::TimestampMicrosecondArray
629    ///
630    /// # Notes
631    ///
632    /// The provided schema must have the same number of columns as the parquet schema and
633    /// the column names must be the same.
634    ///
635    /// # Example
636    /// ```
637    /// # use std::sync::Arc;
638    /// # use bytes::Bytes;
639    /// # use arrow_array::{ArrayRef, Int32Array, RecordBatch};
640    /// # use arrow_schema::{DataType, Field, Schema, TimeUnit};
641    /// # use parquet::arrow::arrow_reader::{ArrowReaderOptions, ParquetRecordBatchReaderBuilder};
642    /// # use parquet::arrow::ArrowWriter;
643    /// // Write data - schema is inferred from the data to be Int32
644    /// let mut file = Vec::new();
645    /// let batch = RecordBatch::try_from_iter(vec![
646    ///     ("col_1", Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef),
647    /// ]).unwrap();
648    /// let mut writer = ArrowWriter::try_new(&mut file, batch.schema(), None).unwrap();
649    /// writer.write(&batch).unwrap();
650    /// writer.close().unwrap();
651    /// let file = Bytes::from(file);
652    ///
653    /// // Read the file back.
654    /// // Supply a schema that interprets the Int32 column as a Timestamp.
655    /// let supplied_schema = Arc::new(Schema::new(vec![
656    ///     Field::new("col_1", DataType::Timestamp(TimeUnit::Nanosecond, None), false)
657    /// ]));
658    /// let options = ArrowReaderOptions::new().with_schema(supplied_schema.clone());
659    /// let mut builder = ParquetRecordBatchReaderBuilder::try_new_with_options(
660    ///     file.clone(),
661    ///     options
662    /// ).expect("Error if the schema is not compatible with the parquet file schema.");
663    ///
664    /// // Create the reader and read the data using the supplied schema.
665    /// let mut reader = builder.build().unwrap();
666    /// let _batch = reader.next().unwrap().unwrap();
667    /// ```
668    ///
669    /// # Example: Preserving Dictionary Encoding
670    ///
671    /// By default, Parquet string columns are read as `Utf8Array` (or `LargeUtf8Array`),
672    /// even if the underlying Parquet data uses dictionary encoding. You can preserve
673    /// the dictionary encoding by specifying a `Dictionary` type in the schema hint:
674    ///
675    /// ```
676    /// # use std::sync::Arc;
677    /// # use bytes::Bytes;
678    /// # use arrow_array::{ArrayRef, RecordBatch, StringArray};
679    /// # use arrow_schema::{DataType, Field, Schema};
680    /// # use parquet::arrow::arrow_reader::{ArrowReaderOptions, ParquetRecordBatchReaderBuilder};
681    /// # use parquet::arrow::ArrowWriter;
682    /// // Write a Parquet file with string data
683    /// let mut file = Vec::new();
684    /// let schema = Arc::new(Schema::new(vec![
685    ///     Field::new("city", DataType::Utf8, false)
686    /// ]));
687    /// let cities = StringArray::from(vec!["Berlin", "Berlin", "Paris", "Berlin", "Paris"]);
688    /// let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(cities)]).unwrap();
689    ///
690    /// let mut writer = ArrowWriter::try_new(&mut file, batch.schema(), None).unwrap();
691    /// writer.write(&batch).unwrap();
692    /// writer.close().unwrap();
693    /// let file = Bytes::from(file);
694    ///
695    /// // Read the file back, requesting dictionary encoding preservation
696    /// let dict_schema = Arc::new(Schema::new(vec![
697    ///     Field::new("city", DataType::Dictionary(
698    ///         Box::new(DataType::Int32),
699    ///         Box::new(DataType::Utf8)
700    ///     ), false)
701    /// ]));
702    /// let options = ArrowReaderOptions::new().with_schema(dict_schema);
703    /// let builder = ParquetRecordBatchReaderBuilder::try_new_with_options(
704    ///     file.clone(),
705    ///     options
706    /// ).unwrap();
707    ///
708    /// let mut reader = builder.build().unwrap();
709    /// let batch = reader.next().unwrap().unwrap();
710    ///
711    /// // The column is now a DictionaryArray
712    /// assert!(matches!(
713    ///     batch.column(0).data_type(),
714    ///     DataType::Dictionary(_, _)
715    /// ));
716    /// ```
717    ///
718    /// **Note**: Dictionary encoding preservation works best when:
719    /// 1. The original column was dictionary encoded (the default for string columns)
720    /// 2. There are a small number of distinct values
721    pub fn with_schema(self, schema: SchemaRef) -> Self {
722        Self {
723            supplied_schema: Some(schema),
724            skip_arrow_metadata: true,
725            ..self
726        }
727    }
728
729    /// Sets the [`PageIndexPolicy`] for both the column and offset indexes.
730    ///
731    /// The `PageIndex` can be used to push down predicates to the parquet scan,
732    /// potentially eliminating unnecessary IO, by some query engines.
733    /// The `PageIndex` consists of two structures: the `ColumnIndex` and `OffsetIndex`.
734    /// This method sets the same policy for both. For fine-grained control, use
735    /// [`Self::with_column_index_policy`] and [`Self::with_offset_index_policy`].
736    pub fn with_page_index_policy(self, policy: PageIndexPolicy) -> Self {
737        self.with_column_index_policy(policy)
738            .with_offset_index_policy(policy)
739    }
740
741    /// Sets the [`PageIndexPolicy`] for the Parquet [ColumnIndex] structure.
742    ///
743    /// The `ColumnIndex` contains min/max statistics for each page, which can be used
744    /// for predicate pushdown and page-level pruning.
745    ///
746    /// [ColumnIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
747    pub fn with_column_index_policy(mut self, policy: PageIndexPolicy) -> Self {
748        self.column_index = policy;
749        self
750    }
751
752    /// Sets the [`PageIndexPolicy`] for the Parquet [OffsetIndex] structure.
753    ///
754    /// The `OffsetIndex` contains the locations and sizes of each page, which enables
755    /// efficient page-level skipping and random access within column chunks.
756    ///
757    /// [OffsetIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
758    pub fn with_offset_index_policy(mut self, policy: PageIndexPolicy) -> Self {
759        self.offset_index = policy;
760        self
761    }
762
763    // TODO: deprecate the following as part of https://github.com/apache/arrow-rs/issues/11395
764
765    /// Sets the same [`ColumnChunkMask`] for both page-index structures.
766    pub fn with_page_index_mask(self, mask: ColumnChunkMask) -> Self {
767        self.with_column_index_mask(mask.clone())
768            .with_offset_index_mask(mask)
769    }
770
771    /// Sets the [`ColumnChunkMask`] for the Parquet [ColumnIndex] structure.
772    ///
773    /// The column index can be costly to decode and store, especially when it is needed
774    /// only for a subset of row groups or columns (such as when filtering by a predicate
775    /// on a single column). Providing a [`ColumnChunkMask`] can greatly decrease
776    /// the time needed to decode this metadata.
777    ///
778    /// The mask applies only if the column-index policy is not [`PageIndexPolicy::Skip`]
779    /// (the default), or an underlying reader is configured to preload the index. It is
780    /// honored by loading APIs such as [`ArrowReaderMetadata::load`];
781    /// [`ArrowReaderMetadata::try_new`] does not load or filter page indexes.
782    ///
783    /// See [`PageIndexProvider`] for more details on customizing page index loading.
784    ///
785    /// [`PageIndexProvider`]: crate::file::metadata::page_index::PageIndexProvider
786    /// [ColumnIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
787    pub fn with_column_index_mask(mut self, mask: ColumnChunkMask) -> Self {
788        self.column_index_mask = mask;
789        self
790    }
791
792    /// Sets the [`ColumnChunkMask`] for the Parquet [OffsetIndex] structure.
793    ///
794    /// The offset index can be costly to decode and store, especially when it is needed
795    /// only for a subset of row groups or columns (such as when projecting a small subset
796    /// of columns). Providing a [`ColumnChunkMask`] can greatly decrease
797    /// the time needed to decode this metadata.
798    ///
799    /// Page pruning also needs the offset index for predicate columns, so callers should
800    /// include those columns in addition to projected columns. The same loading and policy
801    /// qualifications as [`Self::with_column_index_mask`] apply.
802    ///
803    /// See [`PageIndexProvider`] for more details on customizing page index loading.
804    ///
805    /// [`PageIndexProvider`]: crate::file::metadata::page_index::PageIndexProvider
806    /// [OffsetIndex]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
807    pub fn with_offset_index_mask(mut self, mask: ColumnChunkMask) -> Self {
808        self.offset_index_mask = mask;
809        self
810    }
811
812    /// Provide a Parquet schema to use when decoding the metadata. The schema in the Parquet
813    /// footer will be skipped.
814    ///
815    /// This can be used to avoid reparsing the schema from the file when it is
816    /// already known.
817    pub fn with_parquet_schema(mut self, schema: Arc<SchemaDescriptor>) -> Self {
818        self.metadata_options.set_schema(schema);
819        self
820    }
821
822    /// Set whether to convert the [`encoding_stats`] in the Parquet `ColumnMetaData` to a bitmask
823    /// (defaults to `false`).
824    ///
825    /// See [`ColumnChunkMetaData::page_encoding_stats_mask`] for an explanation of why this
826    /// might be desirable.
827    ///
828    /// [`ColumnChunkMetaData::page_encoding_stats_mask`]:
829    /// crate::file::metadata::ColumnChunkMetaData::page_encoding_stats_mask
830    /// [`encoding_stats`]:
831    /// https://github.com/apache/parquet-format/blob/786142e26740487930ddc3ec5e39d780bd930907/src/main/thrift/parquet.thrift#L917
832    pub fn with_encoding_stats_as_mask(mut self, val: bool) -> Self {
833        self.metadata_options.set_encoding_stats_as_mask(val);
834        self
835    }
836
837    /// Sets the decoding policy for [`encoding_stats`] in the Parquet `ColumnMetaData`.
838    ///
839    /// [`encoding_stats`]:
840    /// https://github.com/apache/parquet-format/blob/786142e26740487930ddc3ec5e39d780bd930907/src/main/thrift/parquet.thrift#L917
841    pub fn with_encoding_stats_policy(mut self, policy: ParquetStatisticsPolicy) -> Self {
842        self.metadata_options.set_encoding_stats_policy(policy);
843        self
844    }
845
846    /// Sets the decoding policy for [`statistics`] in the Parquet `ColumnMetaData`.
847    ///
848    /// [`statistics`]:
849    /// https://github.com/apache/parquet-format/blob/786142e26740487930ddc3ec5e39d780bd930907/src/main/thrift/parquet.thrift#L912
850    pub fn with_column_stats_policy(mut self, policy: ParquetStatisticsPolicy) -> Self {
851        self.metadata_options.set_column_stats_policy(policy);
852        self
853    }
854
855    /// Sets the decoding policy for [`size_statistics`] in the Parquet `ColumnMetaData`.
856    ///
857    /// [`size_statistics`]:
858    /// https://github.com/apache/parquet-format/blob/786142e26740487930ddc3ec5e39d780bd930907/src/main/thrift/parquet.thrift#L936
859    pub fn with_size_stats_policy(mut self, policy: ParquetStatisticsPolicy) -> Self {
860        self.metadata_options.set_size_stats_policy(policy);
861        self
862    }
863
864    /// Provide the file decryption properties to use when reading encrypted parquet files.
865    ///
866    /// If encryption is enabled and the file is encrypted, the `file_decryption_properties` must be provided.
867    #[cfg(feature = "encryption")]
868    pub fn with_file_decryption_properties(
869        self,
870        file_decryption_properties: Arc<FileDecryptionProperties>,
871    ) -> Self {
872        Self {
873            file_decryption_properties: Some(file_decryption_properties),
874            ..self
875        }
876    }
877
878    /// Include virtual columns in the output.
879    ///
880    /// Virtual columns are columns that are not part of the Parquet schema, but are added to the output by the reader such as row numbers and row group indices.
881    ///
882    /// # Example
883    /// ```
884    /// # use std::sync::Arc;
885    /// # use bytes::Bytes;
886    /// # use arrow_array::{ArrayRef, Int64Array, RecordBatch};
887    /// # use arrow_schema::{DataType, Field, Schema};
888    /// # use parquet::arrow::{ArrowWriter, RowNumber};
889    /// # use parquet::arrow::arrow_reader::{ArrowReaderOptions, ParquetRecordBatchReaderBuilder};
890    /// #
891    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
892    /// // Create a simple record batch with some data
893    /// let values = Arc::new(Int64Array::from(vec![1, 2, 3])) as ArrayRef;
894    /// let batch = RecordBatch::try_from_iter(vec![("value", values)])?;
895    ///
896    /// // Write the batch to an in-memory buffer
897    /// let mut file = Vec::new();
898    /// let mut writer = ArrowWriter::try_new(
899    ///     &mut file,
900    ///     batch.schema(),
901    ///     None
902    /// )?;
903    /// writer.write(&batch)?;
904    /// writer.close()?;
905    /// let file = Bytes::from(file);
906    ///
907    /// // Create a virtual column for row numbers
908    /// let row_number_field = Arc::new(Field::new("row_number", DataType::Int64, false)
909    ///     .with_extension_type(RowNumber));
910    ///
911    /// // Configure options with virtual columns
912    /// let options = ArrowReaderOptions::new()
913    ///     .with_virtual_columns(vec![row_number_field])?;
914    ///
915    /// // Create a reader with the options
916    /// let mut reader = ParquetRecordBatchReaderBuilder::try_new_with_options(
917    ///     file,
918    ///     options
919    /// )?
920    /// .build()?;
921    ///
922    /// // Read the batch - it will include both the original column and the virtual row_number column
923    /// let result_batch = reader.next().unwrap()?;
924    /// assert_eq!(result_batch.num_columns(), 2); // "value" + "row_number"
925    /// assert_eq!(result_batch.num_rows(), 3);
926    /// #
927    /// # Ok(())
928    /// # }
929    /// ```
930    pub fn with_virtual_columns(self, virtual_columns: Vec<FieldRef>) -> Result<Self> {
931        // Validate that all fields are virtual columns
932        for field in &virtual_columns {
933            if !is_virtual_column(field) {
934                return Err(ParquetError::General(format!(
935                    "Field '{}' is not a virtual column. Virtual columns must have extension type names starting with 'arrow.virtual.'",
936                    field.name()
937                )));
938            }
939        }
940        Ok(Self {
941            virtual_columns,
942            ..self
943        })
944    }
945
946    /// Retrieve the currently set [`PageIndexPolicy`] for the offset index.
947    ///
948    /// This can be set via [`with_offset_index_policy`][Self::with_offset_index_policy]
949    /// or [`with_page_index_policy`][Self::with_page_index_policy].
950    pub fn offset_index_policy(&self) -> PageIndexPolicy {
951        self.offset_index
952    }
953
954    /// Retrieve the currently set [`PageIndexPolicy`] for the column index.
955    ///
956    /// This can be set via [`with_column_index_policy`][Self::with_column_index_policy]
957    /// or [`with_page_index_policy`][Self::with_page_index_policy].
958    pub fn column_index_policy(&self) -> PageIndexPolicy {
959        self.column_index
960    }
961
962    // TODO: deprecate the following as part of https://github.com/apache/arrow-rs/issues/11395
963
964    /// Retrieve the currently set [`ColumnChunkMask`] for the offset index.
965    ///
966    /// This can be set via [`with_offset_index_mask`][Self::with_offset_index_mask].
967    pub fn offset_index_mask(&self) -> &ColumnChunkMask {
968        &self.offset_index_mask
969    }
970
971    /// Retrieve the currently set [`ColumnChunkMask`] for the column index.
972    ///
973    /// This can be set via [`with_column_index_mask`][Self::with_column_index_mask].
974    pub fn column_index_mask(&self) -> &ColumnChunkMask {
975        &self.column_index_mask
976    }
977
978    /// Retrieve the currently set metadata decoding options.
979    pub fn metadata_options(&self) -> &ParquetMetaDataOptions {
980        &self.metadata_options
981    }
982
983    /// Retrieve the currently set file decryption properties.
984    ///
985    /// This can be set via
986    /// [`file_decryption_properties`][Self::with_file_decryption_properties].
987    #[cfg(feature = "encryption")]
988    pub fn file_decryption_properties(&self) -> Option<&Arc<FileDecryptionProperties>> {
989        self.file_decryption_properties.as_ref()
990    }
991}
992
993impl ParquetMetaDataReader {
994    /// Applies the metadata related settings from [`ArrowReaderOptions`],
995    /// such as the [`ParquetMetaDataOptions`], decryption properties, and
996    /// [`PageIndexPolicy`] to this reader.
997    ///
998    /// The page index policies are only applied if at least one of them is not
999    /// [`PageIndexPolicy::Skip`], so policies previously configured on this
1000    /// reader (e.g. from a preload setting) are preserved when the options do
1001    /// not request the page index.
1002    ///
1003    /// This encodes the canonical way to construct a `ParquetMetaDataReader`
1004    /// inside `AsyncFileReader::get_metadata` (available with the `async`
1005    /// feature), so implementations outside this crate do not need to
1006    /// duplicate it.
1007    pub fn with_arrow_reader_options(mut self, options: Option<&ArrowReaderOptions>) -> Self {
1008        let Some(options) = options else { return self };
1009
1010        self = self.with_metadata_options(Some(options.metadata_options().clone()));
1011
1012        #[cfg(feature = "encryption")]
1013        {
1014            self = self.with_decryption_properties(
1015                options.file_decryption_properties.as_ref().map(Arc::clone),
1016            );
1017        }
1018
1019        if options.column_index_policy() != PageIndexPolicy::Skip
1020            || options.offset_index_policy() != PageIndexPolicy::Skip
1021        {
1022            self = self
1023                .with_column_index_policy(options.column_index_policy())
1024                .with_offset_index_policy(options.offset_index_policy());
1025        }
1026
1027        // Preload settings on the underlying reader may enable an index even when the
1028        // corresponding options policy is `Skip`, so apply non-default masks independently.
1029        //
1030        // TODO: https://github.com/apache/arrow-rs/issues/11395 may deprecate this API. Decide
1031        // at that time if masks should be added to ParquetObjectReader.
1032        if !options.column_index_mask().is_all() {
1033            self = self.with_column_index_mask(options.column_index_mask().clone());
1034        }
1035        if !options.offset_index_mask().is_all() {
1036            self = self.with_offset_index_mask(options.offset_index_mask().clone());
1037        }
1038
1039        self
1040    }
1041}
1042
1043/// The metadata necessary to construct a [`ArrowReaderBuilder`]
1044///
1045/// Note this structure is cheaply clone-able as it consists of several arcs.
1046///
1047/// This structure allows
1048///
1049/// 1. Loading metadata for a file once and then using that same metadata to
1050///    construct multiple separate readers, for example, to distribute readers
1051///    across multiple threads
1052///
1053/// 2. Using a cached copy of the [`ParquetMetadata`] rather than reading it
1054///    from the file each time a reader is constructed.
1055///
1056/// [`ParquetMetadata`]: crate::file::metadata::ParquetMetaData
1057#[derive(Debug, Clone)]
1058pub struct ArrowReaderMetadata {
1059    /// The Parquet Metadata, if known aprior
1060    pub(crate) metadata: Arc<ParquetMetaData>,
1061    /// The Arrow Schema
1062    pub(crate) schema: SchemaRef,
1063    /// The Parquet schema (root field)
1064    pub(crate) fields: Option<Arc<ParquetField>>,
1065}
1066
1067impl ArrowReaderMetadata {
1068    /// Create [`ArrowReaderMetadata`] from the provided [`ArrowReaderOptions`]
1069    /// and [`ChunkReader`]
1070    ///
1071    /// See [`ParquetRecordBatchReaderBuilder::new_with_metadata`] for an
1072    /// example of how this can be used
1073    ///
1074    /// # Notes
1075    ///
1076    /// If `options` indicates the page index should be read, but
1077    /// `Self::metadata` is missing the page index, this function will attempt
1078    /// to load the page index by making an object store request.
1079    ///
1080    /// See [`ArrowReaderOptions::with_page_index_policy`] for more information on the page index.
1081    pub fn load<T: ChunkReader>(reader: &T, options: ArrowReaderOptions) -> Result<Self> {
1082        let metadata = ParquetMetaDataReader::new()
1083            .with_column_index_policy(options.column_index_policy())
1084            .with_offset_index_policy(options.offset_index_policy())
1085            .with_column_index_mask(options.column_index_mask().clone())
1086            .with_offset_index_mask(options.offset_index_mask().clone())
1087            .with_metadata_options(Some(options.metadata_options.clone()));
1088        #[cfg(feature = "encryption")]
1089        let metadata = metadata.with_decryption_properties(
1090            options.file_decryption_properties.as_ref().map(Arc::clone),
1091        );
1092        let metadata = metadata.parse_and_finish(reader)?;
1093        Self::try_new(Arc::new(metadata), options)
1094    }
1095
1096    /// Create a new [`ArrowReaderMetadata`] from a pre-existing
1097    /// [`ParquetMetaData`] and [`ArrowReaderOptions`].
1098    ///
1099    /// # Notes
1100    ///
1101    /// This function will not attempt to load the PageIndex if not present in the metadata, regardless
1102    /// of the settings in `options`. See [`Self::load`] to load metadata including the page index if needed.
1103    pub fn try_new(metadata: Arc<ParquetMetaData>, options: ArrowReaderOptions) -> Result<Self> {
1104        match options.supplied_schema {
1105            Some(supplied_schema) => Self::with_supplied_schema(
1106                metadata,
1107                supplied_schema.clone(),
1108                &options.virtual_columns,
1109            ),
1110            None => {
1111                let kv_metadata = match options.skip_arrow_metadata {
1112                    true => None,
1113                    false => metadata.file_metadata().key_value_metadata(),
1114                };
1115
1116                let (schema, fields) = parquet_to_arrow_schema_and_fields(
1117                    metadata.file_metadata().schema_descr(),
1118                    ProjectionMask::all(),
1119                    kv_metadata,
1120                    &options.virtual_columns,
1121                )?;
1122
1123                Ok(Self {
1124                    metadata,
1125                    schema: Arc::new(schema),
1126                    fields: fields.map(Arc::new),
1127                })
1128            }
1129        }
1130    }
1131
1132    fn with_supplied_schema(
1133        metadata: Arc<ParquetMetaData>,
1134        supplied_schema: SchemaRef,
1135        virtual_columns: &[FieldRef],
1136    ) -> Result<Self> {
1137        let parquet_schema = metadata.file_metadata().schema_descr();
1138        let field_levels = parquet_to_arrow_field_levels_with_virtual(
1139            parquet_schema,
1140            ProjectionMask::all(),
1141            Some(supplied_schema.fields()),
1142            virtual_columns,
1143        )?;
1144        let fields = field_levels.fields;
1145        let inferred_len = fields.len();
1146        let supplied_len = supplied_schema.fields().len() + virtual_columns.len();
1147        // Ensure the supplied schema has the same number of columns as the parquet schema.
1148        // parquet_to_arrow_field_levels is expected to throw an error if the schemas have
1149        // different lengths, but we check here to be safe.
1150        if inferred_len != supplied_len {
1151            return Err(arrow_err!(format!(
1152                "Incompatible supplied Arrow schema: expected {} columns received {}",
1153                inferred_len, supplied_len
1154            )));
1155        }
1156
1157        let mut errors = Vec::new();
1158
1159        let field_iter = supplied_schema.fields().iter().zip(fields.iter());
1160
1161        for (field1, field2) in field_iter {
1162            if field1.data_type() != field2.data_type() {
1163                errors.push(format!(
1164                    "data type mismatch for field {}: requested {} but found {}",
1165                    field1.name(),
1166                    field1.data_type(),
1167                    field2.data_type()
1168                ));
1169            }
1170            if field1.is_nullable() != field2.is_nullable() {
1171                errors.push(format!(
1172                    "nullability mismatch for field {}: expected {:?} but found {:?}",
1173                    field1.name(),
1174                    field1.is_nullable(),
1175                    field2.is_nullable()
1176                ));
1177            }
1178            if field1.metadata() != field2.metadata() {
1179                errors.push(format!(
1180                    "metadata mismatch for field {}: expected {:?} but found {:?}",
1181                    field1.name(),
1182                    field1.metadata(),
1183                    field2.metadata()
1184                ));
1185            }
1186        }
1187
1188        if !errors.is_empty() {
1189            let message = errors.join(", ");
1190            return Err(ParquetError::ArrowError(format!(
1191                "Incompatible supplied Arrow schema: {message}",
1192            )));
1193        }
1194
1195        // `fields` is the supplied fields followed by the virtual columns, so the reported
1196        // schema has to be built from it or the virtual columns go missing.
1197        let schema = if virtual_columns.is_empty() {
1198            supplied_schema
1199        } else {
1200            Arc::new(Schema::new_with_metadata(
1201                fields,
1202                supplied_schema.metadata().clone(),
1203            ))
1204        };
1205
1206        Ok(Self {
1207            metadata,
1208            schema,
1209            fields: field_levels.levels.map(Arc::new),
1210        })
1211    }
1212
1213    /// Returns a reference to the [`ParquetMetaData`] for this parquet file
1214    pub fn metadata(&self) -> &Arc<ParquetMetaData> {
1215        &self.metadata
1216    }
1217
1218    /// Returns the parquet [`SchemaDescriptor`] for this parquet file
1219    pub fn parquet_schema(&self) -> &SchemaDescriptor {
1220        self.metadata.file_metadata().schema_descr()
1221    }
1222
1223    /// Returns the arrow [`SchemaRef`] for this parquet file
1224    pub fn schema(&self) -> &SchemaRef {
1225        &self.schema
1226    }
1227}
1228
1229#[doc(hidden)]
1230// A newtype used within `ReaderOptionsBuilder` to distinguish sync readers from async
1231pub struct SyncReader<T: ChunkReader>(T);
1232
1233impl<T: Debug + ChunkReader> Debug for SyncReader<T> {
1234    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
1235        f.debug_tuple("SyncReader").field(&self.0).finish()
1236    }
1237}
1238
1239/// Creates [`ParquetRecordBatchReader`] for reading Parquet files into Arrow [`RecordBatch`]es
1240///
1241/// # See Also
1242/// * [`crate::arrow::async_reader::ParquetRecordBatchStreamBuilder`] for an async API
1243/// * [`crate::arrow::push_decoder::ParquetPushDecoderBuilder`] for a SansIO decoder API
1244/// * [`ArrowReaderBuilder`] for additional member functions
1245pub type ParquetRecordBatchReaderBuilder<T> = ArrowReaderBuilder<SyncReader<T>>;
1246
1247impl<T: ChunkReader + 'static> ParquetRecordBatchReaderBuilder<T> {
1248    /// Create a new [`ParquetRecordBatchReaderBuilder`]
1249    ///
1250    /// ```
1251    /// # use std::sync::Arc;
1252    /// # use bytes::Bytes;
1253    /// # use arrow_array::{Int32Array, RecordBatch};
1254    /// # use arrow_schema::{DataType, Field, Schema};
1255    /// # use parquet::arrow::arrow_reader::{ParquetRecordBatchReader, ParquetRecordBatchReaderBuilder};
1256    /// # use parquet::arrow::ArrowWriter;
1257    /// # let mut file: Vec<u8> = Vec::with_capacity(1024);
1258    /// # let schema = Arc::new(Schema::new(vec![Field::new("i32", DataType::Int32, false)]));
1259    /// # let mut writer = ArrowWriter::try_new(&mut file, schema.clone(), None).unwrap();
1260    /// # let batch = RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]).unwrap();
1261    /// # writer.write(&batch).unwrap();
1262    /// # writer.close().unwrap();
1263    /// # let file = Bytes::from(file);
1264    /// // Build the reader from anything that implements `ChunkReader`
1265    /// // such as a `File`, or `Bytes`
1266    /// let mut builder = ParquetRecordBatchReaderBuilder::try_new(file).unwrap();
1267    /// // The builder has access to ParquetMetaData such
1268    /// // as the number and layout of row groups
1269    /// assert_eq!(builder.metadata().num_row_groups(), 1);
1270    /// // Call build to create the reader
1271    /// let mut reader: ParquetRecordBatchReader = builder.build().unwrap();
1272    /// // Read data
1273    /// while let Some(batch) = reader.next().transpose()? {
1274    ///     println!("Read {} rows", batch.num_rows());
1275    /// }
1276    /// # Ok::<(), parquet::errors::ParquetError>(())
1277    /// ```
1278    pub fn try_new(reader: T) -> Result<Self> {
1279        Self::try_new_with_options(reader, Default::default())
1280    }
1281
1282    /// Create a new [`ParquetRecordBatchReaderBuilder`] with [`ArrowReaderOptions`]
1283    ///
1284    /// Use this method if you want to control the options for reading the
1285    /// [`ParquetMetaData`]
1286    pub fn try_new_with_options(reader: T, options: ArrowReaderOptions) -> Result<Self> {
1287        let metadata = ArrowReaderMetadata::load(&reader, options)?;
1288        Ok(Self::new_with_metadata(reader, metadata))
1289    }
1290
1291    /// Create a [`ParquetRecordBatchReaderBuilder`] from the provided [`ArrowReaderMetadata`]
1292    ///
1293    /// Use this method if you already have [`ParquetMetaData`] for a file.
1294    /// This interface allows:
1295    ///
1296    /// 1. Loading metadata once and using it to create multiple builders with
1297    ///    potentially different settings or run on different threads
1298    ///
1299    /// 2. Using a cached copy of the metadata rather than re-reading it from the
1300    ///    file each time a reader is constructed.
1301    ///
1302    /// See the docs on [`ArrowReaderMetadata`] for more details
1303    ///
1304    /// # Example
1305    /// ```
1306    /// # use std::fs::metadata;
1307    /// # use std::sync::Arc;
1308    /// # use bytes::Bytes;
1309    /// # use arrow_array::{Int32Array, RecordBatch};
1310    /// # use arrow_schema::{DataType, Field, Schema};
1311    /// # use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ParquetRecordBatchReader, ParquetRecordBatchReaderBuilder};
1312    /// # use parquet::arrow::ArrowWriter;
1313    /// #
1314    /// # let mut file: Vec<u8> = Vec::with_capacity(1024);
1315    /// # let schema = Arc::new(Schema::new(vec![Field::new("i32", DataType::Int32, false)]));
1316    /// # let mut writer = ArrowWriter::try_new(&mut file, schema.clone(), None).unwrap();
1317    /// # let batch = RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]).unwrap();
1318    /// # writer.write(&batch).unwrap();
1319    /// # writer.close().unwrap();
1320    /// # let file = Bytes::from(file);
1321    /// #
1322    /// let metadata = ArrowReaderMetadata::load(&file, Default::default()).unwrap();
1323    /// let mut a = ParquetRecordBatchReaderBuilder::new_with_metadata(file.clone(), metadata.clone()).build().unwrap();
1324    /// let mut b = ParquetRecordBatchReaderBuilder::new_with_metadata(file, metadata).build().unwrap();
1325    ///
1326    /// // Should be able to read from both in parallel
1327    /// assert_eq!(a.next().unwrap().unwrap(), b.next().unwrap().unwrap());
1328    /// ```
1329    pub fn new_with_metadata(input: T, metadata: ArrowReaderMetadata) -> Self {
1330        Self::new_builder(SyncReader(input), metadata)
1331    }
1332
1333    /// Read bloom filter for a column in a row group
1334    ///
1335    /// Returns `None` if the column does not have a bloom filter
1336    ///
1337    /// We should call this function after other forms pruning, such as projection and predicate pushdown.
1338    pub fn get_row_group_column_bloom_filter(
1339        &self,
1340        row_group_idx: usize,
1341        column_idx: usize,
1342    ) -> Result<Option<Sbbf>> {
1343        let metadata = self.metadata.row_group(row_group_idx);
1344        let column_metadata = metadata.column(column_idx);
1345
1346        Sbbf::read_from_column_chunk(column_metadata, &self.input.0)
1347    }
1348
1349    /// Build a [`ParquetRecordBatchReader`]
1350    ///
1351    /// Note: this will eagerly evaluate any `RowFilter` before returning
1352    pub fn build(self) -> Result<ParquetRecordBatchReader> {
1353        let Self {
1354            input,
1355            metadata,
1356            schema: _,
1357            fields,
1358            batch_size,
1359            row_group_plan,
1360            projection,
1361            mut filter,
1362            row_selection_policy,
1363            limit,
1364            offset,
1365            metrics,
1366            // Not used for the sync reader, see https://github.com/apache/arrow-rs/issues/8000
1367            max_predicate_cache_size: _,
1368        } = self;
1369
1370        // Try to avoid allocate large buffer
1371        let batch_size = batch_size.min(metadata.file_metadata().num_rows() as usize);
1372
1373        let (row_groups, selection) = row_group_plan.into_global()?;
1374
1375        let row_groups = row_groups.unwrap_or_else(|| (0..metadata.num_row_groups()).collect());
1376
1377        let reader = ReaderRowGroups {
1378            reader: Arc::new(input.0),
1379            metadata,
1380            row_groups,
1381        };
1382
1383        let mut plan_builder = ReadPlanBuilder::new(batch_size)
1384            .with_selection(selection)
1385            .with_row_selection_policy(row_selection_policy);
1386
1387        // Update selection based on any filters
1388        if let Some(filter) = filter.as_mut() {
1389            for predicate in &mut filter.predicates {
1390                // break early if we have ruled out all rows
1391                if !plan_builder.selects_any() {
1392                    break;
1393                }
1394
1395                let mut cache_projection = predicate.projection().clone();
1396                cache_projection.intersect(&projection);
1397
1398                let array_reader = ArrayReaderBuilder::new(&reader, &metrics)
1399                    .with_batch_size(batch_size)
1400                    .with_parquet_metadata(&reader.metadata)
1401                    .build_array_reader(fields.as_deref(), predicate.projection())?;
1402
1403                plan_builder = plan_builder.with_predicate(array_reader, predicate.as_mut())?;
1404            }
1405        }
1406
1407        let array_reader = ArrayReaderBuilder::new(&reader, &metrics)
1408            .with_batch_size(batch_size)
1409            .with_parquet_metadata(&reader.metadata)
1410            .build_array_reader(fields.as_deref(), &projection)?;
1411
1412        let read_plan = plan_builder
1413            .limited(reader.num_rows())
1414            .with_offset(offset)
1415            .with_limit(limit)
1416            .build_limited()
1417            .build();
1418
1419        Ok(ParquetRecordBatchReader::new(array_reader, read_plan))
1420    }
1421}
1422
1423struct ReaderRowGroups<T: ChunkReader> {
1424    reader: Arc<T>,
1425
1426    metadata: Arc<ParquetMetaData>,
1427    /// Optional list of row group indices to scan
1428    row_groups: Vec<usize>,
1429}
1430
1431impl<T: ChunkReader + 'static> RowGroups for ReaderRowGroups<T> {
1432    fn num_rows(&self) -> usize {
1433        let meta = self.metadata.row_groups();
1434        self.row_groups
1435            .iter()
1436            .map(|x| meta[*x].num_rows() as usize)
1437            .sum()
1438    }
1439
1440    fn column_chunks(&self, i: usize) -> Result<Box<dyn PageIterator>> {
1441        Ok(Box::new(ReaderPageIterator {
1442            column_idx: i,
1443            reader: self.reader.clone(),
1444            metadata: self.metadata.clone(),
1445            row_groups: self.row_groups.clone().into_iter(),
1446        }))
1447    }
1448
1449    fn row_groups(&self) -> Box<dyn Iterator<Item = &RowGroupMetaData> + '_> {
1450        Box::new(
1451            self.row_groups
1452                .iter()
1453                .map(move |i| self.metadata.row_group(*i)),
1454        )
1455    }
1456
1457    fn metadata(&self) -> &ParquetMetaData {
1458        self.metadata.as_ref()
1459    }
1460}
1461
1462struct ReaderPageIterator<T: ChunkReader> {
1463    reader: Arc<T>,
1464    column_idx: usize,
1465    row_groups: std::vec::IntoIter<usize>,
1466    metadata: Arc<ParquetMetaData>,
1467}
1468
1469impl<T: ChunkReader + 'static> ReaderPageIterator<T> {
1470    /// Return the next SerializedPageReader
1471    fn next_page_reader(&self, rg_idx: usize) -> Result<SerializedPageReader<T>> {
1472        let rg = self.metadata.row_group(rg_idx);
1473        let column_chunk_metadata = rg.column(self.column_idx);
1474        let page_locations = self
1475            .metadata
1476            .page_index()
1477            .map(|i| i.page_locations(rg_idx, self.column_idx).cloned())
1478            .unwrap_or(None);
1479        let total_rows = rg.num_rows() as usize;
1480        let reader = self.reader.clone();
1481
1482        SerializedPageReader::new(reader, column_chunk_metadata, total_rows, page_locations)?
1483            .add_crypto_context(
1484                rg_idx,
1485                self.column_idx,
1486                self.metadata.as_ref(),
1487                column_chunk_metadata,
1488            )
1489    }
1490}
1491
1492impl<T: ChunkReader + 'static> Iterator for ReaderPageIterator<T> {
1493    type Item = Result<Box<dyn PageReader>>;
1494
1495    fn next(&mut self) -> Option<Self::Item> {
1496        let rg_idx = self.row_groups.next()?;
1497        let page_reader = self
1498            .next_page_reader(rg_idx)
1499            .map(|page_reader| Box::new(page_reader) as _);
1500        Some(page_reader)
1501    }
1502}
1503
1504impl<T: ChunkReader + 'static> PageIterator for ReaderPageIterator<T> {}
1505
1506/// Reads Parquet data as Arrow [`RecordBatch`]es
1507///
1508/// This struct implements the [`RecordBatchReader`] trait and is an
1509/// `Iterator<Item = ArrowResult<RecordBatch>>` that yields [`RecordBatch`]es.
1510///
1511/// Typically, either reads from a file or an in memory buffer [`Bytes`]
1512///
1513/// Created by [`ParquetRecordBatchReaderBuilder`]
1514///
1515/// [`Bytes`]: bytes::Bytes
1516pub struct ParquetRecordBatchReader {
1517    array_reader: Box<dyn ArrayReader>,
1518    schema: SchemaRef,
1519    read_plan: ReadPlan,
1520}
1521
1522/// Accumulates filter masks for decoded chunks in one logical output batch.
1523///
1524/// The first chunk keeps its [`BooleanBuffer`] without copying. A second chunk
1525/// promotes the accumulator to a [`BooleanBufferBuilder`], and later chunks are
1526/// appended to it. For example, chunks `1001` and `1` become `10011`:
1527///
1528/// ```text
1529///           append(1001)             append(1)
1530///   Empty ───────────────▶ Single ───────────────▶ Combined
1531///                          1001                    10011
1532///                          (zero copy)             (promoted to builder)
1533/// ```
1534///
1535/// The combined mask lines up with the rows that [`ArrayReader::read_records`]
1536/// buffered across the decoded chunks (see [`read_mask_batch`]). Consuming the
1537/// buffered batch and filtering it with the accumulated mask yields the output.
1538///
1539/// ```text
1540///   decoded rows:   0 1 2 3   11   <-- buffered by the array reader
1541///   chunk masks:   [1 0 0 1] [1]
1542///   finish():       1 0 0 1   1    <-- filters the whole batch in one pass
1543/// ```
1544#[derive(Default)]
1545enum FilterMaskAccumulator {
1546    #[default]
1547    Empty,
1548    Single(BooleanBuffer),
1549    Combined(BooleanBufferBuilder),
1550}
1551
1552impl FilterMaskAccumulator {
1553    fn append(&mut self, mask: BooleanBuffer) {
1554        *self = match std::mem::take(self) {
1555            Self::Empty => Self::Single(mask),
1556            Self::Single(first) => {
1557                let mut combined = BooleanBufferBuilder::new(first.len() + mask.len());
1558                combined.append_buffer(&first);
1559                combined.append_buffer(&mask);
1560                Self::Combined(combined)
1561            }
1562            Self::Combined(mut combined) => {
1563                combined.append_buffer(&mask);
1564                Self::Combined(combined)
1565            }
1566        };
1567    }
1568
1569    fn finish(self) -> Option<BooleanBuffer> {
1570        match self {
1571            Self::Empty => None,
1572            Self::Single(mask) => Some(mask),
1573            Self::Combined(combined) => Some(combined.build()),
1574        }
1575    }
1576}
1577
1578/// Converts the projection buffered by `array_reader` into a record batch.
1579fn consume_record_batch(array_reader: &mut dyn ArrayReader) -> Result<RecordBatch> {
1580    let array = array_reader.consume_batch()?;
1581    let struct_array = array.as_struct_opt().ok_or_else(|| {
1582        ArrowError::ParquetError("Struct array reader should return struct array".to_string())
1583    })?;
1584    Ok(RecordBatch::from(struct_array))
1585}
1586
1587/// Reads one logical Mask batch, potentially spanning multiple loaded ranges.
1588///
1589/// Each [`MaskCursor`] chunk is safe to decode because it stays within loaded
1590/// pages. Gaps are crossed with [`ArrayReader::skip_records`], while decoded
1591/// arrays and their mask fragments remain buffered. Once `batch_size` selected
1592/// rows have accumulated, this consumes the underlying batch and filters it
1593/// once with the combined mask.
1594fn read_mask_batch(
1595    array_reader: &mut dyn ArrayReader,
1596    mask_cursor: &mut MaskCursor,
1597    batch_size: usize,
1598) -> Result<Option<RecordBatch>> {
1599    let mut selected_rows = 0;
1600    let mut filter_mask = FilterMaskAccumulator::default();
1601
1602    while selected_rows < batch_size && !mask_cursor.is_empty() {
1603        let mask_chunk = mask_cursor.next_chunk(batch_size - selected_rows)?;
1604
1605        if mask_chunk.initial_skip > 0 {
1606            let skipped = array_reader.skip_records(mask_chunk.initial_skip)?;
1607            if skipped != mask_chunk.initial_skip {
1608                return Err(general_err!(
1609                    "failed to skip rows, expected {}, got {}",
1610                    mask_chunk.initial_skip,
1611                    skipped
1612                ));
1613            }
1614        }
1615
1616        let mask = mask_cursor.mask_values_for(&mask_chunk)?;
1617        let read = array_reader.read_records(mask_chunk.chunk_rows)?;
1618        if read == 0 {
1619            return Err(general_err!(
1620                "reached end of column while expecting {} rows",
1621                mask_chunk.chunk_rows
1622            ));
1623        }
1624        if read != mask_chunk.chunk_rows {
1625            return Err(general_err!(
1626                "insufficient rows read from array reader - expected {}, got {}",
1627                mask_chunk.chunk_rows,
1628                read
1629            ));
1630        }
1631
1632        filter_mask.append(mask.values().clone());
1633        selected_rows += mask_chunk.selected_rows;
1634    }
1635
1636    if selected_rows == 0 {
1637        return Ok(None);
1638    }
1639
1640    let filter_mask = filter_mask
1641        .finish()
1642        .ok_or_else(|| general_err!("Internal Error: decoded Mask batch has no filter values"))?;
1643    let batch = consume_record_batch(array_reader)?;
1644    let filtered_batch = filter_record_batch(&batch, &BooleanArray::from(filter_mask))?;
1645    if filtered_batch.num_rows() != selected_rows {
1646        return Err(general_err!(
1647            "filtered rows mismatch selection - expected {}, got {}",
1648            selected_rows,
1649            filtered_batch.num_rows()
1650        ));
1651    }
1652
1653    Ok(Some(filtered_batch))
1654}
1655
1656impl Debug for ParquetRecordBatchReader {
1657    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
1658        f.debug_struct("ParquetRecordBatchReader")
1659            .field("array_reader", &"...")
1660            .field("schema", &self.schema)
1661            .field("read_plan", &self.read_plan)
1662            .finish()
1663    }
1664}
1665
1666impl Iterator for ParquetRecordBatchReader {
1667    type Item = Result<RecordBatch, ArrowError>;
1668
1669    fn next(&mut self) -> Option<Self::Item> {
1670        self.next_inner()
1671            .map_err(|arrow_err| arrow_err.into())
1672            .transpose()
1673    }
1674}
1675
1676impl ParquetRecordBatchReader {
1677    /// Returns the next `RecordBatch` from the reader, or `None` if the reader
1678    /// has reached the end of the file.
1679    ///
1680    /// Returns `Result<Option<..>>` rather than `Option<Result<..>>` to
1681    /// simplify error handling with `?`
1682    fn next_inner(&mut self) -> Result<Option<RecordBatch>> {
1683        let mut read_records = 0;
1684        let batch_size = self.batch_size();
1685        if batch_size == 0 {
1686            return Ok(None);
1687        }
1688        match self.read_plan.row_selection_cursor_mut() {
1689            RowSelectionCursor::Mask(mask_cursor) => {
1690                return read_mask_batch(self.array_reader.as_mut(), mask_cursor, batch_size);
1691            }
1692            RowSelectionCursor::Selectors(selectors_cursor) => {
1693                while read_records < batch_size && !selectors_cursor.is_empty() {
1694                    let front = selectors_cursor.next_selector();
1695                    if front.skip {
1696                        let skipped = self.array_reader.skip_records(front.row_count)?;
1697
1698                        if skipped != front.row_count {
1699                            return Err(general_err!(
1700                                "failed to skip rows, expected {}, got {}",
1701                                front.row_count,
1702                                skipped
1703                            ));
1704                        }
1705                        continue;
1706                    }
1707
1708                    //Currently, when RowSelectors with row_count = 0 are included then its interpreted as end of reader.
1709                    //Fix is to skip such entries. See https://github.com/apache/arrow-rs/issues/2669
1710                    if front.row_count == 0 {
1711                        continue;
1712                    }
1713
1714                    // try to read record
1715                    let need_read = batch_size - read_records;
1716                    let to_read = match front.row_count.checked_sub(need_read) {
1717                        Some(remaining) if remaining != 0 => {
1718                            // if page row count less than batch_size we must set batch size to page row count.
1719                            // add check avoid dead loop
1720                            selectors_cursor.return_selector(RowSelector::select(remaining));
1721                            need_read
1722                        }
1723                        _ => front.row_count,
1724                    };
1725                    match self.array_reader.read_records(to_read)? {
1726                        0 => break,
1727                        rec => read_records += rec,
1728                    }
1729                }
1730            }
1731            RowSelectionCursor::All => {
1732                self.array_reader.read_records(batch_size)?;
1733            }
1734        }
1735
1736        let batch = consume_record_batch(self.array_reader.as_mut())?;
1737        Ok(if batch.num_rows() > 0 {
1738            Some(batch)
1739        } else {
1740            None
1741        })
1742    }
1743}
1744
1745impl RecordBatchReader for ParquetRecordBatchReader {
1746    /// Returns the projected [`SchemaRef`] for reading the parquet file.
1747    ///
1748    /// Note that the schema metadata will be stripped here. See
1749    /// [`ParquetRecordBatchReaderBuilder::schema`] if the metadata is desired.
1750    fn schema(&self) -> SchemaRef {
1751        self.schema.clone()
1752    }
1753}
1754
1755impl ParquetRecordBatchReader {
1756    /// Create a new [`ParquetRecordBatchReader`] from the provided chunk reader
1757    ///
1758    /// See [`ParquetRecordBatchReaderBuilder`] for more options
1759    pub fn try_new<T: ChunkReader + 'static>(reader: T, batch_size: usize) -> Result<Self> {
1760        ParquetRecordBatchReaderBuilder::try_new(reader)?
1761            .with_batch_size(batch_size)
1762            .build()
1763    }
1764
1765    /// Create a new [`ParquetRecordBatchReader`] from the provided [`RowGroups`]
1766    ///
1767    /// Note: this is a low-level interface see [`ParquetRecordBatchReader::try_new`] for a
1768    /// higher-level interface for reading parquet data from a file
1769    pub fn try_new_with_row_groups(
1770        levels: &FieldLevels,
1771        row_groups: &dyn RowGroups,
1772        batch_size: usize,
1773        selection: Option<RowSelection>,
1774    ) -> Result<Self> {
1775        // note metrics are not supported in this API
1776        let metrics = ArrowReaderMetrics::disabled();
1777        let array_reader = ArrayReaderBuilder::new(row_groups, &metrics)
1778            .with_batch_size(batch_size)
1779            .with_parquet_metadata(row_groups.metadata())
1780            .build_array_reader(levels.levels.as_ref(), &ProjectionMask::all())?;
1781
1782        let read_plan = ReadPlanBuilder::new(batch_size)
1783            .with_selection(selection)
1784            .build();
1785
1786        Ok(Self {
1787            array_reader,
1788            schema: Arc::new(Schema::new(levels.fields.clone())),
1789            read_plan,
1790        })
1791    }
1792
1793    /// Create a new [`ParquetRecordBatchReader`] that will read at most `batch_size` rows at
1794    /// a time from [`ArrayReader`] based on the configured `selection`. If `selection` is `None`
1795    /// all rows will be returned
1796    pub(crate) fn new(array_reader: Box<dyn ArrayReader>, read_plan: ReadPlan) -> Self {
1797        let schema = match array_reader.get_data_type() {
1798            ArrowType::Struct(fields) => Schema::new(fields.clone()),
1799            _ => unreachable!("Struct array reader's data type is not struct!"),
1800        };
1801
1802        Self {
1803            array_reader,
1804            schema: Arc::new(schema),
1805            read_plan,
1806        }
1807    }
1808
1809    #[inline(always)]
1810    pub(crate) fn batch_size(&self) -> usize {
1811        self.read_plan.batch_size()
1812    }
1813}
1814
1815#[cfg(test)]
1816pub(crate) mod tests;