Skip to main content

parquet/file/metadata/
parser.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Internal metadata parsing routines
6//!
7//! These functions parse thrift-encoded metadata from a byte slice
8//! into the corresponding Rust structures
9
10use std::ops::Range;
11
12use bytes::Bytes;
13
14use crate::errors::{ParquetError, Result};
15use crate::file::metadata::column_chunk::{ColumnChunkEntries, ColumnChunkKey};
16use crate::file::metadata::page_index::{PageIndex, PageIndexProvider};
17use crate::file::metadata::thrift::parquet_metadata_from_bytes;
18use crate::file::metadata::{
19    ColumnChunkMask, ColumnChunkMetaData, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions,
20};
21
22use crate::file::page_index::column_index::ColumnIndexMetaData;
23use crate::file::page_index::index_reader::{decode_column_index, decode_offset_index};
24use crate::file::page_index::offset_index::OffsetIndexMetaData;
25
26/// Helper struct for metadata parsing
27///
28/// This structure parses thrift-encoded bytes into the correct Rust structs,
29/// such as [`ParquetMetaData`], handling decryption if necessary.
30//
31// Note this structure is used to minimize the number of
32// places to add `#[cfg(feature = "encryption")]` checks.
33pub(crate) use inner::MetadataParser;
34
35#[cfg(feature = "encryption")]
36mod inner {
37    use std::sync::Arc;
38
39    use super::*;
40    use crate::encryption::decrypt::FileDecryptionProperties;
41
42    /// API for decoding metadata that may be encrypted
43    #[derive(Debug, Default)]
44    pub(crate) struct MetadataParser {
45        // the credentials and keys needed to decrypt metadata
46        file_decryption_properties: Option<Arc<FileDecryptionProperties>>,
47        // metadata parsing options
48        metadata_options: Option<Arc<ParquetMetaDataOptions>>,
49    }
50
51    impl MetadataParser {
52        pub(crate) fn new() -> Self {
53            MetadataParser::default()
54        }
55
56        pub(crate) fn with_file_decryption_properties(
57            mut self,
58            file_decryption_properties: Option<Arc<FileDecryptionProperties>>,
59        ) -> Self {
60            self.file_decryption_properties = file_decryption_properties;
61            self
62        }
63
64        pub(crate) fn with_metadata_options(
65            self,
66            options: Option<Arc<ParquetMetaDataOptions>>,
67        ) -> Self {
68            Self {
69                metadata_options: options,
70                ..self
71            }
72        }
73
74        pub(crate) fn decode_metadata(
75            &self,
76            buf: &[u8],
77            encrypted_footer: bool,
78        ) -> Result<ParquetMetaData> {
79            if encrypted_footer || self.file_decryption_properties.is_some() {
80                crate::file::metadata::thrift::encryption::parquet_metadata_with_encryption(
81                    self.file_decryption_properties.as_ref(),
82                    encrypted_footer,
83                    buf,
84                    self.metadata_options.as_deref(),
85                )
86            } else {
87                decode_metadata(buf, self.metadata_options.as_deref())
88            }
89        }
90    }
91
92    pub(super) fn parse_single_column_index(
93        bytes: &[u8],
94        metadata: &ParquetMetaData,
95        column: &ColumnChunkMetaData,
96        row_group_index: usize,
97        col_index: usize,
98    ) -> Result<ColumnIndexMetaData> {
99        use crate::encryption::decrypt::CryptoContext;
100        match &column.column_crypto_metadata {
101            Some(crypto_metadata) => {
102                let file_decryptor = metadata.file_decryptor.as_ref().ok_or_else(|| {
103                    general_err!("Cannot decrypt column index, no file decryptor set")
104                })?;
105                let crypto_context = CryptoContext::for_column(
106                    file_decryptor,
107                    crypto_metadata,
108                    row_group_index,
109                    col_index,
110                )?;
111                let column_decryptor = crypto_context.metadata_decryptor();
112                let aad = crypto_context.create_column_index_aad()?;
113                let plaintext = column_decryptor.decrypt(bytes, &aad)?;
114                decode_column_index(&plaintext, column.column_type())
115            }
116            None => decode_column_index(bytes, column.column_type()),
117        }
118    }
119
120    pub(super) fn parse_single_offset_index(
121        bytes: &[u8],
122        metadata: &ParquetMetaData,
123        column: &ColumnChunkMetaData,
124        row_group_index: usize,
125        col_index: usize,
126    ) -> Result<OffsetIndexMetaData> {
127        use crate::encryption::decrypt::CryptoContext;
128        match &column.column_crypto_metadata {
129            Some(crypto_metadata) => {
130                let file_decryptor = metadata.file_decryptor.as_ref().ok_or_else(|| {
131                    general_err!("Cannot decrypt offset index, no file decryptor set")
132                })?;
133                let crypto_context = CryptoContext::for_column(
134                    file_decryptor,
135                    crypto_metadata,
136                    row_group_index,
137                    col_index,
138                )?;
139                let column_decryptor = crypto_context.metadata_decryptor();
140                let aad = crypto_context.create_offset_index_aad()?;
141                let plaintext = column_decryptor.decrypt(bytes, &aad)?;
142                decode_offset_index(&plaintext)
143            }
144            None => decode_offset_index(bytes),
145        }
146    }
147}
148
149#[cfg(not(feature = "encryption"))]
150mod inner {
151    use super::*;
152    use std::sync::Arc;
153    /// parallel implementation when encryption feature is not enabled
154    ///
155    /// This has the same API as the encryption-enabled version
156    #[derive(Debug, Default)]
157    pub(crate) struct MetadataParser {
158        // metadata parsing options
159        metadata_options: Option<Arc<ParquetMetaDataOptions>>,
160    }
161
162    impl MetadataParser {
163        pub(crate) fn new() -> Self {
164            MetadataParser::default()
165        }
166
167        pub(crate) fn with_metadata_options(
168            self,
169            options: Option<Arc<ParquetMetaDataOptions>>,
170        ) -> Self {
171            Self {
172                metadata_options: options,
173            }
174        }
175
176        pub(crate) fn decode_metadata(
177            &self,
178            buf: &[u8],
179            encrypted_footer: bool,
180        ) -> Result<ParquetMetaData> {
181            if encrypted_footer {
182                Err(general_err!(
183                    "Parquet file has an encrypted footer but the encryption feature is disabled"
184                ))
185            } else {
186                decode_metadata(buf, self.metadata_options.as_deref())
187            }
188        }
189    }
190
191    pub(super) fn parse_single_column_index(
192        bytes: &[u8],
193        _metadata: &ParquetMetaData,
194        column: &ColumnChunkMetaData,
195        _row_group_index: usize,
196        _col_index: usize,
197    ) -> Result<ColumnIndexMetaData> {
198        decode_column_index(bytes, column.column_type())
199    }
200
201    pub(super) fn parse_single_offset_index(
202        bytes: &[u8],
203        _metadata: &ParquetMetaData,
204        _column: &ColumnChunkMetaData,
205        _row_group_index: usize,
206        _col_index: usize,
207    ) -> Result<OffsetIndexMetaData> {
208        decode_offset_index(bytes)
209    }
210}
211
212/// Decodes [`ParquetMetaData`] from the provided bytes.
213///
214/// Typically this is used to decode the metadata from the end of a parquet
215/// file. The format of `buf` is the Thrift compact binary protocol, as specified
216/// by the [Parquet Spec].
217///
218/// [Parquet Spec]: https://github.com/apache/parquet-format#metadata
219pub(crate) fn decode_metadata(
220    buf: &[u8],
221    options: Option<&ParquetMetaDataOptions>,
222) -> Result<ParquetMetaData> {
223    parquet_metadata_from_bytes(buf, options)
224}
225
226/// Parses page indexes from the provided bytes.
227///
228/// Arguments
229/// * `metadata` - The footer metadata describing the page index locations.
230/// * `column_index_policy` - The policy for handling column index parsing (e.g.,
231///   Required, Optional, Skip).
232/// * `offset_index_policy` - The policy for handling offset index parsing (e.g.,
233///   Required, Optional, Skip).
234/// * `column_index_mask` - The row groups and leaf columns whose column indexes are parsed.
235///   Ignored when `column_index_policy` is [`PageIndexPolicy::Skip`].
236/// * `offset_index_mask` - The row groups and leaf columns whose offset indexes are parsed.
237///   Ignored when `offset_index_policy` is [`PageIndexPolicy::Skip`].
238/// * `bytes` - The byte slice containing the page index data.
239/// * `start_offset` - The offset where `bytes` begin in the file.
240pub(crate) fn parse_page_index(
241    metadata: &ParquetMetaData,
242    column_index_policy: PageIndexPolicy,
243    offset_index_policy: PageIndexPolicy,
244    column_index_mask: &ColumnChunkMask,
245    offset_index_mask: &ColumnChunkMask,
246    bytes: &Bytes,
247    start_offset: u64,
248) -> Result<Option<PageIndex>> {
249    let column_indexes = if column_index_policy == PageIndexPolicy::Skip {
250        None
251    } else {
252        Some(parse_column_index(
253            metadata,
254            column_index_mask,
255            bytes,
256            start_offset,
257        )?)
258    };
259    let offset_indexes = if offset_index_policy == PageIndexPolicy::Skip {
260        None
261    } else {
262        Some(parse_offset_index(
263            metadata,
264            offset_index_policy,
265            offset_index_mask,
266            bytes,
267            start_offset,
268        )?)
269    };
270
271    let page_index = PageIndex::from_entries(column_indexes, offset_indexes);
272    if page_index.has_column_indexes() || page_index.has_offset_indexes() {
273        Ok(Some(page_index))
274    } else {
275        Ok(None)
276    }
277}
278
279/// Reserve only for selected chunks that actually have an index range. An extra
280/// metadata-only pass avoids both vector growth for dense reads and reserving a
281/// full rectangular grid for sparse reads or files with missing indexes.
282fn reserve_index_entries<T>(
283    metadata: &ParquetMetaData,
284    mask: &ColumnChunkMask,
285    index_range: impl Fn(&ColumnChunkMetaData) -> Option<Range<u64>>,
286) -> Result<Vec<(ColumnChunkKey, T)>> {
287    let num_entries =
288        mask.row_group_indices(metadata.num_row_groups())
289            .try_fold(0_usize, |count, rg_idx| {
290                let rg = metadata.row_group(rg_idx);
291                let present = mask
292                    .column_indices(rg.num_columns())
293                    .filter(|&col_idx| index_range(rg.column(col_idx)).is_some())
294                    .count();
295                count.checked_add(present).ok_or_else(|| {
296                    ParquetError::General("column chunk entry count exceeds usize::MAX".to_string())
297                })
298            })?;
299    let mut entries = Vec::new();
300    entries.try_reserve_exact(num_entries).map_err(|err| {
301        ParquetError::General(format!(
302            "failed to allocate storage for {num_entries} column chunk entries: {err}"
303        ))
304    })?;
305    Ok(entries)
306}
307
308fn parse_column_index(
309    metadata: &ParquetMetaData,
310    mask: &ColumnChunkMask,
311    bytes: &Bytes,
312    start_offset: u64,
313) -> Result<ColumnChunkEntries<ColumnIndexMetaData>> {
314    let mut entries =
315        reserve_index_entries(metadata, mask, ColumnChunkMetaData::column_index_range)?;
316    // Mask axes are sorted and unique, so present entries remain in row-major
317    // order even when selected chunks have no index.
318    for rg_idx in mask.row_group_indices(metadata.num_row_groups()) {
319        let rg = metadata.row_group(rg_idx);
320        for col_idx in mask.column_indices(rg.num_columns()) {
321            let col = rg.column(col_idx);
322            if let Some(r) = col.column_index_range() {
323                let idx_bytes = get_index_bytes(bytes, start_offset, r)?;
324                let idx =
325                    inner::parse_single_column_index(idx_bytes, metadata, col, rg_idx, col_idx)?;
326                entries.push((ColumnChunkKey::try_new(rg_idx, col_idx)?, idx));
327            }
328        }
329    }
330
331    ColumnChunkEntries::try_from_sorted_entries(
332        metadata.num_row_groups(),
333        metadata.file_metadata().schema_descr().num_columns(),
334        entries,
335    )
336}
337
338fn parse_offset_index(
339    metadata: &ParquetMetaData,
340    offset_index_policy: PageIndexPolicy,
341    mask: &ColumnChunkMask,
342    bytes: &Bytes,
343    start_offset: u64,
344) -> Result<ColumnChunkEntries<OffsetIndexMetaData>> {
345    let mut entries =
346        reserve_index_entries(metadata, mask, ColumnChunkMetaData::offset_index_range)?;
347    for rg_idx in mask.row_group_indices(metadata.num_row_groups()) {
348        let rg = metadata.row_group(rg_idx);
349        for col_idx in mask.column_indices(rg.num_columns()) {
350            let col = rg.column(col_idx);
351            if let Some(r) = col.offset_index_range() {
352                let idx_bytes = get_index_bytes(bytes, start_offset, r)?;
353                let idx =
354                    inner::parse_single_offset_index(idx_bytes, metadata, col, rg_idx, col_idx)?;
355                entries.push((ColumnChunkKey::try_new(rg_idx, col_idx)?, idx));
356            } else if offset_index_policy == PageIndexPolicy::Required {
357                return Err(general_err!("missing offset index"));
358            }
359        }
360    }
361
362    ColumnChunkEntries::try_from_sorted_entries(
363        metadata.num_row_groups(),
364        metadata.file_metadata().schema_descr().num_columns(),
365        entries,
366    )
367}
368
369fn get_index_bytes(bytes: &[u8], start_offset: u64, range: Range<u64>) -> Result<&[u8]> {
370    let start = usize::try_from(range.start - start_offset)?;
371    let end = usize::try_from(range.end - start_offset)?;
372    Ok(&bytes[start..end])
373}