1use std::ops::Range;
11
12use bytes::Bytes;
13
14use crate::errors::{ParquetError, Result};
15use crate::file::metadata::column_chunk::{ColumnChunkEntries, ColumnChunkKey};
16use crate::file::metadata::page_index::{PageIndex, PageIndexProvider};
17use crate::file::metadata::thrift::parquet_metadata_from_bytes;
18use crate::file::metadata::{
19 ColumnChunkMask, ColumnChunkMetaData, PageIndexPolicy, ParquetMetaData, ParquetMetaDataOptions,
20};
21
22use crate::file::page_index::column_index::ColumnIndexMetaData;
23use crate::file::page_index::index_reader::{decode_column_index, decode_offset_index};
24use crate::file::page_index::offset_index::OffsetIndexMetaData;
25
26pub(crate) use inner::MetadataParser;
34
35#[cfg(feature = "encryption")]
36mod inner {
37 use std::sync::Arc;
38
39 use super::*;
40 use crate::encryption::decrypt::FileDecryptionProperties;
41
42 #[derive(Debug, Default)]
44 pub(crate) struct MetadataParser {
45 file_decryption_properties: Option<Arc<FileDecryptionProperties>>,
47 metadata_options: Option<Arc<ParquetMetaDataOptions>>,
49 }
50
51 impl MetadataParser {
52 pub(crate) fn new() -> Self {
53 MetadataParser::default()
54 }
55
56 pub(crate) fn with_file_decryption_properties(
57 mut self,
58 file_decryption_properties: Option<Arc<FileDecryptionProperties>>,
59 ) -> Self {
60 self.file_decryption_properties = file_decryption_properties;
61 self
62 }
63
64 pub(crate) fn with_metadata_options(
65 self,
66 options: Option<Arc<ParquetMetaDataOptions>>,
67 ) -> Self {
68 Self {
69 metadata_options: options,
70 ..self
71 }
72 }
73
74 pub(crate) fn decode_metadata(
75 &self,
76 buf: &[u8],
77 encrypted_footer: bool,
78 ) -> Result<ParquetMetaData> {
79 if encrypted_footer || self.file_decryption_properties.is_some() {
80 crate::file::metadata::thrift::encryption::parquet_metadata_with_encryption(
81 self.file_decryption_properties.as_ref(),
82 encrypted_footer,
83 buf,
84 self.metadata_options.as_deref(),
85 )
86 } else {
87 decode_metadata(buf, self.metadata_options.as_deref())
88 }
89 }
90 }
91
92 pub(super) fn parse_single_column_index(
93 bytes: &[u8],
94 metadata: &ParquetMetaData,
95 column: &ColumnChunkMetaData,
96 row_group_index: usize,
97 col_index: usize,
98 ) -> Result<ColumnIndexMetaData> {
99 use crate::encryption::decrypt::CryptoContext;
100 match &column.column_crypto_metadata {
101 Some(crypto_metadata) => {
102 let file_decryptor = metadata.file_decryptor.as_ref().ok_or_else(|| {
103 general_err!("Cannot decrypt column index, no file decryptor set")
104 })?;
105 let crypto_context = CryptoContext::for_column(
106 file_decryptor,
107 crypto_metadata,
108 row_group_index,
109 col_index,
110 )?;
111 let column_decryptor = crypto_context.metadata_decryptor();
112 let aad = crypto_context.create_column_index_aad()?;
113 let plaintext = column_decryptor.decrypt(bytes, &aad)?;
114 decode_column_index(&plaintext, column.column_type())
115 }
116 None => decode_column_index(bytes, column.column_type()),
117 }
118 }
119
120 pub(super) fn parse_single_offset_index(
121 bytes: &[u8],
122 metadata: &ParquetMetaData,
123 column: &ColumnChunkMetaData,
124 row_group_index: usize,
125 col_index: usize,
126 ) -> Result<OffsetIndexMetaData> {
127 use crate::encryption::decrypt::CryptoContext;
128 match &column.column_crypto_metadata {
129 Some(crypto_metadata) => {
130 let file_decryptor = metadata.file_decryptor.as_ref().ok_or_else(|| {
131 general_err!("Cannot decrypt offset index, no file decryptor set")
132 })?;
133 let crypto_context = CryptoContext::for_column(
134 file_decryptor,
135 crypto_metadata,
136 row_group_index,
137 col_index,
138 )?;
139 let column_decryptor = crypto_context.metadata_decryptor();
140 let aad = crypto_context.create_offset_index_aad()?;
141 let plaintext = column_decryptor.decrypt(bytes, &aad)?;
142 decode_offset_index(&plaintext)
143 }
144 None => decode_offset_index(bytes),
145 }
146 }
147}
148
149#[cfg(not(feature = "encryption"))]
150mod inner {
151 use super::*;
152 use std::sync::Arc;
153 #[derive(Debug, Default)]
157 pub(crate) struct MetadataParser {
158 metadata_options: Option<Arc<ParquetMetaDataOptions>>,
160 }
161
162 impl MetadataParser {
163 pub(crate) fn new() -> Self {
164 MetadataParser::default()
165 }
166
167 pub(crate) fn with_metadata_options(
168 self,
169 options: Option<Arc<ParquetMetaDataOptions>>,
170 ) -> Self {
171 Self {
172 metadata_options: options,
173 }
174 }
175
176 pub(crate) fn decode_metadata(
177 &self,
178 buf: &[u8],
179 encrypted_footer: bool,
180 ) -> Result<ParquetMetaData> {
181 if encrypted_footer {
182 Err(general_err!(
183 "Parquet file has an encrypted footer but the encryption feature is disabled"
184 ))
185 } else {
186 decode_metadata(buf, self.metadata_options.as_deref())
187 }
188 }
189 }
190
191 pub(super) fn parse_single_column_index(
192 bytes: &[u8],
193 _metadata: &ParquetMetaData,
194 column: &ColumnChunkMetaData,
195 _row_group_index: usize,
196 _col_index: usize,
197 ) -> Result<ColumnIndexMetaData> {
198 decode_column_index(bytes, column.column_type())
199 }
200
201 pub(super) fn parse_single_offset_index(
202 bytes: &[u8],
203 _metadata: &ParquetMetaData,
204 _column: &ColumnChunkMetaData,
205 _row_group_index: usize,
206 _col_index: usize,
207 ) -> Result<OffsetIndexMetaData> {
208 decode_offset_index(bytes)
209 }
210}
211
212pub(crate) fn decode_metadata(
220 buf: &[u8],
221 options: Option<&ParquetMetaDataOptions>,
222) -> Result<ParquetMetaData> {
223 parquet_metadata_from_bytes(buf, options)
224}
225
226pub(crate) fn parse_page_index(
241 metadata: &ParquetMetaData,
242 column_index_policy: PageIndexPolicy,
243 offset_index_policy: PageIndexPolicy,
244 column_index_mask: &ColumnChunkMask,
245 offset_index_mask: &ColumnChunkMask,
246 bytes: &Bytes,
247 start_offset: u64,
248) -> Result<Option<PageIndex>> {
249 let column_indexes = if column_index_policy == PageIndexPolicy::Skip {
250 None
251 } else {
252 Some(parse_column_index(
253 metadata,
254 column_index_mask,
255 bytes,
256 start_offset,
257 )?)
258 };
259 let offset_indexes = if offset_index_policy == PageIndexPolicy::Skip {
260 None
261 } else {
262 Some(parse_offset_index(
263 metadata,
264 offset_index_policy,
265 offset_index_mask,
266 bytes,
267 start_offset,
268 )?)
269 };
270
271 let page_index = PageIndex::from_entries(column_indexes, offset_indexes);
272 if page_index.has_column_indexes() || page_index.has_offset_indexes() {
273 Ok(Some(page_index))
274 } else {
275 Ok(None)
276 }
277}
278
279fn reserve_index_entries<T>(
283 metadata: &ParquetMetaData,
284 mask: &ColumnChunkMask,
285 index_range: impl Fn(&ColumnChunkMetaData) -> Option<Range<u64>>,
286) -> Result<Vec<(ColumnChunkKey, T)>> {
287 let num_entries =
288 mask.row_group_indices(metadata.num_row_groups())
289 .try_fold(0_usize, |count, rg_idx| {
290 let rg = metadata.row_group(rg_idx);
291 let present = mask
292 .column_indices(rg.num_columns())
293 .filter(|&col_idx| index_range(rg.column(col_idx)).is_some())
294 .count();
295 count.checked_add(present).ok_or_else(|| {
296 ParquetError::General("column chunk entry count exceeds usize::MAX".to_string())
297 })
298 })?;
299 let mut entries = Vec::new();
300 entries.try_reserve_exact(num_entries).map_err(|err| {
301 ParquetError::General(format!(
302 "failed to allocate storage for {num_entries} column chunk entries: {err}"
303 ))
304 })?;
305 Ok(entries)
306}
307
308fn parse_column_index(
309 metadata: &ParquetMetaData,
310 mask: &ColumnChunkMask,
311 bytes: &Bytes,
312 start_offset: u64,
313) -> Result<ColumnChunkEntries<ColumnIndexMetaData>> {
314 let mut entries =
315 reserve_index_entries(metadata, mask, ColumnChunkMetaData::column_index_range)?;
316 for rg_idx in mask.row_group_indices(metadata.num_row_groups()) {
319 let rg = metadata.row_group(rg_idx);
320 for col_idx in mask.column_indices(rg.num_columns()) {
321 let col = rg.column(col_idx);
322 if let Some(r) = col.column_index_range() {
323 let idx_bytes = get_index_bytes(bytes, start_offset, r)?;
324 let idx =
325 inner::parse_single_column_index(idx_bytes, metadata, col, rg_idx, col_idx)?;
326 entries.push((ColumnChunkKey::try_new(rg_idx, col_idx)?, idx));
327 }
328 }
329 }
330
331 ColumnChunkEntries::try_from_sorted_entries(
332 metadata.num_row_groups(),
333 metadata.file_metadata().schema_descr().num_columns(),
334 entries,
335 )
336}
337
338fn parse_offset_index(
339 metadata: &ParquetMetaData,
340 offset_index_policy: PageIndexPolicy,
341 mask: &ColumnChunkMask,
342 bytes: &Bytes,
343 start_offset: u64,
344) -> Result<ColumnChunkEntries<OffsetIndexMetaData>> {
345 let mut entries =
346 reserve_index_entries(metadata, mask, ColumnChunkMetaData::offset_index_range)?;
347 for rg_idx in mask.row_group_indices(metadata.num_row_groups()) {
348 let rg = metadata.row_group(rg_idx);
349 for col_idx in mask.column_indices(rg.num_columns()) {
350 let col = rg.column(col_idx);
351 if let Some(r) = col.offset_index_range() {
352 let idx_bytes = get_index_bytes(bytes, start_offset, r)?;
353 let idx =
354 inner::parse_single_offset_index(idx_bytes, metadata, col, rg_idx, col_idx)?;
355 entries.push((ColumnChunkKey::try_new(rg_idx, col_idx)?, idx));
356 } else if offset_index_policy == PageIndexPolicy::Required {
357 return Err(general_err!("missing offset index"));
358 }
359 }
360 }
361
362 ColumnChunkEntries::try_from_sorted_entries(
363 metadata.num_row_groups(),
364 metadata.file_metadata().schema_descr().num_columns(),
365 entries,
366 )
367}
368
369fn get_index_bytes(bytes: &[u8], start_offset: u64, range: Range<u64>) -> Result<&[u8]> {
370 let start = usize::try_from(range.start - start_offset)?;
371 let end = usize::try_from(range.end - start_offset)?;
372 Ok(&bytes[start..end])
373}