Skip to main content

parquet/file/metadata/thrift/
encryption.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Encryption support for Thrift serialization
6
7use crate::{
8    encryption::decrypt::{FileDecryptionProperties, FileDecryptor},
9    errors::{ParquetError, Result},
10    file::{
11        column_crypto_metadata::ColumnCryptoMetaData,
12        metadata::{
13            HeapSize, ParquetMetaData, ParquetMetaDataOptions, RowGroupMetaData,
14            thrift::{
15                file_metadata_and_row_groups_from_bytes, read_column_metadata,
16                validate_column_metadata,
17            },
18        },
19    },
20    parquet_thrift::{
21        ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol,
22        ThriftCompactOutputProtocol, ThriftSliceInputProtocol, WriteThrift, WriteThriftField,
23    },
24    thrift_struct, thrift_union,
25};
26use std::io::Write;
27use std::sync::Arc;
28
29thrift_struct!(
30pub(crate) struct AesGcmV1 {
31  /// AAD prefix
32  1: optional binary aad_prefix
33
34  /// Unique file identifier part of AAD suffix
35  2: optional binary aad_file_unique
36
37  /// In files encrypted with AAD prefix without storing it,
38  /// readers must supply the prefix
39  3: optional bool supply_aad_prefix
40}
41);
42
43impl HeapSize for AesGcmV1 {
44    fn heap_size(&self) -> usize {
45        self.aad_prefix.heap_size()
46            + self.aad_file_unique.heap_size()
47            + self.supply_aad_prefix.heap_size()
48    }
49}
50
51thrift_struct!(
52pub(crate) struct AesGcmCtrV1 {
53  /// AAD prefix
54  1: optional binary aad_prefix
55
56  /// Unique file identifier part of AAD suffix
57  2: optional binary aad_file_unique
58
59  /// In files encrypted with AAD prefix without storing it,
60  /// readers must supply the prefix
61  3: optional bool supply_aad_prefix
62}
63);
64
65impl HeapSize for AesGcmCtrV1 {
66    fn heap_size(&self) -> usize {
67        self.aad_prefix.heap_size()
68            + self.aad_file_unique.heap_size()
69            + self.supply_aad_prefix.heap_size()
70    }
71}
72
73thrift_union!(
74union EncryptionAlgorithm {
75  1: (AesGcmV1) AES_GCM_V1
76  2: (AesGcmCtrV1) AES_GCM_CTR_V1
77}
78);
79
80impl HeapSize for EncryptionAlgorithm {
81    fn heap_size(&self) -> usize {
82        match self {
83            Self::AES_GCM_V1(gcm) => gcm.heap_size(),
84            Self::AES_GCM_CTR_V1(gcm_ctr) => gcm_ctr.heap_size(),
85        }
86    }
87}
88
89thrift_struct!(
90/// Crypto metadata for files with encrypted footer
91pub(crate) struct FileCryptoMetaData<'a> {
92  /// Encryption algorithm. This field is only used for files
93  /// with encrypted footer. Files with plaintext footer store algorithm id
94  /// inside footer (FileMetaData structure).
95  1: required EncryptionAlgorithm encryption_algorithm
96
97  /// Retrieval metadata of key used for encryption of footer,
98  /// and (possibly) columns.
99  2: optional binary<'a> key_metadata
100}
101);
102
103fn row_group_from_encrypted_thrift(
104    mut rg: RowGroupMetaData,
105    decryptor: Option<&FileDecryptor>,
106    options: Option<&ParquetMetaDataOptions>,
107) -> Result<RowGroupMetaData> {
108    let schema_descr = rg.schema_descr;
109
110    if schema_descr.num_columns() != rg.columns.len() {
111        return Err(general_err!(
112            "Column count mismatch. Schema has {} columns while Row Group has {}",
113            schema_descr.num_columns(),
114            rg.columns.len()
115        ));
116    }
117    let total_byte_size = rg.total_byte_size;
118    let num_rows = rg.num_rows;
119    let mut columns = vec![];
120
121    for (i, (mut c, d)) in rg
122        .columns
123        .drain(0..)
124        .zip(schema_descr.columns())
125        .enumerate()
126    {
127        // Read encrypted metadata if it's present and we have a decryptor.
128        if let (true, Some(decryptor)) = (c.encrypted_column_metadata.is_some(), decryptor) {
129            let column_decryptor = match c.crypto_metadata() {
130                None => {
131                    return Err(general_err!(
132                        "No crypto_metadata is set for column '{}', which has encrypted metadata",
133                        d.path().string()
134                    ));
135                }
136                Some(ColumnCryptoMetaData::ENCRYPTION_WITH_COLUMN_KEY(crypto_metadata)) => {
137                    let column_name = crypto_metadata.path_in_schema.join(".");
138                    // Try to get the decryptor - if it fails, we don't have the key
139                    match decryptor.get_column_metadata_decryptor(
140                        column_name.as_str(),
141                        crypto_metadata.key_metadata.as_deref(),
142                    ) {
143                        Ok(dec) => dec,
144                        Err(_) => {
145                            // We don't have the key for this column, so we can't decrypt its metadata.
146                            columns.push(c);
147                            continue;
148                        }
149                    }
150                }
151                Some(ColumnCryptoMetaData::ENCRYPTION_WITH_FOOTER_KEY) => {
152                    decryptor.get_footer_decryptor()?
153                }
154            };
155
156            // The ordinal is part of the AAD for encrypted column metadata.
157            // It can be missing here only for files with *mixed* row-group
158            // ordinals, which decode leaves untouched — fail cleanly rather
159            // than panic (see `ensure_row_group_ordinals`).
160            let rg_ordinal = rg.ordinal.ok_or_else(|| {
161                general_err!(
162                    "Row group ordinal is required to decrypt column metadata for \
163                     column '{}', but the file's row-group ordinals are inconsistent",
164                    d.path().string()
165                )
166            })?;
167            // Reject a negative ordinal cleanly rather than sign-extending it into
168            // a bogus AAD via `as usize`.
169            let rg_ordinal = usize::try_from(rg_ordinal).map_err(|_| {
170                general_err!(
171                    "Row group ordinal {rg_ordinal} is invalid (must be non-negative) \
172                     for decrypting column metadata for column '{}'",
173                    d.path().string()
174                )
175            })?;
176            let column_aad = crate::encryption::modules::create_module_aad(
177                decryptor.file_aad(),
178                crate::encryption::modules::ModuleType::ColumnMetaData,
179                rg_ordinal,
180                i,
181                None,
182            )?;
183
184            // Take the encrypted column metadata as it is no longer needed.
185            let encrypted_column_metadata = c.encrypted_column_metadata.take();
186            let buf = encrypted_column_metadata.unwrap();
187            let decrypted_cc_buf = column_decryptor
188                .decrypt(&buf, column_aad.as_ref())
189                .map_err(|_| {
190                    general_err!(
191                        "Unable to decrypt column '{}', perhaps the column key is wrong?",
192                        d.path().string()
193                    )
194                })?;
195
196            // parse decrypted buffer and then replace fields in 'c'
197            let mut prot = ThriftSliceInputProtocol::new(&decrypted_cc_buf);
198            let mask = read_column_metadata(&mut prot, &mut c, i, options)?;
199            validate_column_metadata(mask)?;
200
201            columns.push(c);
202        } else {
203            columns.push(c);
204        }
205    }
206
207    let sorting_columns = rg.sorting_columns;
208    let file_offset = rg.file_offset;
209    let ordinal = rg.ordinal;
210
211    Ok(RowGroupMetaData {
212        columns,
213        num_rows,
214        sorting_columns,
215        total_byte_size,
216        schema_descr,
217        file_offset,
218        ordinal,
219    })
220}
221
222/// Decodes [`ParquetMetaData`] from the provided bytes, handling metadata that may be encrypted.
223///
224/// Typically this is used to decode the metadata from the end of a parquet
225/// file. The format of `buf` is the Thrift compact binary protocol, as specified
226/// by the [Parquet Spec]. Buffer can be encrypted with AES GCM or AES CTR
227/// ciphers as specified in the [Parquet Encryption Spec].
228///
229/// [Parquet Spec]: https://github.com/apache/parquet-format#metadata
230/// [Parquet Encryption Spec]: https://parquet.apache.org/docs/file-format/data-pages/encryption/
231pub(crate) fn parquet_metadata_with_encryption(
232    file_decryption_properties: Option<&Arc<FileDecryptionProperties>>,
233    encrypted_footer: bool,
234    buf: &[u8],
235    options: Option<&ParquetMetaDataOptions>,
236) -> Result<ParquetMetaData> {
237    use crate::file::metadata::ParquetMetaDataBuilder;
238
239    let mut buf = buf;
240    let mut file_decryptor = None;
241    let decrypted_fmd_buf;
242
243    if encrypted_footer {
244        let mut prot = ThriftSliceInputProtocol::new(buf);
245        if let Some(file_decryption_properties) = file_decryption_properties {
246            let t_file_crypto_metadata: FileCryptoMetaData =
247                FileCryptoMetaData::read_thrift(&mut prot)
248                    .map_err(|e| general_err!("Could not parse crypto metadata: {}", e))?;
249            let supply_aad_prefix = match &t_file_crypto_metadata.encryption_algorithm {
250                EncryptionAlgorithm::AES_GCM_V1(algo) => algo.supply_aad_prefix,
251                EncryptionAlgorithm::AES_GCM_CTR_V1(_) => Some(false),
252            }
253            .unwrap_or(false);
254            if supply_aad_prefix && file_decryption_properties.aad_prefix().is_none() {
255                return Err(general_err!(
256                    "Parquet file was encrypted with an AAD prefix that is not stored in the file, \
257                        but no AAD prefix was provided in the file decryption properties"
258                ));
259            }
260            let decryptor = get_file_decryptor(
261                t_file_crypto_metadata.encryption_algorithm,
262                t_file_crypto_metadata.key_metadata,
263                file_decryption_properties,
264            )?;
265            let footer_decryptor = decryptor.get_footer_decryptor();
266            let aad_footer = crate::encryption::modules::create_footer_aad(decryptor.file_aad())?;
267
268            decrypted_fmd_buf = footer_decryptor?
269                .decrypt(prot.as_slice().as_ref(), aad_footer.as_ref())
270                .map_err(|_| {
271                    general_err!(
272                        "Provided footer key and AAD were unable to decrypt parquet footer"
273                    )
274                })?;
275
276            buf = &decrypted_fmd_buf;
277            file_decryptor = Some(decryptor);
278        } else {
279            return Err(general_err!(
280                "Parquet file has an encrypted footer but decryption properties were not provided"
281            ));
282        }
283    }
284
285    let (mut file_metadata, row_groups) = file_metadata_and_row_groups_from_bytes(buf, options)
286        .map_err(|e| general_err!("Could not parse metadata: {}", e))?;
287
288    // Take the encryption algorithm and footer signing key metadata as they are no longer
289    // needed after this.
290    if let (Some(algo), Some(file_decryption_properties)) = (
291        file_metadata.encryption_algorithm.take(),
292        file_decryption_properties,
293    ) {
294        let footer_signing_key_metadata = file_metadata.footer_signing_key_metadata.take();
295
296        // File has a plaintext footer but encryption algorithm is set
297        let file_decryptor_value = get_file_decryptor(
298            *algo,
299            footer_signing_key_metadata.as_deref(),
300            file_decryption_properties,
301        )?;
302        if file_decryption_properties.check_plaintext_footer_integrity() && !encrypted_footer {
303            file_decryptor_value.verify_plaintext_footer_signature(buf)?;
304        }
305        file_decryptor = Some(file_decryptor_value);
306    }
307
308    // decrypt column chunk info
309    let row_groups = row_groups
310        .into_iter()
311        .map(|rg| row_group_from_encrypted_thrift(rg, file_decryptor.as_ref(), options))
312        .collect::<Result<Vec<_>>>()?;
313
314    let metadata = ParquetMetaDataBuilder::new(file_metadata)
315        .set_row_groups(row_groups)
316        .set_file_decryptor(file_decryptor)
317        .build();
318
319    Ok(metadata)
320}
321
322fn get_file_decryptor(
323    encryption_algorithm: EncryptionAlgorithm,
324    footer_key_metadata: Option<&[u8]>,
325    file_decryption_properties: &Arc<FileDecryptionProperties>,
326) -> Result<FileDecryptor> {
327    match encryption_algorithm {
328        EncryptionAlgorithm::AES_GCM_V1(algo) => {
329            let aad_file_unique = algo
330                .aad_file_unique
331                .ok_or_else(|| general_err!("AAD unique file identifier is not set"))?;
332            let aad_prefix = if let Some(aad_prefix) = file_decryption_properties.aad_prefix() {
333                aad_prefix.clone()
334            } else {
335                algo.aad_prefix.unwrap_or_default()
336            };
337
338            FileDecryptor::new(
339                file_decryption_properties,
340                footer_key_metadata,
341                aad_file_unique,
342                aad_prefix,
343            )
344        }
345        EncryptionAlgorithm::AES_GCM_CTR_V1(_) => Err(nyi_err!(
346            "The AES_GCM_CTR_V1 encryption algorithm is not yet supported"
347        )),
348    }
349}