parquet/file/metadata/thrift/
encryption.rs1use crate::{
8 encryption::decrypt::{FileDecryptionProperties, FileDecryptor},
9 errors::{ParquetError, Result},
10 file::{
11 column_crypto_metadata::ColumnCryptoMetaData,
12 metadata::{
13 HeapSize, ParquetMetaData, ParquetMetaDataOptions, RowGroupMetaData,
14 thrift::{
15 file_metadata_and_row_groups_from_bytes, read_column_metadata,
16 validate_column_metadata,
17 },
18 },
19 },
20 parquet_thrift::{
21 ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol,
22 ThriftCompactOutputProtocol, ThriftSliceInputProtocol, WriteThrift, WriteThriftField,
23 },
24 thrift_struct, thrift_union,
25};
26use std::io::Write;
27use std::sync::Arc;
28
29thrift_struct!(
30pub(crate) struct AesGcmV1 {
31 1: optional binary aad_prefix
33
34 2: optional binary aad_file_unique
36
37 3: optional bool supply_aad_prefix
40}
41);
42
43impl HeapSize for AesGcmV1 {
44 fn heap_size(&self) -> usize {
45 self.aad_prefix.heap_size()
46 + self.aad_file_unique.heap_size()
47 + self.supply_aad_prefix.heap_size()
48 }
49}
50
51thrift_struct!(
52pub(crate) struct AesGcmCtrV1 {
53 1: optional binary aad_prefix
55
56 2: optional binary aad_file_unique
58
59 3: optional bool supply_aad_prefix
62}
63);
64
65impl HeapSize for AesGcmCtrV1 {
66 fn heap_size(&self) -> usize {
67 self.aad_prefix.heap_size()
68 + self.aad_file_unique.heap_size()
69 + self.supply_aad_prefix.heap_size()
70 }
71}
72
73thrift_union!(
74union EncryptionAlgorithm {
75 1: (AesGcmV1) AES_GCM_V1
76 2: (AesGcmCtrV1) AES_GCM_CTR_V1
77}
78);
79
80impl HeapSize for EncryptionAlgorithm {
81 fn heap_size(&self) -> usize {
82 match self {
83 Self::AES_GCM_V1(gcm) => gcm.heap_size(),
84 Self::AES_GCM_CTR_V1(gcm_ctr) => gcm_ctr.heap_size(),
85 }
86 }
87}
88
89thrift_struct!(
90pub(crate) struct FileCryptoMetaData<'a> {
92 1: required EncryptionAlgorithm encryption_algorithm
96
97 2: optional binary<'a> key_metadata
100}
101);
102
103fn row_group_from_encrypted_thrift(
104 mut rg: RowGroupMetaData,
105 decryptor: Option<&FileDecryptor>,
106 options: Option<&ParquetMetaDataOptions>,
107) -> Result<RowGroupMetaData> {
108 let schema_descr = rg.schema_descr;
109
110 if schema_descr.num_columns() != rg.columns.len() {
111 return Err(general_err!(
112 "Column count mismatch. Schema has {} columns while Row Group has {}",
113 schema_descr.num_columns(),
114 rg.columns.len()
115 ));
116 }
117 let total_byte_size = rg.total_byte_size;
118 let num_rows = rg.num_rows;
119 let mut columns = vec![];
120
121 for (i, (mut c, d)) in rg
122 .columns
123 .drain(0..)
124 .zip(schema_descr.columns())
125 .enumerate()
126 {
127 if let (true, Some(decryptor)) = (c.encrypted_column_metadata.is_some(), decryptor) {
129 let column_decryptor = match c.crypto_metadata() {
130 None => {
131 return Err(general_err!(
132 "No crypto_metadata is set for column '{}', which has encrypted metadata",
133 d.path().string()
134 ));
135 }
136 Some(ColumnCryptoMetaData::ENCRYPTION_WITH_COLUMN_KEY(crypto_metadata)) => {
137 let column_name = crypto_metadata.path_in_schema.join(".");
138 match decryptor.get_column_metadata_decryptor(
140 column_name.as_str(),
141 crypto_metadata.key_metadata.as_deref(),
142 ) {
143 Ok(dec) => dec,
144 Err(_) => {
145 columns.push(c);
147 continue;
148 }
149 }
150 }
151 Some(ColumnCryptoMetaData::ENCRYPTION_WITH_FOOTER_KEY) => {
152 decryptor.get_footer_decryptor()?
153 }
154 };
155
156 let rg_ordinal = rg.ordinal.ok_or_else(|| {
161 general_err!(
162 "Row group ordinal is required to decrypt column metadata for \
163 column '{}', but the file's row-group ordinals are inconsistent",
164 d.path().string()
165 )
166 })?;
167 let rg_ordinal = usize::try_from(rg_ordinal).map_err(|_| {
170 general_err!(
171 "Row group ordinal {rg_ordinal} is invalid (must be non-negative) \
172 for decrypting column metadata for column '{}'",
173 d.path().string()
174 )
175 })?;
176 let column_aad = crate::encryption::modules::create_module_aad(
177 decryptor.file_aad(),
178 crate::encryption::modules::ModuleType::ColumnMetaData,
179 rg_ordinal,
180 i,
181 None,
182 )?;
183
184 let encrypted_column_metadata = c.encrypted_column_metadata.take();
186 let buf = encrypted_column_metadata.unwrap();
187 let decrypted_cc_buf = column_decryptor
188 .decrypt(&buf, column_aad.as_ref())
189 .map_err(|_| {
190 general_err!(
191 "Unable to decrypt column '{}', perhaps the column key is wrong?",
192 d.path().string()
193 )
194 })?;
195
196 let mut prot = ThriftSliceInputProtocol::new(&decrypted_cc_buf);
198 let mask = read_column_metadata(&mut prot, &mut c, i, options)?;
199 validate_column_metadata(mask)?;
200
201 columns.push(c);
202 } else {
203 columns.push(c);
204 }
205 }
206
207 let sorting_columns = rg.sorting_columns;
208 let file_offset = rg.file_offset;
209 let ordinal = rg.ordinal;
210
211 Ok(RowGroupMetaData {
212 columns,
213 num_rows,
214 sorting_columns,
215 total_byte_size,
216 schema_descr,
217 file_offset,
218 ordinal,
219 })
220}
221
222pub(crate) fn parquet_metadata_with_encryption(
232 file_decryption_properties: Option<&Arc<FileDecryptionProperties>>,
233 encrypted_footer: bool,
234 buf: &[u8],
235 options: Option<&ParquetMetaDataOptions>,
236) -> Result<ParquetMetaData> {
237 use crate::file::metadata::ParquetMetaDataBuilder;
238
239 let mut buf = buf;
240 let mut file_decryptor = None;
241 let decrypted_fmd_buf;
242
243 if encrypted_footer {
244 let mut prot = ThriftSliceInputProtocol::new(buf);
245 if let Some(file_decryption_properties) = file_decryption_properties {
246 let t_file_crypto_metadata: FileCryptoMetaData =
247 FileCryptoMetaData::read_thrift(&mut prot)
248 .map_err(|e| general_err!("Could not parse crypto metadata: {}", e))?;
249 let supply_aad_prefix = match &t_file_crypto_metadata.encryption_algorithm {
250 EncryptionAlgorithm::AES_GCM_V1(algo) => algo.supply_aad_prefix,
251 EncryptionAlgorithm::AES_GCM_CTR_V1(_) => Some(false),
252 }
253 .unwrap_or(false);
254 if supply_aad_prefix && file_decryption_properties.aad_prefix().is_none() {
255 return Err(general_err!(
256 "Parquet file was encrypted with an AAD prefix that is not stored in the file, \
257 but no AAD prefix was provided in the file decryption properties"
258 ));
259 }
260 let decryptor = get_file_decryptor(
261 t_file_crypto_metadata.encryption_algorithm,
262 t_file_crypto_metadata.key_metadata,
263 file_decryption_properties,
264 )?;
265 let footer_decryptor = decryptor.get_footer_decryptor();
266 let aad_footer = crate::encryption::modules::create_footer_aad(decryptor.file_aad())?;
267
268 decrypted_fmd_buf = footer_decryptor?
269 .decrypt(prot.as_slice().as_ref(), aad_footer.as_ref())
270 .map_err(|_| {
271 general_err!(
272 "Provided footer key and AAD were unable to decrypt parquet footer"
273 )
274 })?;
275
276 buf = &decrypted_fmd_buf;
277 file_decryptor = Some(decryptor);
278 } else {
279 return Err(general_err!(
280 "Parquet file has an encrypted footer but decryption properties were not provided"
281 ));
282 }
283 }
284
285 let (mut file_metadata, row_groups) = file_metadata_and_row_groups_from_bytes(buf, options)
286 .map_err(|e| general_err!("Could not parse metadata: {}", e))?;
287
288 if let (Some(algo), Some(file_decryption_properties)) = (
291 file_metadata.encryption_algorithm.take(),
292 file_decryption_properties,
293 ) {
294 let footer_signing_key_metadata = file_metadata.footer_signing_key_metadata.take();
295
296 let file_decryptor_value = get_file_decryptor(
298 *algo,
299 footer_signing_key_metadata.as_deref(),
300 file_decryption_properties,
301 )?;
302 if file_decryption_properties.check_plaintext_footer_integrity() && !encrypted_footer {
303 file_decryptor_value.verify_plaintext_footer_signature(buf)?;
304 }
305 file_decryptor = Some(file_decryptor_value);
306 }
307
308 let row_groups = row_groups
310 .into_iter()
311 .map(|rg| row_group_from_encrypted_thrift(rg, file_decryptor.as_ref(), options))
312 .collect::<Result<Vec<_>>>()?;
313
314 let metadata = ParquetMetaDataBuilder::new(file_metadata)
315 .set_row_groups(row_groups)
316 .set_file_decryptor(file_decryptor)
317 .build();
318
319 Ok(metadata)
320}
321
322fn get_file_decryptor(
323 encryption_algorithm: EncryptionAlgorithm,
324 footer_key_metadata: Option<&[u8]>,
325 file_decryption_properties: &Arc<FileDecryptionProperties>,
326) -> Result<FileDecryptor> {
327 match encryption_algorithm {
328 EncryptionAlgorithm::AES_GCM_V1(algo) => {
329 let aad_file_unique = algo
330 .aad_file_unique
331 .ok_or_else(|| general_err!("AAD unique file identifier is not set"))?;
332 let aad_prefix = if let Some(aad_prefix) = file_decryption_properties.aad_prefix() {
333 aad_prefix.clone()
334 } else {
335 algo.aad_prefix.unwrap_or_default()
336 };
337
338 FileDecryptor::new(
339 file_decryption_properties,
340 footer_key_metadata,
341 aad_file_unique,
342 aad_prefix,
343 )
344 }
345 EncryptionAlgorithm::AES_GCM_CTR_V1(_) => Err(nyi_err!(
346 "The AES_GCM_CTR_V1 encryption algorithm is not yet supported"
347 )),
348 }
349}