Skip to main content

parquet/file/metadata/
writer.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use crate::file::metadata::page_index::PageIndexProvider;
6use crate::file::metadata::thrift::FileMeta;
7use crate::file::metadata::{ColumnChunkMetaData, RowGroupMetaData};
8use crate::schema::types::{SchemaDescPtr, SchemaDescriptor};
9use crate::{
10    basic::ColumnOrder,
11    file::metadata::{FileMetaData, ParquetMetaDataBuilder},
12};
13#[cfg(feature = "encryption")]
14use crate::{
15    encryption::{
16        encrypt::{FileEncryptor, encrypt_thrift_object, write_signed_plaintext_thrift_object},
17        modules::{ModuleType, create_footer_aad, create_module_aad},
18    },
19    file::column_crypto_metadata::ColumnCryptoMetaData,
20    file::metadata::thrift::encryption::{AesGcmV1, EncryptionAlgorithm, FileCryptoMetaData},
21};
22use crate::{errors::Result, file::page_index::column_index::ColumnIndexMetaData};
23
24use crate::{
25    file::writer::{TrackedWrite, get_file_magic},
26    parquet_thrift::WriteThrift,
27};
28use crate::{
29    file::{
30        metadata::{KeyValue, ParquetMetaData},
31        page_index::offset_index::OffsetIndexMetaData,
32    },
33    parquet_thrift::ThriftCompactOutputProtocol,
34};
35use std::io::Write;
36use std::sync::Arc;
37
38/// Writes `crate::file::metadata` structures to a thrift encoded byte stream
39///
40/// See [`ParquetMetaDataWriter`] for background and example.
41pub(crate) struct ThriftMetadataWriter<'a, W: Write> {
42    buf: &'a mut TrackedWrite<W>,
43    schema_descr: &'a SchemaDescPtr,
44    row_groups: Vec<RowGroupMetaData>,
45    page_index: Option<Arc<dyn PageIndexProvider>>,
46    key_value_metadata: Option<Vec<KeyValue>>,
47    created_by: Option<String>,
48    object_writer: MetadataObjectWriter,
49    writer_version: i32,
50    write_path_in_schema: bool,
51    preserve_page_index_locations: bool,
52}
53
54impl<'a, W: Write> ThriftMetadataWriter<'a, W> {
55    /// Serialize all the offset indexes to `self.buf`,
56    ///
57    /// Note: also updates the `ColumnChunk::offset_index_offset` and
58    /// `ColumnChunk::offset_index_length` to reflect the position and length
59    /// of the serialized offset indexes.
60    fn write_offset_indexes(&mut self, page_index: &dyn PageIndexProvider) -> Result<()> {
61        // iter row group
62        // iter each column
63        // write offset index to the file
64        for (row_group_idx, row_group) in self.row_groups.iter_mut().enumerate() {
65            for (column_idx, column_metadata) in row_group.columns.iter_mut().enumerate() {
66                if let Some(offset_index) = page_index.offset_index(row_group_idx, column_idx) {
67                    let start_offset = self.buf.bytes_written();
68                    self.object_writer.write_offset_index(
69                        offset_index,
70                        column_metadata,
71                        row_group_idx,
72                        column_idx,
73                        &mut *self.buf,
74                    )?;
75                    let end_offset = self.buf.bytes_written();
76                    // set offset and index for offset index
77                    column_metadata.offset_index_offset = Some(start_offset as i64);
78                    column_metadata.offset_index_length = Some((end_offset - start_offset) as i32);
79                } else if !self.preserve_page_index_locations {
80                    column_metadata.offset_index_offset = None;
81                    column_metadata.offset_index_length = None;
82                }
83            }
84        }
85        Ok(())
86    }
87
88    /// Serialize all the column indexes to the `self.buf`
89    ///
90    /// Note: also updates the `ColumnChunk::column_index_offset` and
91    /// `ColumnChunk::column_index_length` to reflect the position and length
92    /// of the serialized column indexes.
93    fn write_column_indexes(&mut self, page_index: &dyn PageIndexProvider) -> Result<()> {
94        // iter row group
95        // iter each column
96        // write column index to the file
97        for (row_group_idx, row_group) in self.row_groups.iter_mut().enumerate() {
98            for (column_idx, column_metadata) in row_group.columns.iter_mut().enumerate() {
99                if let Some(column_index) = page_index.column_index(row_group_idx, column_idx) {
100                    let start_offset = self.buf.bytes_written();
101                    // only update column_metadata if the write succeeds
102                    if self.object_writer.write_column_index(
103                        column_index,
104                        column_metadata,
105                        row_group_idx,
106                        column_idx,
107                        &mut *self.buf,
108                    )? {
109                        let end_offset = self.buf.bytes_written();
110                        // set offset and index for offset index
111                        column_metadata.column_index_offset = Some(start_offset as i64);
112                        column_metadata.column_index_length =
113                            Some((end_offset - start_offset) as i32);
114                    }
115                } else if !self.preserve_page_index_locations {
116                    column_metadata.column_index_offset = None;
117                    column_metadata.column_index_length = None;
118                }
119            }
120        }
121        Ok(())
122    }
123
124    fn clear_page_indexes(&mut self) {
125        for row_group in &mut self.row_groups {
126            for column_metadata in &mut row_group.columns {
127                column_metadata.column_index_offset = None;
128                column_metadata.column_index_length = None;
129                column_metadata.offset_index_offset = None;
130                column_metadata.offset_index_length = None;
131            }
132        }
133    }
134
135    /// Assembles and writes the final metadata to self.buf
136    pub(crate) fn finish(mut self) -> Result<ParquetMetaData> {
137        let num_rows = self.row_groups.iter().map(|x| x.num_rows).sum();
138
139        // serialize page indexes and update index locations
140        let page_index = self.page_index.take();
141        if let Some(pi) = page_index.as_deref() {
142            self.write_column_indexes(pi)?;
143            self.write_offset_indexes(pi)?;
144        } else if !self.preserve_page_index_locations {
145            // no page indexes at all, then clear index locations
146            self.clear_page_indexes();
147        }
148
149        // We only include ColumnOrder for leaf nodes.
150        let column_orders = self
151            .schema_descr
152            .columns()
153            .iter()
154            .map(|col| {
155                ColumnOrder::column_order_for_type(
156                    col.logical_type_ref(),
157                    col.converted_type(),
158                    col.physical_type(),
159                )
160            })
161            .collect();
162
163        // This field is optional, perhaps in cases where no min/max fields are set
164        // in any Statistics or ColumnIndex object in the whole file.
165        // But for simplicity we always set this field.
166        let column_orders = Some(column_orders);
167
168        let (row_groups, unencrypted_row_groups) = self
169            .object_writer
170            .apply_row_group_encryption(self.row_groups)?;
171
172        #[cfg(feature = "encryption")]
173        let (encryption_algorithm, footer_signing_key_metadata) =
174            self.object_writer.get_plaintext_footer_crypto_metadata();
175        #[cfg(feature = "encryption")]
176        let file_metadata = FileMetaData::new(
177            self.writer_version,
178            num_rows,
179            self.created_by,
180            self.key_value_metadata,
181            self.schema_descr.clone(),
182            column_orders,
183        )
184        .with_encryption_algorithm(encryption_algorithm)
185        .with_footer_signing_key_metadata(footer_signing_key_metadata);
186
187        #[cfg(not(feature = "encryption"))]
188        let file_metadata = FileMetaData::new(
189            self.writer_version,
190            num_rows,
191            self.created_by,
192            self.key_value_metadata,
193            self.schema_descr.clone(),
194            column_orders,
195        );
196
197        let file_meta = FileMeta {
198            file_metadata: &file_metadata,
199            row_groups: &row_groups,
200            write_path_in_schema: self.write_path_in_schema,
201        };
202
203        // Write file metadata
204        let start_pos = self.buf.bytes_written();
205        self.object_writer
206            .write_file_metadata(&file_meta, &mut *self.buf)?;
207        let end_pos = self.buf.bytes_written();
208
209        // Write footer
210        let metadata_len = (end_pos - start_pos) as u32;
211
212        self.buf.write_all(&metadata_len.to_le_bytes())?;
213        self.buf.write_all(self.object_writer.get_file_magic())?;
214
215        // Note: we simply pass the original reference counted page index provider to the
216        // builder. Either it came from ParquetMetaDataWriter where it was cloned, and this
217        // reference will go away when ParquetMetaDataWriter::finish drops the newly created
218        // metadata, or it was passed an owned version from SerializedFileWriter which then
219        // takes it back via the returned metadata.
220        let builder = ParquetMetaDataBuilder::new(file_metadata).set_page_index(page_index);
221
222        // If row group metadata was encrypted, we replace the encrypted row groups with
223        // unencrypted metadata before it is returned to users. This allows the metadata
224        // to be usable for retrieving the row group statistics for example, without users
225        // needing to decrypt the metadata.
226        Ok(match unencrypted_row_groups {
227            Some(rg) => builder.set_row_groups(rg).build(),
228            None => builder.set_row_groups(row_groups).build(),
229        })
230    }
231
232    pub(crate) fn new(
233        buf: &'a mut TrackedWrite<W>,
234        schema_descr: &'a SchemaDescPtr,
235        row_groups: Vec<RowGroupMetaData>,
236        created_by: Option<String>,
237        writer_version: i32,
238        write_path_in_schema: bool,
239        preserve_page_index_locations: bool,
240    ) -> Self {
241        Self {
242            buf,
243            schema_descr,
244            row_groups,
245            page_index: None,
246            key_value_metadata: None,
247            created_by,
248            object_writer: Default::default(),
249            writer_version,
250            write_path_in_schema,
251            preserve_page_index_locations,
252        }
253    }
254
255    pub(crate) fn with_page_index(mut self, page_index: Arc<dyn PageIndexProvider>) -> Self {
256        self.page_index = Some(page_index);
257        self
258    }
259
260    pub(crate) fn with_key_value_metadata(mut self, key_value_metadata: Vec<KeyValue>) -> Self {
261        self.key_value_metadata = Some(key_value_metadata);
262        self
263    }
264
265    #[cfg(feature = "encryption")]
266    pub(crate) fn with_file_encryptor(
267        mut self,
268        file_encryptor: Option<Arc<FileEncryptor>>,
269    ) -> Self {
270        self.object_writer = self.object_writer.with_file_encryptor(file_encryptor);
271        self
272    }
273}
274
275/// Writes [`ParquetMetaData`] to a byte stream
276///
277/// This structure handles the details of writing the various parts of Parquet
278/// metadata into a byte stream. It is used to write the metadata into a parquet
279/// file and can also write metadata into other locations (such as a store of
280/// bytes).
281///
282/// # Discussion
283///
284/// The process of writing Parquet metadata is tricky because the
285/// metadata is not stored as a single inline thrift structure. It can have
286/// several "out of band" structures such as the [`OffsetIndex`] and
287/// BloomFilters stored in separate structures whose locations are stored as
288/// offsets from the beginning of the file.
289///
290/// Note: this writer does not directly write BloomFilters. In order to write
291/// BloomFilters, write the bloom filters into the buffer before creating the
292/// metadata writer. Then set the corresponding `bloom_filter_offset` and
293/// `bloom_filter_length` on [`ColumnChunkMetaData`] passed to this writer.
294///
295/// The writer serializes the page index of any [`PageIndexProvider`] attached
296/// to the input [`ParquetMetaData`]. By default, if the provider has no index
297/// for a column chunk, the offset and length of that index in the [`ColumnChunkMetaData`]
298/// are preserved in the output metadata. This is done so one can store the footer
299/// metadata externally but keep the page indexes in the original file. If this
300/// behavior is not desired, then [`Self::with_preserve_page_index_locations`] should
301/// be set to `false`.
302///
303/// # Output Format
304///
305/// The format of the metadata is as follows:
306///
307/// 1. Optional [`ColumnIndex`] (thrift encoded)
308/// 2. Optional [`OffsetIndex`] (thrift encoded)
309/// 3. [`FileMetaData`] (thrift encoded)
310/// 4. Length of encoded `FileMetaData` (4 bytes, little endian)
311/// 5. Parquet Magic Bytes (4 bytes)
312///
313/// [`FileMetaData`]: https://github.com/apache/parquet-format/tree/master?tab=readme-ov-file#metadata
314/// [`ColumnChunkMetaData`]: crate::file::metadata::ColumnChunkMetaData
315/// [`ColumnIndex`]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
316/// [`OffsetIndex`]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
317/// [`PageIndexProvider`]: crate::file::metadata::page_index::PageIndexProvider
318///
319/// ```text
320/// ┌──────────────────────┐
321/// │                      │
322/// │         ...          │
323/// │                      │
324/// │┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┐ │
325/// │     ColumnIndex     ◀│─ ─ ─
326/// ││    (Optional)     │ │     │
327/// │ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─  │
328/// │┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┐ │     │ FileMetadata
329/// │     OffsetIndex      │       contains embedded
330/// ││    (Optional)     │◀┼ ─   │ offsets to
331/// │ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─  │  │    ColumnIndex and
332/// │╔═══════════════════╗ │     │ OffsetIndex
333/// │║                   ║ │  │
334/// │║                   ║ ┼ ─   │
335/// │║   FileMetadata    ║ │
336/// │║                   ║ ┼ ─ ─ ┘
337/// │║                   ║ │
338/// │╚═══════════════════╝ │
339/// │┌───────────────────┐ │
340/// ││  metadata length  │ │ length of FileMetadata  (only)
341/// │└───────────────────┘ │
342/// │┌───────────────────┐ │
343/// ││      'PAR1'       │ │ Parquet Magic Bytes
344/// │└───────────────────┘ │
345/// └──────────────────────┘
346///      Output Buffer
347/// ```
348///
349/// # Example
350/// ```no_run
351/// # use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataWriter};
352/// # fn get_metadata() -> ParquetMetaData { unimplemented!(); }
353/// // write parquet metadata to an in-memory buffer
354/// let mut buffer = vec![];
355/// let metadata: ParquetMetaData = get_metadata();
356/// let writer = ParquetMetaDataWriter::new(&mut buffer, &metadata);
357/// // write the metadata to the buffer
358/// writer.finish().unwrap();
359/// assert!(!buffer.is_empty());
360/// ```
361pub struct ParquetMetaDataWriter<'a, W: Write> {
362    buf: TrackedWrite<W>,
363    metadata: &'a ParquetMetaData,
364    write_path_in_schema: bool,
365    preserve_page_index_locations: bool,
366}
367
368impl<'a, W: Write> ParquetMetaDataWriter<'a, W> {
369    /// Create a new `ParquetMetaDataWriter` to write to `buf`
370    ///
371    /// Note any embedded offsets in the metadata will be written assuming the
372    /// metadata is at the start of the buffer. If the metadata is being written
373    /// to a location other than the start of the buffer, see [`Self::new_with_tracked`]
374    ///
375    /// See example on the struct level documentation
376    pub fn new(buf: W, metadata: &'a ParquetMetaData) -> Self {
377        Self::new_with_tracked(TrackedWrite::new(buf), metadata)
378    }
379
380    /// Create a new ParquetMetaDataWriter to write to `buf`
381    ///
382    /// This method is used when the metadata is being written to a location other
383    /// than the start of the buffer.
384    ///
385    /// See example on the struct level documentation
386    pub fn new_with_tracked(buf: TrackedWrite<W>, metadata: &'a ParquetMetaData) -> Self {
387        Self {
388            buf,
389            metadata,
390            write_path_in_schema: true,
391            preserve_page_index_locations: true,
392        }
393    }
394
395    /// Set whether or not to write the `path_in_schema` field in the Thrift `ColumnMetaData`
396    /// struct.
397    pub fn with_write_path_in_schema(self, val: bool) -> Self {
398        Self {
399            write_path_in_schema: val,
400            ..self
401        }
402    }
403
404    /// Set whether or not to preserve the page index location metadata in the Thrift
405    /// `ColumnMetaData` (defaults to `true`).
406    ///
407    /// Because this struct is often used to externalize the footer metadata, it is
408    /// usually desirable to preserve this location information, even when the
409    /// page indexes are not duplicated (for instance if the provided `ParquetMetaData`
410    /// returns no `PageIndexProvider`). As such, this defaults to `true`.
411    ///
412    /// Set this to `false` to reset the location metadata if no page indexes are
413    /// present in the provided [`ParquetMetaData`].
414    pub fn with_preserve_page_index_locations(self, val: bool) -> Self {
415        Self {
416            preserve_page_index_locations: val,
417            ..self
418        }
419    }
420
421    /// Write the metadata to the buffer
422    pub fn finish(mut self) -> Result<()> {
423        let file_metadata = self.metadata.file_metadata();
424
425        let schema = Arc::new(file_metadata.schema().clone());
426        let schema_descr = Arc::new(SchemaDescriptor::new(schema.clone()));
427        let created_by = file_metadata.created_by().map(str::to_string);
428
429        // need to clone because ThriftMetadataWriter wants to update page
430        // index pointers. we throw this clone away upon calling encoder.finish()
431        let row_groups = self.metadata.row_groups.as_ref().to_vec();
432
433        let key_value_metadata = file_metadata.key_value_metadata().cloned();
434
435        let mut encoder = ThriftMetadataWriter::new(
436            &mut self.buf,
437            &schema_descr,
438            row_groups,
439            created_by,
440            file_metadata.version(),
441            self.write_path_in_schema,
442            self.preserve_page_index_locations,
443        );
444
445        if let Some(page_index_arc) = self.metadata.page_index.clone() {
446            encoder = encoder.with_page_index(page_index_arc);
447        }
448
449        if let Some(key_value_metadata) = key_value_metadata {
450            encoder = encoder.with_key_value_metadata(key_value_metadata);
451        }
452        encoder.finish()?;
453
454        Ok(())
455    }
456}
457
458#[derive(Debug, Default)]
459struct MetadataObjectWriter {
460    #[cfg(feature = "encryption")]
461    file_encryptor: Option<Arc<FileEncryptor>>,
462}
463
464impl MetadataObjectWriter {
465    #[inline]
466    fn write_thrift_object(object: &impl WriteThrift, sink: impl Write) -> Result<()> {
467        let mut protocol = ThriftCompactOutputProtocol::new(sink);
468        object.write_thrift(&mut protocol)?;
469        Ok(())
470    }
471}
472
473/// Implementations of [`MetadataObjectWriter`] methods for when encryption is disabled
474#[cfg(not(feature = "encryption"))]
475impl MetadataObjectWriter {
476    /// Write [`FileMetaData`] in Thrift format
477    ///
478    /// [`FileMetaData`]: https://github.com/apache/parquet-format/tree/master?tab=readme-ov-file#metadata
479    fn write_file_metadata(&self, file_metadata: &FileMeta, sink: impl Write) -> Result<()> {
480        Self::write_thrift_object(file_metadata, sink)
481    }
482
483    /// Write a column [`OffsetIndex`] in Thrift format
484    ///
485    /// [`OffsetIndex`]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
486    fn write_offset_index(
487        &self,
488        offset_index: &OffsetIndexMetaData,
489        _column_chunk: &ColumnChunkMetaData,
490        _row_group_idx: usize,
491        _column_idx: usize,
492        sink: impl Write,
493    ) -> Result<()> {
494        Self::write_thrift_object(offset_index, sink)
495    }
496
497    /// Write a column [`ColumnIndex`] in Thrift format
498    ///
499    /// Returns `true` unless there is an error.
500    ///
501    /// [`ColumnIndex`]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
502    fn write_column_index(
503        &self,
504        column_index: &ColumnIndexMetaData,
505        _column_chunk: &ColumnChunkMetaData,
506        _row_group_idx: usize,
507        _column_idx: usize,
508        sink: impl Write,
509    ) -> Result<bool> {
510        Self::write_thrift_object(column_index, sink)?;
511        Ok(true)
512    }
513
514    /// No-op implementation of row-group metadata encryption
515    fn apply_row_group_encryption(
516        &self,
517        row_groups: Vec<RowGroupMetaData>,
518    ) -> Result<(Vec<RowGroupMetaData>, Option<Vec<RowGroupMetaData>>)> {
519        Ok((row_groups, None))
520    }
521
522    /// Get the "magic" bytes identifying the file type
523    pub fn get_file_magic(&self) -> &[u8; 4] {
524        get_file_magic()
525    }
526}
527
528/// Implementations of [`MetadataObjectWriter`] methods that rely on encryption being enabled
529#[cfg(feature = "encryption")]
530impl MetadataObjectWriter {
531    /// Set the file encryptor to use
532    fn with_file_encryptor(mut self, encryptor: Option<Arc<FileEncryptor>>) -> Self {
533        self.file_encryptor = encryptor;
534        self
535    }
536
537    /// Write [`FileMetaData`] in Thrift format, possibly encrypting it if required
538    ///
539    /// [`FileMetaData`]: https://github.com/apache/parquet-format/tree/master?tab=readme-ov-file#metadata
540    fn write_file_metadata(&self, file_metadata: &FileMeta, mut sink: impl Write) -> Result<()> {
541        match self.file_encryptor.as_ref() {
542            Some(file_encryptor) if file_encryptor.properties().encrypt_footer() => {
543                // First write FileCryptoMetadata
544                let crypto_metadata = Self::file_crypto_metadata(file_encryptor)?;
545                let mut protocol = ThriftCompactOutputProtocol::new(&mut sink);
546                crypto_metadata.write_thrift(&mut protocol)?;
547
548                // Then write encrypted footer
549                let aad = create_footer_aad(file_encryptor.file_aad())?;
550                let mut encryptor = file_encryptor.get_footer_encryptor()?;
551                encrypt_thrift_object(file_metadata, &mut encryptor, &mut sink, &aad)
552            }
553            Some(file_encryptor) if file_metadata.file_metadata.encryption_algorithm.is_some() => {
554                let aad = create_footer_aad(file_encryptor.file_aad())?;
555                let mut encryptor = file_encryptor.get_footer_encryptor()?;
556                write_signed_plaintext_thrift_object(file_metadata, &mut encryptor, &mut sink, &aad)
557            }
558            _ => Self::write_thrift_object(file_metadata, &mut sink),
559        }
560    }
561
562    /// Write a column [`OffsetIndex`] in Thrift format, possibly encrypting it if required
563    ///
564    /// [`OffsetIndex`]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
565    fn write_offset_index(
566        &self,
567        offset_index: &OffsetIndexMetaData,
568        column_chunk: &ColumnChunkMetaData,
569        row_group_idx: usize,
570        column_idx: usize,
571        sink: impl Write,
572    ) -> Result<()> {
573        match &self.file_encryptor {
574            Some(file_encryptor) => Self::write_thrift_object_with_encryption(
575                offset_index,
576                sink,
577                file_encryptor,
578                column_chunk,
579                ModuleType::OffsetIndex,
580                row_group_idx,
581                column_idx,
582            ),
583            None => Self::write_thrift_object(offset_index, sink),
584        }
585    }
586
587    /// Write a column [`ColumnIndex`] in Thrift format, possibly encrypting it if required
588    ///
589    /// Returns `true` unless there is an error.
590    ///
591    /// [`ColumnIndex`]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
592    fn write_column_index(
593        &self,
594        column_index: &ColumnIndexMetaData,
595        column_chunk: &ColumnChunkMetaData,
596        row_group_idx: usize,
597        column_idx: usize,
598        sink: impl Write,
599    ) -> Result<bool> {
600        match &self.file_encryptor {
601            Some(file_encryptor) => Self::write_thrift_object_with_encryption(
602                column_index,
603                sink,
604                file_encryptor,
605                column_chunk,
606                ModuleType::ColumnIndex,
607                row_group_idx,
608                column_idx,
609            )?,
610            None => Self::write_thrift_object(column_index, sink)?,
611        }
612        Ok(true)
613    }
614
615    /// If encryption is enabled and configured, encrypt row group metadata.
616    /// Returns a tuple of the row group metadata to write,
617    /// and possibly unencrypted metadata to be returned to clients if data was encrypted.
618    fn apply_row_group_encryption(
619        &self,
620        row_groups: Vec<RowGroupMetaData>,
621    ) -> Result<(Vec<RowGroupMetaData>, Option<Vec<RowGroupMetaData>>)> {
622        match &self.file_encryptor {
623            Some(file_encryptor) => {
624                let unencrypted_row_groups = row_groups.clone();
625                let encrypted_row_groups = Self::encrypt_row_groups(row_groups, file_encryptor)?;
626                Ok((encrypted_row_groups, Some(unencrypted_row_groups)))
627            }
628            None => Ok((row_groups, None)),
629        }
630    }
631
632    /// Get the "magic" bytes identifying the file type
633    fn get_file_magic(&self) -> &[u8; 4] {
634        get_file_magic(
635            self.file_encryptor
636                .as_ref()
637                .map(|encryptor| encryptor.properties()),
638        )
639    }
640
641    fn write_thrift_object_with_encryption(
642        object: &impl WriteThrift,
643        mut sink: impl Write,
644        file_encryptor: &FileEncryptor,
645        column_metadata: &ColumnChunkMetaData,
646        module_type: ModuleType,
647        row_group_index: usize,
648        column_index: usize,
649    ) -> Result<()> {
650        let column_path_vec = column_metadata.column_path().as_ref();
651
652        let joined_column_path;
653        let column_path = if column_path_vec.len() == 1 {
654            &column_path_vec[0]
655        } else {
656            joined_column_path = column_path_vec.join(".");
657            &joined_column_path
658        };
659
660        if file_encryptor.is_column_encrypted(column_path) {
661            use crate::encryption::encrypt::encrypt_thrift_object;
662
663            let aad = create_module_aad(
664                file_encryptor.file_aad(),
665                module_type,
666                row_group_index,
667                column_index,
668                None,
669            )?;
670            let mut encryptor = file_encryptor.get_column_encryptor(column_path)?;
671            encrypt_thrift_object(object, &mut encryptor, &mut sink, &aad)
672        } else {
673            Self::write_thrift_object(object, sink)
674        }
675    }
676
677    fn get_plaintext_footer_crypto_metadata(
678        &self,
679    ) -> (Option<EncryptionAlgorithm>, Option<Vec<u8>>) {
680        // Only plaintext footers may contain encryption algorithm and footer key metadata.
681        if let Some(file_encryptor) = self.file_encryptor.as_ref() {
682            let encryption_properties = file_encryptor.properties();
683            if !encryption_properties.encrypt_footer() {
684                return (
685                    Some(Self::encryption_algorithm_from_encryptor(file_encryptor)),
686                    encryption_properties.footer_key_metadata().cloned(),
687                );
688            }
689        }
690        (None, None)
691    }
692
693    fn encryption_algorithm_from_encryptor(file_encryptor: &FileEncryptor) -> EncryptionAlgorithm {
694        let supply_aad_prefix = file_encryptor
695            .properties()
696            .aad_prefix()
697            .map(|_| !file_encryptor.properties().store_aad_prefix());
698        let aad_prefix = if file_encryptor.properties().store_aad_prefix() {
699            file_encryptor.properties().aad_prefix()
700        } else {
701            None
702        };
703        EncryptionAlgorithm::AES_GCM_V1(AesGcmV1 {
704            aad_prefix: aad_prefix.cloned(),
705            aad_file_unique: Some(file_encryptor.aad_file_unique().clone()),
706            supply_aad_prefix,
707        })
708    }
709
710    fn file_crypto_metadata(file_encryptor: &'_ FileEncryptor) -> Result<FileCryptoMetaData<'_>> {
711        let properties = file_encryptor.properties();
712        Ok(FileCryptoMetaData {
713            encryption_algorithm: Self::encryption_algorithm_from_encryptor(file_encryptor),
714            key_metadata: properties.footer_key_metadata().map(|v| v.as_slice()),
715        })
716    }
717
718    fn encrypt_row_groups(
719        row_groups: Vec<RowGroupMetaData>,
720        file_encryptor: &Arc<FileEncryptor>,
721    ) -> Result<Vec<RowGroupMetaData>> {
722        row_groups
723            .into_iter()
724            .enumerate()
725            .map(|(rg_idx, mut rg)| {
726                let cols: Result<Vec<ColumnChunkMetaData>> = rg
727                    .columns
728                    .into_iter()
729                    .enumerate()
730                    .map(|(col_idx, c)| {
731                        Self::encrypt_column_chunk(c, file_encryptor, rg_idx, col_idx)
732                    })
733                    .collect();
734                rg.columns = cols?;
735                Ok(rg)
736            })
737            .collect()
738    }
739
740    /// Apply column encryption to column chunk metadata
741    fn encrypt_column_chunk(
742        mut column_chunk: ColumnChunkMetaData,
743        file_encryptor: &Arc<FileEncryptor>,
744        row_group_index: usize,
745        column_index: usize,
746    ) -> Result<ColumnChunkMetaData> {
747        // Column crypto metadata should have already been set when the column was created.
748        // Here we apply the encryption by encrypting the column metadata if required.
749        let encryptor = match column_chunk.column_crypto_metadata.as_deref() {
750            None => None,
751            Some(ColumnCryptoMetaData::ENCRYPTION_WITH_FOOTER_KEY) => {
752                let is_footer_encrypted = file_encryptor.properties().encrypt_footer();
753
754                // When uniform encryption is used the footer is already encrypted,
755                // so the column chunk does not need additional encryption.
756                // Except if we're in plaintext footer mode, then we need to encrypt
757                // the column metadata here.
758                if !is_footer_encrypted {
759                    Some(file_encryptor.get_footer_encryptor()?)
760                } else {
761                    None
762                }
763            }
764            Some(ColumnCryptoMetaData::ENCRYPTION_WITH_COLUMN_KEY(col_key)) => {
765                let column_path = col_key.path_in_schema.join(".");
766                Some(file_encryptor.get_column_encryptor(&column_path)?)
767            }
768        };
769
770        if let Some(mut encryptor) = encryptor {
771            use crate::file::metadata::thrift::serialize_column_meta_data;
772
773            let aad = create_module_aad(
774                file_encryptor.file_aad(),
775                ModuleType::ColumnMetaData,
776                row_group_index,
777                column_index,
778                None,
779            )?;
780            // create temp ColumnMetaData that we can encrypt
781            let mut buffer: Vec<u8> = vec![];
782            {
783                let mut prot = ThriftCompactOutputProtocol::new(&mut buffer);
784                serialize_column_meta_data(&column_chunk, &mut prot)?;
785            }
786            let ciphertext = encryptor.encrypt(&buffer, &aad)?;
787            column_chunk.encrypted_column_metadata = Some(ciphertext);
788            // Track whether the footer is plaintext, which affects how we serialize
789            // the column metadata (we need to write stripped metadata for backward compatibility)
790            column_chunk.plaintext_footer_mode = !file_encryptor.properties().encrypt_footer();
791        }
792
793        Ok(column_chunk)
794    }
795}