1use crate::file::metadata::page_index::PageIndexProvider;
6use crate::file::metadata::thrift::FileMeta;
7use crate::file::metadata::{ColumnChunkMetaData, RowGroupMetaData};
8use crate::schema::types::{SchemaDescPtr, SchemaDescriptor};
9use crate::{
10 basic::ColumnOrder,
11 file::metadata::{FileMetaData, ParquetMetaDataBuilder},
12};
13#[cfg(feature = "encryption")]
14use crate::{
15 encryption::{
16 encrypt::{FileEncryptor, encrypt_thrift_object, write_signed_plaintext_thrift_object},
17 modules::{ModuleType, create_footer_aad, create_module_aad},
18 },
19 file::column_crypto_metadata::ColumnCryptoMetaData,
20 file::metadata::thrift::encryption::{AesGcmV1, EncryptionAlgorithm, FileCryptoMetaData},
21};
22use crate::{errors::Result, file::page_index::column_index::ColumnIndexMetaData};
23
24use crate::{
25 file::writer::{TrackedWrite, get_file_magic},
26 parquet_thrift::WriteThrift,
27};
28use crate::{
29 file::{
30 metadata::{KeyValue, ParquetMetaData},
31 page_index::offset_index::OffsetIndexMetaData,
32 },
33 parquet_thrift::ThriftCompactOutputProtocol,
34};
35use std::io::Write;
36use std::sync::Arc;
37
38pub(crate) struct ThriftMetadataWriter<'a, W: Write> {
42 buf: &'a mut TrackedWrite<W>,
43 schema_descr: &'a SchemaDescPtr,
44 row_groups: Vec<RowGroupMetaData>,
45 page_index: Option<Arc<dyn PageIndexProvider>>,
46 key_value_metadata: Option<Vec<KeyValue>>,
47 created_by: Option<String>,
48 object_writer: MetadataObjectWriter,
49 writer_version: i32,
50 write_path_in_schema: bool,
51 preserve_page_index_locations: bool,
52}
53
54impl<'a, W: Write> ThriftMetadataWriter<'a, W> {
55 fn write_offset_indexes(&mut self, page_index: &dyn PageIndexProvider) -> Result<()> {
61 for (row_group_idx, row_group) in self.row_groups.iter_mut().enumerate() {
65 for (column_idx, column_metadata) in row_group.columns.iter_mut().enumerate() {
66 if let Some(offset_index) = page_index.offset_index(row_group_idx, column_idx) {
67 let start_offset = self.buf.bytes_written();
68 self.object_writer.write_offset_index(
69 offset_index,
70 column_metadata,
71 row_group_idx,
72 column_idx,
73 &mut *self.buf,
74 )?;
75 let end_offset = self.buf.bytes_written();
76 column_metadata.offset_index_offset = Some(start_offset as i64);
78 column_metadata.offset_index_length = Some((end_offset - start_offset) as i32);
79 } else if !self.preserve_page_index_locations {
80 column_metadata.offset_index_offset = None;
81 column_metadata.offset_index_length = None;
82 }
83 }
84 }
85 Ok(())
86 }
87
88 fn write_column_indexes(&mut self, page_index: &dyn PageIndexProvider) -> Result<()> {
94 for (row_group_idx, row_group) in self.row_groups.iter_mut().enumerate() {
98 for (column_idx, column_metadata) in row_group.columns.iter_mut().enumerate() {
99 if let Some(column_index) = page_index.column_index(row_group_idx, column_idx) {
100 let start_offset = self.buf.bytes_written();
101 if self.object_writer.write_column_index(
103 column_index,
104 column_metadata,
105 row_group_idx,
106 column_idx,
107 &mut *self.buf,
108 )? {
109 let end_offset = self.buf.bytes_written();
110 column_metadata.column_index_offset = Some(start_offset as i64);
112 column_metadata.column_index_length =
113 Some((end_offset - start_offset) as i32);
114 }
115 } else if !self.preserve_page_index_locations {
116 column_metadata.column_index_offset = None;
117 column_metadata.column_index_length = None;
118 }
119 }
120 }
121 Ok(())
122 }
123
124 fn clear_page_indexes(&mut self) {
125 for row_group in &mut self.row_groups {
126 for column_metadata in &mut row_group.columns {
127 column_metadata.column_index_offset = None;
128 column_metadata.column_index_length = None;
129 column_metadata.offset_index_offset = None;
130 column_metadata.offset_index_length = None;
131 }
132 }
133 }
134
135 pub(crate) fn finish(mut self) -> Result<ParquetMetaData> {
137 let num_rows = self.row_groups.iter().map(|x| x.num_rows).sum();
138
139 let page_index = self.page_index.take();
141 if let Some(pi) = page_index.as_deref() {
142 self.write_column_indexes(pi)?;
143 self.write_offset_indexes(pi)?;
144 } else if !self.preserve_page_index_locations {
145 self.clear_page_indexes();
147 }
148
149 let column_orders = self
151 .schema_descr
152 .columns()
153 .iter()
154 .map(|col| {
155 ColumnOrder::column_order_for_type(
156 col.logical_type_ref(),
157 col.converted_type(),
158 col.physical_type(),
159 )
160 })
161 .collect();
162
163 let column_orders = Some(column_orders);
167
168 let (row_groups, unencrypted_row_groups) = self
169 .object_writer
170 .apply_row_group_encryption(self.row_groups)?;
171
172 #[cfg(feature = "encryption")]
173 let (encryption_algorithm, footer_signing_key_metadata) =
174 self.object_writer.get_plaintext_footer_crypto_metadata();
175 #[cfg(feature = "encryption")]
176 let file_metadata = FileMetaData::new(
177 self.writer_version,
178 num_rows,
179 self.created_by,
180 self.key_value_metadata,
181 self.schema_descr.clone(),
182 column_orders,
183 )
184 .with_encryption_algorithm(encryption_algorithm)
185 .with_footer_signing_key_metadata(footer_signing_key_metadata);
186
187 #[cfg(not(feature = "encryption"))]
188 let file_metadata = FileMetaData::new(
189 self.writer_version,
190 num_rows,
191 self.created_by,
192 self.key_value_metadata,
193 self.schema_descr.clone(),
194 column_orders,
195 );
196
197 let file_meta = FileMeta {
198 file_metadata: &file_metadata,
199 row_groups: &row_groups,
200 write_path_in_schema: self.write_path_in_schema,
201 };
202
203 let start_pos = self.buf.bytes_written();
205 self.object_writer
206 .write_file_metadata(&file_meta, &mut *self.buf)?;
207 let end_pos = self.buf.bytes_written();
208
209 let metadata_len = (end_pos - start_pos) as u32;
211
212 self.buf.write_all(&metadata_len.to_le_bytes())?;
213 self.buf.write_all(self.object_writer.get_file_magic())?;
214
215 let builder = ParquetMetaDataBuilder::new(file_metadata).set_page_index(page_index);
221
222 Ok(match unencrypted_row_groups {
227 Some(rg) => builder.set_row_groups(rg).build(),
228 None => builder.set_row_groups(row_groups).build(),
229 })
230 }
231
232 pub(crate) fn new(
233 buf: &'a mut TrackedWrite<W>,
234 schema_descr: &'a SchemaDescPtr,
235 row_groups: Vec<RowGroupMetaData>,
236 created_by: Option<String>,
237 writer_version: i32,
238 write_path_in_schema: bool,
239 preserve_page_index_locations: bool,
240 ) -> Self {
241 Self {
242 buf,
243 schema_descr,
244 row_groups,
245 page_index: None,
246 key_value_metadata: None,
247 created_by,
248 object_writer: Default::default(),
249 writer_version,
250 write_path_in_schema,
251 preserve_page_index_locations,
252 }
253 }
254
255 pub(crate) fn with_page_index(mut self, page_index: Arc<dyn PageIndexProvider>) -> Self {
256 self.page_index = Some(page_index);
257 self
258 }
259
260 pub(crate) fn with_key_value_metadata(mut self, key_value_metadata: Vec<KeyValue>) -> Self {
261 self.key_value_metadata = Some(key_value_metadata);
262 self
263 }
264
265 #[cfg(feature = "encryption")]
266 pub(crate) fn with_file_encryptor(
267 mut self,
268 file_encryptor: Option<Arc<FileEncryptor>>,
269 ) -> Self {
270 self.object_writer = self.object_writer.with_file_encryptor(file_encryptor);
271 self
272 }
273}
274
275pub struct ParquetMetaDataWriter<'a, W: Write> {
362 buf: TrackedWrite<W>,
363 metadata: &'a ParquetMetaData,
364 write_path_in_schema: bool,
365 preserve_page_index_locations: bool,
366}
367
368impl<'a, W: Write> ParquetMetaDataWriter<'a, W> {
369 pub fn new(buf: W, metadata: &'a ParquetMetaData) -> Self {
377 Self::new_with_tracked(TrackedWrite::new(buf), metadata)
378 }
379
380 pub fn new_with_tracked(buf: TrackedWrite<W>, metadata: &'a ParquetMetaData) -> Self {
387 Self {
388 buf,
389 metadata,
390 write_path_in_schema: true,
391 preserve_page_index_locations: true,
392 }
393 }
394
395 pub fn with_write_path_in_schema(self, val: bool) -> Self {
398 Self {
399 write_path_in_schema: val,
400 ..self
401 }
402 }
403
404 pub fn with_preserve_page_index_locations(self, val: bool) -> Self {
415 Self {
416 preserve_page_index_locations: val,
417 ..self
418 }
419 }
420
421 pub fn finish(mut self) -> Result<()> {
423 let file_metadata = self.metadata.file_metadata();
424
425 let schema = Arc::new(file_metadata.schema().clone());
426 let schema_descr = Arc::new(SchemaDescriptor::new(schema.clone()));
427 let created_by = file_metadata.created_by().map(str::to_string);
428
429 let row_groups = self.metadata.row_groups.as_ref().to_vec();
432
433 let key_value_metadata = file_metadata.key_value_metadata().cloned();
434
435 let mut encoder = ThriftMetadataWriter::new(
436 &mut self.buf,
437 &schema_descr,
438 row_groups,
439 created_by,
440 file_metadata.version(),
441 self.write_path_in_schema,
442 self.preserve_page_index_locations,
443 );
444
445 if let Some(page_index_arc) = self.metadata.page_index.clone() {
446 encoder = encoder.with_page_index(page_index_arc);
447 }
448
449 if let Some(key_value_metadata) = key_value_metadata {
450 encoder = encoder.with_key_value_metadata(key_value_metadata);
451 }
452 encoder.finish()?;
453
454 Ok(())
455 }
456}
457
458#[derive(Debug, Default)]
459struct MetadataObjectWriter {
460 #[cfg(feature = "encryption")]
461 file_encryptor: Option<Arc<FileEncryptor>>,
462}
463
464impl MetadataObjectWriter {
465 #[inline]
466 fn write_thrift_object(object: &impl WriteThrift, sink: impl Write) -> Result<()> {
467 let mut protocol = ThriftCompactOutputProtocol::new(sink);
468 object.write_thrift(&mut protocol)?;
469 Ok(())
470 }
471}
472
473#[cfg(not(feature = "encryption"))]
475impl MetadataObjectWriter {
476 fn write_file_metadata(&self, file_metadata: &FileMeta, sink: impl Write) -> Result<()> {
480 Self::write_thrift_object(file_metadata, sink)
481 }
482
483 fn write_offset_index(
487 &self,
488 offset_index: &OffsetIndexMetaData,
489 _column_chunk: &ColumnChunkMetaData,
490 _row_group_idx: usize,
491 _column_idx: usize,
492 sink: impl Write,
493 ) -> Result<()> {
494 Self::write_thrift_object(offset_index, sink)
495 }
496
497 fn write_column_index(
503 &self,
504 column_index: &ColumnIndexMetaData,
505 _column_chunk: &ColumnChunkMetaData,
506 _row_group_idx: usize,
507 _column_idx: usize,
508 sink: impl Write,
509 ) -> Result<bool> {
510 Self::write_thrift_object(column_index, sink)?;
511 Ok(true)
512 }
513
514 fn apply_row_group_encryption(
516 &self,
517 row_groups: Vec<RowGroupMetaData>,
518 ) -> Result<(Vec<RowGroupMetaData>, Option<Vec<RowGroupMetaData>>)> {
519 Ok((row_groups, None))
520 }
521
522 pub fn get_file_magic(&self) -> &[u8; 4] {
524 get_file_magic()
525 }
526}
527
528#[cfg(feature = "encryption")]
530impl MetadataObjectWriter {
531 fn with_file_encryptor(mut self, encryptor: Option<Arc<FileEncryptor>>) -> Self {
533 self.file_encryptor = encryptor;
534 self
535 }
536
537 fn write_file_metadata(&self, file_metadata: &FileMeta, mut sink: impl Write) -> Result<()> {
541 match self.file_encryptor.as_ref() {
542 Some(file_encryptor) if file_encryptor.properties().encrypt_footer() => {
543 let crypto_metadata = Self::file_crypto_metadata(file_encryptor)?;
545 let mut protocol = ThriftCompactOutputProtocol::new(&mut sink);
546 crypto_metadata.write_thrift(&mut protocol)?;
547
548 let aad = create_footer_aad(file_encryptor.file_aad())?;
550 let mut encryptor = file_encryptor.get_footer_encryptor()?;
551 encrypt_thrift_object(file_metadata, &mut encryptor, &mut sink, &aad)
552 }
553 Some(file_encryptor) if file_metadata.file_metadata.encryption_algorithm.is_some() => {
554 let aad = create_footer_aad(file_encryptor.file_aad())?;
555 let mut encryptor = file_encryptor.get_footer_encryptor()?;
556 write_signed_plaintext_thrift_object(file_metadata, &mut encryptor, &mut sink, &aad)
557 }
558 _ => Self::write_thrift_object(file_metadata, &mut sink),
559 }
560 }
561
562 fn write_offset_index(
566 &self,
567 offset_index: &OffsetIndexMetaData,
568 column_chunk: &ColumnChunkMetaData,
569 row_group_idx: usize,
570 column_idx: usize,
571 sink: impl Write,
572 ) -> Result<()> {
573 match &self.file_encryptor {
574 Some(file_encryptor) => Self::write_thrift_object_with_encryption(
575 offset_index,
576 sink,
577 file_encryptor,
578 column_chunk,
579 ModuleType::OffsetIndex,
580 row_group_idx,
581 column_idx,
582 ),
583 None => Self::write_thrift_object(offset_index, sink),
584 }
585 }
586
587 fn write_column_index(
593 &self,
594 column_index: &ColumnIndexMetaData,
595 column_chunk: &ColumnChunkMetaData,
596 row_group_idx: usize,
597 column_idx: usize,
598 sink: impl Write,
599 ) -> Result<bool> {
600 match &self.file_encryptor {
601 Some(file_encryptor) => Self::write_thrift_object_with_encryption(
602 column_index,
603 sink,
604 file_encryptor,
605 column_chunk,
606 ModuleType::ColumnIndex,
607 row_group_idx,
608 column_idx,
609 )?,
610 None => Self::write_thrift_object(column_index, sink)?,
611 }
612 Ok(true)
613 }
614
615 fn apply_row_group_encryption(
619 &self,
620 row_groups: Vec<RowGroupMetaData>,
621 ) -> Result<(Vec<RowGroupMetaData>, Option<Vec<RowGroupMetaData>>)> {
622 match &self.file_encryptor {
623 Some(file_encryptor) => {
624 let unencrypted_row_groups = row_groups.clone();
625 let encrypted_row_groups = Self::encrypt_row_groups(row_groups, file_encryptor)?;
626 Ok((encrypted_row_groups, Some(unencrypted_row_groups)))
627 }
628 None => Ok((row_groups, None)),
629 }
630 }
631
632 fn get_file_magic(&self) -> &[u8; 4] {
634 get_file_magic(
635 self.file_encryptor
636 .as_ref()
637 .map(|encryptor| encryptor.properties()),
638 )
639 }
640
641 fn write_thrift_object_with_encryption(
642 object: &impl WriteThrift,
643 mut sink: impl Write,
644 file_encryptor: &FileEncryptor,
645 column_metadata: &ColumnChunkMetaData,
646 module_type: ModuleType,
647 row_group_index: usize,
648 column_index: usize,
649 ) -> Result<()> {
650 let column_path_vec = column_metadata.column_path().as_ref();
651
652 let joined_column_path;
653 let column_path = if column_path_vec.len() == 1 {
654 &column_path_vec[0]
655 } else {
656 joined_column_path = column_path_vec.join(".");
657 &joined_column_path
658 };
659
660 if file_encryptor.is_column_encrypted(column_path) {
661 use crate::encryption::encrypt::encrypt_thrift_object;
662
663 let aad = create_module_aad(
664 file_encryptor.file_aad(),
665 module_type,
666 row_group_index,
667 column_index,
668 None,
669 )?;
670 let mut encryptor = file_encryptor.get_column_encryptor(column_path)?;
671 encrypt_thrift_object(object, &mut encryptor, &mut sink, &aad)
672 } else {
673 Self::write_thrift_object(object, sink)
674 }
675 }
676
677 fn get_plaintext_footer_crypto_metadata(
678 &self,
679 ) -> (Option<EncryptionAlgorithm>, Option<Vec<u8>>) {
680 if let Some(file_encryptor) = self.file_encryptor.as_ref() {
682 let encryption_properties = file_encryptor.properties();
683 if !encryption_properties.encrypt_footer() {
684 return (
685 Some(Self::encryption_algorithm_from_encryptor(file_encryptor)),
686 encryption_properties.footer_key_metadata().cloned(),
687 );
688 }
689 }
690 (None, None)
691 }
692
693 fn encryption_algorithm_from_encryptor(file_encryptor: &FileEncryptor) -> EncryptionAlgorithm {
694 let supply_aad_prefix = file_encryptor
695 .properties()
696 .aad_prefix()
697 .map(|_| !file_encryptor.properties().store_aad_prefix());
698 let aad_prefix = if file_encryptor.properties().store_aad_prefix() {
699 file_encryptor.properties().aad_prefix()
700 } else {
701 None
702 };
703 EncryptionAlgorithm::AES_GCM_V1(AesGcmV1 {
704 aad_prefix: aad_prefix.cloned(),
705 aad_file_unique: Some(file_encryptor.aad_file_unique().clone()),
706 supply_aad_prefix,
707 })
708 }
709
710 fn file_crypto_metadata(file_encryptor: &'_ FileEncryptor) -> Result<FileCryptoMetaData<'_>> {
711 let properties = file_encryptor.properties();
712 Ok(FileCryptoMetaData {
713 encryption_algorithm: Self::encryption_algorithm_from_encryptor(file_encryptor),
714 key_metadata: properties.footer_key_metadata().map(|v| v.as_slice()),
715 })
716 }
717
718 fn encrypt_row_groups(
719 row_groups: Vec<RowGroupMetaData>,
720 file_encryptor: &Arc<FileEncryptor>,
721 ) -> Result<Vec<RowGroupMetaData>> {
722 row_groups
723 .into_iter()
724 .enumerate()
725 .map(|(rg_idx, mut rg)| {
726 let cols: Result<Vec<ColumnChunkMetaData>> = rg
727 .columns
728 .into_iter()
729 .enumerate()
730 .map(|(col_idx, c)| {
731 Self::encrypt_column_chunk(c, file_encryptor, rg_idx, col_idx)
732 })
733 .collect();
734 rg.columns = cols?;
735 Ok(rg)
736 })
737 .collect()
738 }
739
740 fn encrypt_column_chunk(
742 mut column_chunk: ColumnChunkMetaData,
743 file_encryptor: &Arc<FileEncryptor>,
744 row_group_index: usize,
745 column_index: usize,
746 ) -> Result<ColumnChunkMetaData> {
747 let encryptor = match column_chunk.column_crypto_metadata.as_deref() {
750 None => None,
751 Some(ColumnCryptoMetaData::ENCRYPTION_WITH_FOOTER_KEY) => {
752 let is_footer_encrypted = file_encryptor.properties().encrypt_footer();
753
754 if !is_footer_encrypted {
759 Some(file_encryptor.get_footer_encryptor()?)
760 } else {
761 None
762 }
763 }
764 Some(ColumnCryptoMetaData::ENCRYPTION_WITH_COLUMN_KEY(col_key)) => {
765 let column_path = col_key.path_in_schema.join(".");
766 Some(file_encryptor.get_column_encryptor(&column_path)?)
767 }
768 };
769
770 if let Some(mut encryptor) = encryptor {
771 use crate::file::metadata::thrift::serialize_column_meta_data;
772
773 let aad = create_module_aad(
774 file_encryptor.file_aad(),
775 ModuleType::ColumnMetaData,
776 row_group_index,
777 column_index,
778 None,
779 )?;
780 let mut buffer: Vec<u8> = vec![];
782 {
783 let mut prot = ThriftCompactOutputProtocol::new(&mut buffer);
784 serialize_column_meta_data(&column_chunk, &mut prot)?;
785 }
786 let ciphertext = encryptor.encrypt(&buffer, &aad)?;
787 column_chunk.encrypted_column_metadata = Some(ciphertext);
788 column_chunk.plaintext_footer_mode = !file_encryptor.properties().encrypt_footer();
791 }
792
793 Ok(column_chunk)
794 }
795}