Skip to main content

parquet_derive/
lib.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! This crate provides a procedural macro to derive
6//! implementations of a RecordWriter and RecordReader
7//!
8//! # Platform Support
9//!
10//! Only little-endian platforms are officially supported and tested in CI.
11//! Big-endian platforms are not tested in CI and may not work correctly.
12//! Fixes for big-endian platforms are welcome and handled on a best-effort basis,
13//! but compatibility is not guaranteed.
14
15#![doc(
16    html_logo_url = "https://raw.githubusercontent.com/apache/parquet-format/25f05e73d8cd7f5c83532ce51cb4f4de8ba5f2a2/logo/parquet-logos_1.svg",
17    html_favicon_url = "https://raw.githubusercontent.com/apache/parquet-format/25f05e73d8cd7f5c83532ce51cb4f4de8ba5f2a2/logo/parquet-logos_1.svg"
18)]
19#![cfg_attr(docsrs, feature(doc_cfg))]
20#![warn(missing_docs)]
21#![recursion_limit = "128"]
22
23#[macro_use]
24extern crate quote;
25
26use ::syn::{Data, DataStruct, DeriveInput, ext::IdentExt, parse_macro_input};
27
28mod parquet_field;
29
30/// Derive flat, simple RecordWriter implementations.
31///
32/// Works by parsing a struct tagged with `#[derive(ParquetRecordWriter)]` and emitting
33/// the correct writing code for each field of the struct. Column writers
34/// are generated in the order they are defined.
35///
36/// It is up to the programmer to keep the order of the struct
37/// fields lined up with the schema.
38///
39/// Example:
40///
41/// ```rust
42/// use parquet::file::properties::WriterProperties;
43/// use parquet::file::writer::SerializedFileWriter;
44/// use parquet::record::RecordWriter;
45/// use parquet_derive::ParquetRecordWriter;
46/// use std::fs::File;
47/// use std::sync::Arc;
48///
49/// // For reader
50/// use parquet::file::reader::{FileReader, SerializedFileReader};
51/// use parquet::record::RecordReader;
52/// use parquet_derive::ParquetRecordReader;
53///
54/// #[derive(Debug, ParquetRecordWriter, ParquetRecordReader)]
55/// struct ACompleteRecord {
56///     pub a_bool: bool,
57///     pub a_string: String,
58/// }
59///
60/// fn write_some_records() {
61///     let samples = vec![
62///         ACompleteRecord {
63///             a_bool: true,
64///             a_string: "I'm true".into(),
65///         },
66///         ACompleteRecord {
67///             a_bool: false,
68///             a_string: "I'm false".into(),
69///         },
70///     ];
71///
72///     let schema = samples.as_slice().schema().unwrap();
73///
74///     let props = Arc::new(WriterProperties::builder().build());
75///
76///     let file = File::create("example.parquet").unwrap();
77///
78///     let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
79///
80///     let mut row_group = writer.next_row_group().unwrap();
81///
82///     samples
83///         .as_slice()
84///         .write_to_row_group(&mut row_group)
85///         .unwrap();
86///
87///     row_group.close().unwrap();
88///
89///     writer.close().unwrap();
90/// }
91///
92/// fn read_some_records() -> Vec<ACompleteRecord> {
93///     let mut samples: Vec<ACompleteRecord> = Vec::new();
94///     let file = File::open("example.parquet").unwrap();
95///
96///     let reader = SerializedFileReader::new(file).unwrap();
97///     let mut row_group = reader.get_row_group(0).unwrap();
98///     samples.read_from_row_group(&mut *row_group, 2).unwrap();
99///
100///     samples
101/// }
102///
103/// pub fn main() {
104///     write_some_records();
105///
106///     let records = read_some_records();
107///
108///     std::fs::remove_file("example.parquet").unwrap();
109///
110///     assert_eq!(
111///         format!("{:?}", records),
112///         "[ACompleteRecord { a_bool: true, a_string: \"I'm true\" }, ACompleteRecord { a_bool: false, a_string: \"I'm false\" }]"
113///     );
114/// }
115/// ```
116///
117#[proc_macro_derive(ParquetRecordWriter)]
118pub fn parquet_record_writer(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
119    let input: DeriveInput = parse_macro_input!(input as DeriveInput);
120    let fields = match input.data {
121        Data::Struct(DataStruct { fields, .. }) => fields,
122        Data::Enum(_) => unimplemented!("Enum currently is not supported"),
123        Data::Union(_) => unimplemented!("Union currently is not supported"),
124    };
125
126    let field_infos: Vec<_> = fields.iter().map(parquet_field::Field::from).collect();
127
128    let writer_snippets: Vec<proc_macro2::TokenStream> =
129        field_infos.iter().map(|x| x.writer_snippet()).collect();
130
131    let derived_for = input.ident;
132    let generics = input.generics;
133
134    let field_types: Vec<proc_macro2::TokenStream> =
135        field_infos.iter().map(|x| x.parquet_type()).collect();
136
137    (quote! {
138    impl #generics ::parquet::record::RecordWriter<#derived_for #generics> for &[#derived_for #generics] {
139      fn write_to_row_group<W: ::std::io::Write + Send>(
140        &self,
141        row_group_writer: &mut ::parquet::file::writer::SerializedRowGroupWriter<'_, W>
142      ) -> ::std::result::Result<(), ::parquet::errors::ParquetError> {
143        use ::parquet::column::writer::ColumnWriter;
144
145        let mut row_group_writer = row_group_writer;
146        let records = &self; // Used by all the writer snippets to be more clear
147
148        #(
149          {
150              let mut some_column_writer = row_group_writer.next_column().unwrap();
151              if let Some(mut column_writer) = some_column_writer {
152                  #writer_snippets
153                  column_writer.close()?;
154              } else {
155                  return Err(::parquet::errors::ParquetError::General("Failed to get next column".into()))
156              }
157          }
158        );*
159
160        Ok(())
161      }
162
163      fn schema(&self) -> ::std::result::Result<::parquet::schema::types::TypePtr, ::parquet::errors::ParquetError> {
164        use ::parquet::schema::types::Type as ParquetType;
165        use ::parquet::schema::types::TypePtr;
166        use ::parquet::basic::LogicalType;
167
168        let mut fields: ::std::vec::Vec<TypePtr> = ::std::vec::Vec::new();
169        #(
170          #field_types
171        );*;
172        let group = ParquetType::group_type_builder("rust_schema")
173          .with_fields(fields)
174          .build()?;
175        Ok(group.into())
176      }
177    }
178  }).into()
179}
180
181/// Derive flat, simple RecordReader implementations.
182///
183/// Works by parsing a struct tagged with `#[derive(ParquetRecordReader)]` and emitting
184/// the correct writing code for each field of the struct. Column readers
185/// are generated by matching names in the schema to the names in the struct.
186///
187/// It is up to the programmer to ensure the names in the struct
188/// fields line up with the schema.
189///
190/// Example:
191///
192/// ```rust
193/// use parquet::record::RecordReader;
194/// use parquet::file::{serialized_reader::SerializedFileReader, reader::FileReader};
195/// use parquet_derive::{ParquetRecordReader};
196/// use std::fs::File;
197///
198/// #[derive(ParquetRecordReader)]
199/// struct ACompleteRecord {
200///     pub a_bool: bool,
201///     pub a_string: String,
202/// }
203///
204/// pub fn read_some_records() -> Vec<ACompleteRecord> {
205///   let mut samples: Vec<ACompleteRecord> = Vec::new();
206///   let file = File::open("some_file.parquet").unwrap();
207///
208///   let reader = SerializedFileReader::new(file).unwrap();
209///   let mut row_group = reader.get_row_group(0).unwrap();
210///   samples.read_from_row_group(&mut *row_group, 1).unwrap();
211///   samples
212/// }
213/// ```
214///
215#[proc_macro_derive(ParquetRecordReader)]
216pub fn parquet_record_reader(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
217    let input: DeriveInput = parse_macro_input!(input as DeriveInput);
218    let fields = match input.data {
219        Data::Struct(DataStruct { fields, .. }) => fields,
220        Data::Enum(_) => unimplemented!("Enum currently is not supported"),
221        Data::Union(_) => unimplemented!("Union currently is not supported"),
222    };
223
224    let field_infos: Vec<_> = fields.iter().map(parquet_field::Field::from).collect();
225    let field_names: Vec<_> = fields.iter().map(|f| f.ident.clone()).collect();
226    // unraw the identifiers, so raw identifiers like `r#type` are looked
227    // up by their column name `type` in the parquet file
228    let field_names_str: Vec<_> = fields
229        .iter()
230        .map(|f| {
231            f.ident
232                .as_ref()
233                .expect("Only structs with named fields are currently supported")
234                .unraw()
235                .to_string()
236        })
237        .collect();
238    let reader_snippets: Vec<proc_macro2::TokenStream> =
239        field_infos.iter().map(|x| x.reader_snippet()).collect();
240
241    let derived_for = input.ident;
242    let generics = input.generics;
243
244    (quote! {
245
246    impl #generics ::parquet::record::RecordReader<#derived_for #generics> for Vec<#derived_for #generics> {
247      fn read_from_row_group(
248        &mut self,
249        row_group_reader: &mut dyn ::parquet::file::reader::RowGroupReader,
250        num_records: usize,
251      ) -> ::std::result::Result<(), ::parquet::errors::ParquetError> {
252        use ::parquet::column::reader::ColumnReader;
253
254        let mut row_group_reader = row_group_reader;
255
256        // key: parquet file column name, value: column index
257        let mut name_to_index = std::collections::HashMap::new();
258        for (idx, col) in row_group_reader.metadata().schema_descr().columns().iter().enumerate() {
259            name_to_index.insert(col.name().to_string(), idx);
260        }
261
262        for _ in 0..num_records {
263          self.push(#derived_for {
264            #(
265              #field_names: Default::default()
266            ),*
267          })
268        }
269
270        let records = self; // Used by all the reader snippets to be more clear
271
272        #(
273          {
274              let idx: usize = match name_to_index.get(#field_names_str) {
275                Some(&col_idx) => col_idx,
276                None => {
277                  let error_msg = format!("column name '{}' is not found in parquet file!", #field_names_str);
278                  return Err(::parquet::errors::ParquetError::General(error_msg));
279                }
280              };
281              if let Ok(column_reader) = row_group_reader.get_column_reader(idx) {
282                  #reader_snippets
283              } else {
284                  return Err(::parquet::errors::ParquetError::General("Failed to get next column".into()))
285              }
286          }
287        );*
288
289        Ok(())
290      }
291    }
292  }).into()
293}