parquet_derive/lib.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! This crate provides a procedural macro to derive
6//! implementations of a RecordWriter and RecordReader
7//!
8//! # Platform Support
9//!
10//! Only little-endian platforms are officially supported and tested in CI.
11//! Big-endian platforms are not tested in CI and may not work correctly.
12//! Fixes for big-endian platforms are welcome and handled on a best-effort basis,
13//! but compatibility is not guaranteed.
14
15#![doc(
16 html_logo_url = "https://raw.githubusercontent.com/apache/parquet-format/25f05e73d8cd7f5c83532ce51cb4f4de8ba5f2a2/logo/parquet-logos_1.svg",
17 html_favicon_url = "https://raw.githubusercontent.com/apache/parquet-format/25f05e73d8cd7f5c83532ce51cb4f4de8ba5f2a2/logo/parquet-logos_1.svg"
18)]
19#![cfg_attr(docsrs, feature(doc_cfg))]
20#![warn(missing_docs)]
21#![recursion_limit = "128"]
22
23#[macro_use]
24extern crate quote;
25
26use ::syn::{Data, DataStruct, DeriveInput, ext::IdentExt, parse_macro_input};
27
28mod parquet_field;
29
30/// Derive flat, simple RecordWriter implementations.
31///
32/// Works by parsing a struct tagged with `#[derive(ParquetRecordWriter)]` and emitting
33/// the correct writing code for each field of the struct. Column writers
34/// are generated in the order they are defined.
35///
36/// It is up to the programmer to keep the order of the struct
37/// fields lined up with the schema.
38///
39/// Example:
40///
41/// ```rust
42/// use parquet::file::properties::WriterProperties;
43/// use parquet::file::writer::SerializedFileWriter;
44/// use parquet::record::RecordWriter;
45/// use parquet_derive::ParquetRecordWriter;
46/// use std::fs::File;
47/// use std::sync::Arc;
48///
49/// // For reader
50/// use parquet::file::reader::{FileReader, SerializedFileReader};
51/// use parquet::record::RecordReader;
52/// use parquet_derive::ParquetRecordReader;
53///
54/// #[derive(Debug, ParquetRecordWriter, ParquetRecordReader)]
55/// struct ACompleteRecord {
56/// pub a_bool: bool,
57/// pub a_string: String,
58/// }
59///
60/// fn write_some_records() {
61/// let samples = vec![
62/// ACompleteRecord {
63/// a_bool: true,
64/// a_string: "I'm true".into(),
65/// },
66/// ACompleteRecord {
67/// a_bool: false,
68/// a_string: "I'm false".into(),
69/// },
70/// ];
71///
72/// let schema = samples.as_slice().schema().unwrap();
73///
74/// let props = Arc::new(WriterProperties::builder().build());
75///
76/// let file = File::create("example.parquet").unwrap();
77///
78/// let mut writer = SerializedFileWriter::new(file, schema, props).unwrap();
79///
80/// let mut row_group = writer.next_row_group().unwrap();
81///
82/// samples
83/// .as_slice()
84/// .write_to_row_group(&mut row_group)
85/// .unwrap();
86///
87/// row_group.close().unwrap();
88///
89/// writer.close().unwrap();
90/// }
91///
92/// fn read_some_records() -> Vec<ACompleteRecord> {
93/// let mut samples: Vec<ACompleteRecord> = Vec::new();
94/// let file = File::open("example.parquet").unwrap();
95///
96/// let reader = SerializedFileReader::new(file).unwrap();
97/// let mut row_group = reader.get_row_group(0).unwrap();
98/// samples.read_from_row_group(&mut *row_group, 2).unwrap();
99///
100/// samples
101/// }
102///
103/// pub fn main() {
104/// write_some_records();
105///
106/// let records = read_some_records();
107///
108/// std::fs::remove_file("example.parquet").unwrap();
109///
110/// assert_eq!(
111/// format!("{:?}", records),
112/// "[ACompleteRecord { a_bool: true, a_string: \"I'm true\" }, ACompleteRecord { a_bool: false, a_string: \"I'm false\" }]"
113/// );
114/// }
115/// ```
116///
117#[proc_macro_derive(ParquetRecordWriter)]
118pub fn parquet_record_writer(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
119 let input: DeriveInput = parse_macro_input!(input as DeriveInput);
120 let fields = match input.data {
121 Data::Struct(DataStruct { fields, .. }) => fields,
122 Data::Enum(_) => unimplemented!("Enum currently is not supported"),
123 Data::Union(_) => unimplemented!("Union currently is not supported"),
124 };
125
126 let field_infos: Vec<_> = fields.iter().map(parquet_field::Field::from).collect();
127
128 let writer_snippets: Vec<proc_macro2::TokenStream> =
129 field_infos.iter().map(|x| x.writer_snippet()).collect();
130
131 let derived_for = input.ident;
132 let generics = input.generics;
133
134 let field_types: Vec<proc_macro2::TokenStream> =
135 field_infos.iter().map(|x| x.parquet_type()).collect();
136
137 (quote! {
138 impl #generics ::parquet::record::RecordWriter<#derived_for #generics> for &[#derived_for #generics] {
139 fn write_to_row_group<W: ::std::io::Write + Send>(
140 &self,
141 row_group_writer: &mut ::parquet::file::writer::SerializedRowGroupWriter<'_, W>
142 ) -> ::std::result::Result<(), ::parquet::errors::ParquetError> {
143 use ::parquet::column::writer::ColumnWriter;
144
145 let mut row_group_writer = row_group_writer;
146 let records = &self; // Used by all the writer snippets to be more clear
147
148 #(
149 {
150 let mut some_column_writer = row_group_writer.next_column().unwrap();
151 if let Some(mut column_writer) = some_column_writer {
152 #writer_snippets
153 column_writer.close()?;
154 } else {
155 return Err(::parquet::errors::ParquetError::General("Failed to get next column".into()))
156 }
157 }
158 );*
159
160 Ok(())
161 }
162
163 fn schema(&self) -> ::std::result::Result<::parquet::schema::types::TypePtr, ::parquet::errors::ParquetError> {
164 use ::parquet::schema::types::Type as ParquetType;
165 use ::parquet::schema::types::TypePtr;
166 use ::parquet::basic::LogicalType;
167
168 let mut fields: ::std::vec::Vec<TypePtr> = ::std::vec::Vec::new();
169 #(
170 #field_types
171 );*;
172 let group = ParquetType::group_type_builder("rust_schema")
173 .with_fields(fields)
174 .build()?;
175 Ok(group.into())
176 }
177 }
178 }).into()
179}
180
181/// Derive flat, simple RecordReader implementations.
182///
183/// Works by parsing a struct tagged with `#[derive(ParquetRecordReader)]` and emitting
184/// the correct writing code for each field of the struct. Column readers
185/// are generated by matching names in the schema to the names in the struct.
186///
187/// It is up to the programmer to ensure the names in the struct
188/// fields line up with the schema.
189///
190/// Example:
191///
192/// ```rust
193/// use parquet::record::RecordReader;
194/// use parquet::file::{serialized_reader::SerializedFileReader, reader::FileReader};
195/// use parquet_derive::{ParquetRecordReader};
196/// use std::fs::File;
197///
198/// #[derive(ParquetRecordReader)]
199/// struct ACompleteRecord {
200/// pub a_bool: bool,
201/// pub a_string: String,
202/// }
203///
204/// pub fn read_some_records() -> Vec<ACompleteRecord> {
205/// let mut samples: Vec<ACompleteRecord> = Vec::new();
206/// let file = File::open("some_file.parquet").unwrap();
207///
208/// let reader = SerializedFileReader::new(file).unwrap();
209/// let mut row_group = reader.get_row_group(0).unwrap();
210/// samples.read_from_row_group(&mut *row_group, 1).unwrap();
211/// samples
212/// }
213/// ```
214///
215#[proc_macro_derive(ParquetRecordReader)]
216pub fn parquet_record_reader(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
217 let input: DeriveInput = parse_macro_input!(input as DeriveInput);
218 let fields = match input.data {
219 Data::Struct(DataStruct { fields, .. }) => fields,
220 Data::Enum(_) => unimplemented!("Enum currently is not supported"),
221 Data::Union(_) => unimplemented!("Union currently is not supported"),
222 };
223
224 let field_infos: Vec<_> = fields.iter().map(parquet_field::Field::from).collect();
225 let field_names: Vec<_> = fields.iter().map(|f| f.ident.clone()).collect();
226 // unraw the identifiers, so raw identifiers like `r#type` are looked
227 // up by their column name `type` in the parquet file
228 let field_names_str: Vec<_> = fields
229 .iter()
230 .map(|f| {
231 f.ident
232 .as_ref()
233 .expect("Only structs with named fields are currently supported")
234 .unraw()
235 .to_string()
236 })
237 .collect();
238 let reader_snippets: Vec<proc_macro2::TokenStream> =
239 field_infos.iter().map(|x| x.reader_snippet()).collect();
240
241 let derived_for = input.ident;
242 let generics = input.generics;
243
244 (quote! {
245
246 impl #generics ::parquet::record::RecordReader<#derived_for #generics> for Vec<#derived_for #generics> {
247 fn read_from_row_group(
248 &mut self,
249 row_group_reader: &mut dyn ::parquet::file::reader::RowGroupReader,
250 num_records: usize,
251 ) -> ::std::result::Result<(), ::parquet::errors::ParquetError> {
252 use ::parquet::column::reader::ColumnReader;
253
254 let mut row_group_reader = row_group_reader;
255
256 // key: parquet file column name, value: column index
257 let mut name_to_index = std::collections::HashMap::new();
258 for (idx, col) in row_group_reader.metadata().schema_descr().columns().iter().enumerate() {
259 name_to_index.insert(col.name().to_string(), idx);
260 }
261
262 for _ in 0..num_records {
263 self.push(#derived_for {
264 #(
265 #field_names: Default::default()
266 ),*
267 })
268 }
269
270 let records = self; // Used by all the reader snippets to be more clear
271
272 #(
273 {
274 let idx: usize = match name_to_index.get(#field_names_str) {
275 Some(&col_idx) => col_idx,
276 None => {
277 let error_msg = format!("column name '{}' is not found in parquet file!", #field_names_str);
278 return Err(::parquet::errors::ParquetError::General(error_msg));
279 }
280 };
281 if let Ok(column_reader) = row_group_reader.get_column_reader(idx) {
282 #reader_snippets
283 } else {
284 return Err(::parquet::errors::ParquetError::General("Failed to get next column".into()))
285 }
286 }
287 );*
288
289 Ok(())
290 }
291 }
292 }).into()
293}