Skip to main content

parquet_derive/
parquet_field.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use syn::ext::IdentExt;
6
7/// Parquet physical types used while selecting the code emitted by the macro.
8/// Keeping this local avoids compiling `parquet` for the proc-macro host.
9#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10enum PhysicalType {
11    Boolean,
12    Int32,
13    Int64,
14    Float,
15    Double,
16    ByteArray,
17    FixedLenByteArray,
18}
19
20#[derive(Debug, PartialEq)]
21pub struct Field {
22    ident: syn::Ident,
23    ty: Type,
24    is_a_byte_buf: bool,
25    third_party_type: Option<ThirdPartyType>,
26}
27
28/// Use third party libraries, detected
29/// at compile time. These libraries will
30/// be written to parquet as their preferred
31/// physical type.
32///
33///   ChronoNaiveDateTime is written as i64
34///   ChronoNaiveDate is written as i32
35#[derive(Debug, PartialEq)]
36enum ThirdPartyType {
37    ChronoNaiveDateTime,
38    ChronoNaiveDate,
39    Uuid,
40}
41
42impl Field {
43    pub fn from(f: &syn::Field) -> Self {
44        let ty = Type::from(f);
45        let is_a_byte_buf = ty.physical_type() == PhysicalType::ByteArray;
46
47        let third_party_type = match &ty.last_part()[..] {
48            "NaiveDateTime" => Some(ThirdPartyType::ChronoNaiveDateTime),
49            "NaiveDate" => Some(ThirdPartyType::ChronoNaiveDate),
50            "Uuid" => Some(ThirdPartyType::Uuid),
51            _ => None,
52        };
53
54        Field {
55            ident: f
56                .ident
57                .clone()
58                .expect("Only structs with named fields are currently supported"),
59            ty,
60            is_a_byte_buf,
61            third_party_type,
62        }
63    }
64
65    /// Takes the parsed field of the struct and emits a valid
66    /// column writer snippet. Should match exactly what you
67    /// would write by hand.
68    ///
69    /// Can only generate writers for basic structs, for example:
70    ///
71    /// struct Record {
72    ///   a_bool: bool,
73    ///   maybe_a_bool: `Option<bool>`
74    /// }
75    ///
76    /// but not
77    ///
78    /// struct UnsupportedNestedRecord {
79    ///   a_property: bool,
80    ///   nested_record: Record
81    /// }
82    ///
83    /// because this parsing logic is not sophisticated enough for definition
84    /// levels beyond 2.
85    pub fn writer_snippet(&self) -> proc_macro2::TokenStream {
86        let ident = &self.ident;
87        let column_writer = self.ty.column_writer();
88
89        let vals_builder = match &self.ty {
90            Type::TypePath(_) => self.copied_direct_vals(),
91            Type::Option(first_type) => match **first_type {
92                Type::TypePath(_) => self.option_into_vals(),
93                Type::Reference(_, ref second_type) => match **second_type {
94                    Type::TypePath(_) => self.option_into_vals(),
95                    _ => unimplemented!("Unsupported type encountered"),
96                },
97                Type::Vec(ref first_type) => match **first_type {
98                    Type::TypePath(_) => self.option_into_vals(),
99                    _ => unimplemented!("Unsupported type encountered"),
100                },
101                ref f => unimplemented!("Unsupported: {:#?}", f),
102            },
103            Type::Reference(_, first_type) => match **first_type {
104                Type::TypePath(_) => self.copied_direct_vals(),
105                Type::Option(ref second_type) => match **second_type {
106                    Type::TypePath(_) => self.option_into_vals(),
107                    Type::Reference(_, ref second_type) => match **second_type {
108                        Type::TypePath(_) => self.option_into_vals(),
109                        Type::Slice(ref second_type) => match **second_type {
110                            Type::TypePath(_) => self.option_into_vals(),
111                            ref f => unimplemented!("Unsupported: {:#?}", f),
112                        },
113                        _ => unimplemented!("Unsupported type encountered"),
114                    },
115                    Type::Vec(ref first_type) => match **first_type {
116                        Type::TypePath(_) => self.option_into_vals(),
117                        _ => unimplemented!("Unsupported type encountered"),
118                    },
119                    ref f => unimplemented!("Unsupported: {:#?}", f),
120                },
121                Type::Slice(ref second_type) => match **second_type {
122                    Type::TypePath(_) => self.copied_direct_vals(),
123                    ref f => unimplemented!("Unsupported: {:#?}", f),
124                },
125                ref f => unimplemented!("Unsupported: {:#?}", f),
126            },
127            Type::Vec(first_type) => match **first_type {
128                Type::TypePath(_) => self.copied_direct_vals(),
129                ref f => unimplemented!("Unsupported: {:#?}", f),
130            },
131            f => unimplemented!("Unsupported: {:#?}", f),
132        };
133
134        let definition_levels = match &self.ty {
135            Type::TypePath(_) => None,
136            Type::Option(first_type) => match **first_type {
137                Type::TypePath(_) => Some(self.optional_definition_levels()),
138                Type::Option(_) => unimplemented!("Unsupported nesting encountered"),
139                Type::Reference(_, ref second_type)
140                | Type::Vec(ref second_type)
141                | Type::Array(ref second_type, _)
142                | Type::Slice(ref second_type) => match **second_type {
143                    Type::TypePath(_) => Some(self.optional_definition_levels()),
144                    _ => unimplemented!("Unsupported nesting encountered"),
145                },
146            },
147            Type::Reference(_, first_type)
148            | Type::Vec(first_type)
149            | Type::Array(first_type, _)
150            | Type::Slice(first_type) => match **first_type {
151                Type::TypePath(_) => None,
152                Type::Vec(ref second_type)
153                | Type::Array(ref second_type, _)
154                | Type::Slice(ref second_type) => match **second_type {
155                    Type::TypePath(_) => None,
156                    Type::Reference(_, ref third_type) => match **third_type {
157                        Type::TypePath(_) => None,
158                        _ => unimplemented!("Unsupported definition encountered"),
159                    },
160                    _ => unimplemented!("Unsupported definition encountered"),
161                },
162                Type::Reference(_, ref second_type) | Type::Option(ref second_type) => {
163                    match **second_type {
164                        Type::TypePath(_) => Some(self.optional_definition_levels()),
165                        Type::Vec(ref third_type)
166                        | Type::Array(ref third_type, _)
167                        | Type::Slice(ref third_type) => match **third_type {
168                            Type::TypePath(_) => Some(self.optional_definition_levels()),
169                            Type::Reference(_, ref fourth_type) => match **fourth_type {
170                                Type::TypePath(_) => Some(self.optional_definition_levels()),
171                                _ => unimplemented!("Unsupported definition encountered"),
172                            },
173                            _ => unimplemented!("Unsupported definition encountered"),
174                        },
175                        Type::Reference(_, ref third_type) => match **third_type {
176                            Type::TypePath(_) => Some(self.optional_definition_levels()),
177                            Type::Slice(ref fourth_type) => match **fourth_type {
178                                Type::TypePath(_) => Some(self.optional_definition_levels()),
179                                _ => unimplemented!("Unsupported definition encountered"),
180                            },
181                            _ => unimplemented!("Unsupported definition encountered"),
182                        },
183                        Type::Option(_) => unimplemented!("Unsupported definition encountered"),
184                    }
185                }
186            },
187        };
188
189        // "vals" is the run of primitive data being written for the column
190        // "definition_levels" is a vector of bools which controls whether a value is missing or present
191        // this TokenStream is only one part of the code for writing a column and
192        // it relies on values calculated in prior code snippets, namely "definition_levels" and "vals_builder".
193        // All the context is put together in this functions final quote and
194        // this expression just switches between non-nullable and nullable write statements
195        let write_batch_expr = if definition_levels.is_some() {
196            quote! {
197                if let #column_writer(typed) = column_writer.untyped() {
198                    typed.write_batch(&vals[..], Some(&definition_levels[..]), None)?;
199                } else {
200                    panic!("Schema and struct disagree on type for {}", stringify!{#ident})
201                }
202            }
203        } else {
204            quote! {
205                if let #column_writer(typed) = column_writer.untyped() {
206                    typed.write_batch(&vals[..], None, None)?;
207                } else {
208                    panic!("Schema and struct disagree on type for {}", stringify!{#ident})
209                }
210            }
211        };
212
213        quote! {
214            {
215                #definition_levels
216
217                #vals_builder
218
219                #write_batch_expr
220            }
221        }
222    }
223
224    /// Takes the parsed field of the struct and emits a valid
225    /// column reader snippet. Should match exactly what you
226    /// would write by hand.
227    ///
228    /// Can only generate writers for basic structs, for example:
229    ///
230    /// struct Record {
231    ///   a_bool: bool
232    /// }
233    ///
234    /// but not
235    ///
236    /// struct UnsupportedNestedRecord {
237    ///   a_property: bool,
238    ///   nested_record: Record
239    /// }
240    ///
241    /// because this parsing logic is not sophisticated enough for definition
242    /// levels beyond 2.
243    ///
244    /// `Option` types and references not supported, but the column itself can be nullable
245    /// (i.e., def_level==1), as long as the values are all valid.
246    pub fn reader_snippet(&self) -> proc_macro2::TokenStream {
247        let ident = &self.ident;
248        let column_reader = self.ty.column_reader();
249
250        // generate the code to read the column into a vector `vals`
251        let write_batch_expr = quote! {
252            let mut vals = Vec::new();
253            if let #column_reader(mut typed) = column_reader {
254                let mut definition_levels = Vec::new();
255                let (total_num, valid_num, decoded_num) = typed.read_records(
256                    num_records, Some(&mut definition_levels), None, &mut vals)?;
257                if valid_num != decoded_num {
258                    panic!("Support only valid records, found {} null records in column type {}",
259                        decoded_num - valid_num, stringify!{#ident});
260                }
261            } else {
262                panic!("Schema and struct disagree on type for {}", stringify!{#ident});
263            }
264        };
265
266        // generate the code to convert each element of `vals` to the correct type and then write
267        // it to its field in the corresponding struct
268        let vals_writer = match &self.ty {
269            Type::TypePath(_) => self.copied_direct_fields(),
270            Type::Reference(_, first_type) => match **first_type {
271                Type::TypePath(_) => self.copied_direct_fields(),
272                Type::Slice(ref second_type) => match **second_type {
273                    Type::TypePath(_) => self.copied_direct_fields(),
274                    ref f => unimplemented!("Unsupported: {:#?}", f),
275                },
276                ref f => unimplemented!("Unsupported: {:#?}", f),
277            },
278            Type::Vec(first_type) => match **first_type {
279                Type::TypePath(_) => self.copied_direct_fields(),
280                ref f => unimplemented!("Unsupported: {:#?}", f),
281            },
282            f => unimplemented!("Unsupported: {:#?}", f),
283        };
284
285        quote! {
286            {
287                #write_batch_expr
288
289                #vals_writer
290            }
291        }
292    }
293
294    pub fn parquet_type(&self) -> proc_macro2::TokenStream {
295        // TODO: Support group types
296        // TODO: Add length if dealing with fixedlenbinary
297
298        // unraw the identifier, so a raw identifier like `r#type`
299        // becomes a column named `type` in the parquet schema
300        let field_name = self.ident.unraw().to_string();
301        let physical_type = match self.ty.physical_type() {
302            PhysicalType::Boolean => quote! {
303                ::parquet::basic::Type::BOOLEAN
304            },
305            PhysicalType::Int32 => quote! {
306                ::parquet::basic::Type::INT32
307            },
308            PhysicalType::Int64 => quote! {
309                ::parquet::basic::Type::INT64
310            },
311            PhysicalType::Float => quote! {
312                ::parquet::basic::Type::FLOAT
313            },
314            PhysicalType::Double => quote! {
315                ::parquet::basic::Type::DOUBLE
316            },
317            PhysicalType::ByteArray => quote! {
318                ::parquet::basic::Type::BYTE_ARRAY
319            },
320            PhysicalType::FixedLenByteArray => quote! {
321                ::parquet::basic::Type::FIXED_LEN_BYTE_ARRAY
322            },
323        };
324        let logical_type = self.ty.logical_type();
325        let repetition = self.ty.repetition();
326        let converted_type = self.ty.converted_type();
327        let length = self.ty.length();
328
329        let mut builder = quote! {
330            ParquetType::primitive_type_builder(#field_name, #physical_type)
331                .with_logical_type(#logical_type)
332                .with_repetition(#repetition)
333        };
334
335        if let Some(converted_type) = converted_type {
336            builder = quote! { #builder.with_converted_type(#converted_type) };
337        }
338
339        if let Some(length) = length {
340            builder = quote! { #builder.with_length(#length) };
341        }
342
343        quote! {  fields.push(#builder.build().unwrap().into()) }
344    }
345
346    fn option_into_vals(&self) -> proc_macro2::TokenStream {
347        let field_name = &self.ident;
348        let is_a_byte_buf = self.is_a_byte_buf;
349        let is_a_timestamp = self.third_party_type == Some(ThirdPartyType::ChronoNaiveDateTime);
350        let is_a_date = self.third_party_type == Some(ThirdPartyType::ChronoNaiveDate);
351        let is_a_uuid = self.third_party_type == Some(ThirdPartyType::Uuid);
352        let copy_to_vec = !matches!(
353            self.ty.physical_type(),
354            PhysicalType::ByteArray | PhysicalType::FixedLenByteArray
355        );
356
357        let binding = if copy_to_vec {
358            quote! { let Some(inner) = rec.#field_name }
359        } else {
360            quote! { let Some(inner) = &rec.#field_name }
361        };
362
363        let some = if is_a_timestamp {
364            quote! { Some(inner.timestamp_millis()) }
365        } else if is_a_date {
366            quote! { Some(inner.signed_duration_since(::chrono::NaiveDate::from_ymd(1970, 1, 1)).num_days() as i32)  }
367        } else if is_a_uuid {
368            quote! { Some((&inner.to_string()[..]).into()) }
369        } else if is_a_byte_buf {
370            quote! { Some((&inner[..]).into())}
371        } else {
372            // Type might need converting to a physical type
373            match self.ty.physical_type() {
374                PhysicalType::Int32 => quote! { Some(inner as i32) },
375                PhysicalType::Int64 => quote! { Some(inner as i64) },
376                _ => quote! { Some(inner) },
377            }
378        };
379
380        quote! {
381            let vals: Vec<_> = records.iter().filter_map(|rec| {
382                if #binding {
383                    #some
384                } else {
385                    None
386                }
387            }).collect();
388        }
389    }
390
391    // generates code to read `field_name` from each record into a vector `vals`
392    fn copied_direct_vals(&self) -> proc_macro2::TokenStream {
393        let field_name = &self.ident;
394
395        let access = match self.third_party_type {
396            Some(ThirdPartyType::ChronoNaiveDateTime) => {
397                quote! { rec.#field_name.timestamp_millis() }
398            }
399            Some(ThirdPartyType::ChronoNaiveDate) => {
400                quote! { rec.#field_name.signed_duration_since(::chrono::NaiveDate::from_ymd(1970, 1, 1)).num_days() as i32 }
401            }
402            Some(ThirdPartyType::Uuid) => {
403                quote! { rec.#field_name.as_bytes().to_vec().into() }
404            }
405            _ => {
406                if self.is_a_byte_buf {
407                    quote! { (&rec.#field_name[..]).into() }
408                } else {
409                    // Type might need converting to a physical type
410                    match self.ty.physical_type() {
411                        PhysicalType::Int32 => quote! { rec.#field_name as i32 },
412                        PhysicalType::Int64 => quote! { rec.#field_name as i64 },
413                        _ => quote! { rec.#field_name },
414                    }
415                }
416            }
417        };
418
419        quote! {
420            let vals: Vec<_> = records.iter().map(|rec| #access).collect();
421        }
422    }
423
424    // generates code to read a vector `records` into `field_name` for each record
425    fn copied_direct_fields(&self) -> proc_macro2::TokenStream {
426        let field_name = &self.ident;
427
428        let value = match self.third_party_type {
429            Some(ThirdPartyType::ChronoNaiveDateTime) => {
430                quote! { ::chrono::naive::NaiveDateTime::from_timestamp_millis(vals[i]).unwrap() }
431            }
432            Some(ThirdPartyType::ChronoNaiveDate) => {
433                // NaiveDateTime::UNIX_EPOCH.num_days_from_ce() == 719163
434                quote! {
435                    ::chrono::naive::NaiveDate::from_num_days_from_ce_opt(vals[i].saturating_add(719163)).unwrap()
436                }
437            }
438            Some(ThirdPartyType::Uuid) => {
439                quote! { ::uuid::Uuid::from_bytes(vals[i].data().try_into().unwrap()) }
440            }
441            _ => match &self.ty {
442                Type::TypePath(_) => match self.ty.last_part().as_str() {
443                    "String" => quote! { String::from(std::str::from_utf8(vals[i].data())
444                    .expect("invalid UTF-8 sequence")) },
445                    t => {
446                        let s: proc_macro2::TokenStream = t.parse().unwrap();
447                        quote! { vals[i] as #s }
448                    }
449                },
450                Type::Vec(_) => quote! { vals[i].data().to_vec() },
451                f => unimplemented!("Unsupported: {:#?}", f),
452            },
453        };
454
455        quote! {
456            for (i, r) in &mut records[..num_records].iter_mut().enumerate() {
457                r.#field_name = #value;
458            }
459        }
460    }
461
462    fn optional_definition_levels(&self) -> proc_macro2::TokenStream {
463        let field_name = &self.ident;
464
465        quote! {
466            let definition_levels: Vec<i16> = self
467              .iter()
468              .map(|rec| if rec.#field_name.is_some() { 1 } else { 0 })
469              .collect();
470        }
471    }
472}
473
474#[expect(clippy::enum_variant_names)]
475#[derive(Debug, PartialEq)]
476enum Type {
477    Array(Box<Type>, syn::Expr),
478    Option(Box<Type>),
479    Slice(Box<Type>),
480    Vec(Box<Type>),
481    TypePath(syn::Type),
482    Reference(Option<syn::Lifetime>, Box<Type>),
483}
484
485impl Type {
486    /// Takes a rust type and returns the appropriate
487    /// parquet-rs column writer
488    fn column_writer(&self) -> syn::TypePath {
489        match self.physical_type() {
490            PhysicalType::Boolean => {
491                syn::parse_quote!(ColumnWriter::BoolColumnWriter)
492            }
493            PhysicalType::Int32 => syn::parse_quote!(ColumnWriter::Int32ColumnWriter),
494            PhysicalType::Int64 => syn::parse_quote!(ColumnWriter::Int64ColumnWriter),
495            PhysicalType::Float => syn::parse_quote!(ColumnWriter::FloatColumnWriter),
496            PhysicalType::Double => syn::parse_quote!(ColumnWriter::DoubleColumnWriter),
497            PhysicalType::ByteArray => {
498                syn::parse_quote!(ColumnWriter::ByteArrayColumnWriter)
499            }
500            PhysicalType::FixedLenByteArray => {
501                syn::parse_quote!(ColumnWriter::FixedLenByteArrayColumnWriter)
502            }
503        }
504    }
505
506    /// Takes a rust type and returns the appropriate
507    /// parquet-rs column reader
508    fn column_reader(&self) -> syn::TypePath {
509        match self.physical_type() {
510            PhysicalType::Boolean => {
511                syn::parse_quote!(ColumnReader::BoolColumnReader)
512            }
513            PhysicalType::Int32 => syn::parse_quote!(ColumnReader::Int32ColumnReader),
514            PhysicalType::Int64 => syn::parse_quote!(ColumnReader::Int64ColumnReader),
515            PhysicalType::Float => syn::parse_quote!(ColumnReader::FloatColumnReader),
516            PhysicalType::Double => syn::parse_quote!(ColumnReader::DoubleColumnReader),
517            PhysicalType::ByteArray => {
518                syn::parse_quote!(ColumnReader::ByteArrayColumnReader)
519            }
520            PhysicalType::FixedLenByteArray => {
521                syn::parse_quote!(ColumnReader::FixedLenByteArrayColumnReader)
522            }
523        }
524    }
525
526    /// Helper to simplify a nested field definition to its leaf type
527    ///
528    /// Ex:
529    ///   `Option<&String>` => Type::TypePath(String)
530    ///   `&Option<i32>` => Type::TypePath(i32)
531    ///   `Vec<Vec<u8>>` => Type::Vec(u8)
532    ///
533    /// Useful in determining the physical type of a field and the
534    /// definition levels.
535    fn leaf_type_recursive(&self) -> &Type {
536        Type::leaf_type_recursive_helper(self, None)
537    }
538
539    fn leaf_type_recursive_helper<'a>(ty: &'a Type, parent_ty: Option<&'a Type>) -> &'a Type {
540        match ty {
541            Type::TypePath(_) => parent_ty.unwrap_or(ty),
542            Type::Option(first_type)
543            | Type::Vec(first_type)
544            | Type::Array(first_type, _)
545            | Type::Slice(first_type)
546            | Type::Reference(_, first_type) => {
547                Type::leaf_type_recursive_helper(first_type, Some(ty))
548            }
549        }
550    }
551
552    /// Helper method to further unwrap leaf_type() to get inner-most
553    /// type information, useful for determining the physical type
554    /// and normalizing the type paths.
555    fn inner_type(&self) -> &syn::Type {
556        let leaf_type = self.leaf_type_recursive();
557
558        match leaf_type {
559            Type::TypePath(type_) => type_,
560            Type::Option(first_type)
561            | Type::Vec(first_type)
562            | Type::Array(first_type, _)
563            | Type::Slice(first_type)
564            | Type::Reference(_, first_type) => match **first_type {
565                Type::TypePath(ref type_) => type_,
566                _ => unimplemented!("leaf_type() should only return shallow types"),
567            },
568        }
569    }
570
571    /// Helper to normalize a type path by extracting the
572    /// most identifiable part
573    ///
574    /// Ex:
575    ///   std::string::String => String
576    ///   `Vec<u8>` => `Vec<u8>`
577    ///   chrono::NaiveDateTime => NaiveDateTime
578    ///
579    /// Does run the risk of mis-identifying a type if import
580    /// rename is in play. Please note procedural macros always
581    /// run before type resolution so this is a risk the user
582    /// takes on when renaming imports.
583    fn last_part(&self) -> String {
584        let inner_type = self.inner_type();
585        let inner_type_str = (quote! { #inner_type }).to_string();
586
587        inner_type_str
588            .split("::")
589            .last()
590            .unwrap()
591            .trim()
592            .to_string()
593    }
594
595    /// Converts rust types to parquet physical types.
596    ///
597    /// Ex:
598    ///   [u8; 10] => FIXED_LEN_BYTE_ARRAY
599    ///   `Vec<u8>`  => BYTE_ARRAY
600    ///   String => BYTE_ARRAY
601    ///   i32 => INT32
602    fn physical_type(&self) -> PhysicalType {
603        let last_part = self.last_part();
604        let leaf_type = self.leaf_type_recursive();
605
606        match leaf_type {
607            Type::Array(first_type, _length) => {
608                if let Type::TypePath(_) = **first_type
609                    && last_part == "u8"
610                {
611                    return PhysicalType::FixedLenByteArray;
612                }
613            }
614            Type::Vec(first_type) | Type::Slice(first_type) => {
615                if let Type::TypePath(_) = **first_type
616                    && last_part == "u8"
617                {
618                    return PhysicalType::ByteArray;
619                }
620            }
621            _ => (),
622        }
623
624        match last_part.trim() {
625            "bool" => PhysicalType::Boolean,
626            "u8" | "u16" | "u32" => PhysicalType::Int32,
627            "i8" | "i16" | "i32" | "NaiveDate" => PhysicalType::Int32,
628            "u64" | "i64" | "NaiveDateTime" => PhysicalType::Int64,
629            "usize" | "isize" => {
630                if usize::BITS == 64 {
631                    PhysicalType::Int64
632                } else {
633                    PhysicalType::Int32
634                }
635            }
636            "f32" => PhysicalType::Float,
637            "f64" => PhysicalType::Double,
638            "String" | "str" | "Arc < str >" => PhysicalType::ByteArray,
639            "Uuid" => PhysicalType::FixedLenByteArray,
640            f => unimplemented!("{} currently is not supported", f),
641        }
642    }
643
644    fn length(&self) -> Option<syn::Expr> {
645        let last_part = self.last_part();
646        let leaf_type = self.leaf_type_recursive();
647
648        // `[u8; N]` => Some(N)
649        if let Type::Array(first_type, length) = leaf_type
650            && let Type::TypePath(_) = **first_type
651            && last_part == "u8"
652        {
653            return Some(length.clone());
654        }
655
656        match last_part.trim() {
657            // Uuid => [u8; 16] => Some(16)
658            "Uuid" => Some(syn::parse_quote!(16)),
659            _ => None,
660        }
661    }
662
663    fn logical_type(&self) -> proc_macro2::TokenStream {
664        let last_part = self.last_part();
665        let leaf_type = self.leaf_type_recursive();
666
667        match leaf_type {
668            Type::Array(first_type, _length) => {
669                if let Type::TypePath(_) = **first_type
670                    && last_part == "u8"
671                {
672                    return quote! { None };
673                }
674            }
675            Type::Vec(first_type) | Type::Slice(first_type) => {
676                if let Type::TypePath(_) = **first_type
677                    && last_part == "u8"
678                {
679                    return quote! { None };
680                }
681            }
682            _ => (),
683        }
684
685        match last_part.trim() {
686            "bool" => quote! { None },
687            "u8" => quote! { Some(LogicalType::integer(8, false)) },
688            "u16" => quote! { Some(LogicalType::integer(16, false)) },
689            "u32" => quote! { Some(LogicalType::integer(32, false)) },
690            "u64" => quote! { Some(LogicalType::integer(64, false)) },
691            "i8" => quote! { Some(LogicalType::integer(8, true)) },
692            "i16" => quote! { Some(LogicalType::integer(16, true)) },
693            "i32" | "i64" => quote! { None },
694            "usize" => {
695                quote! { Some(LogicalType::integer(usize::BITS as i8, false)) }
696            }
697            "isize" => {
698                quote! { Some(LogicalType::integer(usize::BITS as i8, true)) }
699            }
700            "NaiveDate" => quote! { Some(LogicalType::Date) },
701            "NaiveDateTime" => quote! { None },
702            "f32" | "f64" => quote! { None },
703            "String" | "str" | "Arc < str >" => quote! { Some(LogicalType::String) },
704            "Uuid" => quote! { Some(LogicalType::Uuid) },
705            f => unimplemented!("{} currently is not supported", f),
706        }
707    }
708
709    fn converted_type(&self) -> Option<proc_macro2::TokenStream> {
710        let last_part = self.last_part();
711
712        match last_part.trim() {
713            "NaiveDateTime" => Some(quote! { ::parquet::basic::ConvertedType::TIMESTAMP_MILLIS }),
714            _ => None,
715        }
716    }
717
718    fn repetition(&self) -> proc_macro2::TokenStream {
719        match self {
720            Type::Option(_) => quote! { ::parquet::basic::Repetition::OPTIONAL },
721            Type::Reference(_, ty) => ty.repetition(),
722            _ => quote! { ::parquet::basic::Repetition::REQUIRED },
723        }
724    }
725
726    /// Convert a parsed rust field AST in to a more easy to manipulate
727    /// parquet_derive::Field
728    fn from(f: &syn::Field) -> Self {
729        Type::from_type(f, &f.ty)
730    }
731
732    fn from_type(f: &syn::Field, ty: &syn::Type) -> Self {
733        match ty {
734            syn::Type::Path(p) => Type::from_type_path(f, p),
735            syn::Type::Reference(tr) => Type::from_type_reference(f, tr),
736            syn::Type::Array(ta) => Type::from_type_array(f, ta),
737            syn::Type::Slice(ts) => Type::from_type_slice(f, ts),
738            other => unimplemented!(
739                "Unable to derive {:?} - it is currently an unsupported type\n{:#?}",
740                f.ident.as_ref().unwrap(),
741                other
742            ),
743        }
744    }
745
746    fn from_type_path(f: &syn::Field, p: &syn::TypePath) -> Self {
747        let last_segment = p.path.segments.last().unwrap();
748
749        let is_vec = last_segment.ident == syn::Ident::new("Vec", proc_macro2::Span::call_site());
750        let is_option =
751            last_segment.ident == syn::Ident::new("Option", proc_macro2::Span::call_site());
752
753        if is_vec || is_option {
754            let generic_type = match &last_segment.arguments {
755                syn::PathArguments::AngleBracketed(angle_args) => {
756                    assert_eq!(angle_args.args.len(), 1);
757                    let first_arg = &angle_args.args[0];
758
759                    match first_arg {
760                        syn::GenericArgument::Type(typath) => typath.clone(),
761                        other => unimplemented!("Unsupported: {:#?}", other),
762                    }
763                }
764                other => unimplemented!("Unsupported: {:#?}", other),
765            };
766
767            if is_vec {
768                Type::Vec(Box::new(Type::from_type(f, &generic_type)))
769            } else {
770                Type::Option(Box::new(Type::from_type(f, &generic_type)))
771            }
772        } else {
773            Type::TypePath(syn::Type::Path(p.clone()))
774        }
775    }
776
777    fn from_type_reference(f: &syn::Field, tr: &syn::TypeReference) -> Self {
778        let lifetime = tr.lifetime.clone();
779        let inner_type = Type::from_type(f, tr.elem.as_ref());
780        Type::Reference(lifetime, Box::new(inner_type))
781    }
782
783    fn from_type_array(f: &syn::Field, ta: &syn::TypeArray) -> Self {
784        let inner_type = Type::from_type(f, ta.elem.as_ref());
785        Type::Array(Box::new(inner_type), ta.len.clone())
786    }
787
788    fn from_type_slice(f: &syn::Field, ts: &syn::TypeSlice) -> Self {
789        let inner_type = Type::from_type(f, ts.elem.as_ref());
790        Type::Slice(Box::new(inner_type))
791    }
792}
793
794#[cfg(test)]
795mod test {
796    use super::*;
797    use syn::{Data, DataStruct, DeriveInput};
798
799    fn extract_fields(input: proc_macro2::TokenStream) -> Vec<syn::Field> {
800        let input: DeriveInput = syn::parse2(input).unwrap();
801
802        let Data::Struct(DataStruct { fields, .. }) = input.data else {
803            panic!("Input must be a struct")
804        };
805
806        fields.iter().map(|field| field.to_owned()).collect()
807    }
808
809    #[test]
810    fn test_generating_a_simple_writer_snippet() {
811        let snippet: proc_macro2::TokenStream = quote! {
812          struct ABoringStruct {
813            counter: usize,
814          }
815        };
816
817        let fields = extract_fields(snippet);
818        let counter = Field::from(&fields[0]);
819
820        let snippet = counter.writer_snippet().to_string();
821        assert_eq!(snippet,
822                   (quote!{
823                        {
824                            let vals : Vec < _ > = records . iter ( ) . map ( | rec | rec . counter as i64 ) . collect ( );
825
826                            if let ColumnWriter::Int64ColumnWriter ( typed ) = column_writer.untyped() {
827                                typed . write_batch ( & vals [ .. ] , None , None ) ?;
828                            }  else {
829                                panic!("Schema and struct disagree on type for {}" , stringify!{ counter } )
830                            }
831                        }
832                   }).to_string()
833        )
834    }
835
836    #[test]
837    fn test_generating_a_simple_reader_snippet() {
838        let snippet: proc_macro2::TokenStream = quote! {
839          struct ABoringStruct {
840            counter: usize,
841          }
842        };
843
844        let fields = extract_fields(snippet);
845        let counter = Field::from(&fields[0]);
846
847        let snippet = counter.reader_snippet().to_string();
848        assert_eq!(
849            snippet,
850            (quote! {
851                 {
852                    let mut vals = Vec::new();
853                    if let ColumnReader::Int64ColumnReader(mut typed) = column_reader {
854                        let mut definition_levels = Vec::new();
855                        let (total_num, valid_num, decoded_num) = typed.read_records(
856                            num_records, Some(&mut definition_levels), None, &mut vals)?;
857                        if valid_num != decoded_num {
858                            panic!("Support only valid records, found {} null records in column type {}",
859                                decoded_num - valid_num, stringify!{counter});
860                        }
861                    } else {
862                        panic!("Schema and struct disagree on type for {}", stringify!{counter});
863                    }
864                    for (i, r) in &mut records[..num_records].iter_mut().enumerate() {
865                        r.counter = vals[i] as usize;
866                    }
867                 }
868            })
869            .to_string()
870        )
871    }
872
873    #[test]
874    fn test_parquet_type_with_raw_identifier() {
875        let snippet: proc_macro2::TokenStream = quote! {
876          struct ABoringStruct {
877            r#type: i32,
878          }
879        };
880
881        let fields = extract_fields(snippet);
882        let r#type = Field::from(&fields[0]);
883
884        // the raw identifier `r#type` is named `type` in the parquet schema
885        let snippet = r#type.parquet_type().to_string();
886        assert!(
887            snippet.contains("primitive_type_builder (\"type\""),
888            "{snippet}"
889        );
890    }
891
892    #[test]
893    fn test_optional_to_writer_snippet() {
894        let struct_def: proc_macro2::TokenStream = quote! {
895          struct StringBorrower<'a> {
896            optional_str: Option<&'a str>,
897            optional_string: Option<&String>,
898            optional_dumb_int: Option<&i32>,
899          }
900        };
901
902        let fields = extract_fields(struct_def);
903
904        let optional = Field::from(&fields[0]);
905        let snippet = optional.writer_snippet();
906        assert_eq!(snippet.to_string(),
907          (quote! {
908          {
909                let definition_levels : Vec < i16 > = self . iter ( ) . map ( | rec | if rec . optional_str . is_some ( ) { 1 } else { 0 } ) . collect ( ) ;
910
911                let vals: Vec <_> = records.iter().filter_map( |rec| {
912                    if let Some ( inner ) = &rec . optional_str {
913                        Some ( (&inner[..]).into() )
914                    } else {
915                        None
916                    }
917                }).collect();
918
919                if let ColumnWriter::ByteArrayColumnWriter ( typed ) = column_writer.untyped() {
920                    typed . write_batch ( & vals [ .. ] , Some(&definition_levels[..]) , None ) ? ;
921                } else {
922                    panic!("Schema and struct disagree on type for {}" , stringify ! { optional_str } )
923                }
924           }
925            }
926          ).to_string());
927
928        let optional = Field::from(&fields[1]);
929        let snippet = optional.writer_snippet();
930        assert_eq!(snippet.to_string(),
931                   (quote!{
932                   {
933                        let definition_levels : Vec < i16 > = self . iter ( ) . map ( | rec | if rec . optional_string . is_some ( ) { 1 } else { 0 } ) . collect ( ) ;
934
935                        let vals: Vec <_> = records.iter().filter_map( |rec| {
936                            if let Some ( inner ) = &rec . optional_string {
937                                Some ( (&inner[..]).into() )
938                            } else {
939                                None
940                            }
941                        }).collect();
942
943                        if let ColumnWriter::ByteArrayColumnWriter ( typed ) = column_writer.untyped() {
944                            typed . write_batch ( & vals [ .. ] , Some(&definition_levels[..]) , None ) ? ;
945                        } else {
946                            panic!("Schema and struct disagree on type for {}" , stringify ! { optional_string } )
947                        }
948                    }
949        }).to_string());
950
951        let optional = Field::from(&fields[2]);
952        let snippet = optional.writer_snippet();
953        assert_eq!(snippet.to_string(),
954                   (quote!{
955                    {
956                        let definition_levels : Vec < i16 > = self . iter ( ) . map ( | rec | if rec . optional_dumb_int . is_some ( ) { 1 } else { 0 } ) . collect ( ) ;
957
958                        let vals: Vec <_> = records.iter().filter_map( |rec| {
959                            if let Some ( inner ) = rec . optional_dumb_int {
960                                Some ( inner as i32 )
961                            } else {
962                                None
963                            }
964                        }).collect();
965
966                        if let ColumnWriter::Int32ColumnWriter ( typed ) = column_writer.untyped() {
967                            typed . write_batch ( & vals [ .. ] , Some(&definition_levels[..]) , None ) ? ;
968                        }  else {
969                            panic!("Schema and struct disagree on type for {}" , stringify ! { optional_dumb_int } )
970                        }
971                    }
972        }).to_string());
973    }
974
975    #[test]
976    fn test_converting_to_column_writer_type() {
977        let snippet: proc_macro2::TokenStream = quote! {
978          struct ABasicStruct {
979            yes_no: bool,
980            name: String,
981          }
982        };
983
984        let fields = extract_fields(snippet);
985        let processed: Vec<_> = fields.iter().map(Field::from).collect();
986
987        let column_writers: Vec<_> = processed
988            .iter()
989            .map(|field| field.ty.column_writer())
990            .collect();
991
992        assert_eq!(
993            column_writers,
994            vec![
995                syn::parse_quote!(ColumnWriter::BoolColumnWriter),
996                syn::parse_quote!(ColumnWriter::ByteArrayColumnWriter)
997            ]
998        );
999    }
1000
1001    #[test]
1002    fn test_converting_to_column_reader_type() {
1003        let snippet: proc_macro2::TokenStream = quote! {
1004          struct ABasicStruct {
1005            yes_no: bool,
1006            name: String,
1007          }
1008        };
1009
1010        let fields = extract_fields(snippet);
1011        let processed: Vec<_> = fields.iter().map(Field::from).collect();
1012
1013        let column_readers: Vec<_> = processed
1014            .iter()
1015            .map(|field| field.ty.column_reader())
1016            .collect();
1017
1018        assert_eq!(
1019            column_readers,
1020            vec![
1021                syn::parse_quote!(ColumnReader::BoolColumnReader),
1022                syn::parse_quote!(ColumnReader::ByteArrayColumnReader)
1023            ]
1024        );
1025    }
1026
1027    #[test]
1028    fn convert_basic_struct() {
1029        let snippet: proc_macro2::TokenStream = quote! {
1030          struct ABasicStruct {
1031            yes_no: bool,
1032            name: String,
1033            length: usize
1034          }
1035        };
1036
1037        let fields = extract_fields(snippet);
1038        let processed: Vec<_> = fields.iter().map(Field::from).collect();
1039        assert_eq!(processed.len(), 3);
1040
1041        assert_eq!(
1042            processed,
1043            vec![
1044                Field {
1045                    ident: syn::Ident::new("yes_no", proc_macro2::Span::call_site()),
1046                    ty: Type::TypePath(syn::parse_quote!(bool)),
1047                    is_a_byte_buf: false,
1048                    third_party_type: None,
1049                },
1050                Field {
1051                    ident: syn::Ident::new("name", proc_macro2::Span::call_site()),
1052                    ty: Type::TypePath(syn::parse_quote!(String)),
1053                    is_a_byte_buf: true,
1054                    third_party_type: None,
1055                },
1056                Field {
1057                    ident: syn::Ident::new("length", proc_macro2::Span::call_site()),
1058                    ty: Type::TypePath(syn::parse_quote!(usize)),
1059                    is_a_byte_buf: false,
1060                    third_party_type: None,
1061                }
1062            ]
1063        )
1064    }
1065
1066    #[test]
1067    fn test_get_inner_type() {
1068        let snippet: proc_macro2::TokenStream = quote! {
1069          struct LotsOfInnerTypes {
1070            a_vec: Vec<u8>,
1071            a_option: ::std::option::Option<bool>,
1072            a_silly_string: ::std::string::String,
1073            a_complicated_thing: ::std::option::Option<::std::result::Result<(),()>>,
1074          }
1075        };
1076
1077        let fields = extract_fields(snippet);
1078        let converted_fields: Vec<_> = fields.iter().map(Type::from).collect();
1079        let inner_types: Vec<_> = converted_fields
1080            .iter()
1081            .map(|field| field.inner_type())
1082            .collect();
1083        let inner_types_strs: Vec<_> = inner_types
1084            .iter()
1085            .map(|ty| (quote! { #ty }).to_string())
1086            .collect();
1087
1088        assert_eq!(
1089            inner_types_strs,
1090            vec![
1091                "u8",
1092                "bool",
1093                ":: std :: string :: String",
1094                ":: std :: result :: Result < () , () >"
1095            ]
1096        )
1097    }
1098
1099    #[test]
1100    fn test_physical_type() {
1101        let snippet: proc_macro2::TokenStream = quote! {
1102          struct LotsOfInnerTypes {
1103            a_buf: ::std::vec::Vec<u8>,
1104            a_number: i32,
1105            a_verbose_option: ::std::option::Option<bool>,
1106            a_silly_string: String,
1107            a_fix_byte_buf: [u8; 10],
1108            a_complex_option: ::std::option::Option<&Vec<u8>>,
1109            a_complex_vec: &::std::vec::Vec<&Option<u8>>,
1110            a_uuid: ::uuid::Uuid,
1111          }
1112        };
1113
1114        let fields = extract_fields(snippet);
1115        let converted_fields: Vec<_> = fields.iter().map(Type::from).collect();
1116        let physical_types: Vec<_> = converted_fields
1117            .iter()
1118            .map(|ty| ty.physical_type())
1119            .collect();
1120
1121        assert_eq!(
1122            physical_types,
1123            vec![
1124                PhysicalType::ByteArray,
1125                PhysicalType::Int32,
1126                PhysicalType::Boolean,
1127                PhysicalType::ByteArray,
1128                PhysicalType::FixedLenByteArray,
1129                PhysicalType::ByteArray,
1130                PhysicalType::Int32,
1131                PhysicalType::FixedLenByteArray,
1132            ]
1133        )
1134    }
1135
1136    #[test]
1137    fn test_type_length() {
1138        let snippet: proc_macro2::TokenStream = quote! {
1139          struct LotsOfInnerTypes {
1140            a_buf: ::std::vec::Vec<u8>,
1141            a_number: i32,
1142            a_verbose_option: ::std::option::Option<bool>,
1143            a_silly_string: String,
1144            a_fix_byte_buf: [u8; 10],
1145            a_complex_option: ::std::option::Option<&Vec<u8>>,
1146            a_complex_vec: &::std::vec::Vec<&Option<u8>>,
1147            a_uuid: ::uuid::Uuid,
1148          }
1149        };
1150
1151        let fields = extract_fields(snippet);
1152        let converted_fields: Vec<_> = fields.iter().map(Type::from).collect();
1153        let lengths: Vec<_> = converted_fields.iter().map(|ty| ty.length()).collect();
1154
1155        assert_eq!(
1156            lengths,
1157            vec![
1158                None,
1159                None,
1160                None,
1161                None,
1162                Some(syn::parse_quote!(10)),
1163                None,
1164                None,
1165                Some(syn::parse_quote!(16)),
1166            ]
1167        )
1168    }
1169
1170    #[test]
1171    fn test_convert_comprehensive_owned_struct() {
1172        let snippet: proc_macro2::TokenStream = quote! {
1173          struct VecHolder {
1174            a_vec: ::std::vec::Vec<u8>,
1175            a_option: ::std::option::Option<bool>,
1176            a_silly_string: ::std::string::String,
1177            a_complicated_thing: ::std::option::Option<::std::result::Result<(),()>>,
1178          }
1179        };
1180
1181        let fields = extract_fields(snippet);
1182        let converted_fields: Vec<_> = fields.iter().map(Type::from).collect();
1183
1184        assert_eq!(
1185            converted_fields,
1186            vec![
1187                Type::Vec(Box::new(Type::TypePath(syn::parse_quote!(u8)))),
1188                Type::Option(Box::new(Type::TypePath(syn::parse_quote!(bool)))),
1189                Type::TypePath(syn::parse_quote!(::std::string::String)),
1190                Type::Option(Box::new(Type::TypePath(
1191                    syn::parse_quote!(::std::result::Result<(),()>)
1192                ))),
1193            ]
1194        );
1195    }
1196
1197    #[test]
1198    fn test_convert_borrowed_struct() {
1199        let snippet: proc_macro2::TokenStream = quote! {
1200          struct Borrower<'a> {
1201            a_str: &'a str,
1202            a_borrowed_option: &'a Option<bool>,
1203            so_many_borrows: &'a Option<&'a str>,
1204          }
1205        };
1206
1207        let fields = extract_fields(snippet);
1208        let types: Vec<_> = fields.iter().map(Type::from).collect();
1209
1210        assert_eq!(
1211            types,
1212            vec![
1213                Type::Reference(
1214                    Some(syn::Lifetime::new("'a", proc_macro2::Span::call_site())),
1215                    Box::new(Type::TypePath(syn::parse_quote!(str)))
1216                ),
1217                Type::Reference(
1218                    Some(syn::Lifetime::new("'a", proc_macro2::Span::call_site())),
1219                    Box::new(Type::Option(Box::new(Type::TypePath(syn::parse_quote!(
1220                        bool
1221                    )))))
1222                ),
1223                Type::Reference(
1224                    Some(syn::Lifetime::new("'a", proc_macro2::Span::call_site())),
1225                    Box::new(Type::Option(Box::new(Type::Reference(
1226                        Some(syn::Lifetime::new("'a", proc_macro2::Span::call_site())),
1227                        Box::new(Type::TypePath(syn::parse_quote!(str)))
1228                    ))))
1229                ),
1230            ]
1231        );
1232    }
1233
1234    #[test]
1235    fn test_chrono_timestamp_millis_write() {
1236        let snippet: proc_macro2::TokenStream = quote! {
1237          struct ATimestampStruct {
1238            henceforth: chrono::NaiveDateTime,
1239            maybe_happened: Option<&chrono::NaiveDateTime>,
1240          }
1241        };
1242
1243        let fields = extract_fields(snippet);
1244        let when = Field::from(&fields[0]);
1245        assert_eq!(when.writer_snippet().to_string(),(quote!{
1246            {
1247                let vals : Vec<_> = records.iter().map(|rec| rec.henceforth.timestamp_millis() ).collect();
1248                if let ColumnWriter::Int64ColumnWriter(typed) = column_writer.untyped() {
1249                    typed.write_batch(&vals[..], None, None) ?;
1250                } else {
1251                    panic!("Schema and struct disagree on type for {}" , stringify!{ henceforth })
1252                }
1253            }
1254        }).to_string());
1255
1256        let maybe_happened = Field::from(&fields[1]);
1257        assert_eq!(maybe_happened.writer_snippet().to_string(),(quote!{
1258            {
1259                let definition_levels : Vec<i16> = self.iter().map(|rec| if rec.maybe_happened.is_some() { 1 } else { 0 }).collect();
1260                let vals : Vec<_> = records.iter().filter_map(|rec| {
1261                    if let Some(inner) = rec.maybe_happened {
1262                        Some(inner.timestamp_millis())
1263                    } else {
1264                        None
1265                    }
1266                }).collect();
1267
1268                if let ColumnWriter::Int64ColumnWriter(typed) = column_writer.untyped() {
1269                    typed.write_batch(&vals[..], Some(&definition_levels[..]), None) ?;
1270                } else {
1271                    panic!("Schema and struct disagree on type for {}" , stringify!{ maybe_happened })
1272                }
1273            }
1274        }).to_string());
1275    }
1276
1277    #[test]
1278    fn test_chrono_timestamp_millis_read() {
1279        let snippet: proc_macro2::TokenStream = quote! {
1280          struct ATimestampStruct {
1281            henceforth: chrono::NaiveDateTime,
1282          }
1283        };
1284
1285        let fields = extract_fields(snippet);
1286        let when = Field::from(&fields[0]);
1287        assert_eq!(when.reader_snippet().to_string(),(quote!{
1288            {
1289                let mut vals = Vec::new();
1290                if let ColumnReader::Int64ColumnReader(mut typed) = column_reader {
1291                    let mut definition_levels = Vec::new();
1292                    let (total_num, valid_num, decoded_num) = typed.read_records(
1293                        num_records, Some(&mut definition_levels), None, &mut vals)?;
1294                    if valid_num != decoded_num {
1295                        panic!("Support only valid records, found {} null records in column type {}",
1296                            decoded_num - valid_num, stringify!{henceforth});
1297                    }
1298                } else {
1299                    panic!("Schema and struct disagree on type for {}", stringify!{ henceforth });
1300                }
1301                for (i, r) in &mut records[..num_records].iter_mut().enumerate() {
1302                    r.henceforth = ::chrono::naive::NaiveDateTime::from_timestamp_millis(vals[i]).unwrap();
1303                }
1304            }
1305        }).to_string());
1306    }
1307
1308    #[test]
1309    fn test_chrono_date_write() {
1310        let snippet: proc_macro2::TokenStream = quote! {
1311          struct ATimestampStruct {
1312            henceforth: chrono::NaiveDate,
1313            maybe_happened: Option<&chrono::NaiveDate>,
1314          }
1315        };
1316
1317        let fields = extract_fields(snippet);
1318        let when = Field::from(&fields[0]);
1319        assert_eq!(when.writer_snippet().to_string(),(quote!{
1320            {
1321                let vals : Vec<_> = records.iter().map(|rec| rec.henceforth.signed_duration_since(::chrono::NaiveDate::from_ymd(1970, 1, 1)).num_days() as i32).collect();
1322                if let ColumnWriter::Int32ColumnWriter(typed) = column_writer.untyped() {
1323                    typed.write_batch(&vals[..], None, None) ?;
1324                } else {
1325                    panic!("Schema and struct disagree on type for {}" , stringify!{ henceforth })
1326                }
1327            }
1328        }).to_string());
1329
1330        let maybe_happened = Field::from(&fields[1]);
1331        assert_eq!(maybe_happened.writer_snippet().to_string(),(quote!{
1332            {
1333                let definition_levels : Vec<i16> = self.iter().map(|rec| if rec.maybe_happened.is_some() { 1 } else { 0 }).collect();
1334                let vals : Vec<_> = records.iter().filter_map(|rec| {
1335                    if let Some(inner) = rec.maybe_happened {
1336                        Some(inner.signed_duration_since(::chrono::NaiveDate::from_ymd(1970, 1, 1)).num_days() as i32)
1337                    } else {
1338                        None
1339                    }
1340                }).collect();
1341
1342                if let ColumnWriter::Int32ColumnWriter(typed) = column_writer.untyped() {
1343                    typed.write_batch(&vals[..], Some(&definition_levels[..]), None) ?;
1344                } else {
1345                    panic!("Schema and struct disagree on type for {}" , stringify!{ maybe_happened })
1346                }
1347            }
1348        }).to_string());
1349    }
1350
1351    #[test]
1352    fn test_chrono_date_read() {
1353        let snippet: proc_macro2::TokenStream = quote! {
1354          struct ATimestampStruct {
1355            henceforth: chrono::NaiveDate,
1356          }
1357        };
1358
1359        let fields = extract_fields(snippet);
1360        let when = Field::from(&fields[0]);
1361        assert_eq!(when.reader_snippet().to_string(),(quote!{
1362            {
1363                let mut vals = Vec::new();
1364                if let ColumnReader::Int32ColumnReader(mut typed) = column_reader {
1365                    let mut definition_levels = Vec::new();
1366                    let (total_num, valid_num, decoded_num) = typed.read_records(
1367                        num_records, Some(&mut definition_levels), None, &mut vals)?;
1368                    if valid_num != decoded_num {
1369                        panic!("Support only valid records, found {} null records in column type {}",
1370                            decoded_num - valid_num, stringify!{henceforth});
1371                    }
1372                } else {
1373                    panic!("Schema and struct disagree on type for {}", stringify!{ henceforth });
1374                }
1375                for (i, r) in &mut records[..num_records].iter_mut().enumerate() {
1376                    r.henceforth = ::chrono::naive::NaiveDate::from_num_days_from_ce_opt(vals[i].saturating_add(719163)).unwrap();
1377                }
1378            }
1379        }).to_string());
1380    }
1381
1382    #[test]
1383    fn test_uuid_write() {
1384        let snippet: proc_macro2::TokenStream = quote! {
1385          struct AUuidStruct {
1386            unique_id: uuid::Uuid,
1387            maybe_unique_id: Option<&uuid::Uuid>,
1388          }
1389        };
1390
1391        let fields = extract_fields(snippet);
1392        let when = Field::from(&fields[0]);
1393        assert_eq!(when.writer_snippet().to_string(),(quote!{
1394            {
1395                let vals : Vec<_> = records.iter().map(|rec| rec.unique_id.as_bytes().to_vec().into() ).collect();
1396                if let ColumnWriter::FixedLenByteArrayColumnWriter(typed) = column_writer.untyped() {
1397                    typed.write_batch(&vals[..], None, None) ?;
1398                } else {
1399                    panic!("Schema and struct disagree on type for {}" , stringify!{ unique_id })
1400                }
1401            }
1402        }).to_string());
1403
1404        let maybe_happened = Field::from(&fields[1]);
1405        assert_eq!(maybe_happened.writer_snippet().to_string(),(quote!{
1406            {
1407                let definition_levels : Vec<i16> = self.iter().map(|rec| if rec.maybe_unique_id.is_some() { 1 } else { 0 }).collect();
1408                let vals : Vec<_> = records.iter().filter_map(|rec| {
1409                    if let Some(inner) = &rec.maybe_unique_id {
1410                        Some((&inner.to_string()[..]).into())
1411                    } else {
1412                        None
1413                    }
1414                }).collect();
1415
1416                if let ColumnWriter::FixedLenByteArrayColumnWriter(typed) = column_writer.untyped() {
1417                    typed.write_batch(&vals[..], Some(&definition_levels[..]), None) ?;
1418                } else {
1419                    panic!("Schema and struct disagree on type for {}" , stringify!{ maybe_unique_id })
1420                }
1421            }
1422        }).to_string());
1423    }
1424
1425    #[test]
1426    fn test_uuid_read() {
1427        let snippet: proc_macro2::TokenStream = quote! {
1428          struct AUuidStruct {
1429            unique_id: uuid::Uuid,
1430          }
1431        };
1432
1433        let fields = extract_fields(snippet);
1434        let when = Field::from(&fields[0]);
1435        assert_eq!(when.reader_snippet().to_string(),(quote!{
1436            {
1437                let mut vals = Vec::new();
1438                if let ColumnReader::FixedLenByteArrayColumnReader(mut typed) = column_reader {
1439                    let mut definition_levels = Vec::new();
1440                    let (total_num, valid_num, decoded_num) = typed.read_records(
1441                        num_records, Some(&mut definition_levels), None, &mut vals)?;
1442                    if valid_num != decoded_num {
1443                        panic!("Support only valid records, found {} null records in column type {}",
1444                            decoded_num - valid_num, stringify!{unique_id});
1445                    }
1446                } else {
1447                    panic!("Schema and struct disagree on type for {}", stringify!{ unique_id });
1448                }
1449                for (i, r) in &mut records[..num_records].iter_mut().enumerate() {
1450                    r.unique_id = ::uuid::Uuid::from_bytes(vals[i].data().try_into().unwrap());
1451                }
1452            }
1453        }).to_string());
1454    }
1455
1456    #[test]
1457    fn test_converted_type() {
1458        let snippet: proc_macro2::TokenStream = quote! {
1459          struct ATimeStruct {
1460            time: chrono::NaiveDateTime,
1461          }
1462        };
1463
1464        let fields = extract_fields(snippet);
1465
1466        let time = Field::from(&fields[0]);
1467
1468        let converted_type = time.ty.converted_type();
1469        assert_eq!(
1470            converted_type.unwrap().to_string(),
1471            quote! { ::parquet::basic::ConvertedType::TIMESTAMP_MILLIS }.to_string()
1472        );
1473    }
1474}