Skip to main content

parquet/
variant.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! ⚠️ Experimental Support for reading and writing [`Variant`]s to / from Parquet files ⚠️
6//!
7//! This is a 🚧 Work In Progress
8//!
9//! Note: Requires the `variant_experimental` feature of the `parquet` crate to be enabled.
10//!
11//! # Features
12//! * Representation of [`Variant`], and [`VariantArray`] for working with
13//!   Variant values (see [`parquet_variant`] for more details)
14//! * Kernels for working with arrays of Variant values
15//!   such as conversion between `Variant` and JSON, and shredding/unshredding
16//!   (see [`parquet_variant_compute`] for more details)
17//!
18//! # Example: Writing a Parquet file with Variant column
19//! ```rust
20//! # use parquet::variant::{VariantArray, VariantType, VariantArrayBuilder, VariantBuilderExt};
21//! # use std::sync::Arc;
22//! # use arrow_array::{Array, ArrayRef, RecordBatch};
23//! # use arrow_schema::{DataType, Field, Schema};
24//! # use parquet::arrow::ArrowWriter;
25//! # fn main() -> Result<(), parquet::errors::ParquetError> {
26//!  // Use the VariantArrayBuilder to build a VariantArray
27//!  let mut builder = VariantArrayBuilder::new(3);
28//!  builder.new_object().with_field("name", "Alice").finish(); // row 1: {"name": "Alice"}
29//!  builder.append_value("such wow"); // row 2: "such wow" (a string)
30//!  let array = builder.build();
31//!
32//!  // Since VariantArray is an ExtensionType, it needs to be converted
33//!  // to an ArrayRef and Field with the appropriate metadata
34//!  // before it can be written to a Parquet file
35//!  let field = array.field("data");
36//!  let array = ArrayRef::from(array);
37//!  // create a RecordBatch with the VariantArray
38//!  let schema = Schema::new(vec![field]);
39//!  let batch = RecordBatch::try_new(Arc::new(schema), vec![array])?;
40//!
41//!  // Now you can write the RecordBatch to the Parquet file, as normal
42//!  let file = std::fs::File::create("variant.parquet")?;
43//!  let mut writer = ArrowWriter::try_new(file, batch.schema(), None)?;
44//!  writer.write(&batch)?;
45//!  writer.close()?;
46//!
47//! # std::fs::remove_file("variant.parquet")?;
48//! # Ok(())
49//! # }
50//! ```
51//!
52//! # Example: Writing JSON into a Parquet file with Variant column
53//! ```rust
54//! # use std::sync::Arc;
55//! # use arrow_array::{ArrayRef, RecordBatch, StringArray};
56//! # use arrow_schema::Schema;
57//! # use parquet::variant::{json_to_variant, VariantArray};
58//! # use parquet::arrow::ArrowWriter;
59//! # fn main() -> Result<(), parquet::errors::ParquetError> {
60//!  // Create an array of JSON strings, simulating a column of JSON data
61//!  let input_array: ArrayRef = Arc::new(StringArray::from(vec![
62//!   Some(r#"{"name": "Alice", "age": 30}"#),
63//!   Some(r#"{"name": "Bob", "age": 25, "address": {"city": "New York"}}"#),
64//!   None,
65//!   Some("{}"),
66//!  ]));
67//!
68//!  // Convert the JSON strings to a VariantArray
69//!  let array: VariantArray = json_to_variant(&input_array)?;
70//!  // create a RecordBatch with the VariantArray
71//!  let schema = Schema::new(vec![array.field("data")]);
72//!  let batch = RecordBatch::try_new(Arc::new(schema), vec![ArrayRef::from(array)])?;
73//!
74//!  // write the RecordBatch to a Parquet file as normal
75//!  let file = std::fs::File::create("variant-json.parquet")?;
76//!  let mut writer = ArrowWriter::try_new(file, batch.schema(), None)?;
77//!  writer.write(&batch)?;
78//!  writer.close()?;
79//! # std::fs::remove_file("variant-json.parquet")?;
80//! # Ok(())
81//! # }
82//! ```
83//!
84//! # Example: Reading a Parquet file with Variant column
85//!
86//! Use the [`VariantType`] extension type to find the Variant column:
87//!
88//! ```
89//! # use std::sync::Arc;
90//! # use std::path::PathBuf;
91//! # use arrow_array::{ArrayRef, RecordBatch, RecordBatchReader};
92//! # use parquet::variant::{Variant, VariantArray, VariantType};
93//! # use parquet::arrow::arrow_reader::ArrowReaderBuilder;
94//! # fn main() -> Result<(), parquet::errors::ParquetError> {
95//! # use arrow_array::StructArray;
96//! # fn file_path() -> PathBuf { // return a testing file path
97//! #    PathBuf::from(arrow::util::test_util::parquet_test_data())
98//! #   .join("..")
99//! #   .join("shredded_variant")
100//! #   .join("case-075.parquet")
101//! # }
102//! // Read the Parquet file using standard Arrow Parquet reader.
103//! // Note this file has 2 columns: "id", "var", and the "var" column
104//  // contains a variant that looks like this:
105//  // "Variant(metadata=VariantMetadata(dict={}), value=Variant(type=STRING, value=iceberg))"
106//! let file = std::fs::File::open(file_path())?;
107//! let mut reader = ArrowReaderBuilder::try_new(file)?.build()?;
108//!
109//! // You can check if a column contains a Variant using
110//! // the VariantType extension type
111//! let schema = reader.schema();
112//! let field = schema.field_with_name("var")?;
113//! assert!(field.has_valid_extension_type::<VariantType>());
114//!
115//! // The reader will yield RecordBatches with a StructArray
116//! // to convert them to VariantArray, use VariantArray::try_new
117//! let batch = reader.next().unwrap().unwrap();
118//!
119//! let col = batch.column_by_name("var").unwrap();
120//! let var_array = VariantArray::try_new(col)?;
121//! assert_eq!(var_array.len(), 1);
122//! let var_value: Variant = var_array.value(0);
123//! assert_eq!(var_value, Variant::from("iceberg")); // the value in case-075.parquet
124//! # Ok(())
125//! # }
126//! ```
127pub use parquet_variant::*;
128pub use parquet_variant_compute::*;
129
130#[cfg(test)]
131mod tests {
132    use crate::arrow::ArrowWriter;
133    use crate::arrow::arrow_reader::ArrowReaderBuilder;
134    use crate::file::metadata::{ParquetMetaData, ParquetMetaDataReader};
135    use crate::file::reader::ChunkReader;
136    use arrow::util::test_util::parquet_test_data;
137    use arrow_array::{
138        Array, ArrayRef, BinaryViewArray, FixedSizeListArray, Int64Array, RecordBatch, StructArray,
139        new_null_array,
140    };
141    use arrow_schema::{DataType, Field, Fields, Schema};
142    use bytes::Bytes;
143    use parquet_variant::{EMPTY_VARIANT_METADATA_BYTES, Variant, VariantBuilderExt};
144    use parquet_variant_compute::{
145        VariantArray, VariantArrayBuilder, VariantType, unshred_variant,
146    };
147    use std::path::PathBuf;
148    use std::sync::Arc;
149
150    #[test]
151    fn roundtrip_basic() {
152        roundtrip(variant_array());
153    }
154
155    /// Ensure a file with Variant LogicalType, written by another writer in
156    /// parquet-testing, can be read as a VariantArray
157    #[test]
158    fn read_logical_type() {
159        // Note: case-075 2 columns ("id", "var")
160        // The variant looks like this:
161        // "Variant(metadata=VariantMetadata(dict={}), value=Variant(type=STRING, value=iceberg))"
162        let batch = read_shredded_variant_test_case("case-075.parquet");
163
164        assert_variant_metadata(&batch, "var");
165        let var_column = batch.column_by_name("var").expect("expected var column");
166        let var_array =
167            VariantArray::try_new(&var_column).expect("expected var column to be a VariantArray");
168
169        // verify the value
170        assert_eq!(var_array.len(), 1);
171        assert!(var_array.is_valid(0));
172        let var_value = var_array.value(0);
173        assert_eq!(var_value, Variant::from("iceberg"));
174    }
175
176    #[test]
177    fn read_fixed_size_list_typed_value_as_list() {
178        let element_values: ArrayRef = Arc::new(Int64Array::from(vec![1, 2, 3, 4]));
179        let element_value = new_null_array(&DataType::BinaryView, 4);
180        let element_fields = Fields::from(vec![
181            Field::new("value", DataType::BinaryView, true),
182            Field::new("typed_value", DataType::Int64, true),
183        ]);
184        let elements: ArrayRef = Arc::new(StructArray::new(
185            element_fields,
186            vec![element_value, element_values],
187            None,
188        ));
189        let item_field = Arc::new(Field::new("item", elements.data_type().clone(), true));
190        let typed_value: ArrayRef =
191            Arc::new(FixedSizeListArray::new(item_field, 2, elements, None));
192        let metadata: ArrayRef = Arc::new(BinaryViewArray::from_iter_values(std::iter::repeat_n(
193            EMPTY_VARIANT_METADATA_BYTES,
194            2,
195        )));
196        let value = new_null_array(&DataType::BinaryView, 2);
197        let fields = Fields::from(vec![
198            Field::new("metadata", DataType::BinaryView, false),
199            Field::new("value", DataType::BinaryView, true),
200            Field::new("typed_value", typed_value.data_type().clone(), true),
201        ]);
202        let source = StructArray::new(fields, vec![metadata, value, typed_value], None);
203        let field = Field::new("data", source.data_type().clone(), false);
204        let batch =
205            RecordBatch::try_new(Arc::new(Schema::new(vec![field])), vec![Arc::new(source)])
206                .unwrap();
207
208        let buffer = write_to_buffer(&batch);
209        let result = read_to_batch(Bytes::from(buffer));
210        let column = result.column_by_name("data").unwrap();
211        let variant = VariantArray::try_new(column).unwrap();
212        assert!(matches!(
213            variant.typed_value_column().unwrap().data_type(),
214            DataType::List(_)
215        ));
216
217        let unshredded = unshred_variant(&variant).unwrap();
218        assert!(unshredded.typed_value_column().is_none());
219        assert_eq!(unshredded.len(), 2);
220    }
221
222    /// Writes a variant to a parquet file and ensures the parquet logical type
223    /// annotation is correct
224    #[test]
225    fn write_logical_type() {
226        let array = variant_array();
227        let batch = variant_array_to_batch(array);
228        let buffer = write_to_buffer(&batch);
229
230        // read the parquet file's metadata and verify the logical type
231        let metadata = read_metadata(&Bytes::from(buffer));
232        let schema = metadata.file_metadata().schema_descr();
233        let fields = schema.root_schema().get_fields();
234        assert_eq!(fields.len(), 1);
235        let field = &fields[0];
236        assert_eq!(field.name(), "data");
237        // data should have been written with the Variant logical type, with the
238        // specification version set (required by the Parquet spec and by readers
239        // such as Spark, which only accept `VARIANT(1)`)
240        assert_eq!(
241            field.get_basic_info().logical_type_ref(),
242            Some(&crate::basic::LogicalType::variant(Some(1)))
243        );
244    }
245
246    /// Return a VariantArray with 3 rows:
247    ///
248    /// 1. `{"name": "Alice"}`
249    /// 2. `"such wow"` (a string)
250    /// 3. `null`
251    fn variant_array() -> VariantArray {
252        let mut builder = VariantArrayBuilder::new(3);
253        // row 1: {"name": "Alice"}
254        builder.new_object().with_field("name", "Alice").finish();
255        // row 2: "such wow" (a string)
256        builder.append_value("such wow");
257        // row 3: null
258        builder.append_null();
259        builder.build()
260    }
261
262    /// Writes a VariantArray to a parquet file and reads it back, verifying that
263    /// the data is the same
264    fn roundtrip(array: VariantArray) {
265        let source_batch = variant_array_to_batch(array);
266        assert_variant_metadata(&source_batch, "data");
267
268        let buffer = write_to_buffer(&source_batch);
269        let result_batch = read_to_batch(Bytes::from(buffer));
270        assert_variant_metadata(&result_batch, "data");
271        assert_eq!(result_batch, source_batch); // NB this also checks the schemas
272    }
273
274    /// creates a RecordBatch with a single column "data" from a VariantArray,
275    fn variant_array_to_batch(array: VariantArray) -> RecordBatch {
276        let field = array.field("data");
277        let schema = Schema::new(vec![field]);
278        RecordBatch::try_new(Arc::new(schema), vec![ArrayRef::from(array)]).unwrap()
279    }
280
281    /// writes a RecordBatch to memory buffer and returns the buffer
282    fn write_to_buffer(batch: &RecordBatch) -> Vec<u8> {
283        let mut buffer = vec![];
284        let mut writer = ArrowWriter::try_new(&mut buffer, batch.schema(), None).unwrap();
285        writer.write(batch).unwrap();
286        writer.close().unwrap();
287        buffer
288    }
289
290    /// Reads the Parquet metadata
291    fn read_metadata<T: ChunkReader + 'static>(input: &T) -> ParquetMetaData {
292        let mut reader = ParquetMetaDataReader::new();
293        reader.try_parse(input).unwrap();
294        reader.finish().unwrap()
295    }
296
297    /// Reads a RecordBatch from a reader (e.g. Vec or File)
298    fn read_to_batch<T: ChunkReader + 'static>(reader: T) -> RecordBatch {
299        let reader = ArrowReaderBuilder::try_new(reader)
300            .unwrap()
301            .build()
302            .unwrap();
303        let mut batches: Vec<RecordBatch> = reader.collect::<Result<Vec<_>, _>>().unwrap();
304        assert_eq!(batches.len(), 1);
305        batches.swap_remove(0)
306    }
307
308    /// Verifies the variant metadata is present in the schema for the specified
309    /// field name.
310    fn assert_variant_metadata(batch: &RecordBatch, field_name: &str) {
311        let schema = batch.schema();
312        let field = schema
313            .field_with_name(field_name)
314            .expect("could not find expected field");
315
316        // explicitly check the metadata so it is clear in the tests what the
317        // names are
318        let metadata_value = field
319            .metadata()
320            .get("ARROW:extension:name")
321            .expect("metadata does not exist");
322
323        assert_eq!(metadata_value, "arrow.parquet.variant");
324
325        // verify that `VariantType` also correctly finds the metadata
326        assert!(field.has_valid_extension_type::<VariantType>());
327    }
328
329    /// Read the specified test case filename from parquet-testing
330    /// See parquet-testing/shredded_variant/cases.json for more details
331    fn read_shredded_variant_test_case(name: &str) -> RecordBatch {
332        let case_file = PathBuf::from(parquet_test_data())
333            .join("..") // go up from data/ to parquet-testing/
334            .join("shredded_variant")
335            .join(name);
336        let case_file = std::fs::File::open(case_file).unwrap();
337        read_to_batch(case_file)
338    }
339}