parquet/variant.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! ⚠️ Experimental Support for reading and writing [`Variant`]s to / from Parquet files ⚠️
6//!
7//! This is a 🚧 Work In Progress
8//!
9//! Note: Requires the `variant_experimental` feature of the `parquet` crate to be enabled.
10//!
11//! # Features
12//! * Representation of [`Variant`], and [`VariantArray`] for working with
13//! Variant values (see [`parquet_variant`] for more details)
14//! * Kernels for working with arrays of Variant values
15//! such as conversion between `Variant` and JSON, and shredding/unshredding
16//! (see [`parquet_variant_compute`] for more details)
17//!
18//! # Example: Writing a Parquet file with Variant column
19//! ```rust
20//! # use parquet::variant::{VariantArray, VariantType, VariantArrayBuilder, VariantBuilderExt};
21//! # use std::sync::Arc;
22//! # use arrow_array::{Array, ArrayRef, RecordBatch};
23//! # use arrow_schema::{DataType, Field, Schema};
24//! # use parquet::arrow::ArrowWriter;
25//! # fn main() -> Result<(), parquet::errors::ParquetError> {
26//! // Use the VariantArrayBuilder to build a VariantArray
27//! let mut builder = VariantArrayBuilder::new(3);
28//! builder.new_object().with_field("name", "Alice").finish(); // row 1: {"name": "Alice"}
29//! builder.append_value("such wow"); // row 2: "such wow" (a string)
30//! let array = builder.build();
31//!
32//! // Since VariantArray is an ExtensionType, it needs to be converted
33//! // to an ArrayRef and Field with the appropriate metadata
34//! // before it can be written to a Parquet file
35//! let field = array.field("data");
36//! let array = ArrayRef::from(array);
37//! // create a RecordBatch with the VariantArray
38//! let schema = Schema::new(vec![field]);
39//! let batch = RecordBatch::try_new(Arc::new(schema), vec![array])?;
40//!
41//! // Now you can write the RecordBatch to the Parquet file, as normal
42//! let file = std::fs::File::create("variant.parquet")?;
43//! let mut writer = ArrowWriter::try_new(file, batch.schema(), None)?;
44//! writer.write(&batch)?;
45//! writer.close()?;
46//!
47//! # std::fs::remove_file("variant.parquet")?;
48//! # Ok(())
49//! # }
50//! ```
51//!
52//! # Example: Writing JSON into a Parquet file with Variant column
53//! ```rust
54//! # use std::sync::Arc;
55//! # use arrow_array::{ArrayRef, RecordBatch, StringArray};
56//! # use arrow_schema::Schema;
57//! # use parquet::variant::{json_to_variant, VariantArray};
58//! # use parquet::arrow::ArrowWriter;
59//! # fn main() -> Result<(), parquet::errors::ParquetError> {
60//! // Create an array of JSON strings, simulating a column of JSON data
61//! let input_array: ArrayRef = Arc::new(StringArray::from(vec![
62//! Some(r#"{"name": "Alice", "age": 30}"#),
63//! Some(r#"{"name": "Bob", "age": 25, "address": {"city": "New York"}}"#),
64//! None,
65//! Some("{}"),
66//! ]));
67//!
68//! // Convert the JSON strings to a VariantArray
69//! let array: VariantArray = json_to_variant(&input_array)?;
70//! // create a RecordBatch with the VariantArray
71//! let schema = Schema::new(vec![array.field("data")]);
72//! let batch = RecordBatch::try_new(Arc::new(schema), vec![ArrayRef::from(array)])?;
73//!
74//! // write the RecordBatch to a Parquet file as normal
75//! let file = std::fs::File::create("variant-json.parquet")?;
76//! let mut writer = ArrowWriter::try_new(file, batch.schema(), None)?;
77//! writer.write(&batch)?;
78//! writer.close()?;
79//! # std::fs::remove_file("variant-json.parquet")?;
80//! # Ok(())
81//! # }
82//! ```
83//!
84//! # Example: Reading a Parquet file with Variant column
85//!
86//! Use the [`VariantType`] extension type to find the Variant column:
87//!
88//! ```
89//! # use std::sync::Arc;
90//! # use std::path::PathBuf;
91//! # use arrow_array::{ArrayRef, RecordBatch, RecordBatchReader};
92//! # use parquet::variant::{Variant, VariantArray, VariantType};
93//! # use parquet::arrow::arrow_reader::ArrowReaderBuilder;
94//! # fn main() -> Result<(), parquet::errors::ParquetError> {
95//! # use arrow_array::StructArray;
96//! # fn file_path() -> PathBuf { // return a testing file path
97//! # PathBuf::from(arrow::util::test_util::parquet_test_data())
98//! # .join("..")
99//! # .join("shredded_variant")
100//! # .join("case-075.parquet")
101//! # }
102//! // Read the Parquet file using standard Arrow Parquet reader.
103//! // Note this file has 2 columns: "id", "var", and the "var" column
104// // contains a variant that looks like this:
105// // "Variant(metadata=VariantMetadata(dict={}), value=Variant(type=STRING, value=iceberg))"
106//! let file = std::fs::File::open(file_path())?;
107//! let mut reader = ArrowReaderBuilder::try_new(file)?.build()?;
108//!
109//! // You can check if a column contains a Variant using
110//! // the VariantType extension type
111//! let schema = reader.schema();
112//! let field = schema.field_with_name("var")?;
113//! assert!(field.has_valid_extension_type::<VariantType>());
114//!
115//! // The reader will yield RecordBatches with a StructArray
116//! // to convert them to VariantArray, use VariantArray::try_new
117//! let batch = reader.next().unwrap().unwrap();
118//!
119//! let col = batch.column_by_name("var").unwrap();
120//! let var_array = VariantArray::try_new(col)?;
121//! assert_eq!(var_array.len(), 1);
122//! let var_value: Variant = var_array.value(0);
123//! assert_eq!(var_value, Variant::from("iceberg")); // the value in case-075.parquet
124//! # Ok(())
125//! # }
126//! ```
127pub use parquet_variant::*;
128pub use parquet_variant_compute::*;
129
130#[cfg(test)]
131mod tests {
132 use crate::arrow::ArrowWriter;
133 use crate::arrow::arrow_reader::ArrowReaderBuilder;
134 use crate::file::metadata::{ParquetMetaData, ParquetMetaDataReader};
135 use crate::file::reader::ChunkReader;
136 use arrow::util::test_util::parquet_test_data;
137 use arrow_array::{
138 Array, ArrayRef, BinaryViewArray, FixedSizeListArray, Int64Array, RecordBatch, StructArray,
139 new_null_array,
140 };
141 use arrow_schema::{DataType, Field, Fields, Schema};
142 use bytes::Bytes;
143 use parquet_variant::{EMPTY_VARIANT_METADATA_BYTES, Variant, VariantBuilderExt};
144 use parquet_variant_compute::{
145 VariantArray, VariantArrayBuilder, VariantType, unshred_variant,
146 };
147 use std::path::PathBuf;
148 use std::sync::Arc;
149
150 #[test]
151 fn roundtrip_basic() {
152 roundtrip(variant_array());
153 }
154
155 /// Ensure a file with Variant LogicalType, written by another writer in
156 /// parquet-testing, can be read as a VariantArray
157 #[test]
158 fn read_logical_type() {
159 // Note: case-075 2 columns ("id", "var")
160 // The variant looks like this:
161 // "Variant(metadata=VariantMetadata(dict={}), value=Variant(type=STRING, value=iceberg))"
162 let batch = read_shredded_variant_test_case("case-075.parquet");
163
164 assert_variant_metadata(&batch, "var");
165 let var_column = batch.column_by_name("var").expect("expected var column");
166 let var_array =
167 VariantArray::try_new(&var_column).expect("expected var column to be a VariantArray");
168
169 // verify the value
170 assert_eq!(var_array.len(), 1);
171 assert!(var_array.is_valid(0));
172 let var_value = var_array.value(0);
173 assert_eq!(var_value, Variant::from("iceberg"));
174 }
175
176 #[test]
177 fn read_fixed_size_list_typed_value_as_list() {
178 let element_values: ArrayRef = Arc::new(Int64Array::from(vec![1, 2, 3, 4]));
179 let element_value = new_null_array(&DataType::BinaryView, 4);
180 let element_fields = Fields::from(vec![
181 Field::new("value", DataType::BinaryView, true),
182 Field::new("typed_value", DataType::Int64, true),
183 ]);
184 let elements: ArrayRef = Arc::new(StructArray::new(
185 element_fields,
186 vec![element_value, element_values],
187 None,
188 ));
189 let item_field = Arc::new(Field::new("item", elements.data_type().clone(), true));
190 let typed_value: ArrayRef =
191 Arc::new(FixedSizeListArray::new(item_field, 2, elements, None));
192 let metadata: ArrayRef = Arc::new(BinaryViewArray::from_iter_values(std::iter::repeat_n(
193 EMPTY_VARIANT_METADATA_BYTES,
194 2,
195 )));
196 let value = new_null_array(&DataType::BinaryView, 2);
197 let fields = Fields::from(vec![
198 Field::new("metadata", DataType::BinaryView, false),
199 Field::new("value", DataType::BinaryView, true),
200 Field::new("typed_value", typed_value.data_type().clone(), true),
201 ]);
202 let source = StructArray::new(fields, vec![metadata, value, typed_value], None);
203 let field = Field::new("data", source.data_type().clone(), false);
204 let batch =
205 RecordBatch::try_new(Arc::new(Schema::new(vec![field])), vec![Arc::new(source)])
206 .unwrap();
207
208 let buffer = write_to_buffer(&batch);
209 let result = read_to_batch(Bytes::from(buffer));
210 let column = result.column_by_name("data").unwrap();
211 let variant = VariantArray::try_new(column).unwrap();
212 assert!(matches!(
213 variant.typed_value_column().unwrap().data_type(),
214 DataType::List(_)
215 ));
216
217 let unshredded = unshred_variant(&variant).unwrap();
218 assert!(unshredded.typed_value_column().is_none());
219 assert_eq!(unshredded.len(), 2);
220 }
221
222 /// Writes a variant to a parquet file and ensures the parquet logical type
223 /// annotation is correct
224 #[test]
225 fn write_logical_type() {
226 let array = variant_array();
227 let batch = variant_array_to_batch(array);
228 let buffer = write_to_buffer(&batch);
229
230 // read the parquet file's metadata and verify the logical type
231 let metadata = read_metadata(&Bytes::from(buffer));
232 let schema = metadata.file_metadata().schema_descr();
233 let fields = schema.root_schema().get_fields();
234 assert_eq!(fields.len(), 1);
235 let field = &fields[0];
236 assert_eq!(field.name(), "data");
237 // data should have been written with the Variant logical type, with the
238 // specification version set (required by the Parquet spec and by readers
239 // such as Spark, which only accept `VARIANT(1)`)
240 assert_eq!(
241 field.get_basic_info().logical_type_ref(),
242 Some(&crate::basic::LogicalType::variant(Some(1)))
243 );
244 }
245
246 /// Return a VariantArray with 3 rows:
247 ///
248 /// 1. `{"name": "Alice"}`
249 /// 2. `"such wow"` (a string)
250 /// 3. `null`
251 fn variant_array() -> VariantArray {
252 let mut builder = VariantArrayBuilder::new(3);
253 // row 1: {"name": "Alice"}
254 builder.new_object().with_field("name", "Alice").finish();
255 // row 2: "such wow" (a string)
256 builder.append_value("such wow");
257 // row 3: null
258 builder.append_null();
259 builder.build()
260 }
261
262 /// Writes a VariantArray to a parquet file and reads it back, verifying that
263 /// the data is the same
264 fn roundtrip(array: VariantArray) {
265 let source_batch = variant_array_to_batch(array);
266 assert_variant_metadata(&source_batch, "data");
267
268 let buffer = write_to_buffer(&source_batch);
269 let result_batch = read_to_batch(Bytes::from(buffer));
270 assert_variant_metadata(&result_batch, "data");
271 assert_eq!(result_batch, source_batch); // NB this also checks the schemas
272 }
273
274 /// creates a RecordBatch with a single column "data" from a VariantArray,
275 fn variant_array_to_batch(array: VariantArray) -> RecordBatch {
276 let field = array.field("data");
277 let schema = Schema::new(vec![field]);
278 RecordBatch::try_new(Arc::new(schema), vec![ArrayRef::from(array)]).unwrap()
279 }
280
281 /// writes a RecordBatch to memory buffer and returns the buffer
282 fn write_to_buffer(batch: &RecordBatch) -> Vec<u8> {
283 let mut buffer = vec![];
284 let mut writer = ArrowWriter::try_new(&mut buffer, batch.schema(), None).unwrap();
285 writer.write(batch).unwrap();
286 writer.close().unwrap();
287 buffer
288 }
289
290 /// Reads the Parquet metadata
291 fn read_metadata<T: ChunkReader + 'static>(input: &T) -> ParquetMetaData {
292 let mut reader = ParquetMetaDataReader::new();
293 reader.try_parse(input).unwrap();
294 reader.finish().unwrap()
295 }
296
297 /// Reads a RecordBatch from a reader (e.g. Vec or File)
298 fn read_to_batch<T: ChunkReader + 'static>(reader: T) -> RecordBatch {
299 let reader = ArrowReaderBuilder::try_new(reader)
300 .unwrap()
301 .build()
302 .unwrap();
303 let mut batches: Vec<RecordBatch> = reader.collect::<Result<Vec<_>, _>>().unwrap();
304 assert_eq!(batches.len(), 1);
305 batches.swap_remove(0)
306 }
307
308 /// Verifies the variant metadata is present in the schema for the specified
309 /// field name.
310 fn assert_variant_metadata(batch: &RecordBatch, field_name: &str) {
311 let schema = batch.schema();
312 let field = schema
313 .field_with_name(field_name)
314 .expect("could not find expected field");
315
316 // explicitly check the metadata so it is clear in the tests what the
317 // names are
318 let metadata_value = field
319 .metadata()
320 .get("ARROW:extension:name")
321 .expect("metadata does not exist");
322
323 assert_eq!(metadata_value, "arrow.parquet.variant");
324
325 // verify that `VariantType` also correctly finds the metadata
326 assert!(field.has_valid_extension_type::<VariantType>());
327 }
328
329 /// Read the specified test case filename from parquet-testing
330 /// See parquet-testing/shredded_variant/cases.json for more details
331 fn read_shredded_variant_test_case(name: &str) -> RecordBatch {
332 let case_file = PathBuf::from(parquet_test_data())
333 .join("..") // go up from data/ to parquet-testing/
334 .join("shredded_variant")
335 .join(name);
336 let case_file = std::fs::File::open(case_file).unwrap();
337 read_to_batch(case_file)
338 }
339}