Skip to main content

parquet/column/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Low level column reader and writer APIs.
6//!
7//! This API is designed for reading and writing column values, definition and repetition
8//! levels directly.
9//!
10//! # Example of writing and reading data
11//!
12//! Data has the following format:
13//! ```text
14//! +---------------+
15//! |         values|
16//! +---------------+
17//! |[1, 2]         |
18//! |[3, null, null]|
19//! +---------------+
20//! ```
21//!
22//! The example uses column writer and reader APIs to write raw values, definition and
23//! repetition levels and read them to verify write/read correctness.
24//!
25//! ```rust,no_run
26//! # use std::{fs, path::Path, sync::Arc};
27//! #
28//! # use parquet::{
29//! #    column::{reader::ColumnReader, writer::ColumnWriter},
30//! #    data_type::Int32Type,
31//! #    file::{
32//! #        properties::WriterProperties,
33//! #        reader::{FileReader, SerializedFileReader},
34//! #        writer::SerializedFileWriter,
35//! #    },
36//! #    schema::parser::parse_message_type,
37//! # };
38//!
39//! let path = Path::new("/path/to/column_sample.parquet");
40//!
41//! // Writing data using column writer API.
42//!
43//! let message_type = "
44//!   message schema {
45//!     optional group values (LIST) {
46//!       repeated group list {
47//!         optional INT32 element;
48//!       }
49//!     }
50//!   }
51//! ";
52//! let schema = Arc::new(parse_message_type(message_type).unwrap());
53//! let file = fs::File::create(path).unwrap();
54//! let mut writer = SerializedFileWriter::new(file, schema, Default::default()).unwrap();
55//!
56//! let mut row_group_writer = writer.next_row_group().unwrap();
57//! while let Some(mut col_writer) = row_group_writer.next_column().unwrap() {
58//!     col_writer
59//!         .typed::<Int32Type>()
60//!         .write_batch(&[1, 2, 3], Some(&[3, 3, 3, 2, 2]), Some(&[0, 1, 0, 1, 1]))
61//!         .unwrap();
62//!     col_writer.close().unwrap();
63//! }
64//! row_group_writer.close().unwrap();
65//!
66//! writer.close().unwrap();
67//!
68//! // Reading data using column reader API.
69//!
70//! let file = fs::File::open(path).unwrap();
71//! let reader = SerializedFileReader::new(file).unwrap();
72//! let metadata = reader.metadata();
73//!
74//! let mut values = vec![];
75//! let mut def_levels = vec![];
76//! let mut rep_levels = vec![];
77//!
78//! for i in 0..metadata.num_row_groups() {
79//!     let row_group_reader = reader.get_row_group(i).unwrap();
80//!     let row_group_metadata = metadata.row_group(i);
81//!
82//!     for j in 0..row_group_metadata.num_columns() {
83//!         let mut column_reader = row_group_reader.get_column_reader(j).unwrap();
84//!         match column_reader {
85//!             // You can also use `get_typed_column_reader` method to extract typed reader.
86//!             ColumnReader::Int32ColumnReader(ref mut typed_reader) => {
87//!                 let (records, values, levels) = typed_reader.read_records(
88//!                     8, // maximum records to read
89//!                     Some(&mut def_levels),
90//!                     Some(&mut rep_levels),
91//!                     &mut values,
92//!                 ).unwrap();
93//!                 assert_eq!(records, 2);
94//!                 assert_eq!(levels, 5);
95//!                 assert_eq!(values, 3);
96//!             }
97//!             _ => {}
98//!         }
99//!     }
100//! }
101//!
102//! assert_eq!(values, vec![1, 2, 3]);
103//! assert_eq!(def_levels, vec![3, 3, 3, 2, 2]);
104//! assert_eq!(rep_levels, vec![0, 1, 0, 1, 1]);
105//! ```
106
107#[cfg(feature = "arrow")]
108pub(crate) mod chunker;
109pub mod page;
110#[cfg(feature = "encryption")]
111pub(crate) mod page_encryption;
112#[cfg(not(feature = "encryption"))]
113#[path = "page_encryption_disabled.rs"]
114pub(crate) mod page_encryption;
115pub mod page_store;
116pub mod reader;
117pub mod writer;