parquet/column/mod.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Low level column reader and writer APIs.
6//!
7//! This API is designed for reading and writing column values, definition and repetition
8//! levels directly.
9//!
10//! # Example of writing and reading data
11//!
12//! Data has the following format:
13//! ```text
14//! +---------------+
15//! | values|
16//! +---------------+
17//! |[1, 2] |
18//! |[3, null, null]|
19//! +---------------+
20//! ```
21//!
22//! The example uses column writer and reader APIs to write raw values, definition and
23//! repetition levels and read them to verify write/read correctness.
24//!
25//! ```rust,no_run
26//! # use std::{fs, path::Path, sync::Arc};
27//! #
28//! # use parquet::{
29//! # column::{reader::ColumnReader, writer::ColumnWriter},
30//! # data_type::Int32Type,
31//! # file::{
32//! # properties::WriterProperties,
33//! # reader::{FileReader, SerializedFileReader},
34//! # writer::SerializedFileWriter,
35//! # },
36//! # schema::parser::parse_message_type,
37//! # };
38//!
39//! let path = Path::new("/path/to/column_sample.parquet");
40//!
41//! // Writing data using column writer API.
42//!
43//! let message_type = "
44//! message schema {
45//! optional group values (LIST) {
46//! repeated group list {
47//! optional INT32 element;
48//! }
49//! }
50//! }
51//! ";
52//! let schema = Arc::new(parse_message_type(message_type).unwrap());
53//! let file = fs::File::create(path).unwrap();
54//! let mut writer = SerializedFileWriter::new(file, schema, Default::default()).unwrap();
55//!
56//! let mut row_group_writer = writer.next_row_group().unwrap();
57//! while let Some(mut col_writer) = row_group_writer.next_column().unwrap() {
58//! col_writer
59//! .typed::<Int32Type>()
60//! .write_batch(&[1, 2, 3], Some(&[3, 3, 3, 2, 2]), Some(&[0, 1, 0, 1, 1]))
61//! .unwrap();
62//! col_writer.close().unwrap();
63//! }
64//! row_group_writer.close().unwrap();
65//!
66//! writer.close().unwrap();
67//!
68//! // Reading data using column reader API.
69//!
70//! let file = fs::File::open(path).unwrap();
71//! let reader = SerializedFileReader::new(file).unwrap();
72//! let metadata = reader.metadata();
73//!
74//! let mut values = vec![];
75//! let mut def_levels = vec![];
76//! let mut rep_levels = vec![];
77//!
78//! for i in 0..metadata.num_row_groups() {
79//! let row_group_reader = reader.get_row_group(i).unwrap();
80//! let row_group_metadata = metadata.row_group(i);
81//!
82//! for j in 0..row_group_metadata.num_columns() {
83//! let mut column_reader = row_group_reader.get_column_reader(j).unwrap();
84//! match column_reader {
85//! // You can also use `get_typed_column_reader` method to extract typed reader.
86//! ColumnReader::Int32ColumnReader(ref mut typed_reader) => {
87//! let (records, values, levels) = typed_reader.read_records(
88//! 8, // maximum records to read
89//! Some(&mut def_levels),
90//! Some(&mut rep_levels),
91//! &mut values,
92//! ).unwrap();
93//! assert_eq!(records, 2);
94//! assert_eq!(levels, 5);
95//! assert_eq!(values, 3);
96//! }
97//! _ => {}
98//! }
99//! }
100//! }
101//!
102//! assert_eq!(values, vec![1, 2, 3]);
103//! assert_eq!(def_levels, vec![3, 3, 3, 2, 2]);
104//! assert_eq!(rep_levels, vec![0, 1, 0, 1, 1]);
105//! ```
106
107#[cfg(feature = "arrow")]
108pub(crate) mod chunker;
109pub mod page;
110#[cfg(feature = "encryption")]
111pub(crate) mod page_encryption;
112#[cfg(not(feature = "encryption"))]
113#[path = "page_encryption_disabled.rs"]
114pub(crate) mod page_encryption;
115pub mod page_store;
116pub mod reader;
117pub mod writer;