parquet/column/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Low level column reader and writer APIs.
19//!
20//! This API is designed for reading and writing column values, definition and repetition
21//! levels directly.
22//!
23//! # Example of writing and reading data
24//!
25//! Data has the following format:
26//! ```text
27//! +---------------+
28//! |         values|
29//! +---------------+
30//! |[1, 2]         |
31//! |[3, null, null]|
32//! +---------------+
33//! ```
34//!
35//! The example uses column writer and reader APIs to write raw values, definition and
36//! repetition levels and read them to verify write/read correctness.
37//!
38//! ```rust,no_run
39//! # use std::{fs, path::Path, sync::Arc};
40//! #
41//! # use parquet::{
42//! #    column::{reader::ColumnReader, writer::ColumnWriter},
43//! #    data_type::Int32Type,
44//! #    file::{
45//! #        properties::WriterProperties,
46//! #        reader::{FileReader, SerializedFileReader},
47//! #        writer::SerializedFileWriter,
48//! #    },
49//! #    schema::parser::parse_message_type,
50//! # };
51//!
52//! let path = Path::new("/path/to/column_sample.parquet");
53//!
54//! // Writing data using column writer API.
55//!
56//! let message_type = "
57//!   message schema {
58//!     optional group values (LIST) {
59//!       repeated group list {
60//!         optional INT32 element;
61//!       }
62//!     }
63//!   }
64//! ";
65//! let schema = Arc::new(parse_message_type(message_type).unwrap());
66//! let file = fs::File::create(path).unwrap();
67//! let mut writer = SerializedFileWriter::new(file, schema, Default::default()).unwrap();
68//!
69//! let mut row_group_writer = writer.next_row_group().unwrap();
70//! while let Some(mut col_writer) = row_group_writer.next_column().unwrap() {
71//!     col_writer
72//!         .typed::<Int32Type>()
73//!         .write_batch(&[1, 2, 3], Some(&[3, 3, 3, 2, 2]), Some(&[0, 1, 0, 1, 1]))
74//!         .unwrap();
75//!     col_writer.close().unwrap();
76//! }
77//! row_group_writer.close().unwrap();
78//!
79//! writer.close().unwrap();
80//!
81//! // Reading data using column reader API.
82//!
83//! let file = fs::File::open(path).unwrap();
84//! let reader = SerializedFileReader::new(file).unwrap();
85//! let metadata = reader.metadata();
86//!
87//! let mut values = vec![];
88//! let mut def_levels = vec![];
89//! let mut rep_levels = vec![];
90//!
91//! for i in 0..metadata.num_row_groups() {
92//!     let row_group_reader = reader.get_row_group(i).unwrap();
93//!     let row_group_metadata = metadata.row_group(i);
94//!
95//!     for j in 0..row_group_metadata.num_columns() {
96//!         let mut column_reader = row_group_reader.get_column_reader(j).unwrap();
97//!         match column_reader {
98//!             // You can also use `get_typed_column_reader` method to extract typed reader.
99//!             ColumnReader::Int32ColumnReader(ref mut typed_reader) => {
100//!                 let (records, values, levels) = typed_reader.read_records(
101//!                     8, // maximum records to read
102//!                     Some(&mut def_levels),
103//!                     Some(&mut rep_levels),
104//!                     &mut values,
105//!                 ).unwrap();
106//!                 assert_eq!(records, 2);
107//!                 assert_eq!(levels, 5);
108//!                 assert_eq!(values, 3);
109//!             }
110//!             _ => {}
111//!         }
112//!     }
113//! }
114//!
115//! assert_eq!(values, vec![1, 2, 3]);
116//! assert_eq!(def_levels, vec![3, 3, 3, 2, 2]);
117//! assert_eq!(rep_levels, vec![0, 1, 0, 1, 1]);
118//! ```
119
120pub mod page;
121pub mod reader;
122pub mod writer;
parquet/column/mod.rs

parquet/column/
mod.rs
