Skip to main content

parquet_layout/
parquet-layout.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Binary that prints the physical layout of a parquet file
6//!
7//! Alternatives to this binary include [`parquet-cli`] and [`parquet-viewer`].
8//!
9//! # Install
10//!
11//! `parquet-layout` can be installed using `cargo`:
12//! ```
13//! cargo install parquet --features=cli
14//! ```
15//! After this `parquet-layout` should be available:
16//! ```
17//! parquet-layout XYZ.parquet
18//! ```
19//!
20//! The binary can also be built from the source code and run as follows:
21//! ```
22//! cargo run --features=cli --bin parquet-layout XYZ.parquet
23//! ```
24//!
25//! [`parquet-cli`]: https://github.com/apache/parquet-java/tree/master/parquet-cli
26//! [`parquet-viewer`]: https://github.com/xiangpenghao/parquet-viewer
27
28use std::fs::File;
29
30use clap::Parser;
31use parquet::file::metadata::ParquetMetaDataReader;
32use serde::{Serialize, Serializer};
33
34use parquet::basic::{CompressionCodec, Encoding};
35use parquet::errors::Result;
36use parquet::file::reader::ChunkReader;
37
38#[derive(Serialize, Debug)]
39struct Index {
40    offset: i64,
41    length: Option<i32>,
42}
43
44#[derive(Serialize, Debug)]
45struct Footer {
46    metadata_size: Option<usize>,
47}
48
49#[derive(Serialize, Debug)]
50struct ParquetFile {
51    row_groups: Vec<RowGroup>,
52    footer: Footer,
53}
54
55#[derive(Serialize, Debug)]
56struct RowGroup {
57    columns: Vec<ColumnChunk>,
58    row_count: i64,
59}
60
61#[derive(Serialize, Debug)]
62struct ColumnChunk {
63    path: String,
64    has_offset_index: bool,
65    has_column_index: bool,
66    has_bloom_filter: bool,
67    offset_index: Option<Index>,
68    column_index: Option<Index>,
69    bloom_filter: Option<Index>,
70    compression: DebugSerialize<CompressionCodec>,
71    encodings: Vec<DebugSerialize<Encoding>>,
72}
73
74#[derive(Debug)]
75struct DebugSerialize<T: std::fmt::Debug>(T);
76
77impl<T: std::fmt::Debug> Serialize for DebugSerialize<T> {
78    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
79    where
80        S: Serializer,
81    {
82        serializer.serialize_str(&format!("{:?}", self.0))
83    }
84}
85
86fn do_layout<C: ChunkReader>(reader: &C) -> Result<ParquetFile> {
87    let mut metadata_reader = ParquetMetaDataReader::new();
88    metadata_reader.try_parse(reader)?;
89    let metadata_size = metadata_reader.metadata_size();
90    let metadata = metadata_reader.finish()?;
91    let schema = metadata.file_metadata().schema_descr();
92
93    let row_groups = (0..metadata.num_row_groups())
94        .map(|row_group_idx| {
95            let row_group = metadata.row_group(row_group_idx);
96            let columns = row_group
97                .columns()
98                .iter()
99                .zip(schema.columns())
100                .map(|(column, column_schema)| {
101                    let compression = DebugSerialize(column.compression_codec());
102                    let encodings = column.encodings().map(DebugSerialize).collect();
103
104                    Ok(ColumnChunk {
105                        path: column_schema.path().parts().join("."),
106                        has_offset_index: column.offset_index_offset().is_some(),
107                        has_column_index: column.column_index_offset().is_some(),
108                        has_bloom_filter: column.bloom_filter_offset().is_some(),
109                        offset_index: column.offset_index_offset().map(|offset| Index {
110                            offset,
111                            length: column.offset_index_length(),
112                        }),
113                        column_index: column.column_index_offset().map(|offset| Index {
114                            offset,
115                            length: column.column_index_length(),
116                        }),
117                        bloom_filter: column.bloom_filter_offset().map(|offset| Index {
118                            offset,
119                            length: column.bloom_filter_length(),
120                        }),
121                        compression,
122                        encodings,
123                    })
124                })
125                .collect::<Result<Vec<_>>>()?;
126
127            Ok(RowGroup {
128                columns,
129                row_count: row_group.num_rows(),
130            })
131        })
132        .collect::<Result<Vec<_>>>()?;
133
134    Ok(ParquetFile {
135        row_groups,
136        footer: Footer { metadata_size },
137    })
138}
139
140#[derive(Debug, Parser)]
141#[clap(author, version, about("Prints the physical layout of a parquet file"), long_about = None)]
142struct Args {
143    #[clap(help("Path to a parquet file"))]
144    file: String,
145}
146
147impl Args {
148    fn run(&self) -> Result<()> {
149        let file = File::open(&self.file)?;
150        let layout = do_layout(&file)?;
151
152        let out = std::io::stdout();
153        let writer = out.lock();
154
155        serde_json::to_writer_pretty(writer, &layout).unwrap();
156        Ok(())
157    }
158}
159
160fn main() -> Result<()> {
161    Args::parse().run()
162}