parquet_layout/
parquet-layout.rs1use std::fs::File;
29
30use clap::Parser;
31use parquet::file::metadata::ParquetMetaDataReader;
32use serde::{Serialize, Serializer};
33
34use parquet::basic::{CompressionCodec, Encoding};
35use parquet::errors::Result;
36use parquet::file::reader::ChunkReader;
37
38#[derive(Serialize, Debug)]
39struct Index {
40 offset: i64,
41 length: Option<i32>,
42}
43
44#[derive(Serialize, Debug)]
45struct Footer {
46 metadata_size: Option<usize>,
47}
48
49#[derive(Serialize, Debug)]
50struct ParquetFile {
51 row_groups: Vec<RowGroup>,
52 footer: Footer,
53}
54
55#[derive(Serialize, Debug)]
56struct RowGroup {
57 columns: Vec<ColumnChunk>,
58 row_count: i64,
59}
60
61#[derive(Serialize, Debug)]
62struct ColumnChunk {
63 path: String,
64 has_offset_index: bool,
65 has_column_index: bool,
66 has_bloom_filter: bool,
67 offset_index: Option<Index>,
68 column_index: Option<Index>,
69 bloom_filter: Option<Index>,
70 compression: DebugSerialize<CompressionCodec>,
71 encodings: Vec<DebugSerialize<Encoding>>,
72}
73
74#[derive(Debug)]
75struct DebugSerialize<T: std::fmt::Debug>(T);
76
77impl<T: std::fmt::Debug> Serialize for DebugSerialize<T> {
78 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
79 where
80 S: Serializer,
81 {
82 serializer.serialize_str(&format!("{:?}", self.0))
83 }
84}
85
86fn do_layout<C: ChunkReader>(reader: &C) -> Result<ParquetFile> {
87 let mut metadata_reader = ParquetMetaDataReader::new();
88 metadata_reader.try_parse(reader)?;
89 let metadata_size = metadata_reader.metadata_size();
90 let metadata = metadata_reader.finish()?;
91 let schema = metadata.file_metadata().schema_descr();
92
93 let row_groups = (0..metadata.num_row_groups())
94 .map(|row_group_idx| {
95 let row_group = metadata.row_group(row_group_idx);
96 let columns = row_group
97 .columns()
98 .iter()
99 .zip(schema.columns())
100 .map(|(column, column_schema)| {
101 let compression = DebugSerialize(column.compression_codec());
102 let encodings = column.encodings().map(DebugSerialize).collect();
103
104 Ok(ColumnChunk {
105 path: column_schema.path().parts().join("."),
106 has_offset_index: column.offset_index_offset().is_some(),
107 has_column_index: column.column_index_offset().is_some(),
108 has_bloom_filter: column.bloom_filter_offset().is_some(),
109 offset_index: column.offset_index_offset().map(|offset| Index {
110 offset,
111 length: column.offset_index_length(),
112 }),
113 column_index: column.column_index_offset().map(|offset| Index {
114 offset,
115 length: column.column_index_length(),
116 }),
117 bloom_filter: column.bloom_filter_offset().map(|offset| Index {
118 offset,
119 length: column.bloom_filter_length(),
120 }),
121 compression,
122 encodings,
123 })
124 })
125 .collect::<Result<Vec<_>>>()?;
126
127 Ok(RowGroup {
128 columns,
129 row_count: row_group.num_rows(),
130 })
131 })
132 .collect::<Result<Vec<_>>>()?;
133
134 Ok(ParquetFile {
135 row_groups,
136 footer: Footer { metadata_size },
137 })
138}
139
140#[derive(Debug, Parser)]
141#[clap(author, version, about("Prints the physical layout of a parquet file"), long_about = None)]
142struct Args {
143 #[clap(help("Path to a parquet file"))]
144 file: String,
145}
146
147impl Args {
148 fn run(&self) -> Result<()> {
149 let file = File::open(&self.file)?;
150 let layout = do_layout(&file)?;
151
152 let out = std::io::stdout();
153 let writer = out.lock();
154
155 serde_json::to_writer_pretty(writer, &layout).unwrap();
156 Ok(())
157 }
158}
159
160fn main() -> Result<()> {
161 Args::parse().run()
162}