parquet/file/metadata/
memory.rs1use crate::basic::{BoundaryOrder, ColumnOrder, CompressionCodec, Encoding, PageType};
9use crate::data_type::private::ParquetValueType;
10use crate::file::metadata::{
11 ColumnChunkMetaData, FileMetaData, KeyValue, PageEncodingStats, ParquetPageEncodingStats,
12 RowGroupMetaData, SortingColumn,
13};
14use crate::file::page_index::column_index::{
15 ByteArrayColumnIndex, ColumnIndex, ColumnIndexMetaData, PrimitiveColumnIndex,
16};
17use crate::file::page_index::offset_index::{OffsetIndexMetaData, PageLocation};
18use crate::file::statistics::{Statistics, ValueStatistics};
19use std::collections::HashMap;
20use std::sync::Arc;
21
22pub trait HeapSize {
24 fn heap_size(&self) -> usize;
30}
31
32impl<T: HeapSize> HeapSize for Vec<T> {
33 fn heap_size(&self) -> usize {
34 let item_size = std::mem::size_of::<T>();
35 (self.capacity() * item_size) +
37 self.iter().map(|t| t.heap_size()).sum::<usize>()
39 }
40}
41
42impl<K: HeapSize, V: HeapSize> HeapSize for HashMap<K, V> {
43 fn heap_size(&self) -> usize {
44 let capacity = self.capacity();
45 if capacity == 0 {
46 return 0;
47 }
48
49 let key_val_size = std::mem::size_of::<(K, V)>();
53 let group_size = 16;
55 let metadata_size = 1;
57
58 let buckets = if capacity < 15 {
60 let min_cap = match key_val_size {
61 0..=1 => 14,
62 2..=3 => 7,
63 _ => 3,
64 };
65 let cap = min_cap.max(capacity);
66 if cap < 4 {
67 4
68 } else if cap < 8 {
69 8
70 } else {
71 16
72 }
73 } else {
74 (capacity.saturating_mul(8) / 7).next_power_of_two()
75 };
76
77 group_size
78 + (buckets * (key_val_size + metadata_size))
79 + self.keys().map(|k| k.heap_size()).sum::<usize>()
80 + self.values().map(|v| v.heap_size()).sum::<usize>()
81 }
82}
83
84impl<T: HeapSize> HeapSize for Arc<T> {
85 fn heap_size(&self) -> usize {
86 2 * std::mem::size_of::<usize>() + std::mem::size_of::<T>() + self.as_ref().heap_size()
88 }
89}
90
91impl HeapSize for Arc<dyn HeapSize> {
92 fn heap_size(&self) -> usize {
93 2 * std::mem::size_of::<usize>()
94 + std::mem::size_of_val(self.as_ref())
95 + self.as_ref().heap_size()
96 }
97}
98
99impl<T: HeapSize> HeapSize for Box<T> {
100 fn heap_size(&self) -> usize {
101 std::mem::size_of::<T>() + self.as_ref().heap_size()
102 }
103}
104
105impl<T: HeapSize> HeapSize for Option<T> {
106 fn heap_size(&self) -> usize {
107 self.as_ref().map(|inner| inner.heap_size()).unwrap_or(0)
108 }
109}
110
111impl HeapSize for String {
112 fn heap_size(&self) -> usize {
113 self.capacity()
114 }
115}
116
117impl HeapSize for FileMetaData {
118 fn heap_size(&self) -> usize {
119 #[cfg(feature = "encryption")]
120 let encryption_heap_size =
121 self.encryption_algorithm.heap_size() + self.footer_signing_key_metadata.heap_size();
122 #[cfg(not(feature = "encryption"))]
123 let encryption_heap_size = 0;
124
125 self.created_by.heap_size()
126 + self.key_value_metadata.heap_size()
127 + self.schema_descr.heap_size()
128 + self.column_orders.heap_size()
129 + encryption_heap_size
130 }
131}
132
133impl HeapSize for KeyValue {
134 fn heap_size(&self) -> usize {
135 self.key.heap_size() + self.value.heap_size()
136 }
137}
138
139impl HeapSize for RowGroupMetaData {
140 fn heap_size(&self) -> usize {
141 self.columns.heap_size() + self.sorting_columns.heap_size()
144 }
145}
146
147impl HeapSize for ColumnChunkMetaData {
148 fn heap_size(&self) -> usize {
149 #[cfg(feature = "encryption")]
150 let encryption_heap_size =
151 self.column_crypto_metadata.heap_size() + self.encrypted_column_metadata.heap_size();
152 #[cfg(not(feature = "encryption"))]
153 let encryption_heap_size = 0;
154
155 self.encodings.heap_size()
158 + self.file_path.heap_size()
159 + self.compression.heap_size()
160 + self.statistics.heap_size()
161 + self.encoding_stats.heap_size()
162 + self.unencoded_byte_array_data_bytes.heap_size()
163 + self.repetition_level_histogram.heap_size()
164 + self.definition_level_histogram.heap_size()
165 + self.geo_statistics.heap_size()
166 + encryption_heap_size
167 }
168}
169
170impl HeapSize for Encoding {
171 fn heap_size(&self) -> usize {
172 0 }
174}
175
176impl HeapSize for ParquetPageEncodingStats {
177 fn heap_size(&self) -> usize {
178 match self {
179 Self::Full(v) => v.heap_size(),
180 Self::Mask(_) => 0,
181 }
182 }
183}
184
185impl HeapSize for PageEncodingStats {
186 fn heap_size(&self) -> usize {
187 self.page_type.heap_size() + self.encoding.heap_size()
188 }
189}
190
191impl HeapSize for SortingColumn {
192 fn heap_size(&self) -> usize {
193 0 }
195}
196impl HeapSize for CompressionCodec {
197 fn heap_size(&self) -> usize {
198 0 }
200}
201
202impl HeapSize for PageType {
203 fn heap_size(&self) -> usize {
204 0 }
206}
207
208impl HeapSize for Statistics {
209 fn heap_size(&self) -> usize {
210 match self {
211 Statistics::Boolean(value_statistics) => value_statistics.heap_size(),
212 Statistics::Int32(value_statistics) => value_statistics.heap_size(),
213 Statistics::Int64(value_statistics) => value_statistics.heap_size(),
214 Statistics::Int96(value_statistics) => value_statistics.heap_size(),
215 Statistics::Float(value_statistics) => value_statistics.heap_size(),
216 Statistics::Double(value_statistics) => value_statistics.heap_size(),
217 Statistics::ByteArray(value_statistics) => value_statistics.heap_size(),
218 Statistics::FixedLenByteArray(value_statistics) => value_statistics.heap_size(),
219 }
220 }
221}
222
223impl HeapSize for OffsetIndexMetaData {
224 fn heap_size(&self) -> usize {
225 self.page_locations.heap_size() + self.unencoded_byte_array_data_bytes.heap_size()
226 }
227}
228
229impl HeapSize for ColumnIndexMetaData {
230 fn heap_size(&self) -> usize {
231 match self {
232 Self::BOOLEAN(native_index) => native_index.heap_size(),
233 Self::INT32(native_index) => native_index.heap_size(),
234 Self::INT64(native_index) => native_index.heap_size(),
235 Self::INT96(native_index) => native_index.heap_size(),
236 Self::FLOAT(native_index) => native_index.heap_size(),
237 Self::DOUBLE(native_index) => native_index.heap_size(),
238 Self::BYTE_ARRAY(native_index) => native_index.heap_size(),
239 Self::FIXED_LEN_BYTE_ARRAY(native_index) => native_index.heap_size(),
240 }
241 }
242}
243
244impl HeapSize for ColumnIndex {
245 fn heap_size(&self) -> usize {
246 self.null_pages.heap_size()
247 + self.boundary_order.heap_size()
248 + self.null_counts.heap_size()
249 + self.definition_level_histograms.heap_size()
250 + self.repetition_level_histograms.heap_size()
251 }
252}
253
254impl<T: ParquetValueType> HeapSize for PrimitiveColumnIndex<T> {
255 fn heap_size(&self) -> usize {
256 self.column_index.heap_size() + self.min_values.heap_size() + self.max_values.heap_size()
257 }
258}
259
260impl HeapSize for ByteArrayColumnIndex {
261 fn heap_size(&self) -> usize {
262 self.column_index.heap_size()
263 + self.min_bytes.heap_size()
264 + self.min_offsets.heap_size()
265 + self.max_bytes.heap_size()
266 + self.max_offsets.heap_size()
267 }
268}
269
270impl<T: ParquetValueType> HeapSize for ValueStatistics<T> {
271 fn heap_size(&self) -> usize {
272 self.min_opt().map(T::heap_size).unwrap_or(0)
273 + self.max_opt().map(T::heap_size).unwrap_or(0)
274 }
275}
276impl HeapSize for bool {
277 fn heap_size(&self) -> usize {
278 0 }
280}
281impl HeapSize for u8 {
282 fn heap_size(&self) -> usize {
283 0 }
285}
286impl HeapSize for i32 {
287 fn heap_size(&self) -> usize {
288 0 }
290}
291impl HeapSize for i64 {
292 fn heap_size(&self) -> usize {
293 0 }
295}
296
297impl HeapSize for f32 {
298 fn heap_size(&self) -> usize {
299 0 }
301}
302impl HeapSize for f64 {
303 fn heap_size(&self) -> usize {
304 0 }
306}
307
308impl HeapSize for usize {
309 fn heap_size(&self) -> usize {
310 0 }
312}
313
314impl HeapSize for BoundaryOrder {
315 fn heap_size(&self) -> usize {
316 0 }
318}
319
320impl HeapSize for PageLocation {
321 fn heap_size(&self) -> usize {
322 0 }
324}
325
326impl HeapSize for ColumnOrder {
327 fn heap_size(&self) -> usize {
328 0 }
330}