Skip to main content

parquet/file/metadata/
memory.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Memory calculations for [`ParquetMetadata::memory_size`]
6//!
7//! [`ParquetMetadata::memory_size`]: crate::file::metadata::ParquetMetaData::memory_size
8use crate::basic::{BoundaryOrder, ColumnOrder, CompressionCodec, Encoding, PageType};
9use crate::data_type::private::ParquetValueType;
10use crate::file::metadata::{
11    ColumnChunkMetaData, FileMetaData, KeyValue, PageEncodingStats, ParquetPageEncodingStats,
12    RowGroupMetaData, SortingColumn,
13};
14use crate::file::page_index::column_index::{
15    ByteArrayColumnIndex, ColumnIndex, ColumnIndexMetaData, PrimitiveColumnIndex,
16};
17use crate::file::page_index::offset_index::{OffsetIndexMetaData, PageLocation};
18use crate::file::statistics::{Statistics, ValueStatistics};
19use std::collections::HashMap;
20use std::sync::Arc;
21
22/// Trait for calculating the size of various containers
23pub trait HeapSize {
24    /// Return the size of any bytes allocated on the heap by this object,
25    /// including heap memory in those structures
26    ///
27    /// Note that the size of the type itself is not included in the result --
28    /// instead, that size is added by the caller (e.g. container).
29    fn heap_size(&self) -> usize;
30}
31
32impl<T: HeapSize> HeapSize for Vec<T> {
33    fn heap_size(&self) -> usize {
34        let item_size = std::mem::size_of::<T>();
35        // account for the contents of the Vec
36        (self.capacity() * item_size) +
37        // add any heap allocations by contents
38        self.iter().map(|t| t.heap_size()).sum::<usize>()
39    }
40}
41
42impl<K: HeapSize, V: HeapSize> HeapSize for HashMap<K, V> {
43    fn heap_size(&self) -> usize {
44        let capacity = self.capacity();
45        if capacity == 0 {
46            return 0;
47        }
48
49        // HashMap doesn't provide a way to get its heap size, so this is an approximation based on
50        // the behavior of hashbrown::HashMap as at version 0.16.0, and may become inaccurate
51        // if the implementation changes.
52        let key_val_size = std::mem::size_of::<(K, V)>();
53        // Overhead for the control tags group, which may be smaller depending on architecture
54        let group_size = 16;
55        // 1 byte of metadata stored per bucket.
56        let metadata_size = 1;
57
58        // Compute the number of buckets for the capacity. Based on hashbrown's capacity_to_buckets
59        let buckets = if capacity < 15 {
60            let min_cap = match key_val_size {
61                0..=1 => 14,
62                2..=3 => 7,
63                _ => 3,
64            };
65            let cap = min_cap.max(capacity);
66            if cap < 4 {
67                4
68            } else if cap < 8 {
69                8
70            } else {
71                16
72            }
73        } else {
74            (capacity.saturating_mul(8) / 7).next_power_of_two()
75        };
76
77        group_size
78            + (buckets * (key_val_size + metadata_size))
79            + self.keys().map(|k| k.heap_size()).sum::<usize>()
80            + self.values().map(|v| v.heap_size()).sum::<usize>()
81    }
82}
83
84impl<T: HeapSize> HeapSize for Arc<T> {
85    fn heap_size(&self) -> usize {
86        // Arc stores weak and strong counts on the heap alongside an instance of T
87        2 * std::mem::size_of::<usize>() + std::mem::size_of::<T>() + self.as_ref().heap_size()
88    }
89}
90
91impl HeapSize for Arc<dyn HeapSize> {
92    fn heap_size(&self) -> usize {
93        2 * std::mem::size_of::<usize>()
94            + std::mem::size_of_val(self.as_ref())
95            + self.as_ref().heap_size()
96    }
97}
98
99impl<T: HeapSize> HeapSize for Box<T> {
100    fn heap_size(&self) -> usize {
101        std::mem::size_of::<T>() + self.as_ref().heap_size()
102    }
103}
104
105impl<T: HeapSize> HeapSize for Option<T> {
106    fn heap_size(&self) -> usize {
107        self.as_ref().map(|inner| inner.heap_size()).unwrap_or(0)
108    }
109}
110
111impl HeapSize for String {
112    fn heap_size(&self) -> usize {
113        self.capacity()
114    }
115}
116
117impl HeapSize for FileMetaData {
118    fn heap_size(&self) -> usize {
119        #[cfg(feature = "encryption")]
120        let encryption_heap_size =
121            self.encryption_algorithm.heap_size() + self.footer_signing_key_metadata.heap_size();
122        #[cfg(not(feature = "encryption"))]
123        let encryption_heap_size = 0;
124
125        self.created_by.heap_size()
126            + self.key_value_metadata.heap_size()
127            + self.schema_descr.heap_size()
128            + self.column_orders.heap_size()
129            + encryption_heap_size
130    }
131}
132
133impl HeapSize for KeyValue {
134    fn heap_size(&self) -> usize {
135        self.key.heap_size() + self.value.heap_size()
136    }
137}
138
139impl HeapSize for RowGroupMetaData {
140    fn heap_size(&self) -> usize {
141        // don't count schema_descr here because it is already
142        // counted in FileMetaData
143        self.columns.heap_size() + self.sorting_columns.heap_size()
144    }
145}
146
147impl HeapSize for ColumnChunkMetaData {
148    fn heap_size(&self) -> usize {
149        #[cfg(feature = "encryption")]
150        let encryption_heap_size =
151            self.column_crypto_metadata.heap_size() + self.encrypted_column_metadata.heap_size();
152        #[cfg(not(feature = "encryption"))]
153        let encryption_heap_size = 0;
154
155        // don't count column_descr here because it is already counted in
156        // FileMetaData
157        self.encodings.heap_size()
158            + self.file_path.heap_size()
159            + self.compression.heap_size()
160            + self.statistics.heap_size()
161            + self.encoding_stats.heap_size()
162            + self.unencoded_byte_array_data_bytes.heap_size()
163            + self.repetition_level_histogram.heap_size()
164            + self.definition_level_histogram.heap_size()
165            + self.geo_statistics.heap_size()
166            + encryption_heap_size
167    }
168}
169
170impl HeapSize for Encoding {
171    fn heap_size(&self) -> usize {
172        0 // no heap allocations
173    }
174}
175
176impl HeapSize for ParquetPageEncodingStats {
177    fn heap_size(&self) -> usize {
178        match self {
179            Self::Full(v) => v.heap_size(),
180            Self::Mask(_) => 0,
181        }
182    }
183}
184
185impl HeapSize for PageEncodingStats {
186    fn heap_size(&self) -> usize {
187        self.page_type.heap_size() + self.encoding.heap_size()
188    }
189}
190
191impl HeapSize for SortingColumn {
192    fn heap_size(&self) -> usize {
193        0 // no heap allocations
194    }
195}
196impl HeapSize for CompressionCodec {
197    fn heap_size(&self) -> usize {
198        0 // no heap allocations
199    }
200}
201
202impl HeapSize for PageType {
203    fn heap_size(&self) -> usize {
204        0 // no heap allocations
205    }
206}
207
208impl HeapSize for Statistics {
209    fn heap_size(&self) -> usize {
210        match self {
211            Statistics::Boolean(value_statistics) => value_statistics.heap_size(),
212            Statistics::Int32(value_statistics) => value_statistics.heap_size(),
213            Statistics::Int64(value_statistics) => value_statistics.heap_size(),
214            Statistics::Int96(value_statistics) => value_statistics.heap_size(),
215            Statistics::Float(value_statistics) => value_statistics.heap_size(),
216            Statistics::Double(value_statistics) => value_statistics.heap_size(),
217            Statistics::ByteArray(value_statistics) => value_statistics.heap_size(),
218            Statistics::FixedLenByteArray(value_statistics) => value_statistics.heap_size(),
219        }
220    }
221}
222
223impl HeapSize for OffsetIndexMetaData {
224    fn heap_size(&self) -> usize {
225        self.page_locations.heap_size() + self.unencoded_byte_array_data_bytes.heap_size()
226    }
227}
228
229impl HeapSize for ColumnIndexMetaData {
230    fn heap_size(&self) -> usize {
231        match self {
232            Self::BOOLEAN(native_index) => native_index.heap_size(),
233            Self::INT32(native_index) => native_index.heap_size(),
234            Self::INT64(native_index) => native_index.heap_size(),
235            Self::INT96(native_index) => native_index.heap_size(),
236            Self::FLOAT(native_index) => native_index.heap_size(),
237            Self::DOUBLE(native_index) => native_index.heap_size(),
238            Self::BYTE_ARRAY(native_index) => native_index.heap_size(),
239            Self::FIXED_LEN_BYTE_ARRAY(native_index) => native_index.heap_size(),
240        }
241    }
242}
243
244impl HeapSize for ColumnIndex {
245    fn heap_size(&self) -> usize {
246        self.null_pages.heap_size()
247            + self.boundary_order.heap_size()
248            + self.null_counts.heap_size()
249            + self.definition_level_histograms.heap_size()
250            + self.repetition_level_histograms.heap_size()
251    }
252}
253
254impl<T: ParquetValueType> HeapSize for PrimitiveColumnIndex<T> {
255    fn heap_size(&self) -> usize {
256        self.column_index.heap_size() + self.min_values.heap_size() + self.max_values.heap_size()
257    }
258}
259
260impl HeapSize for ByteArrayColumnIndex {
261    fn heap_size(&self) -> usize {
262        self.column_index.heap_size()
263            + self.min_bytes.heap_size()
264            + self.min_offsets.heap_size()
265            + self.max_bytes.heap_size()
266            + self.max_offsets.heap_size()
267    }
268}
269
270impl<T: ParquetValueType> HeapSize for ValueStatistics<T> {
271    fn heap_size(&self) -> usize {
272        self.min_opt().map(T::heap_size).unwrap_or(0)
273            + self.max_opt().map(T::heap_size).unwrap_or(0)
274    }
275}
276impl HeapSize for bool {
277    fn heap_size(&self) -> usize {
278        0 // no heap allocations
279    }
280}
281impl HeapSize for u8 {
282    fn heap_size(&self) -> usize {
283        0 // no heap allocations
284    }
285}
286impl HeapSize for i32 {
287    fn heap_size(&self) -> usize {
288        0 // no heap allocations
289    }
290}
291impl HeapSize for i64 {
292    fn heap_size(&self) -> usize {
293        0 // no heap allocations
294    }
295}
296
297impl HeapSize for f32 {
298    fn heap_size(&self) -> usize {
299        0 // no heap allocations
300    }
301}
302impl HeapSize for f64 {
303    fn heap_size(&self) -> usize {
304        0 // no heap allocations
305    }
306}
307
308impl HeapSize for usize {
309    fn heap_size(&self) -> usize {
310        0 // no heap allocations
311    }
312}
313
314impl HeapSize for BoundaryOrder {
315    fn heap_size(&self) -> usize {
316        0 // no heap allocations
317    }
318}
319
320impl HeapSize for PageLocation {
321    fn heap_size(&self) -> usize {
322        0 // no heap allocations
323    }
324}
325
326impl HeapSize for ColumnOrder {
327    fn heap_size(&self) -> usize {
328        0 // no heap allocations in ColumnOrder
329    }
330}