parquet/column/page_store.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Pluggable storage for completed, serialized page blobs.
6//!
7//! While a row group is being written the [`ArrowWriter`] must buffer every
8//! column's encoded pages, because Parquet requires each column chunk to be
9//! contiguous in the file while record batches arrive with all columns interleaved.
10//! By default that buffer lives on the heap, so the writer's peak memory grows
11//! with the row group size. A [`PageStore`] lets the buffer live somewhere else
12//! — a local temp file, object storage, etc. — bounding peak write memory
13//! independently of the row group size.
14//!
15//! [`ArrowWriter`]: crate::arrow::arrow_writer::ArrowWriter
16
17use std::fmt::Debug;
18
19use bytes::Bytes;
20
21use crate::errors::{ParquetError, Result};
22use crate::schema::types::ColumnDescriptor;
23
24/// An opaque, store-allocated handle to a blob held by a [`PageStore`].
25///
26/// Handles are allocated by the store — densely and sequentially — and are only
27/// meaningful to the store that produced them. The caller treats them as opaque
28/// tokens.
29#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
30pub struct PageKey(u64);
31
32impl PageKey {
33 /// Create a handle wrapping `raw`.
34 ///
35 /// A [`PageStore`] implementation calls this to mint the handle it returns
36 /// from [`put`](PageStore::put). The value is opaque to the caller, so a
37 /// store is free to use a dense counter, a packed locator, or anything else
38 /// it can later resolve in [`take`](PageStore::take).
39 pub const fn new(raw: u64) -> Self {
40 Self(raw)
41 }
42
43 /// The raw value passed to [`new`](Self::new).
44 pub const fn get(self) -> u64 {
45 self.0
46 }
47}
48
49/// A pluggable store for completed, serialized page blobs.
50///
51/// The store is intentionally "dumb": it only maps an opaque [`PageKey`] to a
52/// blob of bytes. It knows nothing about pages, dictionaries, ordering, or
53/// offsets. The caller keeps the handles it gets back from [`put`](Self::put)
54/// and decides what they mean.
55///
56/// Each store instance is owned by a single column writer and mutated by one
57/// thread at a time (both methods take `&mut self`), so it needs no internal
58/// synchronization — hence only `Send`, not `Sync`.
59///
60/// The default ([`InMemoryPageStore`]) keeps blobs in memory on the heap.
61///
62/// For an example of configuring the Parquet writer to use an alternate
63/// `PageStore` see the [`ArrowWriterOptions::with_page_store_factory`] API.
64///
65/// [`ArrowWriterOptions::with_page_store_factory`]: crate::arrow::arrow_writer::ArrowWriterOptions::with_page_store_factory
66pub trait PageStore: Send {
67 /// Store `value`, returning a handle that can later be passed to
68 /// [`take`](Self::take).
69 fn put(&mut self, value: Bytes) -> Result<PageKey>;
70
71 /// Take back the blob previously stored under `key`.
72 ///
73 /// The caller takes ownership of the returned bytes and will **not** request
74 /// `key` again, so the store may release any resources backing it — eagerly
75 /// here, or when the store is dropped.
76 fn take(&mut self, key: PageKey) -> Result<Bytes>;
77
78 /// The number of bytes this store currently holds **in memory** (resident
79 /// on the heap), used to report the writer's memory footprint.
80 ///
81 /// The default is `0`, which is exactly right for a backend that moves
82 /// every blob off-heap (a temp file, object storage): the bytes it has been
83 /// handed no longer occupy heap. The in-memory backend overrides this to
84 /// report its resident blobs. A backend that keeps a partial in-memory
85 /// buffer should report that buffer's size.
86 fn memory_size(&self) -> usize {
87 0
88 }
89}
90
91/// Context for a single [`PageStoreFactory::create`] call.
92///
93/// Describes the leaf column chunk the store will buffer. It is held by
94/// reference for the duration of the call; a backend reads only what it needs.
95/// More fields may be added in future releases without breaking existing
96/// implementations — the type is constructed only by the writer, so an
97/// implementer only ever receives one and calls its accessors.
98pub struct PageStoreArgs<'a> {
99 column_index: usize,
100 column_descriptor: &'a ColumnDescriptor,
101}
102
103impl PageStoreArgs<'_> {
104 // Constructed only by the Arrow writer; without that feature there is no caller.
105 #[cfg(feature = "arrow")]
106 pub(crate) fn new(
107 column_index: usize,
108 column_descriptor: &ColumnDescriptor,
109 ) -> PageStoreArgs<'_> {
110 PageStoreArgs {
111 column_index,
112 column_descriptor,
113 }
114 }
115
116 /// Index of the leaf column within the row group.
117 ///
118 /// A backend may use this to e.g. name spill files or shard across a bounded
119 /// pool; it carries no ordering or coordination requirement.
120 pub fn column_index(&self) -> usize {
121 self.column_index
122 }
123
124 /// Descriptor for the leaf column: physical/logical type, path, and max
125 /// definition/repetition levels.
126 ///
127 /// Lets a backend tailor buffering to the column — for example spilling only
128 /// large `BYTE_ARRAY` columns while keeping small fixed-width ones on the
129 /// heap.
130 pub fn column_descriptor(&self) -> &ColumnDescriptor {
131 self.column_descriptor
132 }
133}
134
135/// Creates a fresh [`PageStore`] for each column chunk.
136///
137/// See
138/// [`ArrowWriterOptions::with_page_store_factory`](crate::arrow::arrow_writer::ArrowWriterOptions::with_page_store_factory).
139pub trait PageStoreFactory: Send + Sync + Debug {
140 /// Create a new, empty [`PageStore`] for the leaf column described by `args`.
141 fn create(&self, args: &PageStoreArgs<'_>) -> Result<Box<dyn PageStore>>;
142}
143
144/// The default [`PageStore`], holding blobs on the heap in a `Vec<Bytes>`.
145///
146/// Peak memory grows with the row group size; use a spilling backend to bound
147/// it.
148#[derive(Debug, Default)]
149pub struct InMemoryPageStore {
150 blobs: Vec<Bytes>,
151 /// Running total of resident blob bytes, kept in step with `put`/`take`.
152 resident: usize,
153}
154
155impl PageStore for InMemoryPageStore {
156 fn put(&mut self, value: Bytes) -> Result<PageKey> {
157 let key = PageKey(self.blobs.len() as u64);
158 self.resident += value.len();
159 self.blobs.push(value);
160 Ok(key)
161 }
162
163 fn take(&mut self, key: PageKey) -> Result<Bytes> {
164 // Replace the slot with an empty `Bytes` so the stored blob is released
165 // as soon as it is taken, keeping memory bounded while the chunk is
166 // streamed into the output file.
167 let blob = self
168 .blobs
169 .get_mut(key.0 as usize)
170 .map(std::mem::take)
171 .ok_or_else(|| ParquetError::General(format!("invalid page key {}", key.0)))?;
172 self.resident -= blob.len();
173 Ok(blob)
174 }
175
176 fn memory_size(&self) -> usize {
177 self.resident
178 }
179}
180
181/// Factory for [`InMemoryPageStore`] — the default used by
182/// [`ArrowWriter`](crate::arrow::arrow_writer::ArrowWriter).
183#[derive(Debug, Default)]
184pub struct InMemoryPageStoreFactory;
185
186impl PageStoreFactory for InMemoryPageStoreFactory {
187 fn create(&self, _args: &PageStoreArgs<'_>) -> Result<Box<dyn PageStore>> {
188 Ok(Box::new(InMemoryPageStore::default()))
189 }
190}
191
192#[cfg(test)]
193mod tests {
194 use super::*;
195
196 #[test]
197 fn in_memory_round_trips_blobs_in_handle_order() {
198 let mut store = InMemoryPageStore::default();
199 let k0 = store.put(Bytes::from_static(b"hello")).unwrap();
200 let k1 = store.put(Bytes::from_static(b"world")).unwrap();
201 assert_ne!(k0, k1);
202 assert_eq!(&store.take(k0).unwrap()[..], b"hello");
203 assert_eq!(&store.take(k1).unwrap()[..], b"world");
204 }
205
206 #[test]
207 fn in_memory_take_releases_the_slot() {
208 let mut store = InMemoryPageStore::default();
209 let k = store.put(Bytes::from_static(b"abc")).unwrap();
210 assert_eq!(&store.take(k).unwrap()[..], b"abc");
211 // A second take yields the emptied placeholder rather than the blob,
212 // confirming the bytes were released on the first take.
213 assert!(store.take(k).unwrap().is_empty());
214 }
215
216 #[test]
217 fn in_memory_invalid_key_errors() {
218 let mut store = InMemoryPageStore::default();
219 assert!(store.take(PageKey(99)).is_err());
220 }
221
222 #[test]
223 fn in_memory_reports_resident_bytes() {
224 let mut store = InMemoryPageStore::default();
225 assert_eq!(store.memory_size(), 0);
226 let k0 = store.put(Bytes::from_static(b"hello")).unwrap();
227 let k1 = store.put(Bytes::from_static(b"!")).unwrap();
228 assert_eq!(store.memory_size(), 6);
229 store.take(k0).unwrap();
230 assert_eq!(store.memory_size(), 1);
231 store.take(k1).unwrap();
232 assert_eq!(store.memory_size(), 0);
233 }
234
235 #[test]
236 fn default_store_memory_size_is_zero() {
237 // A spilling backend that does not override `memory_size` reports 0,
238 // reflecting that its blobs no longer occupy the heap.
239 struct OffHeap;
240 impl PageStore for OffHeap {
241 fn put(&mut self, _value: Bytes) -> Result<PageKey> {
242 Ok(PageKey::new(0))
243 }
244 fn take(&mut self, _key: PageKey) -> Result<Bytes> {
245 Ok(Bytes::new())
246 }
247 }
248 assert_eq!(OffHeap.memory_size(), 0);
249 }
250}