Skip to main content

parquet/column/
page_store.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Pluggable storage for completed, serialized page blobs.
6//!
7//! While a row group is being written the [`ArrowWriter`] must buffer every
8//! column's encoded pages, because Parquet requires each column chunk to be
9//! contiguous in the file while record batches arrive with all columns interleaved.
10//! By default that buffer lives on the heap, so the writer's peak memory grows
11//! with the row group size. A [`PageStore`] lets the buffer live somewhere else
12//! — a local temp file, object storage, etc. — bounding peak write memory
13//! independently of the row group size.
14//!
15//! [`ArrowWriter`]: crate::arrow::arrow_writer::ArrowWriter
16
17use std::fmt::Debug;
18
19use bytes::Bytes;
20
21use crate::errors::{ParquetError, Result};
22use crate::schema::types::ColumnDescriptor;
23
24/// An opaque, store-allocated handle to a blob held by a [`PageStore`].
25///
26/// Handles are allocated by the store — densely and sequentially — and are only
27/// meaningful to the store that produced them. The caller treats them as opaque
28/// tokens.
29#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
30pub struct PageKey(u64);
31
32impl PageKey {
33    /// Create a handle wrapping `raw`.
34    ///
35    /// A [`PageStore`] implementation calls this to mint the handle it returns
36    /// from [`put`](PageStore::put). The value is opaque to the caller, so a
37    /// store is free to use a dense counter, a packed locator, or anything else
38    /// it can later resolve in [`take`](PageStore::take).
39    pub const fn new(raw: u64) -> Self {
40        Self(raw)
41    }
42
43    /// The raw value passed to [`new`](Self::new).
44    pub const fn get(self) -> u64 {
45        self.0
46    }
47}
48
49/// A pluggable store for completed, serialized page blobs.
50///
51/// The store is intentionally "dumb": it only maps an opaque [`PageKey`] to a
52/// blob of bytes. It knows nothing about pages, dictionaries, ordering, or
53/// offsets. The caller keeps the handles it gets back from [`put`](Self::put)
54/// and decides what they mean.
55///
56/// Each store instance is owned by a single column writer and mutated by one
57/// thread at a time (both methods take `&mut self`), so it needs no internal
58/// synchronization — hence only `Send`, not `Sync`.
59///
60/// The default ([`InMemoryPageStore`]) keeps blobs in memory on the heap.
61///
62/// For an example of configuring the Parquet writer to use an alternate
63/// `PageStore` see the [`ArrowWriterOptions::with_page_store_factory`] API.
64///
65/// [`ArrowWriterOptions::with_page_store_factory`]: crate::arrow::arrow_writer::ArrowWriterOptions::with_page_store_factory
66pub trait PageStore: Send {
67    /// Store `value`, returning a handle that can later be passed to
68    /// [`take`](Self::take).
69    fn put(&mut self, value: Bytes) -> Result<PageKey>;
70
71    /// Take back the blob previously stored under `key`.
72    ///
73    /// The caller takes ownership of the returned bytes and will **not** request
74    /// `key` again, so the store may release any resources backing it — eagerly
75    /// here, or when the store is dropped.
76    fn take(&mut self, key: PageKey) -> Result<Bytes>;
77
78    /// The number of bytes this store currently holds **in memory** (resident
79    /// on the heap), used to report the writer's memory footprint.
80    ///
81    /// The default is `0`, which is exactly right for a backend that moves
82    /// every blob off-heap (a temp file, object storage): the bytes it has been
83    /// handed no longer occupy heap. The in-memory backend overrides this to
84    /// report its resident blobs. A backend that keeps a partial in-memory
85    /// buffer should report that buffer's size.
86    fn memory_size(&self) -> usize {
87        0
88    }
89}
90
91/// Context for a single [`PageStoreFactory::create`] call.
92///
93/// Describes the leaf column chunk the store will buffer. It is held by
94/// reference for the duration of the call; a backend reads only what it needs.
95/// More fields may be added in future releases without breaking existing
96/// implementations — the type is constructed only by the writer, so an
97/// implementer only ever receives one and calls its accessors.
98pub struct PageStoreArgs<'a> {
99    column_index: usize,
100    column_descriptor: &'a ColumnDescriptor,
101}
102
103impl PageStoreArgs<'_> {
104    // Constructed only by the Arrow writer; without that feature there is no caller.
105    #[cfg(feature = "arrow")]
106    pub(crate) fn new(
107        column_index: usize,
108        column_descriptor: &ColumnDescriptor,
109    ) -> PageStoreArgs<'_> {
110        PageStoreArgs {
111            column_index,
112            column_descriptor,
113        }
114    }
115
116    /// Index of the leaf column within the row group.
117    ///
118    /// A backend may use this to e.g. name spill files or shard across a bounded
119    /// pool; it carries no ordering or coordination requirement.
120    pub fn column_index(&self) -> usize {
121        self.column_index
122    }
123
124    /// Descriptor for the leaf column: physical/logical type, path, and max
125    /// definition/repetition levels.
126    ///
127    /// Lets a backend tailor buffering to the column — for example spilling only
128    /// large `BYTE_ARRAY` columns while keeping small fixed-width ones on the
129    /// heap.
130    pub fn column_descriptor(&self) -> &ColumnDescriptor {
131        self.column_descriptor
132    }
133}
134
135/// Creates a fresh [`PageStore`] for each column chunk.
136///
137/// See
138/// [`ArrowWriterOptions::with_page_store_factory`](crate::arrow::arrow_writer::ArrowWriterOptions::with_page_store_factory).
139pub trait PageStoreFactory: Send + Sync + Debug {
140    /// Create a new, empty [`PageStore`] for the leaf column described by `args`.
141    fn create(&self, args: &PageStoreArgs<'_>) -> Result<Box<dyn PageStore>>;
142}
143
144/// The default [`PageStore`], holding blobs on the heap in a `Vec<Bytes>`.
145///
146/// Peak memory grows with the row group size; use a spilling backend to bound
147/// it.
148#[derive(Debug, Default)]
149pub struct InMemoryPageStore {
150    blobs: Vec<Bytes>,
151    /// Running total of resident blob bytes, kept in step with `put`/`take`.
152    resident: usize,
153}
154
155impl PageStore for InMemoryPageStore {
156    fn put(&mut self, value: Bytes) -> Result<PageKey> {
157        let key = PageKey(self.blobs.len() as u64);
158        self.resident += value.len();
159        self.blobs.push(value);
160        Ok(key)
161    }
162
163    fn take(&mut self, key: PageKey) -> Result<Bytes> {
164        // Replace the slot with an empty `Bytes` so the stored blob is released
165        // as soon as it is taken, keeping memory bounded while the chunk is
166        // streamed into the output file.
167        let blob = self
168            .blobs
169            .get_mut(key.0 as usize)
170            .map(std::mem::take)
171            .ok_or_else(|| ParquetError::General(format!("invalid page key {}", key.0)))?;
172        self.resident -= blob.len();
173        Ok(blob)
174    }
175
176    fn memory_size(&self) -> usize {
177        self.resident
178    }
179}
180
181/// Factory for [`InMemoryPageStore`] — the default used by
182/// [`ArrowWriter`](crate::arrow::arrow_writer::ArrowWriter).
183#[derive(Debug, Default)]
184pub struct InMemoryPageStoreFactory;
185
186impl PageStoreFactory for InMemoryPageStoreFactory {
187    fn create(&self, _args: &PageStoreArgs<'_>) -> Result<Box<dyn PageStore>> {
188        Ok(Box::new(InMemoryPageStore::default()))
189    }
190}
191
192#[cfg(test)]
193mod tests {
194    use super::*;
195
196    #[test]
197    fn in_memory_round_trips_blobs_in_handle_order() {
198        let mut store = InMemoryPageStore::default();
199        let k0 = store.put(Bytes::from_static(b"hello")).unwrap();
200        let k1 = store.put(Bytes::from_static(b"world")).unwrap();
201        assert_ne!(k0, k1);
202        assert_eq!(&store.take(k0).unwrap()[..], b"hello");
203        assert_eq!(&store.take(k1).unwrap()[..], b"world");
204    }
205
206    #[test]
207    fn in_memory_take_releases_the_slot() {
208        let mut store = InMemoryPageStore::default();
209        let k = store.put(Bytes::from_static(b"abc")).unwrap();
210        assert_eq!(&store.take(k).unwrap()[..], b"abc");
211        // A second take yields the emptied placeholder rather than the blob,
212        // confirming the bytes were released on the first take.
213        assert!(store.take(k).unwrap().is_empty());
214    }
215
216    #[test]
217    fn in_memory_invalid_key_errors() {
218        let mut store = InMemoryPageStore::default();
219        assert!(store.take(PageKey(99)).is_err());
220    }
221
222    #[test]
223    fn in_memory_reports_resident_bytes() {
224        let mut store = InMemoryPageStore::default();
225        assert_eq!(store.memory_size(), 0);
226        let k0 = store.put(Bytes::from_static(b"hello")).unwrap();
227        let k1 = store.put(Bytes::from_static(b"!")).unwrap();
228        assert_eq!(store.memory_size(), 6);
229        store.take(k0).unwrap();
230        assert_eq!(store.memory_size(), 1);
231        store.take(k1).unwrap();
232        assert_eq!(store.memory_size(), 0);
233    }
234
235    #[test]
236    fn default_store_memory_size_is_zero() {
237        // A spilling backend that does not override `memory_size` reports 0,
238        // reflecting that its blobs no longer occupy the heap.
239        struct OffHeap;
240        impl PageStore for OffHeap {
241            fn put(&mut self, _value: Bytes) -> Result<PageKey> {
242                Ok(PageKey::new(0))
243            }
244            fn take(&mut self, _key: PageKey) -> Result<Bytes> {
245                Ok(Bytes::new())
246            }
247        }
248        assert_eq!(OffHeap.memory_size(), 0);
249    }
250}