Skip to main content

parquet/column/
page.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Contains Parquet Page definitions and page reader interface.
6
7use bytes::Bytes;
8
9use crate::basic::{Encoding, PageType};
10use crate::errors::{ParquetError, Result};
11use crate::file::metadata::thrift::{
12    DataPageHeader, DataPageHeaderV2, DictionaryPageHeader, PageHeader,
13};
14use crate::file::statistics::{Statistics, page_stats_to_thrift};
15
16/// Parquet Page definition.
17///
18/// List of supported pages.
19/// These are 1-to-1 mapped from the equivalent Thrift definitions, except `buf` which
20/// used to store uncompressed bytes of the page.
21#[derive(Clone, Debug)]
22pub enum Page {
23    /// Data page Parquet format v1.
24    DataPage {
25        /// The underlying data buffer
26        buf: Bytes,
27        /// Number of values in this page
28        num_values: u32,
29        /// Encoding for values in this page
30        encoding: Encoding,
31        /// Definition level encoding
32        def_level_encoding: Encoding,
33        /// Repetition level encoding
34        rep_level_encoding: Encoding,
35        /// Optional statistics for this page
36        statistics: Option<Statistics>,
37    },
38    /// Data page Parquet format v2.
39    DataPageV2 {
40        /// The underlying data buffer
41        buf: Bytes,
42        /// Number of values in this page
43        num_values: u32,
44        /// Encoding for values in this page
45        encoding: Encoding,
46        /// Number of null values in this page
47        num_nulls: u32,
48        /// Number of rows in this page
49        num_rows: u32,
50        /// Length of definition levels
51        def_levels_byte_len: u32,
52        /// Length of repetition levels
53        rep_levels_byte_len: u32,
54        /// Is this page compressed
55        is_compressed: bool,
56        /// Optional statistics for this page
57        statistics: Option<Statistics>,
58    },
59    /// Dictionary page.
60    DictionaryPage {
61        /// The underlying data buffer
62        buf: Bytes,
63        /// Number of values in this page
64        num_values: u32,
65        /// Encoding for values in this page
66        encoding: Encoding,
67        /// Is dictionary page sorted
68        is_sorted: bool,
69    },
70}
71
72impl Page {
73    /// Returns [`PageType`] for this page.
74    pub fn page_type(&self) -> PageType {
75        match self {
76            Page::DataPage { .. } => PageType::DATA_PAGE,
77            Page::DataPageV2 { .. } => PageType::DATA_PAGE_V2,
78            Page::DictionaryPage { .. } => PageType::DICTIONARY_PAGE,
79        }
80    }
81
82    /// Returns whether this page is any version of a data page
83    pub fn is_data_page(&self) -> bool {
84        matches!(self, Page::DataPage { .. } | Page::DataPageV2 { .. })
85    }
86
87    /// Returns whether this page is a dictionary page
88    pub fn is_dictionary_page(&self) -> bool {
89        matches!(self, Page::DictionaryPage { .. })
90    }
91
92    /// Returns internal byte buffer reference for this page.
93    pub fn buffer(&self) -> &Bytes {
94        match self {
95            Page::DataPage { buf, .. } => buf,
96            Page::DataPageV2 { buf, .. } => buf,
97            Page::DictionaryPage { buf, .. } => buf,
98        }
99    }
100
101    /// Returns number of values in this page.
102    pub fn num_values(&self) -> u32 {
103        match self {
104            Page::DataPage { num_values, .. } => *num_values,
105            Page::DataPageV2 { num_values, .. } => *num_values,
106            Page::DictionaryPage { num_values, .. } => *num_values,
107        }
108    }
109
110    /// Returns this page [`Encoding`].
111    pub fn encoding(&self) -> Encoding {
112        match self {
113            Page::DataPage { encoding, .. } => *encoding,
114            Page::DataPageV2 { encoding, .. } => *encoding,
115            Page::DictionaryPage { encoding, .. } => *encoding,
116        }
117    }
118
119    /// Returns optional [`Statistics`].
120    pub fn statistics(&self) -> Option<&Statistics> {
121        match self {
122            Page::DataPage { statistics, .. } => statistics.as_ref(),
123            Page::DataPageV2 { statistics, .. } => statistics.as_ref(),
124            Page::DictionaryPage { .. } => None,
125        }
126    }
127}
128
129/// Helper struct to represent pages with potentially compressed buffer (data page v1) or
130/// compressed and concatenated buffer (def levels + rep levels + compressed values for
131/// data page v2).
132///
133/// The difference with `Page` is that `Page` buffer is always uncompressed.
134pub struct CompressedPage {
135    compressed_page: Page,
136    uncompressed_size: usize,
137}
138
139impl CompressedPage {
140    /// Creates `CompressedPage` from a page with potentially compressed buffer and
141    /// uncompressed size.
142    pub fn new(compressed_page: Page, uncompressed_size: usize) -> Self {
143        Self {
144            compressed_page,
145            uncompressed_size,
146        }
147    }
148
149    /// Returns page type.
150    pub fn page_type(&self) -> PageType {
151        self.compressed_page.page_type()
152    }
153
154    /// Returns underlying page with potentially compressed buffer.
155    pub fn compressed_page(&self) -> &Page {
156        &self.compressed_page
157    }
158
159    /// Returns uncompressed size in bytes.
160    pub fn uncompressed_size(&self) -> usize {
161        self.uncompressed_size
162    }
163
164    /// Returns compressed size in bytes.
165    ///
166    /// Note that it is assumed that buffer is compressed, but it may not be. In this
167    /// case compressed size will be equal to uncompressed size.
168    pub fn compressed_size(&self) -> usize {
169        self.compressed_page.buffer().len()
170    }
171
172    /// Number of values in page.
173    pub fn num_values(&self) -> u32 {
174        self.compressed_page.num_values()
175    }
176
177    /// Returns encoding for values in page.
178    pub fn encoding(&self) -> Encoding {
179        self.compressed_page.encoding()
180    }
181
182    /// Returns slice of compressed buffer in the page.
183    pub fn data(&self) -> &[u8] {
184        self.compressed_page.buffer()
185    }
186
187    /// Returns the number of heap bytes this page currently holds.
188    ///
189    /// This is the page's compressed buffer (the embedded [`Bytes`]); use it to
190    /// account for a buffered page's memory footprint rather than reaching for
191    /// `data().len()` at each call site.
192    pub fn memory_usage(&self) -> usize {
193        self.compressed_page.buffer().len()
194    }
195
196    /// Returns the thrift page header
197    pub(crate) fn to_thrift_header(&self) -> Result<PageHeader> {
198        let uncompressed_size = self.uncompressed_size();
199        let compressed_size = self.compressed_size();
200        if uncompressed_size > i32::MAX as usize {
201            return Err(general_err!(
202                "Page uncompressed size overflow: {}",
203                uncompressed_size
204            ));
205        }
206        if compressed_size > i32::MAX as usize {
207            return Err(general_err!(
208                "Page compressed size overflow: {}",
209                compressed_size
210            ));
211        }
212        let num_values = self.num_values();
213        let encoding = self.encoding();
214        let page_type = self.page_type();
215
216        let mut page_header = PageHeader {
217            r#type: page_type,
218            uncompressed_page_size: uncompressed_size as i32,
219            compressed_page_size: compressed_size as i32,
220            // TODO: Add support for crc checksum
221            crc: None,
222            data_page_header: None,
223            index_page_header: None,
224            dictionary_page_header: None,
225            data_page_header_v2: None,
226        };
227
228        match self.compressed_page {
229            Page::DataPage {
230                def_level_encoding,
231                rep_level_encoding,
232                ref statistics,
233                ..
234            } => {
235                let data_page_header = DataPageHeader {
236                    num_values: num_values as i32,
237                    encoding,
238                    definition_level_encoding: def_level_encoding,
239                    repetition_level_encoding: rep_level_encoding,
240                    statistics: page_stats_to_thrift(statistics.as_ref()),
241                };
242                page_header.data_page_header = Some(data_page_header);
243            }
244            Page::DataPageV2 {
245                num_nulls,
246                num_rows,
247                def_levels_byte_len,
248                rep_levels_byte_len,
249                is_compressed,
250                ref statistics,
251                ..
252            } => {
253                let data_page_header_v2 = DataPageHeaderV2 {
254                    num_values: num_values as i32,
255                    num_nulls: num_nulls as i32,
256                    num_rows: num_rows as i32,
257                    encoding,
258                    definition_levels_byte_length: def_levels_byte_len as i32,
259                    repetition_levels_byte_length: rep_levels_byte_len as i32,
260                    is_compressed: Some(is_compressed),
261                    statistics: page_stats_to_thrift(statistics.as_ref()),
262                };
263                page_header.data_page_header_v2 = Some(data_page_header_v2);
264            }
265            Page::DictionaryPage { is_sorted, .. } => {
266                let dictionary_page_header = DictionaryPageHeader {
267                    num_values: num_values as i32,
268                    encoding,
269                    is_sorted: Some(is_sorted),
270                };
271                page_header.dictionary_page_header = Some(dictionary_page_header);
272            }
273        }
274        Ok(page_header)
275    }
276
277    /// Update the compressed buffer for a page.
278    /// This might be required when encrypting page data for example.
279    /// The size of uncompressed data must not change.
280    #[cfg(feature = "encryption")]
281    pub(crate) fn with_new_compressed_buffer(mut self, new_buffer: Bytes) -> Self {
282        match &mut self.compressed_page {
283            Page::DataPage { buf, .. } => {
284                *buf = new_buffer;
285            }
286            Page::DataPageV2 { buf, .. } => {
287                *buf = new_buffer;
288            }
289            Page::DictionaryPage { buf, .. } => {
290                *buf = new_buffer;
291            }
292        }
293        self
294    }
295}
296
297/// Contains page write metrics.
298pub struct PageWriteSpec {
299    /// The type of page being written
300    pub page_type: PageType,
301    /// The total size of the page, before compression
302    pub uncompressed_size: usize,
303    /// The compressed size of the page
304    pub compressed_size: usize,
305    /// The number of values in the page
306    pub num_values: u32,
307    /// The offset of the page in the column chunk
308    pub offset: u64,
309    /// The number of bytes written to the underlying sink
310    pub bytes_written: u64,
311}
312
313impl Default for PageWriteSpec {
314    fn default() -> Self {
315        Self::new()
316    }
317}
318
319impl PageWriteSpec {
320    /// Creates new spec with default page write metrics.
321    pub fn new() -> Self {
322        Self {
323            page_type: PageType::DATA_PAGE,
324            uncompressed_size: 0,
325            compressed_size: 0,
326            num_values: 0,
327            offset: 0,
328            bytes_written: 0,
329        }
330    }
331}
332
333/// Contains metadata for a page
334#[derive(Clone)]
335pub struct PageMetadata {
336    /// The number of rows within the page if known
337    pub num_rows: Option<usize>,
338    /// The number of levels within the page if known
339    pub num_levels: Option<usize>,
340    /// Returns true if the page is a dictionary page
341    pub is_dict: bool,
342}
343
344impl TryFrom<&crate::file::metadata::thrift::PageHeader> for PageMetadata {
345    type Error = ParquetError;
346
347    fn try_from(
348        value: &crate::file::metadata::thrift::PageHeader,
349    ) -> std::result::Result<Self, Self::Error> {
350        match value.r#type {
351            PageType::DATA_PAGE => {
352                let header = value.data_page_header.as_ref().unwrap();
353                Ok(PageMetadata {
354                    num_rows: None,
355                    num_levels: Some(header.num_values as _),
356                    is_dict: false,
357                })
358            }
359            PageType::DICTIONARY_PAGE => Ok(PageMetadata {
360                num_rows: None,
361                num_levels: None,
362                is_dict: true,
363            }),
364            PageType::DATA_PAGE_V2 => {
365                let header = value.data_page_header_v2.as_ref().unwrap();
366                Ok(PageMetadata {
367                    num_rows: Some(header.num_rows as _),
368                    num_levels: Some(header.num_values as _),
369                    is_dict: false,
370                })
371            }
372            other @ PageType::INDEX_PAGE => Err(ParquetError::General(format!(
373                "page type {other:?} cannot be converted to PageMetadata"
374            ))),
375        }
376    }
377}
378
379/// API for reading pages from a column chunk.
380/// This offers a iterator like API to get the next page.
381pub trait PageReader: Iterator<Item = Result<Page>> + Send {
382    /// Gets the next page in the column chunk associated with this reader.
383    /// Returns `None` if there are no pages left.
384    fn get_next_page(&mut self) -> Result<Option<Page>>;
385
386    /// Gets metadata about the next page, returns an error if no
387    /// column index information
388    fn peek_next_page(&mut self) -> Result<Option<PageMetadata>>;
389
390    /// Skips reading the next page, returns an error if no
391    /// column index information
392    fn skip_next_page(&mut self) -> Result<()>;
393
394    /// Returns whether skipped dictionary pages can later be recovered with
395    /// [`Self::take_deferred_dictionary`].
396    ///
397    /// Implementations returning `true` must retain dictionary pages consumed by
398    /// [`Self::skip_next_page`].
399    fn supports_deferred_dictionary(&self) -> bool {
400        false
401    }
402
403    /// Returns the decoded dictionary [`Page`] this reader has previously
404    /// skipped past, if any, removing it from the reader.
405    ///
406    /// [`Self::skip_next_page`] may skip a dictionary page without decoding
407    /// it, since skipping rows never needs dictionary contents. If decoding
408    /// later reaches a dictionary-encoded data page, the reader is asked for
409    /// the dictionary through this method, which pays the deferred
410    /// decompression exactly once. A chunk skipped end to end never pays it.
411    ///
412    /// The default implementation returns `Ok(None)`, meaning the reader
413    /// never defers.
414    fn take_deferred_dictionary(&mut self) -> Result<Option<Page>> {
415        Ok(None)
416    }
417
418    /// Returns `true` if the next page can be assumed to contain the start of a new record
419    ///
420    /// Prior to parquet V2 the specification was ambiguous as to whether a single record
421    /// could be split across multiple pages, and prior to [(#4327)] the Rust writer would do
422    /// this in certain situations. However, correctly interpreting the offset index relies on
423    /// this assumption holding [(#4943)], and so this mechanism is provided for a [`PageReader`]
424    /// to signal this to the calling context
425    ///
426    /// [(#4327)]: https://github.com/apache/arrow-rs/pull/4327
427    /// [(#4943)]: https://github.com/apache/arrow-rs/pull/4943
428    fn at_record_boundary(&mut self) -> Result<bool> {
429        match self.peek_next_page()? {
430            // Last page in the column chunk - always a record boundary
431            None => Ok(true),
432            // A V2 data page is required by the parquet spec to start at a
433            // record boundary, so the current page ends at one.  V2 pages
434            // are identified by having `num_rows` set in their header.
435            Some(metadata) => Ok(metadata.num_rows.is_some()),
436        }
437    }
438}
439
440/// API for writing pages in a column chunk.
441///
442/// It is reasonable to assume that all pages will be written in the correct order, e.g.
443/// dictionary page followed by data pages, or a set of data pages, etc.
444pub trait PageWriter: Send {
445    /// Writes a page into the output stream/sink.
446    /// Returns `PageWriteSpec` that contains information about written page metrics,
447    /// including number of bytes, size, number of values, offset, etc.
448    ///
449    /// This method is called for every compressed page we write into underlying buffer,
450    /// either data page or dictionary page.
451    fn write_page(&mut self, page: CompressedPage) -> Result<PageWriteSpec>;
452
453    /// **Unstable, not public API.** This is an internal protocol between
454    /// [`GenericColumnWriter`] and its in-crate page writers; it is hidden from
455    /// the rendered docs and may change or be removed without a major version
456    /// bump. External `PageWriter` implementations should not override it. See
457    /// the page-spilling cleanup tracked in
458    /// <https://github.com/apache/arrow-rs/pull/10020>.
459    ///
460    /// Whether this writer resolves the final page layout itself (at flush)
461    /// rather than committing bytes to their final position as pages arrive.
462    ///
463    /// The dictionary page of a column chunk must be written *first*, but it is
464    /// not finalized until every value has been seen. A writer that commits
465    /// bytes live (e.g. straight to a file) therefore relies on the column
466    /// writer buffering the dictionary-encoded data pages in memory until the
467    /// dictionary page is ready — see [`GenericColumnWriter`]'s `data_pages`.
468    ///
469    /// A writer that instead buffers the whole chunk and splices it later (the
470    /// [`ArrowWriter`] path) can accept data pages *before* the dictionary page
471    /// and order them itself at flush. Returning `true` tells the column writer
472    /// to skip that in-memory buffering and stream dictionary-column data pages
473    /// straight through, bounding the column writer's memory.
474    ///
475    /// [`GenericColumnWriter`]: crate::column::writer::GenericColumnWriter
476    /// [`ArrowWriter`]: crate::arrow::arrow_writer::ArrowWriter
477    #[doc(hidden)]
478    fn defers_dictionary_ordering(&self) -> bool {
479        false
480    }
481
482    /// **Unstable, not public API.** Companion to
483    /// [`defers_dictionary_ordering`](Self::defers_dictionary_ordering): an
484    /// internal hook for the column writer's memory accounting, hidden from the
485    /// rendered docs and subject to change or removal without a major version
486    /// bump. External `PageWriter` implementations should not override it.
487    ///
488    /// The number of bytes this writer is currently holding **in memory** for
489    /// pages it has been handed (i.e. completed pages not yet committed to their
490    /// final destination).
491    ///
492    /// Used by the column writer to report its memory footprint. The default is
493    /// `0`: a writer that streams pages straight to their destination retains
494    /// nothing. A writer that buffers pages should report what it actually holds
495    /// on the heap — which, when it spills to a backing store, can be far less
496    /// than the bytes written.
497    #[doc(hidden)]
498    fn buffered_memory_size(&self) -> usize {
499        0
500    }
501
502    /// Closes resources and flushes underlying sink.
503    /// Page writer should not be used after this method is called.
504    fn close(&mut self) -> Result<()>;
505}
506
507/// An iterator over pages of one specific column in a parquet file.
508pub trait PageIterator: Iterator<Item = Result<Box<dyn PageReader>>> + Send {}
509
510#[cfg(test)]
511mod tests {
512    use super::*;
513
514    #[test]
515    fn test_page() {
516        let data_page = Page::DataPage {
517            buf: Bytes::from(vec![0, 1, 2]),
518            num_values: 10,
519            encoding: Encoding::PLAIN,
520            def_level_encoding: Encoding::RLE,
521            rep_level_encoding: Encoding::RLE,
522            statistics: Some(Statistics::int32(Some(1), Some(2), None, Some(1), true)),
523        };
524        assert_eq!(data_page.page_type(), PageType::DATA_PAGE);
525        assert_eq!(data_page.buffer(), vec![0, 1, 2].as_slice());
526        assert_eq!(data_page.num_values(), 10);
527        assert_eq!(data_page.encoding(), Encoding::PLAIN);
528        assert_eq!(
529            data_page.statistics(),
530            Some(&Statistics::int32(Some(1), Some(2), None, Some(1), true))
531        );
532
533        let data_page_v2 = Page::DataPageV2 {
534            buf: Bytes::from(vec![0, 1, 2]),
535            num_values: 10,
536            encoding: Encoding::PLAIN,
537            num_nulls: 5,
538            num_rows: 20,
539            def_levels_byte_len: 30,
540            rep_levels_byte_len: 40,
541            is_compressed: false,
542            statistics: Some(Statistics::int32(Some(1), Some(2), None, Some(1), true)),
543        };
544        assert_eq!(data_page_v2.page_type(), PageType::DATA_PAGE_V2);
545        assert_eq!(data_page_v2.buffer(), vec![0, 1, 2].as_slice());
546        assert_eq!(data_page_v2.num_values(), 10);
547        assert_eq!(data_page_v2.encoding(), Encoding::PLAIN);
548        assert_eq!(
549            data_page_v2.statistics(),
550            Some(&Statistics::int32(Some(1), Some(2), None, Some(1), true))
551        );
552
553        let dict_page = Page::DictionaryPage {
554            buf: Bytes::from(vec![0, 1, 2]),
555            num_values: 10,
556            encoding: Encoding::PLAIN,
557            is_sorted: false,
558        };
559        assert_eq!(dict_page.page_type(), PageType::DICTIONARY_PAGE);
560        assert_eq!(dict_page.buffer(), vec![0, 1, 2].as_slice());
561        assert_eq!(dict_page.num_values(), 10);
562        assert_eq!(dict_page.encoding(), Encoding::PLAIN);
563        assert_eq!(dict_page.statistics(), None);
564    }
565
566    #[test]
567    fn test_compressed_page() {
568        let data_page = Page::DataPage {
569            buf: Bytes::from(vec![0, 1, 2]),
570            num_values: 10,
571            encoding: Encoding::PLAIN,
572            def_level_encoding: Encoding::RLE,
573            rep_level_encoding: Encoding::RLE,
574            statistics: Some(Statistics::int32(Some(1), Some(2), None, Some(1), true)),
575        };
576
577        let cpage = CompressedPage::new(data_page, 5);
578
579        assert_eq!(cpage.page_type(), PageType::DATA_PAGE);
580        assert_eq!(cpage.uncompressed_size(), 5);
581        assert_eq!(cpage.compressed_size(), 3);
582        assert_eq!(cpage.num_values(), 10);
583        assert_eq!(cpage.encoding(), Encoding::PLAIN);
584        assert_eq!(cpage.data(), &[0, 1, 2]);
585    }
586
587    #[test]
588    fn test_compressed_page_uncompressed_size_overflow() {
589        // Test that to_thrift_header fails when uncompressed size exceeds i32::MAX
590        let data_page = Page::DataPage {
591            buf: Bytes::from(vec![0, 1, 2]),
592            num_values: 10,
593            encoding: Encoding::PLAIN,
594            def_level_encoding: Encoding::RLE,
595            rep_level_encoding: Encoding::RLE,
596            statistics: None,
597        };
598
599        // Create a CompressedPage with uncompressed size larger than i32::MAX
600        let uncompressed_size = (i32::MAX as usize) + 1;
601        let cpage = CompressedPage::new(data_page, uncompressed_size);
602
603        // Verify that to_thrift_header returns an error
604        let result = cpage.to_thrift_header();
605        assert!(result.is_err());
606
607        let error_msg = result.unwrap_err().to_string();
608        assert!(error_msg.contains("Page uncompressed size overflow"));
609    }
610}