parquet/column/page.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Contains Parquet Page definitions and page reader interface.
6
7use bytes::Bytes;
8
9use crate::basic::{Encoding, PageType};
10use crate::errors::{ParquetError, Result};
11use crate::file::metadata::thrift::{
12 DataPageHeader, DataPageHeaderV2, DictionaryPageHeader, PageHeader,
13};
14use crate::file::statistics::{Statistics, page_stats_to_thrift};
15
16/// Parquet Page definition.
17///
18/// List of supported pages.
19/// These are 1-to-1 mapped from the equivalent Thrift definitions, except `buf` which
20/// used to store uncompressed bytes of the page.
21#[derive(Clone, Debug)]
22pub enum Page {
23 /// Data page Parquet format v1.
24 DataPage {
25 /// The underlying data buffer
26 buf: Bytes,
27 /// Number of values in this page
28 num_values: u32,
29 /// Encoding for values in this page
30 encoding: Encoding,
31 /// Definition level encoding
32 def_level_encoding: Encoding,
33 /// Repetition level encoding
34 rep_level_encoding: Encoding,
35 /// Optional statistics for this page
36 statistics: Option<Statistics>,
37 },
38 /// Data page Parquet format v2.
39 DataPageV2 {
40 /// The underlying data buffer
41 buf: Bytes,
42 /// Number of values in this page
43 num_values: u32,
44 /// Encoding for values in this page
45 encoding: Encoding,
46 /// Number of null values in this page
47 num_nulls: u32,
48 /// Number of rows in this page
49 num_rows: u32,
50 /// Length of definition levels
51 def_levels_byte_len: u32,
52 /// Length of repetition levels
53 rep_levels_byte_len: u32,
54 /// Is this page compressed
55 is_compressed: bool,
56 /// Optional statistics for this page
57 statistics: Option<Statistics>,
58 },
59 /// Dictionary page.
60 DictionaryPage {
61 /// The underlying data buffer
62 buf: Bytes,
63 /// Number of values in this page
64 num_values: u32,
65 /// Encoding for values in this page
66 encoding: Encoding,
67 /// Is dictionary page sorted
68 is_sorted: bool,
69 },
70}
71
72impl Page {
73 /// Returns [`PageType`] for this page.
74 pub fn page_type(&self) -> PageType {
75 match self {
76 Page::DataPage { .. } => PageType::DATA_PAGE,
77 Page::DataPageV2 { .. } => PageType::DATA_PAGE_V2,
78 Page::DictionaryPage { .. } => PageType::DICTIONARY_PAGE,
79 }
80 }
81
82 /// Returns whether this page is any version of a data page
83 pub fn is_data_page(&self) -> bool {
84 matches!(self, Page::DataPage { .. } | Page::DataPageV2 { .. })
85 }
86
87 /// Returns whether this page is a dictionary page
88 pub fn is_dictionary_page(&self) -> bool {
89 matches!(self, Page::DictionaryPage { .. })
90 }
91
92 /// Returns internal byte buffer reference for this page.
93 pub fn buffer(&self) -> &Bytes {
94 match self {
95 Page::DataPage { buf, .. } => buf,
96 Page::DataPageV2 { buf, .. } => buf,
97 Page::DictionaryPage { buf, .. } => buf,
98 }
99 }
100
101 /// Returns number of values in this page.
102 pub fn num_values(&self) -> u32 {
103 match self {
104 Page::DataPage { num_values, .. } => *num_values,
105 Page::DataPageV2 { num_values, .. } => *num_values,
106 Page::DictionaryPage { num_values, .. } => *num_values,
107 }
108 }
109
110 /// Returns this page [`Encoding`].
111 pub fn encoding(&self) -> Encoding {
112 match self {
113 Page::DataPage { encoding, .. } => *encoding,
114 Page::DataPageV2 { encoding, .. } => *encoding,
115 Page::DictionaryPage { encoding, .. } => *encoding,
116 }
117 }
118
119 /// Returns optional [`Statistics`].
120 pub fn statistics(&self) -> Option<&Statistics> {
121 match self {
122 Page::DataPage { statistics, .. } => statistics.as_ref(),
123 Page::DataPageV2 { statistics, .. } => statistics.as_ref(),
124 Page::DictionaryPage { .. } => None,
125 }
126 }
127}
128
129/// Helper struct to represent pages with potentially compressed buffer (data page v1) or
130/// compressed and concatenated buffer (def levels + rep levels + compressed values for
131/// data page v2).
132///
133/// The difference with `Page` is that `Page` buffer is always uncompressed.
134pub struct CompressedPage {
135 compressed_page: Page,
136 uncompressed_size: usize,
137}
138
139impl CompressedPage {
140 /// Creates `CompressedPage` from a page with potentially compressed buffer and
141 /// uncompressed size.
142 pub fn new(compressed_page: Page, uncompressed_size: usize) -> Self {
143 Self {
144 compressed_page,
145 uncompressed_size,
146 }
147 }
148
149 /// Returns page type.
150 pub fn page_type(&self) -> PageType {
151 self.compressed_page.page_type()
152 }
153
154 /// Returns underlying page with potentially compressed buffer.
155 pub fn compressed_page(&self) -> &Page {
156 &self.compressed_page
157 }
158
159 /// Returns uncompressed size in bytes.
160 pub fn uncompressed_size(&self) -> usize {
161 self.uncompressed_size
162 }
163
164 /// Returns compressed size in bytes.
165 ///
166 /// Note that it is assumed that buffer is compressed, but it may not be. In this
167 /// case compressed size will be equal to uncompressed size.
168 pub fn compressed_size(&self) -> usize {
169 self.compressed_page.buffer().len()
170 }
171
172 /// Number of values in page.
173 pub fn num_values(&self) -> u32 {
174 self.compressed_page.num_values()
175 }
176
177 /// Returns encoding for values in page.
178 pub fn encoding(&self) -> Encoding {
179 self.compressed_page.encoding()
180 }
181
182 /// Returns slice of compressed buffer in the page.
183 pub fn data(&self) -> &[u8] {
184 self.compressed_page.buffer()
185 }
186
187 /// Returns the number of heap bytes this page currently holds.
188 ///
189 /// This is the page's compressed buffer (the embedded [`Bytes`]); use it to
190 /// account for a buffered page's memory footprint rather than reaching for
191 /// `data().len()` at each call site.
192 pub fn memory_usage(&self) -> usize {
193 self.compressed_page.buffer().len()
194 }
195
196 /// Returns the thrift page header
197 pub(crate) fn to_thrift_header(&self) -> Result<PageHeader> {
198 let uncompressed_size = self.uncompressed_size();
199 let compressed_size = self.compressed_size();
200 if uncompressed_size > i32::MAX as usize {
201 return Err(general_err!(
202 "Page uncompressed size overflow: {}",
203 uncompressed_size
204 ));
205 }
206 if compressed_size > i32::MAX as usize {
207 return Err(general_err!(
208 "Page compressed size overflow: {}",
209 compressed_size
210 ));
211 }
212 let num_values = self.num_values();
213 let encoding = self.encoding();
214 let page_type = self.page_type();
215
216 let mut page_header = PageHeader {
217 r#type: page_type,
218 uncompressed_page_size: uncompressed_size as i32,
219 compressed_page_size: compressed_size as i32,
220 // TODO: Add support for crc checksum
221 crc: None,
222 data_page_header: None,
223 index_page_header: None,
224 dictionary_page_header: None,
225 data_page_header_v2: None,
226 };
227
228 match self.compressed_page {
229 Page::DataPage {
230 def_level_encoding,
231 rep_level_encoding,
232 ref statistics,
233 ..
234 } => {
235 let data_page_header = DataPageHeader {
236 num_values: num_values as i32,
237 encoding,
238 definition_level_encoding: def_level_encoding,
239 repetition_level_encoding: rep_level_encoding,
240 statistics: page_stats_to_thrift(statistics.as_ref()),
241 };
242 page_header.data_page_header = Some(data_page_header);
243 }
244 Page::DataPageV2 {
245 num_nulls,
246 num_rows,
247 def_levels_byte_len,
248 rep_levels_byte_len,
249 is_compressed,
250 ref statistics,
251 ..
252 } => {
253 let data_page_header_v2 = DataPageHeaderV2 {
254 num_values: num_values as i32,
255 num_nulls: num_nulls as i32,
256 num_rows: num_rows as i32,
257 encoding,
258 definition_levels_byte_length: def_levels_byte_len as i32,
259 repetition_levels_byte_length: rep_levels_byte_len as i32,
260 is_compressed: Some(is_compressed),
261 statistics: page_stats_to_thrift(statistics.as_ref()),
262 };
263 page_header.data_page_header_v2 = Some(data_page_header_v2);
264 }
265 Page::DictionaryPage { is_sorted, .. } => {
266 let dictionary_page_header = DictionaryPageHeader {
267 num_values: num_values as i32,
268 encoding,
269 is_sorted: Some(is_sorted),
270 };
271 page_header.dictionary_page_header = Some(dictionary_page_header);
272 }
273 }
274 Ok(page_header)
275 }
276
277 /// Update the compressed buffer for a page.
278 /// This might be required when encrypting page data for example.
279 /// The size of uncompressed data must not change.
280 #[cfg(feature = "encryption")]
281 pub(crate) fn with_new_compressed_buffer(mut self, new_buffer: Bytes) -> Self {
282 match &mut self.compressed_page {
283 Page::DataPage { buf, .. } => {
284 *buf = new_buffer;
285 }
286 Page::DataPageV2 { buf, .. } => {
287 *buf = new_buffer;
288 }
289 Page::DictionaryPage { buf, .. } => {
290 *buf = new_buffer;
291 }
292 }
293 self
294 }
295}
296
297/// Contains page write metrics.
298pub struct PageWriteSpec {
299 /// The type of page being written
300 pub page_type: PageType,
301 /// The total size of the page, before compression
302 pub uncompressed_size: usize,
303 /// The compressed size of the page
304 pub compressed_size: usize,
305 /// The number of values in the page
306 pub num_values: u32,
307 /// The offset of the page in the column chunk
308 pub offset: u64,
309 /// The number of bytes written to the underlying sink
310 pub bytes_written: u64,
311}
312
313impl Default for PageWriteSpec {
314 fn default() -> Self {
315 Self::new()
316 }
317}
318
319impl PageWriteSpec {
320 /// Creates new spec with default page write metrics.
321 pub fn new() -> Self {
322 Self {
323 page_type: PageType::DATA_PAGE,
324 uncompressed_size: 0,
325 compressed_size: 0,
326 num_values: 0,
327 offset: 0,
328 bytes_written: 0,
329 }
330 }
331}
332
333/// Contains metadata for a page
334#[derive(Clone)]
335pub struct PageMetadata {
336 /// The number of rows within the page if known
337 pub num_rows: Option<usize>,
338 /// The number of levels within the page if known
339 pub num_levels: Option<usize>,
340 /// Returns true if the page is a dictionary page
341 pub is_dict: bool,
342}
343
344impl TryFrom<&crate::file::metadata::thrift::PageHeader> for PageMetadata {
345 type Error = ParquetError;
346
347 fn try_from(
348 value: &crate::file::metadata::thrift::PageHeader,
349 ) -> std::result::Result<Self, Self::Error> {
350 match value.r#type {
351 PageType::DATA_PAGE => {
352 let header = value.data_page_header.as_ref().unwrap();
353 Ok(PageMetadata {
354 num_rows: None,
355 num_levels: Some(header.num_values as _),
356 is_dict: false,
357 })
358 }
359 PageType::DICTIONARY_PAGE => Ok(PageMetadata {
360 num_rows: None,
361 num_levels: None,
362 is_dict: true,
363 }),
364 PageType::DATA_PAGE_V2 => {
365 let header = value.data_page_header_v2.as_ref().unwrap();
366 Ok(PageMetadata {
367 num_rows: Some(header.num_rows as _),
368 num_levels: Some(header.num_values as _),
369 is_dict: false,
370 })
371 }
372 other @ PageType::INDEX_PAGE => Err(ParquetError::General(format!(
373 "page type {other:?} cannot be converted to PageMetadata"
374 ))),
375 }
376 }
377}
378
379/// API for reading pages from a column chunk.
380/// This offers a iterator like API to get the next page.
381pub trait PageReader: Iterator<Item = Result<Page>> + Send {
382 /// Gets the next page in the column chunk associated with this reader.
383 /// Returns `None` if there are no pages left.
384 fn get_next_page(&mut self) -> Result<Option<Page>>;
385
386 /// Gets metadata about the next page, returns an error if no
387 /// column index information
388 fn peek_next_page(&mut self) -> Result<Option<PageMetadata>>;
389
390 /// Skips reading the next page, returns an error if no
391 /// column index information
392 fn skip_next_page(&mut self) -> Result<()>;
393
394 /// Returns whether skipped dictionary pages can later be recovered with
395 /// [`Self::take_deferred_dictionary`].
396 ///
397 /// Implementations returning `true` must retain dictionary pages consumed by
398 /// [`Self::skip_next_page`].
399 fn supports_deferred_dictionary(&self) -> bool {
400 false
401 }
402
403 /// Returns the decoded dictionary [`Page`] this reader has previously
404 /// skipped past, if any, removing it from the reader.
405 ///
406 /// [`Self::skip_next_page`] may skip a dictionary page without decoding
407 /// it, since skipping rows never needs dictionary contents. If decoding
408 /// later reaches a dictionary-encoded data page, the reader is asked for
409 /// the dictionary through this method, which pays the deferred
410 /// decompression exactly once. A chunk skipped end to end never pays it.
411 ///
412 /// The default implementation returns `Ok(None)`, meaning the reader
413 /// never defers.
414 fn take_deferred_dictionary(&mut self) -> Result<Option<Page>> {
415 Ok(None)
416 }
417
418 /// Returns `true` if the next page can be assumed to contain the start of a new record
419 ///
420 /// Prior to parquet V2 the specification was ambiguous as to whether a single record
421 /// could be split across multiple pages, and prior to [(#4327)] the Rust writer would do
422 /// this in certain situations. However, correctly interpreting the offset index relies on
423 /// this assumption holding [(#4943)], and so this mechanism is provided for a [`PageReader`]
424 /// to signal this to the calling context
425 ///
426 /// [(#4327)]: https://github.com/apache/arrow-rs/pull/4327
427 /// [(#4943)]: https://github.com/apache/arrow-rs/pull/4943
428 fn at_record_boundary(&mut self) -> Result<bool> {
429 match self.peek_next_page()? {
430 // Last page in the column chunk - always a record boundary
431 None => Ok(true),
432 // A V2 data page is required by the parquet spec to start at a
433 // record boundary, so the current page ends at one. V2 pages
434 // are identified by having `num_rows` set in their header.
435 Some(metadata) => Ok(metadata.num_rows.is_some()),
436 }
437 }
438}
439
440/// API for writing pages in a column chunk.
441///
442/// It is reasonable to assume that all pages will be written in the correct order, e.g.
443/// dictionary page followed by data pages, or a set of data pages, etc.
444pub trait PageWriter: Send {
445 /// Writes a page into the output stream/sink.
446 /// Returns `PageWriteSpec` that contains information about written page metrics,
447 /// including number of bytes, size, number of values, offset, etc.
448 ///
449 /// This method is called for every compressed page we write into underlying buffer,
450 /// either data page or dictionary page.
451 fn write_page(&mut self, page: CompressedPage) -> Result<PageWriteSpec>;
452
453 /// **Unstable, not public API.** This is an internal protocol between
454 /// [`GenericColumnWriter`] and its in-crate page writers; it is hidden from
455 /// the rendered docs and may change or be removed without a major version
456 /// bump. External `PageWriter` implementations should not override it. See
457 /// the page-spilling cleanup tracked in
458 /// <https://github.com/apache/arrow-rs/pull/10020>.
459 ///
460 /// Whether this writer resolves the final page layout itself (at flush)
461 /// rather than committing bytes to their final position as pages arrive.
462 ///
463 /// The dictionary page of a column chunk must be written *first*, but it is
464 /// not finalized until every value has been seen. A writer that commits
465 /// bytes live (e.g. straight to a file) therefore relies on the column
466 /// writer buffering the dictionary-encoded data pages in memory until the
467 /// dictionary page is ready — see [`GenericColumnWriter`]'s `data_pages`.
468 ///
469 /// A writer that instead buffers the whole chunk and splices it later (the
470 /// [`ArrowWriter`] path) can accept data pages *before* the dictionary page
471 /// and order them itself at flush. Returning `true` tells the column writer
472 /// to skip that in-memory buffering and stream dictionary-column data pages
473 /// straight through, bounding the column writer's memory.
474 ///
475 /// [`GenericColumnWriter`]: crate::column::writer::GenericColumnWriter
476 /// [`ArrowWriter`]: crate::arrow::arrow_writer::ArrowWriter
477 #[doc(hidden)]
478 fn defers_dictionary_ordering(&self) -> bool {
479 false
480 }
481
482 /// **Unstable, not public API.** Companion to
483 /// [`defers_dictionary_ordering`](Self::defers_dictionary_ordering): an
484 /// internal hook for the column writer's memory accounting, hidden from the
485 /// rendered docs and subject to change or removal without a major version
486 /// bump. External `PageWriter` implementations should not override it.
487 ///
488 /// The number of bytes this writer is currently holding **in memory** for
489 /// pages it has been handed (i.e. completed pages not yet committed to their
490 /// final destination).
491 ///
492 /// Used by the column writer to report its memory footprint. The default is
493 /// `0`: a writer that streams pages straight to their destination retains
494 /// nothing. A writer that buffers pages should report what it actually holds
495 /// on the heap — which, when it spills to a backing store, can be far less
496 /// than the bytes written.
497 #[doc(hidden)]
498 fn buffered_memory_size(&self) -> usize {
499 0
500 }
501
502 /// Closes resources and flushes underlying sink.
503 /// Page writer should not be used after this method is called.
504 fn close(&mut self) -> Result<()>;
505}
506
507/// An iterator over pages of one specific column in a parquet file.
508pub trait PageIterator: Iterator<Item = Result<Box<dyn PageReader>>> + Send {}
509
510#[cfg(test)]
511mod tests {
512 use super::*;
513
514 #[test]
515 fn test_page() {
516 let data_page = Page::DataPage {
517 buf: Bytes::from(vec![0, 1, 2]),
518 num_values: 10,
519 encoding: Encoding::PLAIN,
520 def_level_encoding: Encoding::RLE,
521 rep_level_encoding: Encoding::RLE,
522 statistics: Some(Statistics::int32(Some(1), Some(2), None, Some(1), true)),
523 };
524 assert_eq!(data_page.page_type(), PageType::DATA_PAGE);
525 assert_eq!(data_page.buffer(), vec![0, 1, 2].as_slice());
526 assert_eq!(data_page.num_values(), 10);
527 assert_eq!(data_page.encoding(), Encoding::PLAIN);
528 assert_eq!(
529 data_page.statistics(),
530 Some(&Statistics::int32(Some(1), Some(2), None, Some(1), true))
531 );
532
533 let data_page_v2 = Page::DataPageV2 {
534 buf: Bytes::from(vec![0, 1, 2]),
535 num_values: 10,
536 encoding: Encoding::PLAIN,
537 num_nulls: 5,
538 num_rows: 20,
539 def_levels_byte_len: 30,
540 rep_levels_byte_len: 40,
541 is_compressed: false,
542 statistics: Some(Statistics::int32(Some(1), Some(2), None, Some(1), true)),
543 };
544 assert_eq!(data_page_v2.page_type(), PageType::DATA_PAGE_V2);
545 assert_eq!(data_page_v2.buffer(), vec![0, 1, 2].as_slice());
546 assert_eq!(data_page_v2.num_values(), 10);
547 assert_eq!(data_page_v2.encoding(), Encoding::PLAIN);
548 assert_eq!(
549 data_page_v2.statistics(),
550 Some(&Statistics::int32(Some(1), Some(2), None, Some(1), true))
551 );
552
553 let dict_page = Page::DictionaryPage {
554 buf: Bytes::from(vec![0, 1, 2]),
555 num_values: 10,
556 encoding: Encoding::PLAIN,
557 is_sorted: false,
558 };
559 assert_eq!(dict_page.page_type(), PageType::DICTIONARY_PAGE);
560 assert_eq!(dict_page.buffer(), vec![0, 1, 2].as_slice());
561 assert_eq!(dict_page.num_values(), 10);
562 assert_eq!(dict_page.encoding(), Encoding::PLAIN);
563 assert_eq!(dict_page.statistics(), None);
564 }
565
566 #[test]
567 fn test_compressed_page() {
568 let data_page = Page::DataPage {
569 buf: Bytes::from(vec![0, 1, 2]),
570 num_values: 10,
571 encoding: Encoding::PLAIN,
572 def_level_encoding: Encoding::RLE,
573 rep_level_encoding: Encoding::RLE,
574 statistics: Some(Statistics::int32(Some(1), Some(2), None, Some(1), true)),
575 };
576
577 let cpage = CompressedPage::new(data_page, 5);
578
579 assert_eq!(cpage.page_type(), PageType::DATA_PAGE);
580 assert_eq!(cpage.uncompressed_size(), 5);
581 assert_eq!(cpage.compressed_size(), 3);
582 assert_eq!(cpage.num_values(), 10);
583 assert_eq!(cpage.encoding(), Encoding::PLAIN);
584 assert_eq!(cpage.data(), &[0, 1, 2]);
585 }
586
587 #[test]
588 fn test_compressed_page_uncompressed_size_overflow() {
589 // Test that to_thrift_header fails when uncompressed size exceeds i32::MAX
590 let data_page = Page::DataPage {
591 buf: Bytes::from(vec![0, 1, 2]),
592 num_values: 10,
593 encoding: Encoding::PLAIN,
594 def_level_encoding: Encoding::RLE,
595 rep_level_encoding: Encoding::RLE,
596 statistics: None,
597 };
598
599 // Create a CompressedPage with uncompressed size larger than i32::MAX
600 let uncompressed_size = (i32::MAX as usize) + 1;
601 let cpage = CompressedPage::new(data_page, uncompressed_size);
602
603 // Verify that to_thrift_header returns an error
604 let result = cpage.to_thrift_header();
605 assert!(result.is_err());
606
607 let error_msg = result.unwrap_err().to_string();
608 assert!(error_msg.contains("Page uncompressed size overflow"));
609 }
610}