Skip to main content

parquet/column/chunker/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Content-defined chunking (CDC) for Parquet data pages.
6//!
7//! CDC creates data page boundaries based on content rather than fixed sizes,
8//! enabling efficient deduplication in content-addressable storage (CAS) systems.
9//! See [`CdcOptions`](crate::file::properties::CdcOptions) for configuration.
10
11mod cdc;
12mod cdc_generated;
13
14pub(crate) use cdc::ContentDefinedChunker;
15
16/// A chunk of data with level and value offsets for record-shredded nested data.
17#[derive(Debug, Clone, Copy)]
18pub(crate) struct CdcChunk {
19    /// The start offset of this chunk inside the given levels.
20    pub level_offset: usize,
21    /// The number of levels in this chunk.
22    pub num_levels: usize,
23    /// The start index into `non_null_indices` for this chunk.
24    pub value_offset: usize,
25    /// The number of `non_null_indices` entries in this chunk.
26    pub num_values: usize,
27}