parquet/column/chunker/mod.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Content-defined chunking (CDC) for Parquet data pages.
6//!
7//! CDC creates data page boundaries based on content rather than fixed sizes,
8//! enabling efficient deduplication in content-addressable storage (CAS) systems.
9//! See [`CdcOptions`](crate::file::properties::CdcOptions) for configuration.
10
11mod cdc;
12mod cdc_generated;
13
14pub(crate) use cdc::ContentDefinedChunker;
15
16/// A chunk of data with level and value offsets for record-shredded nested data.
17#[derive(Debug, Clone, Copy)]
18pub(crate) struct CdcChunk {
19 /// The start offset of this chunk inside the given levels.
20 pub level_offset: usize,
21 /// The number of levels in this chunk.
22 pub num_levels: usize,
23 /// The start index into `non_null_indices` for this chunk.
24 pub value_offset: usize,
25 /// The number of `non_null_indices` entries in this chunk.
26 pub num_values: usize,
27}