Skip to main content

arrow_avro/
compression.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use crate::errors::AvroError;
6use arrow_schema::ArrowError;
7#[cfg(any(
8    feature = "deflate",
9    feature = "zstd",
10    feature = "bzip2",
11    feature = "xz"
12))]
13use std::io::{Read, Write};
14
15/// The metadata key used for storing the JSON encoded [`CompressionCodec`]
16pub const CODEC_METADATA_KEY: &str = "avro.codec";
17
18#[derive(Debug, Copy, Clone, Eq, PartialEq)]
19/// Supported compression codecs for Avro data
20///
21/// Avro supports multiple compression formats for data blocks.
22/// This enum represents the compression codecs available in this implementation.
23pub enum CompressionCodec {
24    /// Deflate compression (RFC 1951)
25    Deflate,
26    /// Snappy compression
27    Snappy,
28    /// ZStandard compression
29    ZStandard,
30    /// Bzip2 compression
31    Bzip2,
32    /// Xz compression
33    Xz,
34}
35
36impl CompressionCodec {
37    #[cfg_attr(
38        not(any(
39            feature = "deflate",
40            feature = "snappy",
41            feature = "zstd",
42            feature = "bzip2",
43            feature = "xz"
44        )),
45        expect(unused_variables)
46    )]
47    pub(crate) fn decompress(&self, block: &[u8]) -> Result<Vec<u8>, AvroError> {
48        match self {
49            #[cfg(feature = "deflate")]
50            CompressionCodec::Deflate => {
51                let mut decoder = flate2::read::DeflateDecoder::new(block);
52                let mut out = Vec::new();
53                decoder.read_to_end(&mut out)?;
54                Ok(out)
55            }
56            #[cfg(not(feature = "deflate"))]
57            CompressionCodec::Deflate => Err(AvroError::ParseError(
58                "Deflate codec requires deflate feature".to_string(),
59            )),
60            #[cfg(feature = "snappy")]
61            CompressionCodec::Snappy => {
62                // Each compressed block is followed by the 4-byte, big-endian CRC32
63                // checksum of the uncompressed data in the block.
64                let crc = &block[block.len() - 4..];
65                let block = &block[..block.len() - 4];
66
67                let mut decoder = snap::raw::Decoder::new();
68                let decoded = decoder
69                    .decompress_vec(block)
70                    .map_err(|e| AvroError::External(Box::new(e)))?;
71
72                let checksum = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC).checksum(&decoded);
73                if checksum != u32::from_be_bytes(crc.try_into().unwrap()) {
74                    return Err(AvroError::ParseError("Snappy CRC mismatch".to_string()));
75                }
76                Ok(decoded)
77            }
78            #[cfg(not(feature = "snappy"))]
79            CompressionCodec::Snappy => Err(AvroError::ParseError(
80                "Snappy codec requires snappy feature".to_string(),
81            )),
82
83            #[cfg(feature = "zstd")]
84            CompressionCodec::ZStandard => {
85                let mut decoder = zstd::Decoder::new(block)?;
86                let mut out = Vec::new();
87                decoder
88                    .read_to_end(&mut out)
89                    .map_err(|e| AvroError::External(Box::new(e)))?;
90                Ok(out)
91            }
92            #[cfg(not(feature = "zstd"))]
93            CompressionCodec::ZStandard => Err(AvroError::ParseError(
94                "ZStandard codec requires zstd feature".to_string(),
95            )),
96            #[cfg(feature = "bzip2")]
97            CompressionCodec::Bzip2 => {
98                let mut decoder = bzip2::read::BzDecoder::new(block);
99                let mut out = Vec::new();
100                decoder
101                    .read_to_end(&mut out)
102                    .map_err(|e| AvroError::External(Box::new(e)))?;
103                Ok(out)
104            }
105            #[cfg(not(feature = "bzip2"))]
106            CompressionCodec::Bzip2 => Err(AvroError::ParseError(
107                "Bzip2 codec requires bzip2 feature".to_string(),
108            )),
109            #[cfg(feature = "xz")]
110            CompressionCodec::Xz => {
111                let mut decoder = xz::read::XzDecoder::new(block);
112                let mut out = Vec::new();
113                decoder
114                    .read_to_end(&mut out)
115                    .map_err(|e| AvroError::External(Box::new(e)))?;
116                Ok(out)
117            }
118            #[cfg(not(feature = "xz"))]
119            CompressionCodec::Xz => Err(AvroError::ParseError(
120                "XZ codec requires xz feature".to_string(),
121            )),
122        }
123    }
124
125    #[cfg_attr(
126        not(any(
127            feature = "deflate",
128            feature = "snappy",
129            feature = "zstd",
130            feature = "bzip2",
131            feature = "xz"
132        )),
133        expect(unused_variables)
134    )]
135    pub(crate) fn compress(&self, data: &[u8]) -> Result<Vec<u8>, ArrowError> {
136        match self {
137            #[cfg(feature = "deflate")]
138            CompressionCodec::Deflate => {
139                let mut encoder =
140                    flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default());
141                encoder.write_all(data)?;
142                let compressed = encoder.finish()?;
143                Ok(compressed)
144            }
145            #[cfg(not(feature = "deflate"))]
146            CompressionCodec::Deflate => Err(ArrowError::ParseError(
147                "Deflate codec requires deflate feature".to_string(),
148            )),
149
150            #[cfg(feature = "snappy")]
151            CompressionCodec::Snappy => {
152                let mut encoder = snap::raw::Encoder::new();
153                // Allocate and compress in one step for efficiency
154                let mut compressed = encoder
155                    .compress_vec(data)
156                    .map_err(|e| ArrowError::ExternalError(Box::new(e)))?;
157                // Compute CRC32 (ISO‑HDLC poly) of **uncompressed** data
158                let crc_val = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC).checksum(data);
159                compressed.extend_from_slice(&crc_val.to_be_bytes());
160                Ok(compressed)
161            }
162            #[cfg(not(feature = "snappy"))]
163            CompressionCodec::Snappy => Err(ArrowError::ParseError(
164                "Snappy codec requires snappy feature".to_string(),
165            )),
166
167            #[cfg(feature = "zstd")]
168            CompressionCodec::ZStandard => {
169                let mut encoder = zstd::Encoder::new(Vec::new(), 0)
170                    .map_err(|e| ArrowError::ExternalError(Box::new(e)))?;
171                encoder.write_all(data)?;
172                let compressed = encoder
173                    .finish()
174                    .map_err(|e| ArrowError::ExternalError(Box::new(e)))?;
175                Ok(compressed)
176            }
177            #[cfg(not(feature = "zstd"))]
178            CompressionCodec::ZStandard => Err(ArrowError::ParseError(
179                "ZStandard codec requires zstd feature".to_string(),
180            )),
181
182            #[cfg(feature = "bzip2")]
183            CompressionCodec::Bzip2 => {
184                let mut encoder =
185                    bzip2::write::BzEncoder::new(Vec::new(), bzip2::Compression::default());
186                encoder.write_all(data)?;
187                let compressed = encoder.finish()?;
188                Ok(compressed)
189            }
190            #[cfg(not(feature = "bzip2"))]
191            CompressionCodec::Bzip2 => Err(ArrowError::ParseError(
192                "Bzip2 codec requires bzip2 feature".to_string(),
193            )),
194            #[cfg(feature = "xz")]
195            CompressionCodec::Xz => {
196                let mut encoder = xz::write::XzEncoder::new(Vec::new(), 6);
197                encoder.write_all(data)?;
198                let compressed = encoder.finish()?;
199                Ok(compressed)
200            }
201            #[cfg(not(feature = "xz"))]
202            CompressionCodec::Xz => Err(ArrowError::ParseError(
203                "XZ codec requires xz feature".to_string(),
204            )),
205        }
206    }
207}