arrow_avro/
compression.rs1use crate::errors::AvroError;
6use arrow_schema::ArrowError;
7#[cfg(any(
8 feature = "deflate",
9 feature = "zstd",
10 feature = "bzip2",
11 feature = "xz"
12))]
13use std::io::{Read, Write};
14
15pub const CODEC_METADATA_KEY: &str = "avro.codec";
17
18#[derive(Debug, Copy, Clone, Eq, PartialEq)]
19pub enum CompressionCodec {
24 Deflate,
26 Snappy,
28 ZStandard,
30 Bzip2,
32 Xz,
34}
35
36impl CompressionCodec {
37 #[cfg_attr(
38 not(any(
39 feature = "deflate",
40 feature = "snappy",
41 feature = "zstd",
42 feature = "bzip2",
43 feature = "xz"
44 )),
45 expect(unused_variables)
46 )]
47 pub(crate) fn decompress(&self, block: &[u8]) -> Result<Vec<u8>, AvroError> {
48 match self {
49 #[cfg(feature = "deflate")]
50 CompressionCodec::Deflate => {
51 let mut decoder = flate2::read::DeflateDecoder::new(block);
52 let mut out = Vec::new();
53 decoder.read_to_end(&mut out)?;
54 Ok(out)
55 }
56 #[cfg(not(feature = "deflate"))]
57 CompressionCodec::Deflate => Err(AvroError::ParseError(
58 "Deflate codec requires deflate feature".to_string(),
59 )),
60 #[cfg(feature = "snappy")]
61 CompressionCodec::Snappy => {
62 let crc = &block[block.len() - 4..];
65 let block = &block[..block.len() - 4];
66
67 let mut decoder = snap::raw::Decoder::new();
68 let decoded = decoder
69 .decompress_vec(block)
70 .map_err(|e| AvroError::External(Box::new(e)))?;
71
72 let checksum = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC).checksum(&decoded);
73 if checksum != u32::from_be_bytes(crc.try_into().unwrap()) {
74 return Err(AvroError::ParseError("Snappy CRC mismatch".to_string()));
75 }
76 Ok(decoded)
77 }
78 #[cfg(not(feature = "snappy"))]
79 CompressionCodec::Snappy => Err(AvroError::ParseError(
80 "Snappy codec requires snappy feature".to_string(),
81 )),
82
83 #[cfg(feature = "zstd")]
84 CompressionCodec::ZStandard => {
85 let mut decoder = zstd::Decoder::new(block)?;
86 let mut out = Vec::new();
87 decoder
88 .read_to_end(&mut out)
89 .map_err(|e| AvroError::External(Box::new(e)))?;
90 Ok(out)
91 }
92 #[cfg(not(feature = "zstd"))]
93 CompressionCodec::ZStandard => Err(AvroError::ParseError(
94 "ZStandard codec requires zstd feature".to_string(),
95 )),
96 #[cfg(feature = "bzip2")]
97 CompressionCodec::Bzip2 => {
98 let mut decoder = bzip2::read::BzDecoder::new(block);
99 let mut out = Vec::new();
100 decoder
101 .read_to_end(&mut out)
102 .map_err(|e| AvroError::External(Box::new(e)))?;
103 Ok(out)
104 }
105 #[cfg(not(feature = "bzip2"))]
106 CompressionCodec::Bzip2 => Err(AvroError::ParseError(
107 "Bzip2 codec requires bzip2 feature".to_string(),
108 )),
109 #[cfg(feature = "xz")]
110 CompressionCodec::Xz => {
111 let mut decoder = xz::read::XzDecoder::new(block);
112 let mut out = Vec::new();
113 decoder
114 .read_to_end(&mut out)
115 .map_err(|e| AvroError::External(Box::new(e)))?;
116 Ok(out)
117 }
118 #[cfg(not(feature = "xz"))]
119 CompressionCodec::Xz => Err(AvroError::ParseError(
120 "XZ codec requires xz feature".to_string(),
121 )),
122 }
123 }
124
125 #[cfg_attr(
126 not(any(
127 feature = "deflate",
128 feature = "snappy",
129 feature = "zstd",
130 feature = "bzip2",
131 feature = "xz"
132 )),
133 expect(unused_variables)
134 )]
135 pub(crate) fn compress(&self, data: &[u8]) -> Result<Vec<u8>, ArrowError> {
136 match self {
137 #[cfg(feature = "deflate")]
138 CompressionCodec::Deflate => {
139 let mut encoder =
140 flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default());
141 encoder.write_all(data)?;
142 let compressed = encoder.finish()?;
143 Ok(compressed)
144 }
145 #[cfg(not(feature = "deflate"))]
146 CompressionCodec::Deflate => Err(ArrowError::ParseError(
147 "Deflate codec requires deflate feature".to_string(),
148 )),
149
150 #[cfg(feature = "snappy")]
151 CompressionCodec::Snappy => {
152 let mut encoder = snap::raw::Encoder::new();
153 let mut compressed = encoder
155 .compress_vec(data)
156 .map_err(|e| ArrowError::ExternalError(Box::new(e)))?;
157 let crc_val = crc::Crc::<u32>::new(&crc::CRC_32_ISO_HDLC).checksum(data);
159 compressed.extend_from_slice(&crc_val.to_be_bytes());
160 Ok(compressed)
161 }
162 #[cfg(not(feature = "snappy"))]
163 CompressionCodec::Snappy => Err(ArrowError::ParseError(
164 "Snappy codec requires snappy feature".to_string(),
165 )),
166
167 #[cfg(feature = "zstd")]
168 CompressionCodec::ZStandard => {
169 let mut encoder = zstd::Encoder::new(Vec::new(), 0)
170 .map_err(|e| ArrowError::ExternalError(Box::new(e)))?;
171 encoder.write_all(data)?;
172 let compressed = encoder
173 .finish()
174 .map_err(|e| ArrowError::ExternalError(Box::new(e)))?;
175 Ok(compressed)
176 }
177 #[cfg(not(feature = "zstd"))]
178 CompressionCodec::ZStandard => Err(ArrowError::ParseError(
179 "ZStandard codec requires zstd feature".to_string(),
180 )),
181
182 #[cfg(feature = "bzip2")]
183 CompressionCodec::Bzip2 => {
184 let mut encoder =
185 bzip2::write::BzEncoder::new(Vec::new(), bzip2::Compression::default());
186 encoder.write_all(data)?;
187 let compressed = encoder.finish()?;
188 Ok(compressed)
189 }
190 #[cfg(not(feature = "bzip2"))]
191 CompressionCodec::Bzip2 => Err(ArrowError::ParseError(
192 "Bzip2 codec requires bzip2 feature".to_string(),
193 )),
194 #[cfg(feature = "xz")]
195 CompressionCodec::Xz => {
196 let mut encoder = xz::write::XzEncoder::new(Vec::new(), 6);
197 encoder.write_all(data)?;
198 let compressed = encoder.finish()?;
199 Ok(compressed)
200 }
201 #[cfg(not(feature = "xz"))]
202 CompressionCodec::Xz => Err(ArrowError::ParseError(
203 "XZ codec requires xz feature".to_string(),
204 )),
205 }
206 }
207}