arrow_avro/writer/
format.rs1use crate::compression::{CODEC_METADATA_KEY, CompressionCodec};
8use crate::errors::AvroError;
9use crate::schema::{AvroSchema, AvroSchemaOptions, SCHEMA_METADATA_KEY};
10use crate::writer::encoder::write_long;
11use arrow_schema::Schema;
12use rand::Rng;
13use std::fmt::Debug;
14use std::io::Write;
15
16pub trait AvroFormat: Debug + Default {
18 const NEEDS_PREFIX: bool;
22
23 fn start_stream<W: Write>(
27 &mut self,
28 writer: &mut W,
29 schema: &Schema,
30 compression: Option<CompressionCodec>,
31 ) -> Result<(), AvroError>;
32
33 fn sync_marker(&self) -> Option<&[u8; 16]>;
35}
36
37#[derive(Debug, Default)]
39pub struct AvroOcfFormat {
40 sync_marker: [u8; 16],
41}
42
43impl AvroFormat for AvroOcfFormat {
44 const NEEDS_PREFIX: bool = false;
45 fn start_stream<W: Write>(
46 &mut self,
47 writer: &mut W,
48 schema: &Schema,
49 compression: Option<CompressionCodec>,
50 ) -> Result<(), AvroError> {
51 let mut rng = rand::rng();
52 rng.fill_bytes(&mut self.sync_marker);
53 let avro_schema = AvroSchema::from_arrow_with_options(
57 schema,
58 Some(AvroSchemaOptions {
59 null_order: None,
60 strip_metadata: true,
61 }),
62 )
63 .map_err(|e| AvroError::SchemaError(format!("{e:?}")))?;
64 writer.write_all(b"Obj\x01")?;
66 let codec_str = match compression {
68 Some(CompressionCodec::Deflate) => "deflate",
69 Some(CompressionCodec::Snappy) => "snappy",
70 Some(CompressionCodec::ZStandard) => "zstandard",
71 Some(CompressionCodec::Bzip2) => "bzip2",
72 Some(CompressionCodec::Xz) => "xz",
73 None => "null",
74 };
75 write_long(writer, 2)?;
77 write_string(writer, SCHEMA_METADATA_KEY)?;
78 write_bytes(writer, avro_schema.json_string.as_bytes())?;
79 write_string(writer, CODEC_METADATA_KEY)?;
80 write_bytes(writer, codec_str.as_bytes())?;
81 write_long(writer, 0)?;
82 writer.write_all(&self.sync_marker)?;
84 Ok(())
85 }
86
87 fn sync_marker(&self) -> Option<&[u8; 16]> {
88 Some(&self.sync_marker)
89 }
90}
91
92#[derive(Debug, Default)]
100pub struct AvroSoeFormat {}
101
102impl AvroFormat for AvroSoeFormat {
103 const NEEDS_PREFIX: bool = true;
104 fn start_stream<W: Write>(
105 &mut self,
106 _writer: &mut W,
107 _schema: &Schema,
108 compression: Option<CompressionCodec>,
109 ) -> Result<(), AvroError> {
110 if compression.is_some() {
111 return Err(AvroError::InvalidArgument(
112 "Compression not supported for Avro SOE streaming".to_string(),
113 ));
114 }
115 Ok(())
116 }
117
118 fn sync_marker(&self) -> Option<&[u8; 16]> {
119 None
120 }
121}
122
123#[derive(Debug, Default)]
133pub struct AvroBinaryFormat;
134
135impl AvroFormat for AvroBinaryFormat {
136 const NEEDS_PREFIX: bool = false;
137
138 fn start_stream<W: Write>(
139 &mut self,
140 _writer: &mut W,
141 _schema: &Schema,
142 compression: Option<CompressionCodec>,
143 ) -> Result<(), AvroError> {
144 if compression.is_some() {
145 return Err(AvroError::InvalidArgument(
146 "Compression not supported for Avro binary streaming".to_string(),
147 ));
148 }
149 Ok(())
150 }
151
152 fn sync_marker(&self) -> Option<&[u8; 16]> {
153 None
154 }
155}
156
157#[inline]
158fn write_string<W: Write>(writer: &mut W, s: &str) -> Result<(), AvroError> {
159 write_bytes(writer, s.as_bytes())
160}
161
162#[inline]
163fn write_bytes<W: Write>(writer: &mut W, bytes: &[u8]) -> Result<(), AvroError> {
164 write_long(writer, bytes.len() as i64)?;
165 writer.write_all(bytes)?;
166 Ok(())
167}
168
169#[cfg(test)]
170mod tests {
171 use super::*;
172 use arrow_schema::{DataType, Field, Schema};
173
174 fn test_schema() -> Schema {
175 Schema::new(vec![Field::new("x", DataType::Int32, false)])
176 }
177
178 #[test]
179 fn avro_binary_format_rejects_compression() {
180 let mut format = AvroBinaryFormat;
181 let schema = test_schema();
182 let err = format
183 .start_stream(
184 &mut Vec::<u8>::new(),
185 &schema,
186 Some(CompressionCodec::Snappy),
187 )
188 .unwrap_err();
189 assert!(
190 err.to_string()
191 .contains("Compression not supported for Avro binary streaming")
192 );
193 }
194
195 #[test]
196 fn avro_soe_format_rejects_compression() {
197 let mut format = AvroSoeFormat::default();
198 let schema = test_schema();
199 let err = format
200 .start_stream(
201 &mut Vec::<u8>::new(),
202 &schema,
203 Some(CompressionCodec::Snappy),
204 )
205 .unwrap_err();
206 assert!(
207 err.to_string()
208 .contains("Compression not supported for Avro SOE streaming")
209 );
210 }
211}