Skip to main content

arrow_avro/writer/
format.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Avro Writer Formats for Arrow.
6
7use crate::compression::{CODEC_METADATA_KEY, CompressionCodec};
8use crate::errors::AvroError;
9use crate::schema::{AvroSchema, AvroSchemaOptions, SCHEMA_METADATA_KEY};
10use crate::writer::encoder::write_long;
11use arrow_schema::Schema;
12use rand::Rng;
13use std::fmt::Debug;
14use std::io::Write;
15
16/// Format abstraction implemented by each container‐level writer.
17pub trait AvroFormat: Debug + Default {
18    /// If `true`, the writer for this format will query `single_object_prefix()`
19    /// and write the prefix before each record. If `false`, the writer can
20    /// skip this step. This is a performance hint for the writer.
21    const NEEDS_PREFIX: bool;
22
23    /// Write any bytes required at the very beginning of the output stream
24    /// (file header, etc.).
25    /// Implementations **must not** write any record data.
26    fn start_stream<W: Write>(
27        &mut self,
28        writer: &mut W,
29        schema: &Schema,
30        compression: Option<CompressionCodec>,
31    ) -> Result<(), AvroError>;
32
33    /// Return the 16‑byte sync marker (OCF) or `None` (binary stream).
34    fn sync_marker(&self) -> Option<&[u8; 16]>;
35}
36
37/// Avro Object Container File (OCF) format writer.
38#[derive(Debug, Default)]
39pub struct AvroOcfFormat {
40    sync_marker: [u8; 16],
41}
42
43impl AvroFormat for AvroOcfFormat {
44    const NEEDS_PREFIX: bool = false;
45    fn start_stream<W: Write>(
46        &mut self,
47        writer: &mut W,
48        schema: &Schema,
49        compression: Option<CompressionCodec>,
50    ) -> Result<(), AvroError> {
51        let mut rng = rand::rng();
52        rng.fill_bytes(&mut self.sync_marker);
53        // Choose the Avro schema JSON that the file will advertise.
54        // If `schema.metadata[SCHEMA_METADATA_KEY]` exists, AvroSchema::try_from
55        // uses it verbatim; otherwise it is generated from the Arrow schema.
56        let avro_schema = AvroSchema::from_arrow_with_options(
57            schema,
58            Some(AvroSchemaOptions {
59                null_order: None,
60                strip_metadata: true,
61            }),
62        )
63        .map_err(|e| AvroError::SchemaError(format!("{e:?}")))?;
64        // Magic
65        writer.write_all(b"Obj\x01")?;
66        // File metadata map: { "avro.schema": <json>, "avro.codec": <codec> }
67        let codec_str = match compression {
68            Some(CompressionCodec::Deflate) => "deflate",
69            Some(CompressionCodec::Snappy) => "snappy",
70            Some(CompressionCodec::ZStandard) => "zstandard",
71            Some(CompressionCodec::Bzip2) => "bzip2",
72            Some(CompressionCodec::Xz) => "xz",
73            None => "null",
74        };
75        // Map block: count=2, then key/value pairs, then terminating count=0
76        write_long(writer, 2)?;
77        write_string(writer, SCHEMA_METADATA_KEY)?;
78        write_bytes(writer, avro_schema.json_string.as_bytes())?;
79        write_string(writer, CODEC_METADATA_KEY)?;
80        write_bytes(writer, codec_str.as_bytes())?;
81        write_long(writer, 0)?;
82        // Sync marker (16 bytes)
83        writer.write_all(&self.sync_marker)?;
84        Ok(())
85    }
86
87    fn sync_marker(&self) -> Option<&[u8; 16]> {
88        Some(&self.sync_marker)
89    }
90}
91
92/// Raw Avro binary streaming format using **Single-Object Encoding** per record.
93///
94/// Each record written by the stream writer is framed with a prefix determined
95/// by the schema fingerprinting algorithm.
96///
97/// See: <https://avro.apache.org/docs/1.11.1/specification/#single-object-encoding>
98/// See: <https://docs.confluent.io/platform/current/schema-registry/fundamentals/serdes-develop/index.html#wire-format>
99#[derive(Debug, Default)]
100pub struct AvroSoeFormat {}
101
102impl AvroFormat for AvroSoeFormat {
103    const NEEDS_PREFIX: bool = true;
104    fn start_stream<W: Write>(
105        &mut self,
106        _writer: &mut W,
107        _schema: &Schema,
108        compression: Option<CompressionCodec>,
109    ) -> Result<(), AvroError> {
110        if compression.is_some() {
111            return Err(AvroError::InvalidArgument(
112                "Compression not supported for Avro SOE streaming".to_string(),
113            ));
114        }
115        Ok(())
116    }
117
118    fn sync_marker(&self) -> Option<&[u8; 16]> {
119        None
120    }
121}
122
123/// Unframed Avro binary streaming format ("raw Avro record body bytes (no prefix, no OCF header)").
124///
125/// Each record written by the stream writer contains only the raw Avro
126/// record body bytes (i.e., the Avro binary encoding of the datum) with **no**
127/// per-record prefix and **no** Object Container File (OCF) header.
128///
129/// This format is useful when another transport provides framing (for example,
130/// length-delimited buffers) or when embedding Avro record payloads inside a
131/// larger envelope.
132#[derive(Debug, Default)]
133pub struct AvroBinaryFormat;
134
135impl AvroFormat for AvroBinaryFormat {
136    const NEEDS_PREFIX: bool = false;
137
138    fn start_stream<W: Write>(
139        &mut self,
140        _writer: &mut W,
141        _schema: &Schema,
142        compression: Option<CompressionCodec>,
143    ) -> Result<(), AvroError> {
144        if compression.is_some() {
145            return Err(AvroError::InvalidArgument(
146                "Compression not supported for Avro binary streaming".to_string(),
147            ));
148        }
149        Ok(())
150    }
151
152    fn sync_marker(&self) -> Option<&[u8; 16]> {
153        None
154    }
155}
156
157#[inline]
158fn write_string<W: Write>(writer: &mut W, s: &str) -> Result<(), AvroError> {
159    write_bytes(writer, s.as_bytes())
160}
161
162#[inline]
163fn write_bytes<W: Write>(writer: &mut W, bytes: &[u8]) -> Result<(), AvroError> {
164    write_long(writer, bytes.len() as i64)?;
165    writer.write_all(bytes)?;
166    Ok(())
167}
168
169#[cfg(test)]
170mod tests {
171    use super::*;
172    use arrow_schema::{DataType, Field, Schema};
173
174    fn test_schema() -> Schema {
175        Schema::new(vec![Field::new("x", DataType::Int32, false)])
176    }
177
178    #[test]
179    fn avro_binary_format_rejects_compression() {
180        let mut format = AvroBinaryFormat;
181        let schema = test_schema();
182        let err = format
183            .start_stream(
184                &mut Vec::<u8>::new(),
185                &schema,
186                Some(CompressionCodec::Snappy),
187            )
188            .unwrap_err();
189        assert!(
190            err.to_string()
191                .contains("Compression not supported for Avro binary streaming")
192        );
193    }
194
195    #[test]
196    fn avro_soe_format_rejects_compression() {
197        let mut format = AvroSoeFormat::default();
198        let schema = test_schema();
199        let err = format
200            .start_stream(
201                &mut Vec::<u8>::new(),
202                &schema,
203                Some(CompressionCodec::Snappy),
204            )
205            .unwrap_err();
206        assert!(
207            err.to_string()
208                .contains("Compression not supported for Avro SOE streaming")
209        );
210    }
211}