1use crate::codec::{AvroDataType, AvroField, Codec};
8use crate::errors::AvroError;
9use crate::schema::{Fingerprint, Nullability, Prefix};
10use arrow_array::Float16Array;
11use arrow_array::cast::AsArray;
12use arrow_array::types::{
13 ArrowPrimitiveType, Date32Type, Date64Type, DurationMicrosecondType, DurationMillisecondType,
14 DurationNanosecondType, DurationSecondType, Float16Type, Float32Type, Float64Type, Int8Type,
15 Int16Type, Int32Type, Int64Type, IntervalDayTimeType, IntervalMonthDayNanoType,
16 IntervalYearMonthType, RunEndIndexType, Time32MillisecondType, Time32SecondType,
17 Time64MicrosecondType, Time64NanosecondType, TimestampMicrosecondType,
18 TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, UInt8Type, UInt16Type,
19 UInt32Type, UInt64Type,
20};
21use arrow_array::{
22 Array, BinaryViewArray, Decimal128Array, Decimal256Array, DictionaryArray,
23 FixedSizeBinaryArray, FixedSizeListArray, GenericBinaryArray, GenericListArray,
24 GenericListViewArray, GenericStringArray, LargeListArray, LargeListViewArray, ListArray,
25 ListViewArray, MapArray, OffsetSizeTrait, PrimitiveArray, RecordBatch, RunArray, StringArray,
26 StringViewArray, StructArray, UnionArray,
27};
28#[cfg(feature = "small_decimals")]
29use arrow_array::{Decimal32Array, Decimal64Array};
30use arrow_buffer::{ArrowNativeType, NullBuffer};
31use arrow_schema::{DataType, Field, IntervalUnit, Schema as ArrowSchema, TimeUnit, UnionMode};
32use bytes::{BufMut, BytesMut};
33use std::io::Write;
34use std::sync::Arc;
35use uuid::Uuid;
36
37macro_rules! for_rows_with_prefix {
38 ($n:expr, $prefix:expr, $out:ident, |$row:ident| $body:block) => {{
39 match $prefix {
40 Some(prefix) => {
41 for $row in 0..$n {
42 $out.write_all(prefix)
43 .map_err(|e| AvroError::IoError(format!("write prefix: {e}"), e))?;
44 $body
45 }
46 }
47 None => {
48 for $row in 0..$n {
49 $body
50 }
51 }
52 }
53 }};
54}
55
56#[inline]
60pub(crate) fn write_long<W: Write + ?Sized>(out: &mut W, value: i64) -> Result<(), AvroError> {
61 let mut zz = ((value << 1) ^ (value >> 63)) as u64;
62 let mut buf = [0u8; 10];
64 let mut i = 0;
65 while (zz & !0x7F) != 0 {
66 buf[i] = ((zz & 0x7F) as u8) | 0x80;
67 i += 1;
68 zz >>= 7;
69 }
70 buf[i] = (zz & 0x7F) as u8;
71 i += 1;
72 out.write_all(&buf[..i])
73 .map_err(|e| AvroError::IoError(format!("write long: {e}"), e))
74}
75
76#[inline]
77fn write_int<W: Write + ?Sized>(out: &mut W, value: i32) -> Result<(), AvroError> {
78 write_long(out, value as i64)
79}
80
81#[inline]
82fn write_len_prefixed<W: Write + ?Sized>(out: &mut W, bytes: &[u8]) -> Result<(), AvroError> {
83 write_long(out, bytes.len() as i64)?;
84 out.write_all(bytes)
85 .map_err(|e| AvroError::IoError(format!("write bytes: {e}"), e))
86}
87
88#[inline]
89fn write_bool<W: Write + ?Sized>(out: &mut W, v: bool) -> Result<(), AvroError> {
90 out.write_all(&[u8::from(v)])
91 .map_err(|e| AvroError::IoError(format!("write bool: {e}"), e))
92}
93
94#[inline]
103fn minimal_twos_complement(be: &[u8]) -> &[u8] {
104 if be.is_empty() {
105 return be;
106 }
107 let sign_byte = if (be[0] & 0x80) != 0 { 0xFF } else { 0x00 };
108 let mut k = 0usize;
109 while k < be.len() && be[k] == sign_byte {
110 k += 1;
111 }
112 if k == 0 {
113 return be;
114 }
115 if k == be.len() {
116 return &be[be.len() - 1..];
117 }
118 let drop = if ((be[k] ^ sign_byte) & 0x80) == 0 {
119 k
120 } else {
121 k - 1
122 };
123 &be[drop..]
124}
125
126#[inline]
138fn write_sign_extended<W: Write + ?Sized>(
139 out: &mut W,
140 src_be: &[u8],
141 n: usize,
142) -> Result<(), AvroError> {
143 let len = src_be.len();
144 if len == n {
145 out.write_all(src_be)?;
146 return Ok(());
147 }
148 let sign_byte = if len > 0 && (src_be[0] & 0x80) != 0 {
149 0xFF
150 } else {
151 0x00
152 };
153 if len > n {
154 let extra = len - n;
155 if n == 0 && src_be.iter().all(|&b| b == sign_byte) {
156 return Ok(());
157 }
158 if src_be[..extra].iter().any(|&b| b != sign_byte)
161 || ((src_be[extra] ^ sign_byte) & 0x80) != 0
162 {
163 return Err(AvroError::InvalidArgument(format!(
164 "Decimal value with {len} bytes cannot be represented in {n} bytes without overflow",
165 )));
166 }
167 return out
168 .write_all(&src_be[extra..])
169 .map_err(|e| AvroError::IoError(format!("write decimal fixed: {e}"), e));
170 }
171 let pad_len = n - len;
173 const ZPAD: [u8; 64] = [0x00; 64];
175 const FPAD: [u8; 64] = [0xFF; 64];
176 let pad = if sign_byte == 0x00 {
177 &ZPAD[..]
178 } else {
179 &FPAD[..]
180 };
181 let mut rem = pad_len;
184 while rem >= pad.len() {
185 out.write_all(pad)
186 .map_err(|e| AvroError::IoError(format!("write decimal fixed: {e}"), e))?;
187 rem -= pad.len();
188 }
189 if rem > 0 {
190 out.write_all(&pad[..rem])
191 .map_err(|e| AvroError::IoError(format!("write decimal fixed: {e}"), e))?;
192 }
193 out.write_all(src_be)
194 .map_err(|e| AvroError::IoError(format!("write decimal fixed: {e}"), e))
195}
196
197fn write_optional_index<W: Write + ?Sized>(
203 out: &mut W,
204 is_null: bool,
205 null_order: Nullability,
206) -> Result<(), AvroError> {
207 let byte = union_value_branch_byte(null_order, is_null);
208 out.write_all(&[byte])
209 .map_err(|e| AvroError::IoError(format!("write union branch: {e}"), e))
210}
211
212#[derive(Debug, Clone)]
213enum NullState<'a> {
214 NonNullable,
215 NullableNoNulls {
216 union_value_byte: u8,
217 },
218 Nullable {
219 nulls: &'a NullBuffer,
220 null_order: Nullability,
221 },
222}
223
224pub(crate) struct FieldEncoder<'a> {
228 encoder: Encoder<'a>,
229 null_state: NullState<'a>,
230}
231
232impl<'a> FieldEncoder<'a> {
233 fn make_encoder(
234 array: &'a dyn Array,
235 plan: &FieldPlan,
236 nullability: Option<Nullability>,
237 ) -> Result<Self, AvroError> {
238 let encoder = match plan {
239 FieldPlan::Scalar => match array.data_type() {
240 DataType::Null => Encoder::Null,
241 DataType::Boolean => Encoder::Boolean(BooleanEncoder(array.as_boolean())),
242 DataType::Utf8 => {
243 Encoder::Utf8(Utf8GenericEncoder::<i32>(array.as_string::<i32>()))
244 }
245 DataType::LargeUtf8 => {
246 Encoder::Utf8Large(Utf8GenericEncoder::<i64>(array.as_string::<i64>()))
247 }
248 DataType::Utf8View => {
249 let arr = array
250 .as_any()
251 .downcast_ref::<StringViewArray>()
252 .ok_or_else(|| AvroError::SchemaError("Expected StringViewArray".into()))?;
253 Encoder::Utf8View(Utf8ViewEncoder(arr))
254 }
255 DataType::BinaryView => {
256 let arr = array
257 .as_any()
258 .downcast_ref::<BinaryViewArray>()
259 .ok_or_else(|| AvroError::SchemaError("Expected BinaryViewArray".into()))?;
260 Encoder::BinaryView(BinaryViewEncoder(arr))
261 }
262 DataType::Int32 => Encoder::Int(IntEncoder(array.as_primitive::<Int32Type>())),
263 DataType::Int64 => Encoder::Long(LongEncoder(array.as_primitive::<Int64Type>())),
264 #[cfg(feature = "avro_custom_types")]
265 DataType::Int8 => Encoder::Int8(Int8Encoder(array.as_primitive::<Int8Type>())),
266 #[cfg(not(feature = "avro_custom_types"))]
267 DataType::Int8 => {
268 Encoder::Int8ToInt(Int8ToIntEncoder(array.as_primitive::<Int8Type>()))
269 }
270 #[cfg(feature = "avro_custom_types")]
271 DataType::Int16 => Encoder::Int16(Int16Encoder(array.as_primitive::<Int16Type>())),
272 #[cfg(not(feature = "avro_custom_types"))]
273 DataType::Int16 => {
274 Encoder::Int16ToInt(Int16ToIntEncoder(array.as_primitive::<Int16Type>()))
275 }
276 #[cfg(feature = "avro_custom_types")]
277 DataType::UInt8 => Encoder::UInt8(UInt8Encoder(array.as_primitive::<UInt8Type>())),
278 #[cfg(not(feature = "avro_custom_types"))]
279 DataType::UInt8 => {
280 Encoder::UInt8ToInt(UInt8ToIntEncoder(array.as_primitive::<UInt8Type>()))
281 }
282 #[cfg(feature = "avro_custom_types")]
283 DataType::UInt16 => {
284 Encoder::UInt16(UInt16Encoder(array.as_primitive::<UInt16Type>()))
285 }
286 #[cfg(not(feature = "avro_custom_types"))]
287 DataType::UInt16 => {
288 Encoder::UInt16ToInt(UInt16ToIntEncoder(array.as_primitive::<UInt16Type>()))
289 }
290 #[cfg(feature = "avro_custom_types")]
291 DataType::UInt32 => {
292 Encoder::UInt32(UInt32Encoder(array.as_primitive::<UInt32Type>()))
293 }
294 #[cfg(not(feature = "avro_custom_types"))]
295 DataType::UInt32 => {
296 Encoder::UInt32ToLong(UInt32ToLongEncoder(array.as_primitive::<UInt32Type>()))
297 }
298 #[cfg(feature = "avro_custom_types")]
299 DataType::UInt64 => {
300 Encoder::UInt64Fixed(UInt64FixedEncoder(array.as_primitive::<UInt64Type>()))
301 }
302 #[cfg(not(feature = "avro_custom_types"))]
303 DataType::UInt64 => {
304 Encoder::UInt64ToLong(UInt64ToLongEncoder(array.as_primitive::<UInt64Type>()))
305 }
306 #[cfg(feature = "avro_custom_types")]
307 DataType::Float16 => {
308 Encoder::Float16Fixed(Float16FixedEncoder(array.as_primitive::<Float16Type>()))
309 }
310 #[cfg(not(feature = "avro_custom_types"))]
311 DataType::Float16 => Encoder::Float16ToFloat(Float16ToFloatEncoder(
312 array.as_primitive::<Float16Type>(),
313 )),
314 DataType::Date32 => Encoder::Date32(IntEncoder(array.as_primitive::<Date32Type>())),
315 #[cfg(not(feature = "avro_custom_types"))]
316 DataType::Date64 => {
317 Encoder::Date64ToLong(Date64ToLongEncoder(array.as_primitive::<Date64Type>()))
319 }
320 #[cfg(feature = "avro_custom_types")]
321 DataType::Date64 => {
322 Encoder::Date64(LongEncoder(array.as_primitive::<Date64Type>()))
323 }
324 #[cfg(feature = "avro_custom_types")]
325 DataType::Time32(TimeUnit::Second) => {
326 Encoder::Time32Secs(IntEncoder(array.as_primitive::<Time32SecondType>()))
327 }
328 DataType::Time32(TimeUnit::Millisecond) => {
329 Encoder::Time32Millis(IntEncoder(array.as_primitive::<Time32MillisecondType>()))
330 }
331 DataType::Time32(TimeUnit::Microsecond) => {
332 return Err(AvroError::InvalidArgument(
333 "Arrow Time32 only supports Second or Millisecond. Use Time64 for microseconds."
334 .into(),
335 ));
336 }
337 DataType::Time32(TimeUnit::Nanosecond) => {
338 return Err(AvroError::InvalidArgument(
339 "Arrow Time32 only supports Second or Millisecond. Use Time64 for nanoseconds."
340 .into(),
341 ));
342 }
343 DataType::Time64(TimeUnit::Microsecond) => Encoder::Time64Micros(LongEncoder(
344 array.as_primitive::<Time64MicrosecondType>(),
345 )),
346 #[cfg(not(feature = "avro_custom_types"))]
347 DataType::Time64(TimeUnit::Nanosecond) => {
348 Encoder::Time64NanosToMicros(Time64NanosToMicrosEncoder(
350 array.as_primitive::<Time64NanosecondType>(),
351 ))
352 }
353 #[cfg(feature = "avro_custom_types")]
354 DataType::Time64(TimeUnit::Nanosecond) => {
355 Encoder::Time64Nanos(LongEncoder(array.as_primitive::<Time64NanosecondType>()))
356 }
357 DataType::Time64(TimeUnit::Millisecond) => {
358 return Err(AvroError::InvalidArgument(
359 "Arrow Time64 with millisecond unit is not a valid Arrow type (use Time32 for millis)."
360 .into(),
361 ));
362 }
363 DataType::Time64(TimeUnit::Second) => {
364 return Err(AvroError::InvalidArgument(
365 "Arrow Time64 with second unit is not a valid Arrow type (use Time32 for seconds)."
366 .into(),
367 ));
368 }
369 DataType::Float32 => {
370 Encoder::Float32(F32Encoder(array.as_primitive::<Float32Type>()))
371 }
372 DataType::Float64 => {
373 Encoder::Float64(F64Encoder(array.as_primitive::<Float64Type>()))
374 }
375 DataType::Binary => Encoder::Binary(BinaryEncoder(array.as_binary::<i32>())),
376 DataType::LargeBinary => {
377 Encoder::LargeBinary(BinaryEncoder(array.as_binary::<i64>()))
378 }
379 DataType::FixedSizeBinary(_len) => {
380 let arr = array
381 .as_any()
382 .downcast_ref::<FixedSizeBinaryArray>()
383 .ok_or_else(|| {
384 AvroError::SchemaError("Expected FixedSizeBinaryArray".into())
385 })?;
386 Encoder::Fixed(FixedEncoder(arr))
387 }
388 DataType::Timestamp(unit, _) => match unit {
389 TimeUnit::Second => {
390 #[cfg(not(feature = "avro_custom_types"))]
391 {
392 Encoder::TimestampSecsToMillis(TimestampSecondsToMillisEncoder(
393 array.as_primitive::<TimestampSecondType>(),
394 ))
395 }
396 #[cfg(feature = "avro_custom_types")]
397 {
398 Encoder::TimestampSecs(LongEncoder(
399 array.as_primitive::<TimestampSecondType>(),
400 ))
401 }
402 }
403 TimeUnit::Millisecond => Encoder::TimestampMillis(LongEncoder(
404 array.as_primitive::<TimestampMillisecondType>(),
405 )),
406 TimeUnit::Microsecond => Encoder::TimestampMicros(LongEncoder(
407 array.as_primitive::<TimestampMicrosecondType>(),
408 )),
409 TimeUnit::Nanosecond => Encoder::TimestampNanos(LongEncoder(
410 array.as_primitive::<TimestampNanosecondType>(),
411 )),
412 },
413 #[cfg(feature = "avro_custom_types")]
414 DataType::Interval(unit) => match unit {
415 IntervalUnit::MonthDayNano => {
416 Encoder::IntervalMonthDayNanoFixed(IntervalMonthDayNanoFixedEncoder(
417 array.as_primitive::<IntervalMonthDayNanoType>(),
418 ))
419 }
420 IntervalUnit::YearMonth => {
421 Encoder::IntervalYearMonthFixed(IntervalYearMonthFixedEncoder(
422 array.as_primitive::<IntervalYearMonthType>(),
423 ))
424 }
425 IntervalUnit::DayTime => Encoder::IntervalDayTimeFixed(
426 IntervalDayTimeFixedEncoder(array.as_primitive::<IntervalDayTimeType>()),
427 ),
428 },
429 DataType::Duration(tu) => match tu {
430 TimeUnit::Second => Encoder::DurationSeconds(LongEncoder(
431 array.as_primitive::<DurationSecondType>(),
432 )),
433 TimeUnit::Millisecond => Encoder::DurationMillis(LongEncoder(
434 array.as_primitive::<DurationMillisecondType>(),
435 )),
436 TimeUnit::Microsecond => Encoder::DurationMicros(LongEncoder(
437 array.as_primitive::<DurationMicrosecondType>(),
438 )),
439 TimeUnit::Nanosecond => Encoder::DurationNanos(LongEncoder(
440 array.as_primitive::<DurationNanosecondType>(),
441 )),
442 },
443 other => {
444 return Err(AvroError::NYI(format!(
445 "Avro scalar type not yet supported: {other:?}"
446 )));
447 }
448 },
449 FieldPlan::Struct { bindings } => {
450 let arr = array
451 .as_any()
452 .downcast_ref::<StructArray>()
453 .ok_or_else(|| AvroError::SchemaError("Expected StructArray".into()))?;
454 Encoder::Struct(Box::new(StructEncoder::try_new(arr, bindings)?))
455 }
456 FieldPlan::List {
457 items_nullability,
458 item_plan,
459 } => match array.data_type() {
460 DataType::List(_) => {
461 let arr = array
462 .as_any()
463 .downcast_ref::<ListArray>()
464 .ok_or_else(|| AvroError::SchemaError("Expected ListArray".into()))?;
465 Encoder::List(Box::new(ListEncoder32::try_new(
466 arr,
467 *items_nullability,
468 item_plan.as_ref(),
469 )?))
470 }
471 DataType::LargeList(_) => {
472 let arr = array
473 .as_any()
474 .downcast_ref::<LargeListArray>()
475 .ok_or_else(|| AvroError::SchemaError("Expected LargeListArray".into()))?;
476 Encoder::LargeList(Box::new(ListEncoder64::try_new(
477 arr,
478 *items_nullability,
479 item_plan.as_ref(),
480 )?))
481 }
482 DataType::ListView(_) => {
483 let arr = array
484 .as_any()
485 .downcast_ref::<ListViewArray>()
486 .ok_or_else(|| AvroError::SchemaError("Expected ListViewArray".into()))?;
487 Encoder::ListView(Box::new(ListViewEncoder32::try_new(
488 arr,
489 *items_nullability,
490 item_plan.as_ref(),
491 )?))
492 }
493 DataType::LargeListView(_) => {
494 let arr = array
495 .as_any()
496 .downcast_ref::<LargeListViewArray>()
497 .ok_or_else(|| {
498 AvroError::SchemaError("Expected LargeListViewArray".into())
499 })?;
500 Encoder::LargeListView(Box::new(ListViewEncoder64::try_new(
501 arr,
502 *items_nullability,
503 item_plan.as_ref(),
504 )?))
505 }
506 DataType::FixedSizeList(_, _) => {
507 let arr = array
508 .as_any()
509 .downcast_ref::<FixedSizeListArray>()
510 .ok_or_else(|| {
511 AvroError::SchemaError("Expected FixedSizeListArray".into())
512 })?;
513 Encoder::FixedSizeList(Box::new(FixedSizeListEncoder::try_new(
514 arr,
515 *items_nullability,
516 item_plan.as_ref(),
517 )?))
518 }
519 other => {
520 return Err(AvroError::SchemaError(format!(
521 "Avro array site requires Arrow List/LargeList/ListView/LargeListView/FixedSizeList, found: {other:?}"
522 )));
523 }
524 },
525 FieldPlan::Decimal { size } => match array.data_type() {
526 #[cfg(feature = "small_decimals")]
527 DataType::Decimal32(_, _) => {
528 let arr = array
529 .as_any()
530 .downcast_ref::<Decimal32Array>()
531 .ok_or_else(|| AvroError::SchemaError("Expected Decimal32Array".into()))?;
532 Encoder::Decimal32(DecimalEncoder::<4, Decimal32Array>::new(arr, *size))
533 }
534 #[cfg(feature = "small_decimals")]
535 DataType::Decimal64(_, _) => {
536 let arr = array
537 .as_any()
538 .downcast_ref::<Decimal64Array>()
539 .ok_or_else(|| AvroError::SchemaError("Expected Decimal64Array".into()))?;
540 Encoder::Decimal64(DecimalEncoder::<8, Decimal64Array>::new(arr, *size))
541 }
542 DataType::Decimal128(_, _) => {
543 let arr = array
544 .as_any()
545 .downcast_ref::<Decimal128Array>()
546 .ok_or_else(|| AvroError::SchemaError("Expected Decimal128Array".into()))?;
547 Encoder::Decimal128(DecimalEncoder::<16, Decimal128Array>::new(arr, *size))
548 }
549 DataType::Decimal256(_, _) => {
550 let arr = array
551 .as_any()
552 .downcast_ref::<Decimal256Array>()
553 .ok_or_else(|| AvroError::SchemaError("Expected Decimal256Array".into()))?;
554 Encoder::Decimal256(DecimalEncoder::<32, Decimal256Array>::new(arr, *size))
555 }
556 other => {
557 return Err(AvroError::SchemaError(format!(
558 "Avro decimal site requires Arrow Decimal 32, 64, 128, or 256, found: {other:?}"
559 )));
560 }
561 },
562 FieldPlan::Uuid => {
563 let arr = array
564 .as_any()
565 .downcast_ref::<FixedSizeBinaryArray>()
566 .ok_or_else(|| {
567 AvroError::SchemaError("Expected FixedSizeBinaryArray".into())
568 })?;
569 Encoder::Uuid(UuidEncoder(arr))
570 }
571 FieldPlan::Map {
572 values_nullability,
573 value_plan,
574 } => {
575 let arr = array
576 .as_any()
577 .downcast_ref::<MapArray>()
578 .ok_or_else(|| AvroError::SchemaError("Expected MapArray".into()))?;
579 Encoder::Map(Box::new(MapEncoder::try_new(
580 arr,
581 *values_nullability,
582 value_plan.as_ref(),
583 )?))
584 }
585 FieldPlan::Enum { symbols } => match array.data_type() {
586 DataType::Dictionary(key_dt, value_dt) => {
587 if **key_dt != DataType::Int32 || **value_dt != DataType::Utf8 {
588 return Err(AvroError::SchemaError(
589 "Avro enum requires Dictionary<Int32, Utf8>".into(),
590 ));
591 }
592 let dict = array
593 .as_any()
594 .downcast_ref::<DictionaryArray<Int32Type>>()
595 .ok_or_else(|| {
596 AvroError::SchemaError("Expected DictionaryArray<Int32>".into())
597 })?;
598 let values = dict
599 .values()
600 .as_any()
601 .downcast_ref::<StringArray>()
602 .ok_or_else(|| {
603 AvroError::SchemaError("Dictionary values must be Utf8".into())
604 })?;
605 if values.len() != symbols.len() {
606 return Err(AvroError::SchemaError(format!(
607 "Enum symbol length {} != dictionary size {}",
608 symbols.len(),
609 values.len()
610 )));
611 }
612 for i in 0..values.len() {
613 if values.value(i) != symbols[i].as_str() {
614 return Err(AvroError::SchemaError(format!(
615 "Enum symbol mismatch at {i}: schema='{}' dict='{}'",
616 symbols[i],
617 values.value(i)
618 )));
619 }
620 }
621 let keys = dict.keys();
622 Encoder::Enum(EnumEncoder { keys })
623 }
624 other => {
625 return Err(AvroError::SchemaError(format!(
626 "Avro enum site requires DataType::Dictionary, found: {other:?}"
627 )));
628 }
629 },
630 FieldPlan::Union { bindings } => {
631 let arr = array
632 .as_any()
633 .downcast_ref::<UnionArray>()
634 .ok_or_else(|| AvroError::SchemaError("Expected UnionArray".into()))?;
635
636 Encoder::Union(Box::new(UnionEncoder::try_new(arr, bindings)?))
637 }
638 FieldPlan::RunEndEncoded {
639 values_nullability,
640 value_plan,
641 } => {
642 let build = |run_arr_any: &'a dyn Array| -> Result<Encoder<'a>, AvroError> {
644 if let Some(arr) = run_arr_any.as_any().downcast_ref::<RunArray<Int16Type>>() {
645 return Ok(Encoder::RunEncoded16(Box::new(RunEncodedEncoder::<
646 Int16Type,
647 >::new(
648 arr,
649 FieldEncoder::make_encoder(
650 arr.values().as_ref(),
651 value_plan.as_ref(),
652 *values_nullability,
653 )?,
654 ))));
655 }
656 if let Some(arr) = run_arr_any.as_any().downcast_ref::<RunArray<Int32Type>>() {
657 return Ok(Encoder::RunEncoded32(Box::new(RunEncodedEncoder::<
658 Int32Type,
659 >::new(
660 arr,
661 FieldEncoder::make_encoder(
662 arr.values().as_ref(),
663 value_plan.as_ref(),
664 *values_nullability,
665 )?,
666 ))));
667 }
668 if let Some(arr) = run_arr_any.as_any().downcast_ref::<RunArray<Int64Type>>() {
669 return Ok(Encoder::RunEncoded64(Box::new(RunEncodedEncoder::<
670 Int64Type,
671 >::new(
672 arr,
673 FieldEncoder::make_encoder(
674 arr.values().as_ref(),
675 value_plan.as_ref(),
676 *values_nullability,
677 )?,
678 ))));
679 }
680 Err(AvroError::SchemaError(
681 "Unsupported run-ends index type for RunEndEncoded; expected Int16/Int32/Int64"
682 .into(),
683 ))
684 };
685 build(array)?
686 }
687 FieldPlan::Duration => match array.data_type() {
688 DataType::Interval(IntervalUnit::MonthDayNano) => {
689 Encoder::IntervalMonthDayNanoDuration(DurationEncoder(
690 array.as_primitive::<IntervalMonthDayNanoType>(),
691 ))
692 }
693 DataType::Interval(IntervalUnit::YearMonth) => Encoder::IntervalYearMonthDuration(
694 DurationEncoder(array.as_primitive::<IntervalYearMonthType>()),
695 ),
696 DataType::Interval(IntervalUnit::DayTime) => Encoder::IntervalDayTimeDuration(
697 DurationEncoder(array.as_primitive::<IntervalDayTimeType>()),
698 ),
699 other => {
700 return Err(AvroError::SchemaError(format!(
701 "Avro duration requires Arrow Interval type, found: {other:?}"
702 )));
703 }
704 },
705 FieldPlan::TimeMillisFromSecs => match array.data_type() {
706 DataType::Time32(TimeUnit::Second) => Encoder::Time32SecsToMillis(
707 Time32SecondsToMillisEncoder(array.as_primitive::<Time32SecondType>()),
708 ),
709 other => {
710 return Err(AvroError::SchemaError(format!(
711 "Avro time-millis-from-seconds requires Arrow Time32(Second), found: {other:?}"
712 )));
713 }
714 },
715 };
716 let null_state = match nullability {
718 None => NullState::NonNullable,
719 Some(null_order) => match array.nulls() {
720 Some(nulls) if array.null_count() > 0 => NullState::Nullable { nulls, null_order },
721 _ => NullState::NullableNoNulls {
722 union_value_byte: union_value_branch_byte(null_order, false),
724 },
725 },
726 };
727 Ok(Self {
728 encoder,
729 null_state,
730 })
731 }
732
733 fn encode<W: Write + ?Sized>(&mut self, out: &mut W, idx: usize) -> Result<(), AvroError> {
734 match &self.null_state {
735 NullState::NonNullable => {}
736 NullState::NullableNoNulls { union_value_byte } => out
737 .write_all(&[*union_value_byte])
738 .map_err(|e| AvroError::IoError(format!("write union value branch: {e}"), e))?,
739 NullState::Nullable { nulls, null_order } if nulls.is_null(idx) => {
740 return write_optional_index(out, true, *null_order); }
742 NullState::Nullable { null_order, .. } => {
743 write_optional_index(out, false, *null_order)?;
744 }
745 }
746 self.encoder.encode(out, idx)
747 }
748}
749
750fn union_value_branch_byte(null_order: Nullability, is_null: bool) -> u8 {
751 let nulls_first = null_order == Nullability::default();
752 if nulls_first == is_null { 0x00 } else { 0x02 }
753}
754
755#[derive(Debug, Clone)]
758enum FieldPlan {
759 Scalar,
761 Struct { bindings: Vec<FieldBinding> },
763 List {
765 items_nullability: Option<Nullability>,
766 item_plan: Box<FieldPlan>,
767 },
768 Decimal { size: Option<usize> },
770 Uuid,
772 Map {
774 values_nullability: Option<Nullability>,
775 value_plan: Box<FieldPlan>,
776 },
777 Enum { symbols: Arc<[String]> },
780 Union { bindings: Vec<FieldBinding> },
782 RunEndEncoded {
785 values_nullability: Option<Nullability>,
786 value_plan: Box<FieldPlan>,
787 },
788 Duration,
792 TimeMillisFromSecs,
796}
797
798#[derive(Debug, Clone)]
799struct FieldBinding {
800 arrow_index: usize,
802 nullability: Option<Nullability>,
804 plan: FieldPlan,
806}
807
808#[derive(Debug)]
810pub(crate) struct RecordEncoderBuilder<'a> {
811 avro_root: &'a AvroField,
812 arrow_schema: &'a ArrowSchema,
813 fingerprint: Option<Fingerprint>,
814}
815
816impl<'a> RecordEncoderBuilder<'a> {
817 pub(crate) fn new(avro_root: &'a AvroField, arrow_schema: &'a ArrowSchema) -> Self {
819 Self {
820 avro_root,
821 arrow_schema,
822 fingerprint: None,
823 }
824 }
825
826 pub(crate) fn with_fingerprint(mut self, fingerprint: Option<Fingerprint>) -> Self {
827 self.fingerprint = fingerprint;
828 self
829 }
830
831 pub(crate) fn build(self) -> Result<RecordEncoder, AvroError> {
834 let avro_root_dt = self.avro_root.data_type();
835 let Codec::Struct(root_fields) = avro_root_dt.codec() else {
836 return Err(AvroError::SchemaError(
837 "Top-level Avro schema must be a record/struct".into(),
838 ));
839 };
840 let mut columns = Vec::with_capacity(root_fields.len());
841 for root_field in root_fields.as_ref() {
842 let name = root_field.name();
843 let arrow_index = self.arrow_schema.index_of(name).map_err(|e| {
844 AvroError::SchemaError(format!("Schema mismatch for field '{name}': {e}"))
845 })?;
846 columns.push(FieldBinding {
847 arrow_index,
848 nullability: root_field.data_type().nullability(),
849 plan: FieldPlan::build(
850 root_field.data_type(),
851 self.arrow_schema.field(arrow_index),
852 )?,
853 });
854 }
855 Ok(RecordEncoder {
856 columns,
857 prefix: self.fingerprint.map(|fp| fp.make_prefix()),
858 })
859 }
860}
861
862#[derive(Debug, Clone)]
868pub(crate) struct RecordEncoder {
869 columns: Vec<FieldBinding>,
870 prefix: Option<Prefix>,
872}
873
874impl RecordEncoder {
875 fn prepare_for_batch<'a>(
876 &'a self,
877 batch: &'a RecordBatch,
878 ) -> Result<Vec<FieldEncoder<'a>>, AvroError> {
879 let arrays = batch.columns();
880 let mut out = Vec::with_capacity(self.columns.len());
881 for col_plan in &self.columns {
882 let arrow_index = col_plan.arrow_index;
883 let array = arrays.get(arrow_index).ok_or_else(|| {
884 AvroError::SchemaError(format!("Column index {arrow_index} out of range"))
885 })?;
886 #[cfg(not(feature = "avro_custom_types"))]
887 let site_nullability = match &col_plan.plan {
888 FieldPlan::RunEndEncoded { .. } => None,
889 _ => col_plan.nullability,
890 };
891 #[cfg(feature = "avro_custom_types")]
892 let site_nullability = col_plan.nullability;
893 out.push(FieldEncoder::make_encoder(
894 array.as_ref(),
895 &col_plan.plan,
896 site_nullability,
897 )?);
898 }
899 Ok(out)
900 }
901
902 pub(crate) fn encode<W: Write>(
906 &self,
907 out: &mut W,
908 batch: &RecordBatch,
909 ) -> Result<(), AvroError> {
910 let mut column_encoders = self.prepare_for_batch(batch)?;
911 let n = batch.num_rows();
912 let prefix = self.prefix.as_ref().map(|p| p.as_slice());
913 for_rows_with_prefix!(n, prefix, out, |row| {
914 for enc in &mut column_encoders {
915 enc.encode(out, row)?;
916 }
917 });
918 Ok(())
919 }
920
921 pub(crate) fn encode_rows(
930 &self,
931 batch: &RecordBatch,
932 row_capacity: usize,
933 out: &mut BytesMut,
934 offsets: &mut Vec<usize>,
935 ) -> Result<(), AvroError> {
936 let out_len = out.len();
937 if offsets.first() != Some(&0) || offsets.last() != Some(&out_len) {
938 return Err(AvroError::General(
939 "encode_rows requires offsets to start with 0 and end at out.len()".to_string(),
940 ));
941 }
942 let n = batch.num_rows();
943 if n == 0 {
944 return Ok(());
945 }
946 if offsets.len().checked_add(n).is_none() {
947 return Err(AvroError::General(
948 "encode_rows cannot append offsets: too many rows".to_string(),
949 ));
950 }
951 let mut column_encoders = self.prepare_for_batch(batch)?;
952 offsets.reserve(n);
953 let prefix_bytes = self.prefix.as_ref().map(|p| p.as_slice());
954 let prefix_len = prefix_bytes.map_or(0, |p| p.len());
955 let per_row_hint = row_capacity.max(prefix_len);
956 if let Some(additional) = n
957 .checked_mul(per_row_hint)
958 .filter(|&a| out_len.checked_add(a).is_some())
959 {
960 out.reserve(additional);
961 }
962 let start_out_len = out.len();
963 let start_offsets_len = offsets.len();
964 let res = (|| -> Result<(), AvroError> {
965 let mut w = out.writer();
966 if let [enc0] = column_encoders.as_mut_slice() {
967 for_rows_with_prefix!(n, prefix_bytes, w, |row| {
968 enc0.encode(&mut w, row)?;
969 offsets.push(w.get_ref().len());
970 });
971 } else {
972 for_rows_with_prefix!(n, prefix_bytes, w, |row| {
973 for enc in &mut column_encoders {
974 enc.encode(&mut w, row)?;
975 }
976 offsets.push(w.get_ref().len());
977 });
978 }
979 Ok(())
980 })();
981 if res.is_err() {
982 out.truncate(start_out_len);
983 offsets.truncate(start_offsets_len);
984 } else {
985 debug_assert_eq!(
986 *offsets.last().unwrap(),
987 out.len(),
988 "encode_rows: offsets/out length mismatch after successful encode"
989 );
990 }
991 res
992 }
993}
994
995fn find_struct_child_index(fields: &arrow_schema::Fields, name: &str) -> Option<usize> {
996 fields.iter().position(|f| f.name() == name)
997}
998
999fn find_map_value_field_index(fields: &arrow_schema::Fields) -> Option<usize> {
1000 find_struct_child_index(fields, Field::MAP_VALUE_FIELD_DEFAULT_NAME)
1002 .or_else(|| find_struct_child_index(fields, "values"))
1003 .or_else(|| if fields.len() == 2 { Some(1) } else { None })
1004}
1005
1006impl FieldPlan {
1007 fn build(avro_dt: &AvroDataType, arrow_field: &Field) -> Result<Self, AvroError> {
1008 #[cfg(not(feature = "avro_custom_types"))]
1009 if let DataType::RunEndEncoded(_re_field, values_field) = arrow_field.data_type() {
1010 let values_nullability = avro_dt.nullability();
1011 let value_site_dt: &AvroDataType = match avro_dt.codec() {
1012 Codec::Union(branches, _, _) => branches
1013 .iter()
1014 .find(|b| !matches!(b.codec(), Codec::Null))
1015 .ok_or_else(|| {
1016 AvroError::SchemaError(
1017 "Avro union at RunEndEncoded site has no non-null branch".into(),
1018 )
1019 })?,
1020 _ => avro_dt,
1021 };
1022 return Ok(FieldPlan::RunEndEncoded {
1023 values_nullability,
1024 value_plan: Box::new(FieldPlan::build(value_site_dt, values_field.as_ref())?),
1025 });
1026 }
1027 if let DataType::FixedSizeBinary(len) = arrow_field.data_type() {
1028 let ext_is_uuid = {
1030 #[cfg(feature = "canonical_extension_types")]
1031 {
1032 matches!(
1033 arrow_field.extension_type_name(),
1034 Some("arrow.uuid" | "uuid")
1035 )
1036 }
1037 #[cfg(not(feature = "canonical_extension_types"))]
1038 {
1039 false
1040 }
1041 };
1042 let md_is_uuid = arrow_field
1043 .metadata()
1044 .get("logicalType")
1045 .map(|s| s.as_str())
1046 == Some("uuid");
1047 if ext_is_uuid || md_is_uuid {
1048 if *len != 16 {
1049 return Err(AvroError::InvalidArgument(
1050 "logicalType=uuid requires FixedSizeBinary(16)".into(),
1051 ));
1052 }
1053 return Ok(FieldPlan::Uuid);
1054 }
1055 }
1056 match avro_dt.codec() {
1057 Codec::Struct(avro_fields) => {
1058 let fields = match arrow_field.data_type() {
1059 DataType::Struct(struct_fields) => struct_fields,
1060 other => {
1061 return Err(AvroError::SchemaError(format!(
1062 "Avro struct maps to Arrow Struct, found: {other:?}"
1063 )));
1064 }
1065 };
1066 let mut bindings = Vec::with_capacity(avro_fields.len());
1067 for avro_field in avro_fields.iter() {
1068 let name = avro_field.name().to_string();
1069 let idx = find_struct_child_index(fields, &name).ok_or_else(|| {
1070 AvroError::SchemaError(format!(
1071 "Struct field '{name}' not present in Arrow field '{}'",
1072 arrow_field.name()
1073 ))
1074 })?;
1075 bindings.push(FieldBinding {
1076 arrow_index: idx,
1077 nullability: avro_field.data_type().nullability(),
1078 plan: FieldPlan::build(avro_field.data_type(), fields[idx].as_ref())?,
1079 });
1080 }
1081 Ok(FieldPlan::Struct { bindings })
1082 }
1083 Codec::List(items_dt) => match arrow_field.data_type() {
1084 DataType::List(field_ref)
1085 | DataType::LargeList(field_ref)
1086 | DataType::ListView(field_ref)
1087 | DataType::LargeListView(field_ref) => Ok(FieldPlan::List {
1088 items_nullability: items_dt.nullability(),
1089 item_plan: Box::new(FieldPlan::build(items_dt.as_ref(), field_ref.as_ref())?),
1090 }),
1091 DataType::FixedSizeList(field_ref, _len) => Ok(FieldPlan::List {
1092 items_nullability: items_dt.nullability(),
1093 item_plan: Box::new(FieldPlan::build(items_dt.as_ref(), field_ref.as_ref())?),
1094 }),
1095 other => Err(AvroError::SchemaError(format!(
1096 "Avro array maps to Arrow List/LargeList/ListView/LargeListView/FixedSizeList, found: {other:?}"
1097 ))),
1098 },
1099 Codec::Map(values_dt) => {
1100 let entries_field = match arrow_field.data_type() {
1101 DataType::Map(entries, _sorted) => entries.as_ref(),
1102 other => {
1103 return Err(AvroError::SchemaError(format!(
1104 "Avro map maps to Arrow DataType::Map, found: {other:?}"
1105 )));
1106 }
1107 };
1108 let entries_struct_fields = match entries_field.data_type() {
1109 DataType::Struct(fs) => fs,
1110 other => {
1111 return Err(AvroError::SchemaError(format!(
1112 "Arrow Map entries must be Struct, found: {other:?}"
1113 )));
1114 }
1115 };
1116 let value_idx =
1117 find_map_value_field_index(entries_struct_fields).ok_or_else(|| {
1118 AvroError::SchemaError("Map entries struct missing value field".into())
1119 })?;
1120 let value_field = entries_struct_fields[value_idx].as_ref();
1121 let value_plan = FieldPlan::build(values_dt.as_ref(), value_field)?;
1122 Ok(FieldPlan::Map {
1123 values_nullability: values_dt.nullability(),
1124 value_plan: Box::new(value_plan),
1125 })
1126 }
1127 Codec::Enum(symbols) => match arrow_field.data_type() {
1128 DataType::Dictionary(key_dt, value_dt) => {
1129 if **key_dt != DataType::Int32 {
1130 return Err(AvroError::SchemaError(
1131 "Avro enum requires Dictionary<Int32, Utf8>".into(),
1132 ));
1133 }
1134 if **value_dt != DataType::Utf8 {
1135 return Err(AvroError::SchemaError(
1136 "Avro enum requires Dictionary<Int32, Utf8>".into(),
1137 ));
1138 }
1139 Ok(FieldPlan::Enum {
1140 symbols: symbols.clone(),
1141 })
1142 }
1143 other => Err(AvroError::SchemaError(format!(
1144 "Avro enum maps to Arrow Dictionary<Int32, Utf8>, found: {other:?}"
1145 ))),
1146 },
1147 Codec::Decimal(precision, scale_opt, fixed_size_opt) => {
1149 let (ap, as_) = match arrow_field.data_type() {
1150 #[cfg(feature = "small_decimals")]
1151 DataType::Decimal32(p, s) => (*p as usize, *s as i32),
1152 #[cfg(feature = "small_decimals")]
1153 DataType::Decimal64(p, s) => (*p as usize, *s as i32),
1154 DataType::Decimal128(p, s) => (*p as usize, *s as i32),
1155 DataType::Decimal256(p, s) => (*p as usize, *s as i32),
1156 other => {
1157 return Err(AvroError::SchemaError(format!(
1158 "Avro decimal requires Arrow decimal, got {other:?} for field '{}'",
1159 arrow_field.name()
1160 )));
1161 }
1162 };
1163 let sc = scale_opt.unwrap_or(0) as i32; if ap != *precision || as_ != sc {
1165 return Err(AvroError::SchemaError(format!(
1166 "Decimal precision/scale mismatch for field '{}': Avro({precision},{sc}) vs Arrow({ap},{as_})",
1167 arrow_field.name()
1168 )));
1169 }
1170 Ok(FieldPlan::Decimal {
1171 size: *fixed_size_opt,
1172 })
1173 }
1174 Codec::Interval => match arrow_field.data_type() {
1175 DataType::Interval(
1176 IntervalUnit::MonthDayNano | IntervalUnit::YearMonth | IntervalUnit::DayTime,
1177 ) => Ok(FieldPlan::Duration),
1178 other => Err(AvroError::SchemaError(format!(
1179 "Avro 'duration' logical type requires an Arrow Interval (MonthDayNano, YearMonth, or DayTime), found: {other:?}"
1180 ))),
1181 },
1182 Codec::Union(avro_branches, _, UnionMode::Dense) => {
1183 let arrow_union_fields = match arrow_field.data_type() {
1184 DataType::Union(fields, UnionMode::Dense) => fields,
1185 DataType::Union(_, UnionMode::Sparse) => {
1186 return Err(AvroError::NYI(
1187 "Sparse Arrow unions are not yet supported".to_string(),
1188 ));
1189 }
1190 other => {
1191 return Err(AvroError::SchemaError(format!(
1192 "Avro union maps to Arrow Union, found: {other:?}"
1193 )));
1194 }
1195 };
1196 if avro_branches.len() != arrow_union_fields.len() {
1197 return Err(AvroError::SchemaError(format!(
1198 "Mismatched number of branches between Avro union ({}) and Arrow union ({}) for field '{}'",
1199 avro_branches.len(),
1200 arrow_union_fields.len(),
1201 arrow_field.name()
1202 )));
1203 }
1204 let bindings = avro_branches
1205 .iter()
1206 .zip(arrow_union_fields.iter())
1207 .enumerate()
1208 .map(|(i, (avro_branch, (_, arrow_child_field)))| {
1209 Ok(FieldBinding {
1210 arrow_index: i,
1211 nullability: avro_branch.nullability(),
1212 plan: FieldPlan::build(avro_branch, arrow_child_field)?,
1213 })
1214 })
1215 .collect::<Result<Vec<_>, AvroError>>()?;
1216 Ok(FieldPlan::Union { bindings })
1217 }
1218 Codec::Union(_, _, UnionMode::Sparse) => Err(AvroError::NYI(
1219 "Sparse Arrow unions are not yet supported".to_string(),
1220 )),
1221 #[cfg(feature = "avro_custom_types")]
1222 Codec::RunEndEncoded(values_dt, _width_code) => {
1223 let values_field = match arrow_field.data_type() {
1224 DataType::RunEndEncoded(_run_ends_field, values_field) => values_field.as_ref(),
1225 other => {
1226 return Err(AvroError::SchemaError(format!(
1227 "Avro RunEndEncoded maps to Arrow DataType::RunEndEncoded, found: {other:?}"
1228 )));
1229 }
1230 };
1231 Ok(FieldPlan::RunEndEncoded {
1232 values_nullability: values_dt.nullability(),
1233 value_plan: Box::new(FieldPlan::build(values_dt.as_ref(), values_field)?),
1234 })
1235 }
1236 Codec::TimeMillis => match arrow_field.data_type() {
1237 DataType::Time32(TimeUnit::Second) => Ok(FieldPlan::TimeMillisFromSecs),
1238 _ => Ok(FieldPlan::Scalar),
1239 },
1240 _ => Ok(FieldPlan::Scalar),
1241 }
1242 }
1243}
1244
1245enum Encoder<'a> {
1246 Boolean(BooleanEncoder<'a>),
1247 Int(IntEncoder<'a, Int32Type>),
1248 Long(LongEncoder<'a, Int64Type>),
1249 TimestampMicros(LongEncoder<'a, TimestampMicrosecondType>),
1250 TimestampMillis(LongEncoder<'a, TimestampMillisecondType>),
1251 TimestampNanos(LongEncoder<'a, TimestampNanosecondType>),
1252 #[cfg(not(feature = "avro_custom_types"))]
1253 TimestampSecsToMillis(TimestampSecondsToMillisEncoder<'a>),
1254 Date32(IntEncoder<'a, Date32Type>),
1255 Time32SecsToMillis(Time32SecondsToMillisEncoder<'a>),
1256 Time32Millis(IntEncoder<'a, Time32MillisecondType>),
1257 Time64Micros(LongEncoder<'a, Time64MicrosecondType>),
1258 DurationSeconds(LongEncoder<'a, DurationSecondType>),
1259 DurationMillis(LongEncoder<'a, DurationMillisecondType>),
1260 DurationMicros(LongEncoder<'a, DurationMicrosecondType>),
1261 DurationNanos(LongEncoder<'a, DurationNanosecondType>),
1262 Float32(F32Encoder<'a>),
1263 Float64(F64Encoder<'a>),
1264 Binary(BinaryEncoder<'a, i32>),
1265 LargeBinary(BinaryEncoder<'a, i64>),
1266 Utf8(Utf8Encoder<'a>),
1267 Utf8Large(Utf8LargeEncoder<'a>),
1268 Utf8View(Utf8ViewEncoder<'a>),
1269 BinaryView(BinaryViewEncoder<'a>),
1270 List(Box<ListEncoder32<'a>>),
1271 LargeList(Box<ListEncoder64<'a>>),
1272 ListView(Box<ListViewEncoder32<'a>>),
1273 LargeListView(Box<ListViewEncoder64<'a>>),
1274 FixedSizeList(Box<FixedSizeListEncoder<'a>>),
1275 Struct(Box<StructEncoder<'a>>),
1276 Fixed(FixedEncoder<'a>),
1278 Uuid(UuidEncoder<'a>),
1280 IntervalMonthDayNanoDuration(DurationEncoder<'a, IntervalMonthDayNanoType>),
1282 #[cfg(feature = "avro_custom_types")]
1284 IntervalMonthDayNanoFixed(IntervalMonthDayNanoFixedEncoder<'a>),
1285 IntervalYearMonthDuration(DurationEncoder<'a, IntervalYearMonthType>),
1287 IntervalDayTimeDuration(DurationEncoder<'a, IntervalDayTimeType>),
1289 #[cfg(feature = "small_decimals")]
1290 Decimal32(Decimal32Encoder<'a>),
1291 #[cfg(feature = "small_decimals")]
1292 Decimal64(Decimal64Encoder<'a>),
1293 Decimal128(Decimal128Encoder<'a>),
1294 Decimal256(Decimal256Encoder<'a>),
1295 Enum(EnumEncoder<'a>),
1297 Map(Box<MapEncoder<'a>>),
1298 Union(Box<UnionEncoder<'a>>),
1299 RunEncoded16(Box<RunEncodedEncoder16<'a>>),
1301 RunEncoded32(Box<RunEncodedEncoder32<'a>>),
1302 RunEncoded64(Box<RunEncodedEncoder64<'a>>),
1303 Null,
1304 #[cfg(feature = "avro_custom_types")]
1305 Int8(Int8Encoder<'a>),
1306 #[cfg(feature = "avro_custom_types")]
1307 Int16(Int16Encoder<'a>),
1308 #[cfg(feature = "avro_custom_types")]
1309 UInt8(UInt8Encoder<'a>),
1310 #[cfg(feature = "avro_custom_types")]
1311 UInt16(UInt16Encoder<'a>),
1312 #[cfg(feature = "avro_custom_types")]
1313 UInt32(UInt32Encoder<'a>),
1314 #[cfg(feature = "avro_custom_types")]
1315 UInt64Fixed(UInt64FixedEncoder<'a>),
1316 #[cfg(feature = "avro_custom_types")]
1317 Float16Fixed(Float16FixedEncoder<'a>),
1318 #[cfg(feature = "avro_custom_types")]
1319 Date64(LongEncoder<'a, Date64Type>),
1320 #[cfg(feature = "avro_custom_types")]
1321 Time64Nanos(LongEncoder<'a, Time64NanosecondType>),
1322 #[cfg(feature = "avro_custom_types")]
1323 Time32Secs(IntEncoder<'a, Time32SecondType>),
1324 #[cfg(feature = "avro_custom_types")]
1325 TimestampSecs(LongEncoder<'a, TimestampSecondType>),
1326 #[cfg(feature = "avro_custom_types")]
1327 IntervalYearMonthFixed(IntervalYearMonthFixedEncoder<'a>),
1328 #[cfg(feature = "avro_custom_types")]
1329 IntervalDayTimeFixed(IntervalDayTimeFixedEncoder<'a>),
1330 #[cfg(not(feature = "avro_custom_types"))]
1331 Int8ToInt(Int8ToIntEncoder<'a>),
1332 #[cfg(not(feature = "avro_custom_types"))]
1333 Int16ToInt(Int16ToIntEncoder<'a>),
1334 #[cfg(not(feature = "avro_custom_types"))]
1335 UInt8ToInt(UInt8ToIntEncoder<'a>),
1336 #[cfg(not(feature = "avro_custom_types"))]
1337 UInt16ToInt(UInt16ToIntEncoder<'a>),
1338 #[cfg(not(feature = "avro_custom_types"))]
1339 UInt32ToLong(UInt32ToLongEncoder<'a>),
1340 #[cfg(not(feature = "avro_custom_types"))]
1341 UInt64ToLong(UInt64ToLongEncoder<'a>),
1342 #[cfg(not(feature = "avro_custom_types"))]
1343 Float16ToFloat(Float16ToFloatEncoder<'a>),
1344 #[cfg(not(feature = "avro_custom_types"))]
1345 Date64ToLong(Date64ToLongEncoder<'a>),
1346 #[cfg(not(feature = "avro_custom_types"))]
1347 Time64NanosToMicros(Time64NanosToMicrosEncoder<'a>),
1348}
1349
1350impl Encoder<'_> {
1351 fn encode<W: Write + ?Sized>(&mut self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1353 match self {
1354 Encoder::Boolean(e) => e.encode(out, idx),
1355 Encoder::Int(e) => e.encode(out, idx),
1356 Encoder::Long(e) => e.encode(out, idx),
1357 Encoder::TimestampMicros(e) => e.encode(out, idx),
1358 Encoder::TimestampMillis(e) => e.encode(out, idx),
1359 Encoder::TimestampNanos(e) => e.encode(out, idx),
1360 #[cfg(not(feature = "avro_custom_types"))]
1361 Encoder::TimestampSecsToMillis(e) => e.encode(out, idx),
1362 Encoder::Date32(e) => e.encode(out, idx),
1363 Encoder::Time32SecsToMillis(e) => e.encode(out, idx),
1364 Encoder::Time32Millis(e) => e.encode(out, idx),
1365 Encoder::Time64Micros(e) => e.encode(out, idx),
1366 Encoder::DurationSeconds(e) => e.encode(out, idx),
1367 Encoder::DurationMicros(e) => e.encode(out, idx),
1368 Encoder::DurationMillis(e) => e.encode(out, idx),
1369 Encoder::DurationNanos(e) => e.encode(out, idx),
1370 Encoder::Float32(e) => e.encode(out, idx),
1371 Encoder::Float64(e) => e.encode(out, idx),
1372 Encoder::Binary(e) => e.encode(out, idx),
1373 Encoder::LargeBinary(e) => e.encode(out, idx),
1374 Encoder::Utf8(e) => e.encode(out, idx),
1375 Encoder::Utf8Large(e) => e.encode(out, idx),
1376 Encoder::Utf8View(e) => e.encode(out, idx),
1377 Encoder::BinaryView(e) => e.encode(out, idx),
1378 Encoder::List(e) => e.encode(out, idx),
1379 Encoder::LargeList(e) => e.encode(out, idx),
1380 Encoder::ListView(e) => e.encode(out, idx),
1381 Encoder::LargeListView(e) => e.encode(out, idx),
1382 Encoder::FixedSizeList(e) => e.encode(out, idx),
1383 Encoder::Struct(e) => e.encode(out, idx),
1384 Encoder::Fixed(e) => (e).encode(out, idx),
1385 Encoder::Uuid(e) => (e).encode(out, idx),
1386 Encoder::IntervalMonthDayNanoDuration(e) => e.encode(out, idx),
1387 #[cfg(feature = "avro_custom_types")]
1388 Encoder::IntervalMonthDayNanoFixed(e) => e.encode(out, idx),
1389 Encoder::IntervalYearMonthDuration(e) => e.encode(out, idx),
1390 Encoder::IntervalDayTimeDuration(e) => e.encode(out, idx),
1391 #[cfg(feature = "small_decimals")]
1392 Encoder::Decimal32(e) => (e).encode(out, idx),
1393 #[cfg(feature = "small_decimals")]
1394 Encoder::Decimal64(e) => (e).encode(out, idx),
1395 Encoder::Decimal128(e) => (e).encode(out, idx),
1396 Encoder::Decimal256(e) => (e).encode(out, idx),
1397 Encoder::Map(e) => (e).encode(out, idx),
1398 Encoder::Enum(e) => (e).encode(out, idx),
1399 Encoder::Union(e) => (e).encode(out, idx),
1400 Encoder::RunEncoded16(e) => (e).encode(out, idx),
1401 Encoder::RunEncoded32(e) => (e).encode(out, idx),
1402 Encoder::RunEncoded64(e) => (e).encode(out, idx),
1403 Encoder::Null => Ok(()),
1404 #[cfg(feature = "avro_custom_types")]
1405 Encoder::Int8(e) => e.encode(out, idx),
1406 #[cfg(feature = "avro_custom_types")]
1407 Encoder::Int16(e) => e.encode(out, idx),
1408 #[cfg(feature = "avro_custom_types")]
1409 Encoder::UInt8(e) => e.encode(out, idx),
1410 #[cfg(feature = "avro_custom_types")]
1411 Encoder::UInt16(e) => e.encode(out, idx),
1412 #[cfg(feature = "avro_custom_types")]
1413 Encoder::UInt32(e) => e.encode(out, idx),
1414 #[cfg(feature = "avro_custom_types")]
1415 Encoder::UInt64Fixed(e) => e.encode(out, idx),
1416 #[cfg(feature = "avro_custom_types")]
1417 Encoder::Float16Fixed(e) => e.encode(out, idx),
1418 #[cfg(feature = "avro_custom_types")]
1419 Encoder::Date64(e) => e.encode(out, idx),
1420 #[cfg(feature = "avro_custom_types")]
1421 Encoder::Time64Nanos(e) => e.encode(out, idx),
1422 #[cfg(feature = "avro_custom_types")]
1423 Encoder::Time32Secs(e) => e.encode(out, idx),
1424 #[cfg(feature = "avro_custom_types")]
1425 Encoder::TimestampSecs(e) => e.encode(out, idx),
1426 #[cfg(feature = "avro_custom_types")]
1427 Encoder::IntervalYearMonthFixed(e) => e.encode(out, idx),
1428 #[cfg(feature = "avro_custom_types")]
1429 Encoder::IntervalDayTimeFixed(e) => e.encode(out, idx),
1430 #[cfg(not(feature = "avro_custom_types"))]
1431 Encoder::Int8ToInt(e) => e.encode(out, idx),
1432 #[cfg(not(feature = "avro_custom_types"))]
1433 Encoder::Int16ToInt(e) => e.encode(out, idx),
1434 #[cfg(not(feature = "avro_custom_types"))]
1435 Encoder::UInt8ToInt(e) => e.encode(out, idx),
1436 #[cfg(not(feature = "avro_custom_types"))]
1437 Encoder::UInt16ToInt(e) => e.encode(out, idx),
1438 #[cfg(not(feature = "avro_custom_types"))]
1439 Encoder::UInt32ToLong(e) => e.encode(out, idx),
1440 #[cfg(not(feature = "avro_custom_types"))]
1441 Encoder::UInt64ToLong(e) => e.encode(out, idx),
1442 #[cfg(not(feature = "avro_custom_types"))]
1443 Encoder::Float16ToFloat(e) => e.encode(out, idx),
1444 #[cfg(not(feature = "avro_custom_types"))]
1445 Encoder::Date64ToLong(e) => e.encode(out, idx),
1446 #[cfg(not(feature = "avro_custom_types"))]
1447 Encoder::Time64NanosToMicros(e) => e.encode(out, idx),
1448 }
1449 }
1450}
1451
1452struct BooleanEncoder<'a>(&'a arrow_array::BooleanArray);
1453impl BooleanEncoder<'_> {
1454 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1455 write_bool(out, self.0.value(idx))
1456 }
1457}
1458
1459struct IntEncoder<'a, P: ArrowPrimitiveType<Native = i32>>(&'a PrimitiveArray<P>);
1461impl<P: ArrowPrimitiveType<Native = i32>> IntEncoder<'_, P> {
1462 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1463 write_int(out, self.0.value(idx))
1464 }
1465}
1466
1467struct LongEncoder<'a, P: ArrowPrimitiveType<Native = i64>>(&'a PrimitiveArray<P>);
1469impl<P: ArrowPrimitiveType<Native = i64>> LongEncoder<'_, P> {
1470 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1471 write_long(out, self.0.value(idx))
1472 }
1473}
1474
1475struct Time32SecondsToMillisEncoder<'a>(&'a PrimitiveArray<Time32SecondType>);
1477impl Time32SecondsToMillisEncoder<'_> {
1478 #[inline]
1479 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1480 let secs = self.0.value(idx);
1481 let millis = secs
1482 .checked_mul(1000)
1483 .ok_or_else(|| AvroError::InvalidArgument("time32(secs) * 1000 overflowed".into()))?;
1484 write_int(out, millis)
1485 }
1486}
1487
1488#[cfg(not(feature = "avro_custom_types"))]
1490struct TimestampSecondsToMillisEncoder<'a>(&'a PrimitiveArray<TimestampSecondType>);
1491#[cfg(not(feature = "avro_custom_types"))]
1492impl TimestampSecondsToMillisEncoder<'_> {
1493 #[inline]
1494 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1495 let secs = self.0.value(idx);
1496 let millis = secs.checked_mul(1000).ok_or_else(|| {
1497 AvroError::InvalidArgument("timestamp(secs) * 1000 overflowed".into())
1498 })?;
1499 write_long(out, millis)
1500 }
1501}
1502
1503#[cfg(feature = "avro_custom_types")]
1505struct Int8Encoder<'a>(&'a PrimitiveArray<Int8Type>);
1506#[cfg(feature = "avro_custom_types")]
1507impl Int8Encoder<'_> {
1508 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1509 write_int(out, self.0.value(idx) as i32)
1510 }
1511}
1512
1513#[cfg(feature = "avro_custom_types")]
1515struct Int16Encoder<'a>(&'a PrimitiveArray<Int16Type>);
1516#[cfg(feature = "avro_custom_types")]
1517impl Int16Encoder<'_> {
1518 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1519 write_int(out, self.0.value(idx) as i32)
1520 }
1521}
1522
1523#[cfg(feature = "avro_custom_types")]
1525struct UInt8Encoder<'a>(&'a PrimitiveArray<UInt8Type>);
1526#[cfg(feature = "avro_custom_types")]
1527impl UInt8Encoder<'_> {
1528 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1529 write_int(out, self.0.value(idx) as i32)
1530 }
1531}
1532
1533#[cfg(feature = "avro_custom_types")]
1535struct UInt16Encoder<'a>(&'a PrimitiveArray<UInt16Type>);
1536#[cfg(feature = "avro_custom_types")]
1537impl UInt16Encoder<'_> {
1538 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1539 write_int(out, self.0.value(idx) as i32)
1540 }
1541}
1542
1543#[cfg(feature = "avro_custom_types")]
1545struct UInt32Encoder<'a>(&'a PrimitiveArray<UInt32Type>);
1546#[cfg(feature = "avro_custom_types")]
1547impl UInt32Encoder<'_> {
1548 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1549 write_long(out, self.0.value(idx) as i64)
1550 }
1551}
1552
1553#[cfg(feature = "avro_custom_types")]
1555struct UInt64FixedEncoder<'a>(&'a PrimitiveArray<UInt64Type>);
1556#[cfg(feature = "avro_custom_types")]
1557impl UInt64FixedEncoder<'_> {
1558 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1559 let v = self.0.value(idx);
1560 out.write_all(&v.to_le_bytes())?;
1561 Ok(())
1562 }
1563}
1564
1565#[cfg(feature = "avro_custom_types")]
1567struct Float16FixedEncoder<'a>(&'a Float16Array);
1568#[cfg(feature = "avro_custom_types")]
1569impl Float16FixedEncoder<'_> {
1570 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1571 let v = self.0.value(idx);
1572 out.write_all(&v.to_le_bytes())?;
1573 Ok(())
1574 }
1575}
1576
1577#[cfg(feature = "avro_custom_types")]
1585struct IntervalMonthDayNanoFixedEncoder<'a>(&'a PrimitiveArray<IntervalMonthDayNanoType>);
1586#[cfg(feature = "avro_custom_types")]
1587impl IntervalMonthDayNanoFixedEncoder<'_> {
1588 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1589 let v = self.0.value(idx);
1590 let (months, days, nanos) = IntervalMonthDayNanoType::to_parts(v);
1591 out.write_all(&months.to_le_bytes())?;
1592 out.write_all(&days.to_le_bytes())?;
1593 out.write_all(&nanos.to_le_bytes())?;
1594 Ok(())
1595 }
1596}
1597
1598#[cfg(feature = "avro_custom_types")]
1600struct IntervalYearMonthFixedEncoder<'a>(&'a PrimitiveArray<IntervalYearMonthType>);
1601#[cfg(feature = "avro_custom_types")]
1602impl IntervalYearMonthFixedEncoder<'_> {
1603 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1604 let months = self.0.value(idx);
1605 out.write_all(&months.to_le_bytes())?;
1606 Ok(())
1607 }
1608}
1609
1610#[cfg(feature = "avro_custom_types")]
1612struct IntervalDayTimeFixedEncoder<'a>(&'a PrimitiveArray<IntervalDayTimeType>);
1613#[cfg(feature = "avro_custom_types")]
1614impl IntervalDayTimeFixedEncoder<'_> {
1615 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1616 let dt = self.0.value(idx);
1617 out.write_all(&dt.days.to_le_bytes())?;
1618 out.write_all(&dt.milliseconds.to_le_bytes())?;
1619 Ok(())
1620 }
1621}
1622
1623#[cfg(not(feature = "avro_custom_types"))]
1625struct Int8ToIntEncoder<'a>(&'a PrimitiveArray<Int8Type>);
1626#[cfg(not(feature = "avro_custom_types"))]
1627impl Int8ToIntEncoder<'_> {
1628 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1629 write_int(out, self.0.value(idx) as i32)
1630 }
1631}
1632
1633#[cfg(not(feature = "avro_custom_types"))]
1635struct Int16ToIntEncoder<'a>(&'a PrimitiveArray<Int16Type>);
1636#[cfg(not(feature = "avro_custom_types"))]
1637impl Int16ToIntEncoder<'_> {
1638 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1639 write_int(out, self.0.value(idx) as i32)
1640 }
1641}
1642
1643#[cfg(not(feature = "avro_custom_types"))]
1645struct UInt8ToIntEncoder<'a>(&'a PrimitiveArray<UInt8Type>);
1646#[cfg(not(feature = "avro_custom_types"))]
1647impl UInt8ToIntEncoder<'_> {
1648 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1649 write_int(out, self.0.value(idx) as i32)
1650 }
1651}
1652
1653#[cfg(not(feature = "avro_custom_types"))]
1655struct UInt16ToIntEncoder<'a>(&'a PrimitiveArray<UInt16Type>);
1656#[cfg(not(feature = "avro_custom_types"))]
1657impl UInt16ToIntEncoder<'_> {
1658 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1659 write_int(out, self.0.value(idx) as i32)
1660 }
1661}
1662
1663#[cfg(not(feature = "avro_custom_types"))]
1665struct UInt32ToLongEncoder<'a>(&'a PrimitiveArray<UInt32Type>);
1666#[cfg(not(feature = "avro_custom_types"))]
1667impl UInt32ToLongEncoder<'_> {
1668 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1669 write_long(out, self.0.value(idx) as i64)
1670 }
1671}
1672
1673#[cfg(not(feature = "avro_custom_types"))]
1675struct UInt64ToLongEncoder<'a>(&'a PrimitiveArray<UInt64Type>);
1676#[cfg(not(feature = "avro_custom_types"))]
1677impl UInt64ToLongEncoder<'_> {
1678 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1679 let v = self.0.value(idx);
1680 if v > i64::MAX as u64 {
1681 return Err(AvroError::InvalidArgument(format!(
1682 "UInt64 value {v} exceeds i64::MAX; enable avro_custom_types feature for full UInt64 support",
1683 )));
1684 }
1685 write_long(out, v as i64)
1686 }
1687}
1688
1689#[cfg(not(feature = "avro_custom_types"))]
1691struct Float16ToFloatEncoder<'a>(&'a Float16Array);
1692#[cfg(not(feature = "avro_custom_types"))]
1693impl Float16ToFloatEncoder<'_> {
1694 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1695 out.write_all(&self.0.value(idx).to_f32().to_bits().to_le_bytes())?;
1696 Ok(())
1697 }
1698}
1699
1700#[cfg(not(feature = "avro_custom_types"))]
1702struct Date64ToLongEncoder<'a>(&'a PrimitiveArray<Date64Type>);
1703#[cfg(not(feature = "avro_custom_types"))]
1704impl Date64ToLongEncoder<'_> {
1705 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1706 write_long(out, self.0.value(idx))
1707 }
1708}
1709
1710#[cfg(not(feature = "avro_custom_types"))]
1712struct Time64NanosToMicrosEncoder<'a>(&'a PrimitiveArray<Time64NanosecondType>);
1713#[cfg(not(feature = "avro_custom_types"))]
1714impl Time64NanosToMicrosEncoder<'_> {
1715 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1716 let nanos = self.0.value(idx);
1717 let micros = nanos / 1000;
1718 write_long(out, micros)
1719 }
1720}
1721
1722struct BinaryEncoder<'a, O: OffsetSizeTrait>(&'a GenericBinaryArray<O>);
1724impl<O: OffsetSizeTrait> BinaryEncoder<'_, O> {
1725 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1726 write_len_prefixed(out, self.0.value(idx))
1727 }
1728}
1729
1730struct BinaryViewEncoder<'a>(&'a BinaryViewArray);
1732impl BinaryViewEncoder<'_> {
1733 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1734 write_len_prefixed(out, self.0.value(idx))
1735 }
1736}
1737
1738struct Utf8ViewEncoder<'a>(&'a StringViewArray);
1740impl Utf8ViewEncoder<'_> {
1741 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1742 write_len_prefixed(out, self.0.value(idx).as_bytes())
1743 }
1744}
1745
1746struct F32Encoder<'a>(&'a arrow_array::Float32Array);
1747impl F32Encoder<'_> {
1748 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1749 out.write_all(&self.0.value(idx).to_bits().to_le_bytes())?;
1751 Ok(())
1752 }
1753}
1754
1755struct F64Encoder<'a>(&'a arrow_array::Float64Array);
1756impl F64Encoder<'_> {
1757 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1758 out.write_all(&self.0.value(idx).to_bits().to_le_bytes())
1760 .map_err(Into::into)
1761 }
1762}
1763
1764struct Utf8GenericEncoder<'a, O: OffsetSizeTrait>(&'a GenericStringArray<O>);
1765
1766impl<O: OffsetSizeTrait> Utf8GenericEncoder<'_, O> {
1767 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1768 write_len_prefixed(out, self.0.value(idx).as_bytes())
1769 }
1770}
1771
1772type Utf8Encoder<'a> = Utf8GenericEncoder<'a, i32>;
1773type Utf8LargeEncoder<'a> = Utf8GenericEncoder<'a, i64>;
1774
1775enum KeyKind<'a> {
1777 Utf8(&'a GenericStringArray<i32>),
1778 LargeUtf8(&'a GenericStringArray<i64>),
1779}
1780struct MapEncoder<'a> {
1781 map: &'a MapArray,
1782 keys: KeyKind<'a>,
1783 values: FieldEncoder<'a>,
1784 keys_offset: usize,
1785 values_offset: usize,
1786}
1787
1788impl<'a> MapEncoder<'a> {
1789 fn try_new(
1790 map: &'a MapArray,
1791 values_nullability: Option<Nullability>,
1792 value_plan: &FieldPlan,
1793 ) -> Result<Self, AvroError> {
1794 let keys_arr = map.keys();
1795 let keys_kind = match keys_arr.data_type() {
1796 DataType::Utf8 => KeyKind::Utf8(keys_arr.as_string::<i32>()),
1797 DataType::LargeUtf8 => KeyKind::LargeUtf8(keys_arr.as_string::<i64>()),
1798 other => {
1799 return Err(AvroError::SchemaError(format!(
1800 "Avro map requires string keys; Arrow key type must be Utf8/LargeUtf8, found: {other:?}"
1801 )));
1802 }
1803 };
1804 Ok(Self {
1805 map,
1806 keys: keys_kind,
1807 values: FieldEncoder::make_encoder(
1808 map.values().as_ref(),
1809 value_plan,
1810 values_nullability,
1811 )?,
1812 keys_offset: keys_arr.offset(),
1813 values_offset: map.values().offset(),
1814 })
1815 }
1816
1817 fn encode_map_entries<W, O>(
1818 out: &mut W,
1819 keys: &GenericStringArray<O>,
1820 keys_offset: usize,
1821 start: usize,
1822 end: usize,
1823 mut write_item: impl FnMut(&mut W, usize) -> Result<(), AvroError>,
1824 ) -> Result<(), AvroError>
1825 where
1826 W: Write + ?Sized,
1827 O: OffsetSizeTrait,
1828 {
1829 encode_blocked_range(out, start, end, |out, j| {
1830 let j_key = j.saturating_sub(keys_offset);
1831 write_len_prefixed(out, keys.value(j_key).as_bytes())?;
1832 write_item(out, j)
1833 })
1834 }
1835
1836 fn encode<W: Write + ?Sized>(&mut self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1837 let offsets = self.map.offsets();
1838 let start = offsets[idx] as usize;
1839 let end = offsets[idx + 1] as usize;
1840 let write_item = |out: &mut W, j: usize| {
1841 let j_val = j.saturating_sub(self.values_offset);
1842 self.values.encode(out, j_val)
1843 };
1844 match self.keys {
1845 KeyKind::Utf8(arr) => MapEncoder::<'a>::encode_map_entries(
1846 out,
1847 arr,
1848 self.keys_offset,
1849 start,
1850 end,
1851 write_item,
1852 ),
1853 KeyKind::LargeUtf8(arr) => MapEncoder::<'a>::encode_map_entries(
1854 out,
1855 arr,
1856 self.keys_offset,
1857 start,
1858 end,
1859 write_item,
1860 ),
1861 }
1862 }
1863}
1864
1865struct EnumEncoder<'a> {
1872 keys: &'a PrimitiveArray<Int32Type>,
1873}
1874impl EnumEncoder<'_> {
1875 fn encode<W: Write + ?Sized>(&self, out: &mut W, row: usize) -> Result<(), AvroError> {
1876 write_int(out, self.keys.value(row))
1877 }
1878}
1879
1880struct UnionEncoder<'a> {
1881 encoders: Vec<FieldEncoder<'a>>,
1882 array: &'a UnionArray,
1883 type_id_to_encoder_index: Vec<Option<usize>>,
1884}
1885
1886impl<'a> UnionEncoder<'a> {
1887 fn try_new(array: &'a UnionArray, field_bindings: &[FieldBinding]) -> Result<Self, AvroError> {
1888 let DataType::Union(fields, UnionMode::Dense) = array.data_type() else {
1889 return Err(AvroError::SchemaError("Expected Dense UnionArray".into()));
1890 };
1891 if fields.len() != field_bindings.len() {
1892 return Err(AvroError::SchemaError(format!(
1893 "Mismatched number of union branches between Arrow array ({}) and encoding plan ({})",
1894 fields.len(),
1895 field_bindings.len()
1896 )));
1897 }
1898 let max_type_id = fields.iter().map(|(tid, _)| tid).max().unwrap_or(0);
1899 let mut type_id_to_encoder_index: Vec<Option<usize>> =
1900 vec![None; (max_type_id + 1) as usize];
1901 let mut encoders = Vec::with_capacity(fields.len());
1902 for (i, (type_id, _)) in fields.iter().enumerate() {
1903 let binding = field_bindings
1904 .get(i)
1905 .ok_or_else(|| AvroError::SchemaError("Binding and field mismatch".to_string()))?;
1906 encoders.push(FieldEncoder::make_encoder(
1907 array.child(type_id).as_ref(),
1908 &binding.plan,
1909 binding.nullability,
1910 )?);
1911 type_id_to_encoder_index[type_id as usize] = Some(i);
1912 }
1913 Ok(Self {
1914 encoders,
1915 array,
1916 type_id_to_encoder_index,
1917 })
1918 }
1919
1920 fn encode<W: Write + ?Sized>(&mut self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1921 let type_id = self.array.type_ids()[idx];
1926 let encoder_index = self
1927 .type_id_to_encoder_index
1928 .get(type_id as usize)
1929 .and_then(|opt| *opt)
1930 .ok_or_else(|| AvroError::SchemaError(format!("Invalid type_id {type_id}")))?;
1931 write_int(out, encoder_index as i32)?;
1932 let encoder = self.encoders.get_mut(encoder_index).ok_or_else(|| {
1933 AvroError::SchemaError(format!("Invalid encoder index {encoder_index}"))
1934 })?;
1935 encoder.encode(out, self.array.value_offset(idx))
1936 }
1937}
1938
1939struct StructEncoder<'a> {
1940 encoders: Vec<FieldEncoder<'a>>,
1941}
1942
1943impl<'a> StructEncoder<'a> {
1944 fn try_new(array: &'a StructArray, field_bindings: &[FieldBinding]) -> Result<Self, AvroError> {
1945 let mut encoders = Vec::with_capacity(field_bindings.len());
1946 for field_binding in field_bindings {
1947 let idx = field_binding.arrow_index;
1948 let column = array.columns().get(idx).ok_or_else(|| {
1949 AvroError::SchemaError(format!("Struct child index {idx} out of range"))
1950 })?;
1951 let encoder = FieldEncoder::make_encoder(
1952 column.as_ref(),
1953 &field_binding.plan,
1954 field_binding.nullability,
1955 )?;
1956 encoders.push(encoder);
1957 }
1958 Ok(Self { encoders })
1959 }
1960
1961 fn encode<W: Write + ?Sized>(&mut self, out: &mut W, idx: usize) -> Result<(), AvroError> {
1962 for encoder in &mut self.encoders {
1963 encoder.encode(out, idx)?;
1964 }
1965 Ok(())
1966 }
1967}
1968
1969fn encode_blocked_range<W: Write + ?Sized, F>(
1973 out: &mut W,
1974 start: usize,
1975 end: usize,
1976 mut write_item: F,
1977) -> Result<(), AvroError>
1978where
1979 F: FnMut(&mut W, usize) -> Result<(), AvroError>,
1980{
1981 let len = end.saturating_sub(start);
1982 if len == 0 {
1983 write_long(out, 0)?;
1985 return Ok(());
1986 }
1987 write_long(out, len as i64)?;
1989 for row in start..end {
1990 write_item(out, row)?;
1991 }
1992 write_long(out, 0)?;
1993 Ok(())
1994}
1995
1996struct ListEncoder<'a, O: OffsetSizeTrait> {
1997 list: &'a GenericListArray<O>,
1998 values: FieldEncoder<'a>,
1999 values_offset: usize,
2000}
2001
2002type ListEncoder32<'a> = ListEncoder<'a, i32>;
2003type ListEncoder64<'a> = ListEncoder<'a, i64>;
2004
2005impl<'a, O: OffsetSizeTrait> ListEncoder<'a, O> {
2006 fn try_new(
2007 list: &'a GenericListArray<O>,
2008 items_nullability: Option<Nullability>,
2009 item_plan: &FieldPlan,
2010 ) -> Result<Self, AvroError> {
2011 Ok(Self {
2012 list,
2013 values: FieldEncoder::make_encoder(
2014 list.values().as_ref(),
2015 item_plan,
2016 items_nullability,
2017 )?,
2018 values_offset: list.values().offset(),
2019 })
2020 }
2021
2022 fn encode_list_range<W: Write + ?Sized>(
2023 &mut self,
2024 out: &mut W,
2025 start: usize,
2026 end: usize,
2027 ) -> Result<(), AvroError> {
2028 encode_blocked_range(out, start, end, |out, row| {
2029 self.values
2030 .encode(out, row.saturating_sub(self.values_offset))
2031 })
2032 }
2033
2034 fn encode<W: Write + ?Sized>(&mut self, out: &mut W, idx: usize) -> Result<(), AvroError> {
2035 let offsets = self.list.offsets();
2036 let start = offsets[idx].to_usize().ok_or_else(|| {
2037 AvroError::InvalidArgument(format!("Error converting offset[{idx}] to usize"))
2038 })?;
2039 let end = offsets[idx + 1].to_usize().ok_or_else(|| {
2040 AvroError::InvalidArgument(format!("Error converting offset[{}] to usize", idx + 1))
2041 })?;
2042 self.encode_list_range(out, start, end)
2043 }
2044}
2045
2046struct ListViewEncoder<'a, O: OffsetSizeTrait> {
2048 list: &'a GenericListViewArray<O>,
2049 values: FieldEncoder<'a>,
2050 values_offset: usize,
2051}
2052type ListViewEncoder32<'a> = ListViewEncoder<'a, i32>;
2053type ListViewEncoder64<'a> = ListViewEncoder<'a, i64>;
2054
2055impl<'a, O: OffsetSizeTrait> ListViewEncoder<'a, O> {
2056 fn try_new(
2057 list: &'a GenericListViewArray<O>,
2058 items_nullability: Option<Nullability>,
2059 item_plan: &FieldPlan,
2060 ) -> Result<Self, AvroError> {
2061 Ok(Self {
2062 list,
2063 values: FieldEncoder::make_encoder(
2064 list.values().as_ref(),
2065 item_plan,
2066 items_nullability,
2067 )?,
2068 values_offset: list.values().offset(),
2069 })
2070 }
2071
2072 fn encode<W: Write + ?Sized>(&mut self, out: &mut W, idx: usize) -> Result<(), AvroError> {
2073 let start = self.list.value_offset(idx).to_usize().ok_or_else(|| {
2074 AvroError::InvalidArgument(format!("Error converting value_offset[{idx}] to usize"))
2075 })?;
2076 let len = self.list.value_size(idx).to_usize().ok_or_else(|| {
2077 AvroError::InvalidArgument(format!("Error converting value_size[{idx}] to usize"))
2078 })?;
2079 let start = start + self.values_offset;
2080 let end = start + len;
2081 encode_blocked_range(out, start, end, |out, row| {
2082 self.values
2083 .encode(out, row.saturating_sub(self.values_offset))
2084 })
2085 }
2086}
2087
2088struct FixedSizeListEncoder<'a> {
2090 values: FieldEncoder<'a>,
2091 values_offset: usize,
2092 elem_len: usize,
2093}
2094
2095impl<'a> FixedSizeListEncoder<'a> {
2096 fn try_new(
2097 list: &'a FixedSizeListArray,
2098 items_nullability: Option<Nullability>,
2099 item_plan: &FieldPlan,
2100 ) -> Result<Self, AvroError> {
2101 Ok(Self {
2102 values: FieldEncoder::make_encoder(
2103 list.values().as_ref(),
2104 item_plan,
2105 items_nullability,
2106 )?,
2107 values_offset: list.values().offset(),
2108 elem_len: list.value_length() as usize,
2109 })
2110 }
2111
2112 fn encode<W: Write + ?Sized>(&mut self, out: &mut W, idx: usize) -> Result<(), AvroError> {
2113 let rel = idx * self.elem_len;
2115 let start = self.values_offset + rel;
2116 let end = start + self.elem_len;
2117 encode_blocked_range(out, start, end, |out, row| {
2118 self.values
2119 .encode(out, row.saturating_sub(self.values_offset))
2120 })
2121 }
2122}
2123
2124struct FixedEncoder<'a>(&'a FixedSizeBinaryArray);
2127impl FixedEncoder<'_> {
2128 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
2129 let v = self.0.value(idx); out.write_all(v)?;
2131 Ok(())
2132 }
2133}
2134
2135struct UuidEncoder<'a>(&'a FixedSizeBinaryArray);
2138impl UuidEncoder<'_> {
2139 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
2140 let mut buf = [0u8; 1 + uuid::fmt::Hyphenated::LENGTH];
2141 buf[0] = 0x48;
2142 let v = self.0.value(idx);
2143 let u = Uuid::from_slice(v)
2144 .map_err(|e| AvroError::InvalidArgument(format!("Invalid UUID bytes: {e}")))?;
2145 let _ = u.hyphenated().encode_lower(&mut buf[1..]);
2146 out.write_all(&buf)?;
2147 Ok(())
2148 }
2149}
2150
2151#[derive(Copy, Clone)]
2152struct DurationParts {
2153 months: u32,
2154 days: u32,
2155 millis: u32,
2156}
2157trait IntervalToDurationParts: ArrowPrimitiveType {
2159 fn duration_parts(native: Self::Native) -> Result<DurationParts, AvroError>;
2160}
2161impl IntervalToDurationParts for IntervalMonthDayNanoType {
2162 fn duration_parts(native: Self::Native) -> Result<DurationParts, AvroError> {
2163 let (months, days, nanos) = IntervalMonthDayNanoType::to_parts(native);
2164 if months < 0 || days < 0 || nanos < 0 {
2165 return Err(AvroError::InvalidArgument(
2166 "Avro 'duration' cannot encode negative months/days/nanoseconds; enable `avro_custom_types` to round-trip signed Arrow intervals".into(),
2167 ));
2168 }
2169 if nanos % 1_000_000 != 0 {
2170 return Err(AvroError::InvalidArgument(
2171 "Avro 'duration' requires whole milliseconds; nanoseconds must be divisible by 1_000_000 (enable `avro_custom_types` to preserve nanosecond intervals)"
2172 .into(),
2173 ));
2174 }
2175 let millis = nanos / 1_000_000;
2176 if millis > u32::MAX as i64 {
2177 return Err(AvroError::InvalidArgument(
2178 "Avro 'duration' milliseconds exceed u32::MAX; enable `avro_custom_types` to preserve full Arrow Interval(MonthDayNano) range".into(),
2179 ));
2180 }
2181 Ok(DurationParts {
2182 months: months as u32,
2183 days: days as u32,
2184 millis: millis as u32,
2185 })
2186 }
2187}
2188impl IntervalToDurationParts for IntervalYearMonthType {
2189 fn duration_parts(native: Self::Native) -> Result<DurationParts, AvroError> {
2190 if native < 0 {
2191 return Err(AvroError::InvalidArgument(
2192 "Avro 'duration' cannot encode negative months; enable `avro_custom_types` to round-trip signed Arrow Interval(YearMonth)".into(),
2193 ));
2194 }
2195 Ok(DurationParts {
2196 months: native as u32,
2197 days: 0,
2198 millis: 0,
2199 })
2200 }
2201}
2202impl IntervalToDurationParts for IntervalDayTimeType {
2203 fn duration_parts(native: Self::Native) -> Result<DurationParts, AvroError> {
2204 let (days, millis) = IntervalDayTimeType::to_parts(native);
2205 if days < 0 || millis < 0 {
2206 return Err(AvroError::InvalidArgument(
2207 "Avro 'duration' cannot encode negative days or milliseconds; enable `avro_custom_types` to round-trip signed Arrow Interval(DayTime)".into(),
2208 ));
2209 }
2210 Ok(DurationParts {
2211 months: 0,
2212 days: days as u32,
2213 millis: millis as u32,
2214 })
2215 }
2216}
2217
2218struct DurationEncoder<'a, P: ArrowPrimitiveType + IntervalToDurationParts>(&'a PrimitiveArray<P>);
2221impl<P: ArrowPrimitiveType + IntervalToDurationParts> DurationEncoder<'_, P> {
2222 #[inline(always)]
2223 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
2224 let parts = P::duration_parts(self.0.value(idx))?;
2225 let months = parts.months.to_le_bytes();
2226 let days = parts.days.to_le_bytes();
2227 let ms = parts.millis.to_le_bytes();
2228 let buf = [
2244 months[0], months[1], months[2], months[3], days[0], days[1], days[2], days[3], ms[0],
2245 ms[1], ms[2], ms[3],
2246 ];
2247 out.write_all(&buf)?;
2248 Ok(())
2249 }
2250}
2251
2252trait DecimalBeBytes<const N: usize> {
2255 fn value_be_bytes(&self, idx: usize) -> [u8; N];
2256}
2257#[cfg(feature = "small_decimals")]
2258impl DecimalBeBytes<4> for Decimal32Array {
2259 fn value_be_bytes(&self, idx: usize) -> [u8; 4] {
2260 self.value(idx).to_be_bytes()
2261 }
2262}
2263#[cfg(feature = "small_decimals")]
2264impl DecimalBeBytes<8> for Decimal64Array {
2265 fn value_be_bytes(&self, idx: usize) -> [u8; 8] {
2266 self.value(idx).to_be_bytes()
2267 }
2268}
2269impl DecimalBeBytes<16> for Decimal128Array {
2270 fn value_be_bytes(&self, idx: usize) -> [u8; 16] {
2271 self.value(idx).to_be_bytes()
2272 }
2273}
2274impl DecimalBeBytes<32> for Decimal256Array {
2275 fn value_be_bytes(&self, idx: usize) -> [u8; 32] {
2276 self.value(idx).to_be_bytes()
2278 }
2279}
2280
2281struct DecimalEncoder<'a, const N: usize, A: DecimalBeBytes<N>> {
2287 arr: &'a A,
2288 fixed_size: Option<usize>,
2289}
2290
2291impl<'a, const N: usize, A: DecimalBeBytes<N>> DecimalEncoder<'a, N, A> {
2292 fn new(arr: &'a A, fixed_size: Option<usize>) -> Self {
2293 Self { arr, fixed_size }
2294 }
2295
2296 fn encode<W: Write + ?Sized>(&self, out: &mut W, idx: usize) -> Result<(), AvroError> {
2297 let be = self.arr.value_be_bytes(idx);
2298 match self.fixed_size {
2299 Some(n) => write_sign_extended(out, &be, n),
2300 None => write_len_prefixed(out, minimal_twos_complement(&be)),
2301 }
2302 }
2303}
2304
2305#[cfg(feature = "small_decimals")]
2306type Decimal32Encoder<'a> = DecimalEncoder<'a, 4, Decimal32Array>;
2307#[cfg(feature = "small_decimals")]
2308type Decimal64Encoder<'a> = DecimalEncoder<'a, 8, Decimal64Array>;
2309type Decimal128Encoder<'a> = DecimalEncoder<'a, 16, Decimal128Array>;
2310type Decimal256Encoder<'a> = DecimalEncoder<'a, 32, Decimal256Array>;
2311
2312struct RunEncodedEncoder<'a, R: RunEndIndexType> {
2316 ends_slice: &'a [<R as ArrowPrimitiveType>::Native],
2317 base: usize,
2318 len: usize,
2319 values: FieldEncoder<'a>,
2320 cur_run: usize,
2322 cur_end: usize,
2324}
2325
2326type RunEncodedEncoder16<'a> = RunEncodedEncoder<'a, Int16Type>;
2327type RunEncodedEncoder32<'a> = RunEncodedEncoder<'a, Int32Type>;
2328type RunEncodedEncoder64<'a> = RunEncodedEncoder<'a, Int64Type>;
2329
2330impl<'a, R: RunEndIndexType> RunEncodedEncoder<'a, R> {
2331 fn new(arr: &'a RunArray<R>, values: FieldEncoder<'a>) -> Self {
2332 let ends = arr.run_ends();
2333 let base = ends.get_start_physical_index();
2334 let slice = ends.values();
2335 let len = ends.len();
2336 let cur_end = if len == 0 { 0 } else { slice[base].as_usize() };
2337 Self {
2338 ends_slice: slice,
2339 base,
2340 len,
2341 values,
2342 cur_run: 0,
2343 cur_end,
2344 }
2345 }
2346
2347 #[inline(always)]
2350 fn advance_to_row(&mut self, idx: usize) -> Result<(), AvroError> {
2351 if idx < self.cur_end {
2352 return Ok(());
2353 }
2354 while self.cur_run + 1 < self.len && idx >= self.cur_end {
2356 self.cur_run += 1;
2357 self.cur_end = self.ends_slice[self.base + self.cur_run].as_usize();
2358 }
2359 if idx < self.cur_end {
2360 Ok(())
2361 } else {
2362 Err(AvroError::InvalidArgument(format!(
2363 "row index {idx} out of bounds for run-ends ({} runs)",
2364 self.len
2365 )))
2366 }
2367 }
2368
2369 #[inline(always)]
2370 fn encode<W: Write + ?Sized>(&mut self, out: &mut W, idx: usize) -> Result<(), AvroError> {
2371 self.advance_to_row(idx)?;
2372 self.values.encode(out, self.cur_run)
2375 }
2376}
2377
2378#[cfg(test)]
2379mod tests {
2380 use super::*;
2381 use arrow_array::types::Int32Type;
2382 use arrow_array::{
2383 Array, ArrayRef, BinaryArray, BooleanArray, Float32Array, Float64Array, Int32Array,
2384 Int64Array, LargeBinaryArray, LargeListArray, LargeStringArray, ListArray, NullArray,
2385 StringArray,
2386 };
2387 use arrow_buffer::Buffer;
2388 use arrow_schema::{DataType, Field, Fields, UnionFields};
2389
2390 fn zigzag_i64(v: i64) -> u64 {
2391 ((v << 1) ^ (v >> 63)) as u64
2392 }
2393
2394 fn varint(mut x: u64) -> Vec<u8> {
2395 let mut out = Vec::new();
2396 while (x & !0x7f) != 0 {
2397 out.push(((x & 0x7f) as u8) | 0x80);
2398 x >>= 7;
2399 }
2400 out.push((x & 0x7f) as u8);
2401 out
2402 }
2403
2404 fn avro_long_bytes(v: i64) -> Vec<u8> {
2405 varint(zigzag_i64(v))
2406 }
2407
2408 fn avro_len_prefixed_bytes(payload: &[u8]) -> Vec<u8> {
2409 let mut out = avro_long_bytes(payload.len() as i64);
2410 out.extend_from_slice(payload);
2411 out
2412 }
2413
2414 fn duration_fixed12(months: u32, days: u32, millis: u32) -> [u8; 12] {
2415 let m = months.to_le_bytes();
2416 let d = days.to_le_bytes();
2417 let ms = millis.to_le_bytes();
2418 [
2419 m[0], m[1], m[2], m[3], d[0], d[1], d[2], d[3], ms[0], ms[1], ms[2], ms[3],
2420 ]
2421 }
2422
2423 #[cfg(feature = "avro_custom_types")]
2424 fn interval_mdn_fixed16(months: i32, days: i32, nanos: i64) -> [u8; 16] {
2425 let m = months.to_le_bytes();
2426 let d = days.to_le_bytes();
2427 let n = nanos.to_le_bytes();
2428 [
2429 m[0], m[1], m[2], m[3], d[0], d[1], d[2], d[3], n[0], n[1], n[2], n[3], n[4], n[5],
2430 n[6], n[7],
2431 ]
2432 }
2433
2434 fn encode_all(
2435 array: &dyn Array,
2436 plan: &FieldPlan,
2437 nullability: Option<Nullability>,
2438 ) -> Vec<u8> {
2439 let mut enc = FieldEncoder::make_encoder(array, plan, nullability).unwrap();
2440 let mut out = Vec::new();
2441 for i in 0..array.len() {
2442 enc.encode(&mut out, i).unwrap();
2443 }
2444 out
2445 }
2446
2447 fn assert_bytes_eq(actual: &[u8], expected: &[u8]) {
2448 if actual != expected {
2449 let to_hex = |b: &[u8]| {
2450 b.iter()
2451 .map(|x| format!("{x:02X}"))
2452 .collect::<Vec<_>>()
2453 .join(" ")
2454 };
2455 panic!(
2456 "mismatch\n expected: [{}]\n actual: [{}]",
2457 to_hex(expected),
2458 to_hex(actual)
2459 );
2460 }
2461 }
2462
2463 fn row_slice<'a>(buf: &'a [u8], offsets: &[usize], row: usize) -> &'a [u8] {
2464 let start = offsets[row];
2465 let end = offsets[row + 1];
2466 &buf[start..end]
2467 }
2468
2469 #[test]
2470 fn binary_encoder() {
2471 let values: Vec<&[u8]> = vec![b"", b"ab", b"\x00\xFF"];
2472 let arr = BinaryArray::from_vec(values);
2473 let mut expected = Vec::new();
2474 for payload in [b"" as &[u8], b"ab", b"\x00\xFF"] {
2475 expected.extend(avro_len_prefixed_bytes(payload));
2476 }
2477 let got = encode_all(&arr, &FieldPlan::Scalar, None);
2478 assert_bytes_eq(&got, &expected);
2479 }
2480
2481 #[test]
2482 fn large_binary_encoder() {
2483 let values: Vec<&[u8]> = vec![b"xyz", b""];
2484 let arr = LargeBinaryArray::from_vec(values);
2485 let mut expected = Vec::new();
2486 for payload in [b"xyz" as &[u8], b""] {
2487 expected.extend(avro_len_prefixed_bytes(payload));
2488 }
2489 let got = encode_all(&arr, &FieldPlan::Scalar, None);
2490 assert_bytes_eq(&got, &expected);
2491 }
2492
2493 #[test]
2494 fn utf8_encoder() {
2495 let arr = StringArray::from(vec!["", "A", "BC"]);
2496 let mut expected = Vec::new();
2497 for s in ["", "A", "BC"] {
2498 expected.extend(avro_len_prefixed_bytes(s.as_bytes()));
2499 }
2500 let got = encode_all(&arr, &FieldPlan::Scalar, None);
2501 assert_bytes_eq(&got, &expected);
2502 }
2503
2504 #[test]
2505 fn large_utf8_encoder() {
2506 let arr = LargeStringArray::from(vec!["hello", ""]);
2507 let mut expected = Vec::new();
2508 for s in ["hello", ""] {
2509 expected.extend(avro_len_prefixed_bytes(s.as_bytes()));
2510 }
2511 let got = encode_all(&arr, &FieldPlan::Scalar, None);
2512 assert_bytes_eq(&got, &expected);
2513 }
2514
2515 #[test]
2516 fn list_encoder_int32() {
2517 let values = Int32Array::from(vec![1, 2, 3]);
2519 let offsets = vec![0, 2, 2, 3];
2520 let list = ListArray::new(
2521 Field::new("item", DataType::Int32, true).into(),
2522 arrow_buffer::OffsetBuffer::new(offsets.into()),
2523 Arc::new(values) as ArrayRef,
2524 None,
2525 );
2526 let mut expected = Vec::new();
2528 expected.extend(avro_long_bytes(2));
2530 expected.extend(avro_long_bytes(1));
2531 expected.extend(avro_long_bytes(2));
2532 expected.extend(avro_long_bytes(0));
2533 expected.extend(avro_long_bytes(0));
2535 expected.extend(avro_long_bytes(1));
2537 expected.extend(avro_long_bytes(3));
2538 expected.extend(avro_long_bytes(0));
2539
2540 let plan = FieldPlan::List {
2541 items_nullability: None,
2542 item_plan: Box::new(FieldPlan::Scalar),
2543 };
2544 let got = encode_all(&list, &plan, None);
2545 assert_bytes_eq(&got, &expected);
2546 }
2547
2548 #[test]
2549 fn fixed_size_list_encoder_int32_sliced() {
2550 let values = Int32Array::from(vec![1, 2, 3, 4, 5, 6]);
2552 let list = FixedSizeListArray::new(
2553 Field::new("item", DataType::Int32, true).into(),
2554 2,
2555 Arc::new(values) as ArrayRef,
2556 None,
2557 )
2558 .slice(1, 2);
2559 let mut expected = Vec::new();
2560 for row in [[3, 4], [5, 6]] {
2561 expected.extend(avro_long_bytes(2));
2562 for value in row {
2563 expected.extend(avro_long_bytes(value));
2564 }
2565 expected.extend(avro_long_bytes(0));
2566 }
2567
2568 let plan = FieldPlan::List {
2569 items_nullability: None,
2570 item_plan: Box::new(FieldPlan::Scalar),
2571 };
2572 let got = encode_all(&list, &plan, None);
2573 assert_bytes_eq(&got, &expected);
2574 }
2575
2576 #[test]
2577 fn struct_encoder_two_fields() {
2578 let a = Int32Array::from(vec![1, 2]);
2580 let b = StringArray::from(vec!["x", "y"]);
2581 let fields = Fields::from(vec![
2582 Field::new("a", DataType::Int32, true),
2583 Field::new("b", DataType::Utf8, true),
2584 ]);
2585 let struct_arr = StructArray::new(
2586 fields.clone(),
2587 vec![Arc::new(a) as ArrayRef, Arc::new(b) as ArrayRef],
2588 None,
2589 );
2590 let plan = FieldPlan::Struct {
2591 bindings: vec![
2592 FieldBinding {
2593 arrow_index: 0,
2594 nullability: None,
2595 plan: FieldPlan::Scalar,
2596 },
2597 FieldBinding {
2598 arrow_index: 1,
2599 nullability: None,
2600 plan: FieldPlan::Scalar,
2601 },
2602 ],
2603 };
2604 let got = encode_all(&struct_arr, &plan, None);
2605 let mut expected = Vec::new();
2607 expected.extend(avro_long_bytes(1)); expected.extend(avro_len_prefixed_bytes(b"x")); expected.extend(avro_long_bytes(2)); expected.extend(avro_len_prefixed_bytes(b"y")); assert_bytes_eq(&got, &expected);
2612 }
2613
2614 #[test]
2615 fn enum_encoder_dictionary() {
2616 let dict_values = StringArray::from(vec!["A", "B", "C"]);
2618 let keys = Int32Array::from(vec![2, 0, 1]);
2619 let dict =
2620 DictionaryArray::<Int32Type>::try_new(keys, Arc::new(dict_values) as ArrayRef).unwrap();
2621 let symbols = Arc::<[String]>::from(
2622 vec!["A".to_string(), "B".to_string(), "C".to_string()].into_boxed_slice(),
2623 );
2624 let plan = FieldPlan::Enum { symbols };
2625 let got = encode_all(&dict, &plan, None);
2626 let mut expected = Vec::new();
2627 expected.extend(avro_long_bytes(2));
2628 expected.extend(avro_long_bytes(0));
2629 expected.extend(avro_long_bytes(1));
2630 assert_bytes_eq(&got, &expected);
2631 }
2632
2633 #[test]
2634 fn decimal_bytes_and_fixed() {
2635 let dec = Decimal128Array::from(vec![1i128, -1i128, 0i128])
2637 .with_precision_and_scale(20, 0)
2638 .unwrap();
2639 let plan_bytes = FieldPlan::Decimal { size: None };
2641 let got_bytes = encode_all(&dec, &plan_bytes, None);
2642 let mut expected_bytes = Vec::new();
2644 expected_bytes.extend(avro_len_prefixed_bytes(&[0x01]));
2645 expected_bytes.extend(avro_len_prefixed_bytes(&[0xFF]));
2646 expected_bytes.extend(avro_len_prefixed_bytes(&[0x00]));
2647 assert_bytes_eq(&got_bytes, &expected_bytes);
2648
2649 let plan_fixed = FieldPlan::Decimal { size: Some(16) };
2650 let got_fixed = encode_all(&dec, &plan_fixed, None);
2651 let mut expected_fixed = Vec::new();
2652 expected_fixed.extend_from_slice(&1i128.to_be_bytes());
2653 expected_fixed.extend_from_slice(&(-1i128).to_be_bytes());
2654 expected_fixed.extend_from_slice(&0i128.to_be_bytes());
2655 assert_bytes_eq(&got_fixed, &expected_fixed);
2656 }
2657
2658 #[test]
2659 fn decimal_bytes_256() {
2660 use arrow_buffer::i256;
2661 let dec = Decimal256Array::from(vec![
2663 i256::from_i128(1),
2664 i256::from_i128(-1),
2665 i256::from_i128(0),
2666 ])
2667 .with_precision_and_scale(76, 0)
2668 .unwrap();
2669 let plan_bytes = FieldPlan::Decimal { size: None };
2671 let got_bytes = encode_all(&dec, &plan_bytes, None);
2672 let mut expected_bytes = Vec::new();
2674 expected_bytes.extend(avro_len_prefixed_bytes(&[0x01]));
2675 expected_bytes.extend(avro_len_prefixed_bytes(&[0xFF]));
2676 expected_bytes.extend(avro_len_prefixed_bytes(&[0x00]));
2677 assert_bytes_eq(&got_bytes, &expected_bytes);
2678
2679 let plan_fixed = FieldPlan::Decimal { size: Some(32) };
2681 let got_fixed = encode_all(&dec, &plan_fixed, None);
2682 let mut expected_fixed = Vec::new();
2683 expected_fixed.extend_from_slice(&i256::from_i128(1).to_be_bytes());
2684 expected_fixed.extend_from_slice(&i256::from_i128(-1).to_be_bytes());
2685 expected_fixed.extend_from_slice(&i256::from_i128(0).to_be_bytes());
2686 assert_bytes_eq(&got_fixed, &expected_fixed);
2687 }
2688
2689 #[cfg(feature = "small_decimals")]
2690 #[test]
2691 fn decimal_bytes_and_fixed_32() {
2692 let dec = Decimal32Array::from(vec![1i32, -1i32, 0i32])
2694 .with_precision_and_scale(9, 0)
2695 .unwrap();
2696 let plan_bytes = FieldPlan::Decimal { size: None };
2698 let got_bytes = encode_all(&dec, &plan_bytes, None);
2699 let mut expected_bytes = Vec::new();
2700 expected_bytes.extend(avro_len_prefixed_bytes(&[0x01]));
2701 expected_bytes.extend(avro_len_prefixed_bytes(&[0xFF]));
2702 expected_bytes.extend(avro_len_prefixed_bytes(&[0x00]));
2703 assert_bytes_eq(&got_bytes, &expected_bytes);
2704 let plan_fixed = FieldPlan::Decimal { size: Some(4) };
2706 let got_fixed = encode_all(&dec, &plan_fixed, None);
2707 let mut expected_fixed = Vec::new();
2708 expected_fixed.extend_from_slice(&1i32.to_be_bytes());
2709 expected_fixed.extend_from_slice(&(-1i32).to_be_bytes());
2710 expected_fixed.extend_from_slice(&0i32.to_be_bytes());
2711 assert_bytes_eq(&got_fixed, &expected_fixed);
2712 }
2713
2714 #[cfg(feature = "small_decimals")]
2715 #[test]
2716 fn decimal_bytes_and_fixed_64() {
2717 let dec = Decimal64Array::from(vec![1i64, -1i64, 0i64])
2719 .with_precision_and_scale(18, 0)
2720 .unwrap();
2721 let plan_bytes = FieldPlan::Decimal { size: None };
2723 let got_bytes = encode_all(&dec, &plan_bytes, None);
2724 let mut expected_bytes = Vec::new();
2725 expected_bytes.extend(avro_len_prefixed_bytes(&[0x01]));
2726 expected_bytes.extend(avro_len_prefixed_bytes(&[0xFF]));
2727 expected_bytes.extend(avro_len_prefixed_bytes(&[0x00]));
2728 assert_bytes_eq(&got_bytes, &expected_bytes);
2729 let plan_fixed = FieldPlan::Decimal { size: Some(8) };
2731 let got_fixed = encode_all(&dec, &plan_fixed, None);
2732 let mut expected_fixed = Vec::new();
2733 expected_fixed.extend_from_slice(&1i64.to_be_bytes());
2734 expected_fixed.extend_from_slice(&(-1i64).to_be_bytes());
2735 expected_fixed.extend_from_slice(&0i64.to_be_bytes());
2736 assert_bytes_eq(&got_fixed, &expected_fixed);
2737 }
2738
2739 #[test]
2740 fn float32_and_float64_encoders() {
2741 let f32a = Float32Array::from(vec![0.0f32, -1.5f32, f32::from_bits(0x7fc00000)]); let f64a = Float64Array::from(vec![0.0f64, -2.25f64]);
2743 let mut expected32 = Vec::new();
2745 for v in [0.0f32, -1.5f32, f32::from_bits(0x7fc00000)] {
2746 expected32.extend_from_slice(&v.to_bits().to_le_bytes());
2747 }
2748 let got32 = encode_all(&f32a, &FieldPlan::Scalar, None);
2749 assert_bytes_eq(&got32, &expected32);
2750 let mut expected64 = Vec::new();
2752 for v in [0.0f64, -2.25f64] {
2753 expected64.extend_from_slice(&v.to_bits().to_le_bytes());
2754 }
2755 let got64 = encode_all(&f64a, &FieldPlan::Scalar, None);
2756 assert_bytes_eq(&got64, &expected64);
2757 }
2758
2759 #[test]
2760 fn long_encoder_int64() {
2761 let arr = Int64Array::from(vec![0i64, 1i64, -1i64, 2i64, -2i64, i64::MIN + 1]);
2762 let mut expected = Vec::new();
2763 for v in [0, 1, -1, 2, -2, i64::MIN + 1] {
2764 expected.extend(avro_long_bytes(v));
2765 }
2766 let got = encode_all(&arr, &FieldPlan::Scalar, None);
2767 assert_bytes_eq(&got, &expected);
2768 }
2769
2770 #[test]
2771 fn fixed_encoder_plain() {
2772 let data = [[0xDE, 0xAD, 0xBE, 0xEF], [0x00, 0x01, 0x02, 0x03]];
2774 let values: Vec<Vec<u8>> = data.iter().map(|x| x.to_vec()).collect();
2775 let arr = FixedSizeBinaryArray::try_from_iter(values.into_iter()).unwrap();
2776 let got = encode_all(&arr, &FieldPlan::Scalar, None);
2777 let mut expected = Vec::new();
2778 expected.extend_from_slice(&data[0]);
2779 expected.extend_from_slice(&data[1]);
2780 assert_bytes_eq(&got, &expected);
2781 }
2782
2783 #[test]
2784 fn uuid_encoder_test() {
2785 let u = Uuid::parse_str("00112233-4455-6677-8899-aabbccddeeff").unwrap();
2787 let bytes = *u.as_bytes();
2788 let arr_ok = FixedSizeBinaryArray::try_from_iter(vec![bytes.to_vec()].into_iter()).unwrap();
2789 let mut expected = Vec::new();
2791 expected.push(0x48);
2792 expected.extend_from_slice(u.hyphenated().to_string().as_bytes());
2793 let got = encode_all(&arr_ok, &FieldPlan::Uuid, None);
2794 assert_bytes_eq(&got, &expected);
2795 }
2796
2797 #[test]
2798 fn uuid_encoder_error() {
2799 let arr =
2801 FixedSizeBinaryArray::try_new(10, arrow_buffer::Buffer::from(vec![0u8; 10]), None)
2802 .unwrap();
2803 let plan = FieldPlan::Uuid;
2804 let mut enc = FieldEncoder::make_encoder(&arr, &plan, None).unwrap();
2805 let mut out = Vec::new();
2806 let err = enc.encode(&mut out, 0).unwrap_err();
2807 match err {
2808 AvroError::InvalidArgument(msg) => {
2809 assert!(msg.contains("Invalid UUID bytes"))
2810 }
2811 other => panic!("expected InvalidArgument, got {other:?}"),
2812 }
2813 }
2814
2815 fn test_scalar_primitive_encoding<T>(
2816 non_nullable_data: &[T::Native],
2817 nullable_data: &[Option<T::Native>],
2818 ) where
2819 T: ArrowPrimitiveType,
2820 T::Native: Into<i64> + Copy,
2821 PrimitiveArray<T>: From<Vec<<T as ArrowPrimitiveType>::Native>>,
2822 {
2823 let plan = FieldPlan::Scalar;
2824
2825 let array = PrimitiveArray::<T>::from(non_nullable_data.to_vec());
2826 let got = encode_all(&array, &plan, None);
2827
2828 let mut expected = Vec::new();
2829 for &value in non_nullable_data {
2830 expected.extend(avro_long_bytes(value.into()));
2831 }
2832 assert_bytes_eq(&got, &expected);
2833
2834 let array_nullable: PrimitiveArray<T> = nullable_data.iter().copied().collect();
2835 let got_nullable = encode_all(&array_nullable, &plan, Some(Nullability::NullFirst));
2836
2837 let mut expected_nullable = Vec::new();
2838 for &opt_value in nullable_data {
2839 match opt_value {
2840 Some(value) => {
2841 expected_nullable.extend(avro_long_bytes(1));
2843 expected_nullable.extend(avro_long_bytes(value.into()));
2844 }
2845 None => {
2846 expected_nullable.extend(avro_long_bytes(0));
2848 }
2849 }
2850 }
2851 assert_bytes_eq(&got_nullable, &expected_nullable);
2852 }
2853
2854 #[test]
2855 fn date32_encoder() {
2856 test_scalar_primitive_encoding::<Date32Type>(
2857 &[
2858 19345, 0, -1, ],
2862 &[Some(19345), None],
2863 );
2864 }
2865
2866 #[test]
2867 fn time32_millis_encoder() {
2868 test_scalar_primitive_encoding::<Time32MillisecondType>(
2869 &[
2870 0, 49530123, 86399999, ],
2874 &[None, Some(49530123)],
2875 );
2876 }
2877
2878 #[test]
2879 fn time64_micros_encoder() {
2880 test_scalar_primitive_encoding::<Time64MicrosecondType>(
2881 &[
2882 0, 86399999999, ],
2885 &[Some(86399999999), None],
2886 );
2887 }
2888
2889 #[test]
2890 fn timestamp_millis_encoder() {
2891 test_scalar_primitive_encoding::<TimestampMillisecondType>(
2892 &[
2893 1704067200000, 0, -123456789, ],
2897 &[None, Some(1704067200000)],
2898 );
2899 }
2900
2901 #[test]
2902 fn map_encoder_string_keys_int_values() {
2903 let keys = StringArray::from(vec!["k1", "k2"]);
2907 let values = Int32Array::from(vec![1, 2]);
2908 let entries_fields = Fields::from(vec![
2909 Field::new(Field::MAP_KEY_FIELD_DEFAULT_NAME, DataType::Utf8, false),
2910 Field::new(Field::MAP_VALUE_FIELD_DEFAULT_NAME, DataType::Int32, true),
2911 ]);
2912 let entries = StructArray::new(
2913 entries_fields,
2914 vec![Arc::new(keys) as ArrayRef, Arc::new(values) as ArrayRef],
2915 None,
2916 );
2917 let offsets = arrow_buffer::OffsetBuffer::new(vec![0i32, 2, 2].into());
2918 let map = MapArray::new(
2919 Field::new(
2920 Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
2921 entries.data_type().clone(),
2922 false,
2923 )
2924 .into(),
2925 offsets,
2926 entries,
2927 None,
2928 false,
2929 );
2930 let plan = FieldPlan::Map {
2931 values_nullability: None,
2932 value_plan: Box::new(FieldPlan::Scalar),
2933 };
2934 let got = encode_all(&map, &plan, None);
2935 let mut expected = Vec::new();
2936 expected.extend(avro_long_bytes(2));
2938 expected.extend(avro_len_prefixed_bytes(b"k1"));
2939 expected.extend(avro_long_bytes(1));
2940 expected.extend(avro_len_prefixed_bytes(b"k2"));
2941 expected.extend(avro_long_bytes(2));
2942 expected.extend(avro_long_bytes(0));
2943 expected.extend(avro_long_bytes(0));
2945 assert_bytes_eq(&got, &expected);
2946 }
2947
2948 #[test]
2949 fn union_encoder_string_int() {
2950 let strings = StringArray::from(vec!["hello", "world"]);
2951 let ints = Int32Array::from(vec![10, 20, 30]);
2952
2953 let union_fields = UnionFields::try_new(
2954 vec![0, 1],
2955 vec![
2956 Field::new("v_str", DataType::Utf8, true),
2957 Field::new("v_int", DataType::Int32, true),
2958 ],
2959 )
2960 .unwrap();
2961
2962 let type_ids = Buffer::from_slice_ref([0_i8, 1, 1, 0, 1]);
2963 let offsets = Buffer::from_slice_ref([0_i32, 0, 1, 1, 2]);
2964
2965 let union_array = UnionArray::try_new(
2966 union_fields,
2967 type_ids.into(),
2968 Some(offsets.into()),
2969 vec![Arc::new(strings), Arc::new(ints)],
2970 )
2971 .unwrap();
2972
2973 let plan = FieldPlan::Union {
2974 bindings: vec![
2975 FieldBinding {
2976 arrow_index: 0,
2977 nullability: None,
2978 plan: FieldPlan::Scalar,
2979 },
2980 FieldBinding {
2981 arrow_index: 1,
2982 nullability: None,
2983 plan: FieldPlan::Scalar,
2984 },
2985 ],
2986 };
2987
2988 let got = encode_all(&union_array, &plan, None);
2989
2990 let mut expected = Vec::new();
2991 expected.extend(avro_long_bytes(0));
2992 expected.extend(avro_len_prefixed_bytes(b"hello"));
2993 expected.extend(avro_long_bytes(1));
2994 expected.extend(avro_long_bytes(10));
2995 expected.extend(avro_long_bytes(1));
2996 expected.extend(avro_long_bytes(20));
2997 expected.extend(avro_long_bytes(0));
2998 expected.extend(avro_len_prefixed_bytes(b"world"));
2999 expected.extend(avro_long_bytes(1));
3000 expected.extend(avro_long_bytes(30));
3001
3002 assert_bytes_eq(&got, &expected);
3003 }
3004
3005 #[test]
3006 fn union_encoder_null_string_int() {
3007 let nulls = NullArray::new(1);
3008 let strings = StringArray::from(vec!["hello"]);
3009 let ints = Int32Array::from(vec![10]);
3010
3011 let union_fields = UnionFields::try_new(
3012 vec![0, 1, 2],
3013 vec![
3014 Field::new("v_null", DataType::Null, true),
3015 Field::new("v_str", DataType::Utf8, true),
3016 Field::new("v_int", DataType::Int32, true),
3017 ],
3018 )
3019 .unwrap();
3020
3021 let type_ids = Buffer::from_slice_ref([0_i8, 1, 2]);
3022 let offsets = Buffer::from_slice_ref([0_i32, 0, 0]);
3026
3027 let union_array = UnionArray::try_new(
3028 union_fields,
3029 type_ids.into(),
3030 Some(offsets.into()),
3031 vec![Arc::new(nulls), Arc::new(strings), Arc::new(ints)],
3032 )
3033 .unwrap();
3034
3035 let plan = FieldPlan::Union {
3036 bindings: vec![
3037 FieldBinding {
3038 arrow_index: 0,
3039 nullability: None,
3040 plan: FieldPlan::Scalar,
3041 },
3042 FieldBinding {
3043 arrow_index: 1,
3044 nullability: None,
3045 plan: FieldPlan::Scalar,
3046 },
3047 FieldBinding {
3048 arrow_index: 2,
3049 nullability: None,
3050 plan: FieldPlan::Scalar,
3051 },
3052 ],
3053 };
3054
3055 let got = encode_all(&union_array, &plan, None);
3056
3057 let mut expected = Vec::new();
3058 expected.extend(avro_long_bytes(0));
3059 expected.extend(avro_long_bytes(1));
3060 expected.extend(avro_len_prefixed_bytes(b"hello"));
3061 expected.extend(avro_long_bytes(2));
3062 expected.extend(avro_long_bytes(10));
3063
3064 assert_bytes_eq(&got, &expected);
3065 }
3066
3067 #[test]
3068 fn list64_encoder_int32() {
3069 let values = Int32Array::from(vec![1, 2, 3]);
3071 let offsets: Vec<i64> = vec![0, 3, 3];
3072 let list = LargeListArray::new(
3073 Field::new("item", DataType::Int32, true).into(),
3074 arrow_buffer::OffsetBuffer::new(offsets.into()),
3075 Arc::new(values) as ArrayRef,
3076 None,
3077 );
3078 let plan = FieldPlan::List {
3079 items_nullability: None,
3080 item_plan: Box::new(FieldPlan::Scalar),
3081 };
3082 let got = encode_all(&list, &plan, None);
3083 let mut expected = Vec::new();
3085 expected.extend(avro_long_bytes(3));
3086 expected.extend(avro_long_bytes(1));
3087 expected.extend(avro_long_bytes(2));
3088 expected.extend(avro_long_bytes(3));
3089 expected.extend(avro_long_bytes(0));
3090 expected.extend(avro_long_bytes(0));
3091 assert_bytes_eq(&got, &expected);
3092 }
3093
3094 #[test]
3095 fn int_encoder_test() {
3096 let ints = Int32Array::from(vec![0, -1, 2]);
3097 let mut expected_i = Vec::new();
3098 for v in [0i32, -1, 2] {
3099 expected_i.extend(avro_long_bytes(v as i64));
3100 }
3101 let got_i = encode_all(&ints, &FieldPlan::Scalar, None);
3102 assert_bytes_eq(&got_i, &expected_i);
3103 }
3104
3105 #[test]
3106 fn boolean_encoder_test() {
3107 let bools = BooleanArray::from(vec![true, false]);
3108 let mut expected_b = Vec::new();
3109 expected_b.extend_from_slice(&[1]);
3110 expected_b.extend_from_slice(&[0]);
3111 let got_b = encode_all(&bools, &FieldPlan::Scalar, None);
3112 assert_bytes_eq(&got_b, &expected_b);
3113 }
3114
3115 #[test]
3116 #[cfg(feature = "avro_custom_types")]
3117 fn duration_encoding_seconds() {
3118 let arr: PrimitiveArray<DurationSecondType> = vec![0i64, -1, 2].into();
3119 let mut expected = Vec::new();
3120 for v in [0i64, -1, 2] {
3121 expected.extend_from_slice(&avro_long_bytes(v));
3122 }
3123 let got = encode_all(&arr, &FieldPlan::Scalar, None);
3124 assert_bytes_eq(&got, &expected);
3125 }
3126
3127 #[test]
3128 #[cfg(feature = "avro_custom_types")]
3129 fn duration_encoding_milliseconds() {
3130 let arr: PrimitiveArray<DurationMillisecondType> = vec![1i64, 0, -2].into();
3131 let mut expected = Vec::new();
3132 for v in [1i64, 0, -2] {
3133 expected.extend_from_slice(&avro_long_bytes(v));
3134 }
3135 let got = encode_all(&arr, &FieldPlan::Scalar, None);
3136 assert_bytes_eq(&got, &expected);
3137 }
3138
3139 #[test]
3140 #[cfg(feature = "avro_custom_types")]
3141 fn duration_encoding_microseconds() {
3142 let arr: PrimitiveArray<DurationMicrosecondType> = vec![5i64, -6, 7].into();
3143 let mut expected = Vec::new();
3144 for v in [5i64, -6, 7] {
3145 expected.extend_from_slice(&avro_long_bytes(v));
3146 }
3147 let got = encode_all(&arr, &FieldPlan::Scalar, None);
3148 assert_bytes_eq(&got, &expected);
3149 }
3150
3151 #[test]
3152 #[cfg(feature = "avro_custom_types")]
3153 fn duration_encoding_nanoseconds() {
3154 let arr: PrimitiveArray<DurationNanosecondType> = vec![8i64, 9, -10].into();
3155 let mut expected = Vec::new();
3156 for v in [8i64, 9, -10] {
3157 expected.extend_from_slice(&avro_long_bytes(v));
3158 }
3159 let got = encode_all(&arr, &FieldPlan::Scalar, None);
3160 assert_bytes_eq(&got, &expected);
3161 }
3162
3163 #[test]
3164 fn duration_encoder_year_month_happy_path() {
3165 let arr: PrimitiveArray<IntervalYearMonthType> = vec![0i32, 1i32, 25i32].into();
3166 let mut expected = Vec::new();
3167 for m in [0u32, 1u32, 25u32] {
3168 expected.extend_from_slice(&duration_fixed12(m, 0, 0));
3169 }
3170 let got = encode_all(&arr, &FieldPlan::Duration, None);
3171 assert_bytes_eq(&got, &expected);
3172 }
3173
3174 #[test]
3175 fn duration_encoder_year_month_rejects_negative() {
3176 let arr: PrimitiveArray<IntervalYearMonthType> = vec![-1i32].into();
3177 let mut enc = FieldEncoder::make_encoder(&arr, &FieldPlan::Duration, None).unwrap();
3178 let mut out = Vec::new();
3179 let err = enc.encode(&mut out, 0).unwrap_err();
3180 match err {
3181 AvroError::InvalidArgument(msg) => {
3182 assert!(msg.contains("cannot encode negative months"))
3183 }
3184 other => panic!("expected InvalidArgument, got {other:?}"),
3185 }
3186 }
3187
3188 #[test]
3189 fn duration_encoder_day_time_happy_path() {
3190 let v0 = IntervalDayTimeType::make_value(2, 500); let v1 = IntervalDayTimeType::make_value(0, 0);
3192 let arr: PrimitiveArray<IntervalDayTimeType> = vec![v0, v1].into();
3193 let mut expected = Vec::new();
3194 expected.extend_from_slice(&duration_fixed12(0, 2, 500));
3195 expected.extend_from_slice(&duration_fixed12(0, 0, 0));
3196 let got = encode_all(&arr, &FieldPlan::Duration, None);
3197 assert_bytes_eq(&got, &expected);
3198 }
3199
3200 #[test]
3201 fn duration_encoder_day_time_rejects_negative() {
3202 let bad = IntervalDayTimeType::make_value(-1, 0);
3203 let arr: PrimitiveArray<IntervalDayTimeType> = vec![bad].into();
3204 let mut enc = FieldEncoder::make_encoder(&arr, &FieldPlan::Duration, None).unwrap();
3205 let mut out = Vec::new();
3206 let err = enc.encode(&mut out, 0).unwrap_err();
3207 match err {
3208 AvroError::InvalidArgument(msg) => {
3209 assert!(msg.contains("cannot encode negative days"))
3210 }
3211 other => panic!("expected InvalidArgument, got {other:?}"),
3212 }
3213 }
3214
3215 #[cfg(feature = "avro_custom_types")]
3216 #[test]
3217 fn interval_month_day_nano_fixed_encoder_happy_path() {
3218 let v0 = IntervalMonthDayNanoType::make_value(1, 2, 3); let v1 = IntervalMonthDayNanoType::make_value(-4, -5, -6);
3221 let arr: PrimitiveArray<IntervalMonthDayNanoType> = vec![v0, v1].into();
3222
3223 let got = encode_all(&arr, &FieldPlan::Scalar, None);
3224 let mut expected = Vec::new();
3225 expected.extend_from_slice(&interval_mdn_fixed16(1, 2, 3));
3226 expected.extend_from_slice(&interval_mdn_fixed16(-4, -5, -6));
3227 assert_bytes_eq(&got, &expected);
3228 }
3229
3230 #[test]
3231 fn duration_encoder_month_day_nano_happy_path() {
3232 let v0 = IntervalMonthDayNanoType::make_value(1, 2, 3_000_000); let v1 = IntervalMonthDayNanoType::make_value(0, 0, 0);
3234 let arr: PrimitiveArray<IntervalMonthDayNanoType> = vec![v0, v1].into();
3235 let mut expected = Vec::new();
3236 expected.extend_from_slice(&duration_fixed12(1, 2, 3));
3237 expected.extend_from_slice(&duration_fixed12(0, 0, 0));
3238 let got = encode_all(&arr, &FieldPlan::Duration, None);
3239 assert_bytes_eq(&got, &expected);
3240 }
3241
3242 #[test]
3243 fn duration_encoder_month_day_nano_rejects_non_ms_multiple() {
3244 let bad = IntervalMonthDayNanoType::make_value(0, 0, 1);
3245 let arr: PrimitiveArray<IntervalMonthDayNanoType> = vec![bad].into();
3246 let mut enc = FieldEncoder::make_encoder(&arr, &FieldPlan::Duration, None).unwrap();
3247 let mut out = Vec::new();
3248 let err = enc.encode(&mut out, 0).unwrap_err();
3249 match err {
3250 AvroError::InvalidArgument(msg) => {
3251 assert!(msg.contains("requires whole milliseconds") || msg.contains("divisible"))
3252 }
3253 other => panic!("expected InvalidArgument, got {other:?}"),
3254 }
3255 }
3256
3257 #[test]
3258 fn minimal_twos_complement_test() {
3259 let pos = [0x00, 0x00, 0x01];
3260 assert_eq!(minimal_twos_complement(&pos), &pos[2..]);
3261 let neg = [0xFF, 0xFF, 0x80]; assert_eq!(minimal_twos_complement(&neg), &neg[2..]);
3263 let zero = [0x00, 0x00, 0x00];
3264 assert_eq!(minimal_twos_complement(&zero), &zero[2..]);
3265 }
3266
3267 #[test]
3268 fn write_sign_extend_test() {
3269 let mut out = Vec::new();
3270 write_sign_extended(&mut out, &[0x01], 4).unwrap();
3271 assert_eq!(out, vec![0x00, 0x00, 0x00, 0x01]);
3272 out.clear();
3273 write_sign_extended(&mut out, &[0xFF], 4).unwrap();
3274 assert_eq!(out, vec![0xFF, 0xFF, 0xFF, 0xFF]);
3275 out.clear();
3276 write_sign_extended(&mut out, &[0xFF, 0xFF, 0x80], 2).unwrap();
3278 assert_eq!(out, vec![0xFF, 0x80]);
3279 out.clear();
3280 let err = write_sign_extended(&mut out, &[0x01, 0x00], 1).unwrap_err();
3282 match err {
3283 AvroError::InvalidArgument(_) => {}
3284 _ => panic!("expected InvalidArgument"),
3285 }
3286 }
3287
3288 #[test]
3289 fn duration_month_day_nano_overflow_millis() {
3290 let nanos = ((u64::from(u32::MAX) + 1) * 1_000_000) as i64;
3292 let v = IntervalMonthDayNanoType::make_value(0, 0, nanos);
3293 let arr: PrimitiveArray<IntervalMonthDayNanoType> = vec![v].into();
3294 let mut enc = FieldEncoder::make_encoder(&arr, &FieldPlan::Duration, None).unwrap();
3295 let mut out = Vec::new();
3296 let err = enc.encode(&mut out, 0).unwrap_err();
3297 match err {
3298 AvroError::InvalidArgument(msg) => assert!(msg.contains("exceed u32::MAX")),
3299 _ => panic!("expected InvalidArgument"),
3300 }
3301 }
3302
3303 #[test]
3304 fn fieldplan_decimal_precision_scale_mismatch_errors() {
3305 use crate::codec::Codec;
3307 use std::collections::HashMap;
3308 let arrow_field = Field::new("d", DataType::Decimal128(12, 2), true);
3309 let avro_dt = AvroDataType::new(Codec::Decimal(10, Some(2), None), HashMap::new(), None);
3310 let err = FieldPlan::build(&avro_dt, &arrow_field).unwrap_err();
3311 match err {
3312 AvroError::SchemaError(msg) => {
3313 assert!(msg.contains("Decimal precision/scale mismatch"))
3314 }
3315 _ => panic!("expected SchemaError"),
3316 }
3317 }
3318
3319 #[test]
3320 fn timestamp_micros_encoder() {
3321 test_scalar_primitive_encoding::<TimestampMicrosecondType>(
3323 &[
3324 1_704_067_200_000_000, 0, -123_456_789, ],
3328 &[None, Some(1_704_067_200_000_000)],
3329 );
3330 }
3331
3332 #[test]
3333 fn list_encoder_nullable_items_null_first() {
3334 let values = Int32Array::from(vec![Some(1), None, Some(2)]);
3336 let offsets = arrow_buffer::OffsetBuffer::new(vec![0i32, 3].into());
3337 let list = ListArray::new(
3338 Field::new("item", DataType::Int32, true).into(),
3339 offsets,
3340 Arc::new(values) as ArrayRef,
3341 None,
3342 );
3343
3344 let plan = FieldPlan::List {
3345 items_nullability: Some(Nullability::NullFirst),
3346 item_plan: Box::new(FieldPlan::Scalar),
3347 };
3348
3349 let mut expected = Vec::new();
3352 expected.extend(avro_long_bytes(3)); expected.extend(avro_long_bytes(1)); expected.extend(avro_long_bytes(1)); expected.extend(avro_long_bytes(0)); expected.extend(avro_long_bytes(1)); expected.extend(avro_long_bytes(2)); expected.extend(avro_long_bytes(0)); let got = encode_all(&list, &plan, None);
3361 assert_bytes_eq(&got, &expected);
3362 }
3363
3364 #[test]
3365 fn large_list_encoder_nullable_items_null_first() {
3366 let values = Int32Array::from(vec![Some(10), None]);
3368 let offsets = arrow_buffer::OffsetBuffer::new(vec![0i64, 2].into());
3369 let list = LargeListArray::new(
3370 Field::new("item", DataType::Int32, true).into(),
3371 offsets,
3372 Arc::new(values) as ArrayRef,
3373 None,
3374 );
3375
3376 let plan = FieldPlan::List {
3377 items_nullability: Some(Nullability::NullFirst),
3378 item_plan: Box::new(FieldPlan::Scalar),
3379 };
3380
3381 let mut expected = Vec::new();
3382 expected.extend(avro_long_bytes(2)); expected.extend(avro_long_bytes(1)); expected.extend(avro_long_bytes(10)); expected.extend(avro_long_bytes(0)); expected.extend(avro_long_bytes(0)); let got = encode_all(&list, &plan, None);
3389 assert_bytes_eq(&got, &expected);
3390 }
3391
3392 #[test]
3393 fn map_encoder_string_keys_nullable_int_values_null_first() {
3394 let keys = StringArray::from(vec!["k1", "k2"]);
3396 let values = Int32Array::from(vec![Some(7), None]);
3397
3398 let entries_fields = Fields::from(vec![
3399 Field::new(Field::MAP_KEY_FIELD_DEFAULT_NAME, DataType::Utf8, false),
3400 Field::new(Field::MAP_VALUE_FIELD_DEFAULT_NAME, DataType::Int32, true),
3401 ]);
3402 let entries = StructArray::new(
3403 entries_fields,
3404 vec![Arc::new(keys) as ArrayRef, Arc::new(values) as ArrayRef],
3405 None,
3406 );
3407
3408 let offsets = arrow_buffer::OffsetBuffer::new(vec![0i32, 2].into());
3410 let map = MapArray::new(
3411 Field::new(
3412 Field::MAP_ENTRIES_FIELD_DEFAULT_NAME,
3413 entries.data_type().clone(),
3414 false,
3415 )
3416 .into(),
3417 offsets,
3418 entries,
3419 None,
3420 false,
3421 );
3422
3423 let plan = FieldPlan::Map {
3424 values_nullability: Some(Nullability::NullFirst),
3425 value_plan: Box::new(FieldPlan::Scalar),
3426 };
3427
3428 let mut expected = Vec::new();
3434 expected.extend(avro_long_bytes(2)); expected.extend(avro_len_prefixed_bytes(b"k1")); expected.extend(avro_long_bytes(1)); expected.extend(avro_long_bytes(7)); expected.extend(avro_len_prefixed_bytes(b"k2")); expected.extend(avro_long_bytes(0)); expected.extend(avro_long_bytes(0)); let got = encode_all(&map, &plan, None);
3443 assert_bytes_eq(&got, &expected);
3444 }
3445
3446 #[test]
3447 fn time32_seconds_to_millis_encoder() {
3448 let arr: arrow_array::PrimitiveArray<arrow_array::types::Time32SecondType> =
3450 vec![0i32, 1, -2, 12_345].into();
3451 let got = encode_all(&arr, &FieldPlan::TimeMillisFromSecs, None);
3452 let mut expected = Vec::new();
3453 for secs in [0i32, 1, -2, 12_345] {
3454 let millis = (secs as i64) * 1000;
3455 expected.extend_from_slice(&avro_long_bytes(millis));
3456 }
3457 assert_bytes_eq(&got, &expected);
3458 }
3459
3460 #[test]
3461 fn time32_seconds_to_millis_overflow() {
3462 let overflow_secs: i32 = i32::MAX / 1000 + 1;
3464 let arr: PrimitiveArray<Time32SecondType> = vec![overflow_secs].into();
3465 let mut enc =
3466 FieldEncoder::make_encoder(&arr, &FieldPlan::TimeMillisFromSecs, None).unwrap();
3467 let mut out = Vec::new();
3468 let err = enc.encode(&mut out, 0).unwrap_err();
3469 match err {
3470 AvroError::InvalidArgument(msg) => {
3471 assert!(
3472 msg.contains("overflowed") || msg.contains("overflow"),
3473 "unexpected message: {msg}"
3474 )
3475 }
3476 other => panic!("expected InvalidArgument, got {other:?}"),
3477 }
3478 }
3479
3480 #[test]
3481 fn time32_seconds_to_millis_type_mismatch_returns_schema_error() {
3482 let arr = Int32Array::from(vec![1, 2, 3]);
3483 match FieldEncoder::make_encoder(&arr, &FieldPlan::TimeMillisFromSecs, None) {
3484 Err(AvroError::SchemaError(msg)) => {
3485 assert!(msg.contains("Time32(Second)"), "unexpected message: {msg}");
3486 assert!(msg.contains("Int32"), "unexpected message: {msg}");
3487 }
3488 Ok(_) => panic!("expected SchemaError"),
3489 Err(other) => panic!("expected SchemaError, got {other:?}"),
3490 }
3491 }
3492
3493 #[test]
3494 fn encode_rows_time32_seconds_plan_rejects_millisecond_column() {
3495 let schema = ArrowSchema::new(vec![Field::new(
3496 "t",
3497 DataType::Time32(TimeUnit::Millisecond),
3498 false,
3499 )]);
3500 let arr: PrimitiveArray<Time32MillisecondType> = vec![1i32].into();
3501 let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(arr)]).unwrap();
3502 let encoder = RecordEncoder {
3503 columns: vec![FieldBinding {
3504 arrow_index: 0,
3505 nullability: None,
3506 plan: FieldPlan::TimeMillisFromSecs,
3507 }],
3508 prefix: None,
3509 };
3510
3511 let mut out = BytesMut::new();
3512 let mut offsets = vec![0usize];
3513 let err = encoder
3514 .encode_rows(&batch, 16, &mut out, &mut offsets)
3515 .unwrap_err();
3516 match err {
3517 AvroError::SchemaError(msg) => {
3518 assert!(msg.contains("Time32(Second)"), "unexpected message: {msg}");
3519 assert!(
3520 msg.contains("Time32(Millisecond)"),
3521 "unexpected message: {msg}"
3522 );
3523 }
3524 other => panic!("expected SchemaError, got {other:?}"),
3525 }
3526 }
3527
3528 #[cfg(not(feature = "avro_custom_types"))]
3529 #[test]
3530 fn timestamp_seconds_to_millis_encoder() {
3531 let arr: PrimitiveArray<TimestampSecondType> = vec![0i64, 1, -1, 1_234_567_890].into();
3533 let got = encode_all(&arr, &FieldPlan::Scalar, None);
3534 let mut expected = Vec::new();
3535 for secs in [0i64, 1, -1, 1_234_567_890] {
3536 let millis = secs * 1000;
3537 expected.extend_from_slice(&avro_long_bytes(millis));
3538 }
3539 assert_bytes_eq(&got, &expected);
3540 }
3541
3542 #[cfg(not(feature = "avro_custom_types"))]
3543 #[test]
3544 fn timestamp_seconds_to_millis_overflow() {
3545 let overflow_secs: i64 = i64::MAX / 1000 + 1;
3547 let arr: PrimitiveArray<TimestampSecondType> = vec![overflow_secs].into();
3548 let mut enc = FieldEncoder::make_encoder(&arr, &FieldPlan::Scalar, None).unwrap();
3549 let mut out = Vec::new();
3550 let err = enc.encode(&mut out, 0).unwrap_err();
3551 match err {
3552 AvroError::InvalidArgument(msg) => {
3553 assert!(
3554 msg.contains("overflowed") || msg.contains("overflow"),
3555 "unexpected message: {msg}"
3556 )
3557 }
3558 other => panic!("expected InvalidArgument, got {other:?}"),
3559 }
3560 }
3561
3562 #[test]
3563 fn timestamp_nanos_encoder() {
3564 let arr: PrimitiveArray<TimestampNanosecondType> = vec![0i64, 1, -1, 123].into();
3565 let got = encode_all(&arr, &FieldPlan::Scalar, None);
3566 let mut expected = Vec::new();
3567 for ns in [0i64, 1, -1, 123] {
3568 expected.extend_from_slice(&avro_long_bytes(ns));
3569 }
3570 assert_bytes_eq(&got, &expected);
3571 }
3572
3573 #[test]
3574 fn union_encoder_string_int_nonzero_type_ids() {
3575 let strings = StringArray::from(vec!["hello", "world"]);
3576 let ints = Int32Array::from(vec![10, 20, 30]);
3577 let union_fields = UnionFields::try_new(
3578 vec![2, 5],
3579 vec![
3580 Field::new("v_str", DataType::Utf8, true),
3581 Field::new("v_int", DataType::Int32, true),
3582 ],
3583 )
3584 .unwrap();
3585 let type_ids = Buffer::from_slice_ref([2_i8, 5, 5, 2, 5]);
3586 let offsets = Buffer::from_slice_ref([0_i32, 0, 1, 1, 2]);
3587 let union_array = UnionArray::try_new(
3588 union_fields,
3589 type_ids.into(),
3590 Some(offsets.into()),
3591 vec![Arc::new(strings), Arc::new(ints)],
3592 )
3593 .unwrap();
3594 let plan = FieldPlan::Union {
3595 bindings: vec![
3596 FieldBinding {
3597 arrow_index: 0,
3598 nullability: None,
3599 plan: FieldPlan::Scalar,
3600 },
3601 FieldBinding {
3602 arrow_index: 1,
3603 nullability: None,
3604 plan: FieldPlan::Scalar,
3605 },
3606 ],
3607 };
3608 let got = encode_all(&union_array, &plan, None);
3609 let mut expected = Vec::new();
3610 expected.extend(avro_long_bytes(0));
3611 expected.extend(avro_len_prefixed_bytes(b"hello"));
3612 expected.extend(avro_long_bytes(1));
3613 expected.extend(avro_long_bytes(10));
3614 expected.extend(avro_long_bytes(1));
3615 expected.extend(avro_long_bytes(20));
3616 expected.extend(avro_long_bytes(0));
3617 expected.extend(avro_len_prefixed_bytes(b"world"));
3618 expected.extend(avro_long_bytes(1));
3619 expected.extend(avro_long_bytes(30));
3620 assert_bytes_eq(&got, &expected);
3621 }
3622
3623 #[test]
3624 fn nullable_state_with_null_buffer_and_zero_nulls() {
3625 let values = vec![1i32, 2, 3];
3626 let arr = Int32Array::from_iter_values_with_nulls(values, Some(NullBuffer::new_valid(3)));
3627 assert_eq!(arr.null_count(), 0);
3628 assert!(arr.nulls().is_some());
3629 let plan = FieldPlan::Scalar;
3630 let enc = FieldEncoder::make_encoder(&arr, &plan, Some(Nullability::NullFirst)).unwrap();
3631 match enc.null_state {
3632 NullState::NullableNoNulls { union_value_byte } => {
3633 assert_eq!(
3634 union_value_byte,
3635 union_value_branch_byte(Nullability::NullFirst, false)
3636 );
3637 }
3638 other => panic!("expected NullableNoNulls, got {other:?}"),
3639 }
3640 }
3641
3642 #[test]
3643 fn encode_rows_single_column_int32() {
3644 let schema = ArrowSchema::new(vec![Field::new("x", DataType::Int32, false)]);
3645 let arr = Int32Array::from(vec![1, 2, 3]);
3646 let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(arr)]).unwrap();
3647 let encoder = RecordEncoder {
3648 columns: vec![FieldBinding {
3649 arrow_index: 0,
3650 nullability: None,
3651 plan: FieldPlan::Scalar,
3652 }],
3653 prefix: None,
3654 };
3655 let mut out = BytesMut::new();
3656 let mut offsets: Vec<usize> = vec![0];
3657 encoder
3658 .encode_rows(&batch, 16, &mut out, &mut offsets)
3659 .unwrap();
3660 assert_eq!(offsets.len(), 4);
3661 assert_eq!(*offsets.last().unwrap(), out.len());
3662 assert_bytes_eq(row_slice(&out, &offsets, 0), &avro_long_bytes(1));
3663 assert_bytes_eq(row_slice(&out, &offsets, 1), &avro_long_bytes(2));
3664 assert_bytes_eq(row_slice(&out, &offsets, 2), &avro_long_bytes(3));
3665 }
3666
3667 #[test]
3668 fn encode_rows_multiple_columns() {
3669 let schema = ArrowSchema::new(vec![
3670 Field::new("a", DataType::Int32, false),
3671 Field::new("b", DataType::Utf8, false),
3672 ]);
3673 let int_arr = Int32Array::from(vec![10, 20]);
3674 let str_arr = StringArray::from(vec!["hello", "world"]);
3675 let batch = RecordBatch::try_new(
3676 Arc::new(schema.clone()),
3677 vec![Arc::new(int_arr), Arc::new(str_arr)],
3678 )
3679 .unwrap();
3680 let encoder = RecordEncoder {
3681 columns: vec![
3682 FieldBinding {
3683 arrow_index: 0,
3684 nullability: None,
3685 plan: FieldPlan::Scalar,
3686 },
3687 FieldBinding {
3688 arrow_index: 1,
3689 nullability: None,
3690 plan: FieldPlan::Scalar,
3691 },
3692 ],
3693 prefix: None,
3694 };
3695 let mut out = BytesMut::new();
3696 let mut offsets: Vec<usize> = vec![0];
3697 encoder
3698 .encode_rows(&batch, 32, &mut out, &mut offsets)
3699 .unwrap();
3700 assert_eq!(offsets.len(), 3);
3701 assert_eq!(*offsets.last().unwrap(), out.len());
3702 let mut expected_row0 = Vec::new();
3703 expected_row0.extend(avro_long_bytes(10));
3704 expected_row0.extend(avro_len_prefixed_bytes(b"hello"));
3705 assert_bytes_eq(row_slice(&out, &offsets, 0), &expected_row0);
3706 let mut expected_row1 = Vec::new();
3707 expected_row1.extend(avro_long_bytes(20));
3708 expected_row1.extend(avro_len_prefixed_bytes(b"world"));
3709 assert_bytes_eq(row_slice(&out, &offsets, 1), &expected_row1);
3710 }
3711
3712 #[test]
3713 fn encode_rows_with_prefix() {
3714 use crate::codec::AvroFieldBuilder;
3715 use crate::schema::AvroSchema;
3716 let schema = ArrowSchema::new(vec![Field::new("x", DataType::Int32, false)]);
3717 let arr = Int32Array::from(vec![42]);
3718 let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(arr)]).unwrap();
3719 let avro_schema = AvroSchema::try_from(&schema).unwrap();
3720 let fingerprint = avro_schema
3721 .fingerprint(crate::schema::FingerprintAlgorithm::Rabin)
3722 .unwrap();
3723 let avro_root = AvroFieldBuilder::new(&avro_schema.schema().unwrap())
3724 .build()
3725 .unwrap();
3726 let encoder = RecordEncoderBuilder::new(&avro_root, &schema)
3727 .with_fingerprint(Some(fingerprint))
3728 .build()
3729 .unwrap();
3730 let mut out = BytesMut::new();
3731 let mut offsets: Vec<usize> = vec![0];
3732 encoder
3733 .encode_rows(&batch, 32, &mut out, &mut offsets)
3734 .unwrap();
3735 assert_eq!(offsets.len(), 2);
3736 let row0 = row_slice(&out, &offsets, 0);
3737 assert!(row0.len() > 10, "Row should contain prefix + encoded value");
3738 assert_eq!(row0[0], 0xC3);
3739 assert_eq!(row0[1], 0x01);
3740 }
3741
3742 #[test]
3743 fn encode_rows_empty_batch() {
3744 let schema = ArrowSchema::new(vec![Field::new("x", DataType::Int32, false)]);
3745 let arr = Int32Array::from(Vec::<i32>::new());
3746 let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(arr)]).unwrap();
3747 let encoder = RecordEncoder {
3748 columns: vec![FieldBinding {
3749 arrow_index: 0,
3750 nullability: None,
3751 plan: FieldPlan::Scalar,
3752 }],
3753 prefix: None,
3754 };
3755 let mut out = BytesMut::new();
3756 let mut offsets: Vec<usize> = vec![0];
3757 encoder
3758 .encode_rows(&batch, 16, &mut out, &mut offsets)
3759 .unwrap();
3760 assert_eq!(offsets, vec![0]);
3761 assert!(out.is_empty());
3762 }
3763
3764 #[test]
3765 fn encode_rows_matches_encode_output() {
3766 let schema = ArrowSchema::new(vec![
3767 Field::new("a", DataType::Int64, false),
3768 Field::new("b", DataType::Float64, false),
3769 ]);
3770 let int_arr = Int64Array::from(vec![100i64, 200, 300]);
3771 let float_arr = Float64Array::from(vec![1.5, 2.5, 3.5]);
3772 let batch = RecordBatch::try_new(
3773 Arc::new(schema.clone()),
3774 vec![Arc::new(int_arr), Arc::new(float_arr)],
3775 )
3776 .unwrap();
3777 let encoder = RecordEncoder {
3778 columns: vec![
3779 FieldBinding {
3780 arrow_index: 0,
3781 nullability: None,
3782 plan: FieldPlan::Scalar,
3783 },
3784 FieldBinding {
3785 arrow_index: 1,
3786 nullability: None,
3787 plan: FieldPlan::Scalar,
3788 },
3789 ],
3790 prefix: None,
3791 };
3792 let mut stream_buf = Vec::new();
3793 encoder.encode(&mut stream_buf, &batch).unwrap();
3794 let mut out = BytesMut::new();
3795 let mut offsets: Vec<usize> = vec![0];
3796 encoder
3797 .encode_rows(&batch, 32, &mut out, &mut offsets)
3798 .unwrap();
3799 assert_eq!(offsets.len(), 1 + batch.num_rows());
3800 assert_bytes_eq(&out[..], &stream_buf);
3801 }
3802
3803 #[test]
3804 fn encode_rows_appends_to_existing_buffer() {
3805 let schema = ArrowSchema::new(vec![Field::new("x", DataType::Int32, false)]);
3806 let arr = Int32Array::from(vec![5, 6]);
3807 let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(arr)]).unwrap();
3808 let encoder = RecordEncoder {
3809 columns: vec![FieldBinding {
3810 arrow_index: 0,
3811 nullability: None,
3812 plan: FieldPlan::Scalar,
3813 }],
3814 prefix: None,
3815 };
3816 let mut out = BytesMut::new();
3817 out.extend_from_slice(&[0xAA, 0xBB]);
3818 let mut offsets: Vec<usize> = vec![0, out.len()];
3819 encoder
3820 .encode_rows(&batch, 16, &mut out, &mut offsets)
3821 .unwrap();
3822 assert_eq!(offsets.len(), 4);
3823 assert_eq!(*offsets.last().unwrap(), out.len());
3824 assert_bytes_eq(row_slice(&out, &offsets, 0), &[0xAA, 0xBB]);
3825 assert_bytes_eq(row_slice(&out, &offsets, 1), &avro_long_bytes(5));
3826 assert_bytes_eq(row_slice(&out, &offsets, 2), &avro_long_bytes(6));
3827 }
3828
3829 #[test]
3830 fn encode_rows_nullable_column() {
3831 let schema = ArrowSchema::new(vec![Field::new("x", DataType::Int32, true)]);
3832 let arr = Int32Array::from(vec![Some(1), None, Some(3)]);
3833 let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(arr)]).unwrap();
3834 let encoder = RecordEncoder {
3835 columns: vec![FieldBinding {
3836 arrow_index: 0,
3837 nullability: Some(Nullability::NullFirst),
3838 plan: FieldPlan::Scalar,
3839 }],
3840 prefix: None,
3841 };
3842 let mut out = BytesMut::new();
3843 let mut offsets: Vec<usize> = vec![0];
3844 encoder
3845 .encode_rows(&batch, 16, &mut out, &mut offsets)
3846 .unwrap();
3847 assert_eq!(offsets.len(), 4);
3848 let mut expected_row0 = Vec::new();
3849 expected_row0.extend(avro_long_bytes(1)); expected_row0.extend(avro_long_bytes(1)); assert_bytes_eq(row_slice(&out, &offsets, 0), &expected_row0);
3852 let expected_row1 = avro_long_bytes(0); assert_bytes_eq(row_slice(&out, &offsets, 1), &expected_row1);
3854 let mut expected_row2 = Vec::new();
3855 expected_row2.extend(avro_long_bytes(1)); expected_row2.extend(avro_long_bytes(3)); assert_bytes_eq(row_slice(&out, &offsets, 2), &expected_row2);
3858 }
3859
3860 #[test]
3861 fn encode_prefix_write_error() {
3862 use crate::codec::AvroFieldBuilder;
3863 use crate::schema::{AvroSchema, FingerprintAlgorithm};
3864 use std::io;
3865
3866 struct FailWriter {
3867 failed: bool,
3868 }
3869
3870 impl io::Write for FailWriter {
3871 fn write(&mut self, _buf: &[u8]) -> io::Result<usize> {
3872 if !self.failed {
3873 self.failed = true;
3874 Err(io::Error::other("fail write"))
3875 } else {
3876 Ok(0)
3877 }
3878 }
3879
3880 fn flush(&mut self) -> io::Result<()> {
3881 Ok(())
3882 }
3883 }
3884
3885 let schema = ArrowSchema::new(vec![Field::new("x", DataType::Int32, false)]);
3886 let arr = Int32Array::from(vec![42]);
3887 let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(arr)]).unwrap();
3888 let avro_schema = AvroSchema::try_from(&schema).unwrap();
3889 let fingerprint = avro_schema
3890 .fingerprint(FingerprintAlgorithm::Rabin)
3891 .unwrap();
3892 let avro_root = AvroFieldBuilder::new(&avro_schema.schema().unwrap())
3893 .build()
3894 .unwrap();
3895 let encoder = RecordEncoderBuilder::new(&avro_root, &schema)
3896 .with_fingerprint(Some(fingerprint))
3897 .build()
3898 .unwrap();
3899
3900 let mut writer = FailWriter { failed: false };
3901 let err = encoder.encode(&mut writer, &batch).unwrap_err();
3902 let msg = format!("{err}");
3903 assert!(msg.contains("write prefix"), "unexpected error: {msg}");
3904 }
3905}