1use crate::codec::{
8 AvroDataType, AvroLiteral, Codec, EnumMapping, Promotion, ResolutionInfo, ResolvedField,
9 ResolvedRecord, ResolvedUnion, Tz,
10};
11use crate::errors::AvroError;
12use crate::reader::cursor::AvroCursor;
13use crate::schema::Nullability;
14#[cfg(feature = "small_decimals")]
15use arrow_array::builder::{Decimal32Builder, Decimal64Builder};
16use arrow_array::builder::{Decimal128Builder, Decimal256Builder, IntervalMonthDayNanoBuilder};
17use arrow_array::types::*;
18use arrow_array::*;
19use arrow_buffer::*;
20#[cfg(feature = "small_decimals")]
21use arrow_schema::{DECIMAL32_MAX_PRECISION, DECIMAL64_MAX_PRECISION};
22use arrow_schema::{
23 DECIMAL128_MAX_PRECISION, DECIMAL256_MAX_PRECISION, DataType, Field as ArrowField, FieldRef,
24 Fields, Schema as ArrowSchema, SchemaRef, UnionFields, UnionMode,
25};
26#[cfg(feature = "avro_custom_types")]
27use arrow_select::take::{TakeOptions, take};
28use strum_macros::AsRefStr;
29use uuid::Uuid;
30
31use std::cmp::Ordering;
32use std::mem;
33use std::sync::Arc;
34
35const DEFAULT_CAPACITY: usize = 1024;
36
37macro_rules! decode_decimal {
39 ($size:expr, $buf:expr, $builder:expr, $N:expr, $Int:ty) => {{
40 let bytes = read_decimal_bytes_be::<{ $N }>($buf, *$size)?;
41 $builder.append_value(<$Int>::from_be_bytes(bytes));
42 }};
43}
44
45macro_rules! flush_decimal {
47 ($builder:expr, $precision:expr, $scale:expr, $nulls:expr, $ArrayTy:ty) => {{
48 let (_, vals, _) = $builder.finish().into_parts();
49 let dec = <$ArrayTy>::try_new(vals, $nulls)?
50 .with_precision_and_scale(*$precision as u8, $scale.unwrap_or(0) as i8)?;
51 Arc::new(dec) as ArrayRef
52 }};
53}
54
55macro_rules! append_decimal_default {
58 ($lit:expr, $builder:expr, $N:literal, $Int:ty, $name:literal) => {{
59 match $lit {
60 AvroLiteral::Bytes(b) => {
61 let ext = sign_cast_to::<$N>(b)?;
62 let val = <$Int>::from_be_bytes(ext);
63 $builder.append_value(val);
64 Ok(())
65 }
66 _ => Err(AvroError::InvalidArgument(
67 concat!(
68 "Default for ",
69 $name,
70 " must be bytes (two's-complement big-endian)"
71 )
72 .to_string(),
73 )),
74 }
75 }};
76}
77
78#[derive(Debug)]
80pub(crate) struct RecordDecoder {
81 schema: SchemaRef,
82 fields: Vec<Decoder>,
83 projector: Option<Projector>,
84 row_count: usize,
85}
86
87impl RecordDecoder {
88 pub(crate) fn try_new_with_options(data_type: &AvroDataType) -> Result<Self, AvroError> {
99 match data_type.codec() {
100 Codec::Struct(reader_fields) => {
101 let mut arrow_fields = Vec::with_capacity(reader_fields.len());
103 let mut encodings = Vec::with_capacity(reader_fields.len());
104 let mut field_defaults = Vec::with_capacity(reader_fields.len());
105 for avro_field in reader_fields.iter() {
106 arrow_fields.push(avro_field.field());
107 encodings.push(Decoder::try_new(avro_field.data_type())?);
108
109 if let Some(ResolutionInfo::DefaultValue(lit)) =
110 avro_field.data_type().resolution.as_ref()
111 {
112 field_defaults.push(Some(lit.clone()));
113 } else {
114 field_defaults.push(None);
115 }
116 }
117 let projector = match data_type.resolution.as_ref() {
118 Some(ResolutionInfo::Record(rec)) => {
119 Some(ProjectorBuilder::try_new(rec, &field_defaults).build()?)
120 }
121 _ => None,
122 };
123 Ok(Self {
124 schema: Arc::new(ArrowSchema::new(arrow_fields)),
125 fields: encodings,
126 projector,
127 row_count: 0,
128 })
129 }
130 other => Err(AvroError::ParseError(format!(
131 "Expected record got {other:?}"
132 ))),
133 }
134 }
135
136 pub(crate) fn schema(&self) -> &SchemaRef {
138 &self.schema
139 }
140
141 pub(crate) fn decode(&mut self, buf: &[u8], count: usize) -> Result<usize, AvroError> {
143 let mut cursor = AvroCursor::new(buf);
144 match self.projector.as_mut() {
145 Some(proj) => {
146 for _ in 0..count {
147 proj.project_record(&mut cursor, &mut self.fields)?;
148 }
149 }
150 None => {
151 for _ in 0..count {
152 for field in &mut self.fields {
153 field.decode(&mut cursor)?;
154 }
155 }
156 }
157 }
158 self.row_count += count;
159 Ok(cursor.position())
160 }
161
162 pub(crate) fn flush(&mut self) -> Result<RecordBatch, AvroError> {
164 let arrays = self
165 .fields
166 .iter_mut()
167 .map(|x| x.flush(None))
168 .collect::<Result<Vec<_>, _>>()?;
169 let batch_options = RecordBatchOptions::new().with_row_count(Some(self.row_count));
170 self.row_count = 0;
171 RecordBatch::try_new_with_options(self.schema.clone(), arrays, &batch_options)
172 .map_err(Into::into)
173 }
174}
175
176#[derive(Debug, AsRefStr)]
177enum Decoder {
178 Null(usize),
179 Boolean(BooleanBufferBuilder),
180 Int32(Vec<i32>),
181 Int64(Vec<i64>),
182 #[cfg(feature = "avro_custom_types")]
183 DurationSecond(Vec<i64>),
184 #[cfg(feature = "avro_custom_types")]
185 DurationMillisecond(Vec<i64>),
186 #[cfg(feature = "avro_custom_types")]
187 DurationMicrosecond(Vec<i64>),
188 #[cfg(feature = "avro_custom_types")]
189 DurationNanosecond(Vec<i64>),
190 #[cfg(feature = "avro_custom_types")]
191 Int8(Vec<i8>),
192 #[cfg(feature = "avro_custom_types")]
193 Int16(Vec<i16>),
194 #[cfg(feature = "avro_custom_types")]
195 UInt8(Vec<u8>),
196 #[cfg(feature = "avro_custom_types")]
197 UInt16(Vec<u16>),
198 #[cfg(feature = "avro_custom_types")]
199 UInt32(Vec<u32>),
200 #[cfg(feature = "avro_custom_types")]
201 UInt64(Vec<u64>),
202 #[cfg(feature = "avro_custom_types")]
203 Float16(Vec<u16>), #[cfg(feature = "avro_custom_types")]
205 Date64(Vec<i64>),
206 #[cfg(feature = "avro_custom_types")]
207 TimeNanos(Vec<i64>),
208 #[cfg(feature = "avro_custom_types")]
209 Time32Secs(Vec<i32>),
210 #[cfg(feature = "avro_custom_types")]
211 TimestampSecs(bool, Vec<i64>),
212 #[cfg(feature = "avro_custom_types")]
213 IntervalYearMonth(Vec<i32>),
214 #[cfg(feature = "avro_custom_types")]
215 IntervalMonthDayNano(Vec<IntervalMonthDayNano>),
216 #[cfg(feature = "avro_custom_types")]
217 IntervalDayTime(Vec<IntervalDayTime>),
218 Float32(Vec<f32>),
219 Float64(Vec<f64>),
220 Date32(Vec<i32>),
221 TimeMillis(Vec<i32>),
222 TimeMicros(Vec<i64>),
223 TimestampMillis(Option<Tz>, Vec<i64>),
224 TimestampMicros(Option<Tz>, Vec<i64>),
225 TimestampNanos(Option<Tz>, Vec<i64>),
226 Int32ToInt64(Vec<i64>),
227 Int32ToFloat32(Vec<f32>),
228 Int32ToFloat64(Vec<f64>),
229 Int64ToFloat32(Vec<f32>),
230 Int64ToFloat64(Vec<f64>),
231 Float32ToFloat64(Vec<f64>),
232 BytesToString(OffsetBufferBuilder<i32>, Vec<u8>),
233 StringToBytes(OffsetBufferBuilder<i32>, Vec<u8>),
234 Binary(OffsetBufferBuilder<i32>, Vec<u8>),
235 String(OffsetBufferBuilder<i32>, Vec<u8>),
237 StringView(OffsetBufferBuilder<i32>, Vec<u8>),
239 Array(FieldRef, OffsetBufferBuilder<i32>, Box<Decoder>),
240 Record {
241 fields: Fields,
242 decoders: Vec<Decoder>,
243 defaults: Vec<Option<AvroLiteral>>,
244 projector: Option<Projector>,
245 row_count: usize,
249 },
250 Map(
251 FieldRef,
252 OffsetBufferBuilder<i32>,
253 OffsetBufferBuilder<i32>,
254 Vec<u8>,
255 Box<Decoder>,
256 ),
257 Fixed(i32, Vec<u8>),
258 Enum(Vec<i32>, Arc<[String]>, Option<EnumResolution>),
259 Duration(IntervalMonthDayNanoBuilder),
260 Uuid(Vec<u8>),
261 #[cfg(feature = "small_decimals")]
262 Decimal32(usize, Option<usize>, Option<usize>, Decimal32Builder),
263 #[cfg(feature = "small_decimals")]
264 Decimal64(usize, Option<usize>, Option<usize>, Decimal64Builder),
265 Decimal128(usize, Option<usize>, Option<usize>, Decimal128Builder),
266 Decimal256(usize, Option<usize>, Option<usize>, Decimal256Builder),
267 #[cfg(feature = "avro_custom_types")]
268 RunEndEncoded(u8, usize, Box<Decoder>),
269 Union(UnionDecoder),
270 Nullable(NullableDecoder),
272}
273
274#[derive(Debug)]
275struct NullableDecoder {
276 plan: NullablePlan,
277 validity: NullBufferBuilder,
278 values: Box<Decoder>,
279 pending_nulls: usize,
280}
281
282impl NullableDecoder {
283 fn new(plan: NullablePlan, values: Decoder) -> Self {
284 Self {
285 plan,
286 validity: NullBufferBuilder::new(DEFAULT_CAPACITY),
287 values: Box::new(values),
288 pending_nulls: 0,
289 }
290 }
291
292 #[inline]
293 fn materialize_pending(&mut self) -> Result<(), AvroError> {
294 if self.pending_nulls == 0 {
295 return Ok(());
296 }
297
298 self.values.append_nulls(self.pending_nulls)?;
299 self.validity.append_n_nulls(self.pending_nulls);
300 self.pending_nulls = 0;
301 Ok(())
302 }
303}
304
305impl Decoder {
306 fn try_new(data_type: &AvroDataType) -> Result<Self, AvroError> {
307 if let Some(ResolutionInfo::Union(info)) = data_type.resolution.as_ref()
308 && info.writer_is_union
309 && !info.reader_is_union
310 {
311 let mut clone = data_type.clone();
312 clone.resolution = None; let target = Self::try_new_internal(&clone)?;
314 let decoder = Self::Union(
315 UnionDecoderBuilder::new()
316 .with_resolved_union(info.clone())
317 .with_target(target)
318 .build()?,
319 );
320 return Ok(decoder);
321 }
322 Self::try_new_internal(data_type)
323 }
324
325 fn try_new_internal(data_type: &AvroDataType) -> Result<Self, AvroError> {
326 let promotion = match data_type.resolution.as_ref() {
328 Some(ResolutionInfo::Promotion(p)) => Some(*p),
329 _ => None,
330 };
331 let decoder = match (data_type.codec(), promotion) {
332 (Codec::Int64, Some(Promotion::IntToLong)) => {
333 Self::Int32ToInt64(Vec::with_capacity(DEFAULT_CAPACITY))
334 }
335 (Codec::Float32, Some(Promotion::IntToFloat)) => {
336 Self::Int32ToFloat32(Vec::with_capacity(DEFAULT_CAPACITY))
337 }
338 (Codec::Float64, Some(Promotion::IntToDouble)) => {
339 Self::Int32ToFloat64(Vec::with_capacity(DEFAULT_CAPACITY))
340 }
341 (Codec::Float32, Some(Promotion::LongToFloat)) => {
342 Self::Int64ToFloat32(Vec::with_capacity(DEFAULT_CAPACITY))
343 }
344 (Codec::Float64, Some(Promotion::LongToDouble)) => {
345 Self::Int64ToFloat64(Vec::with_capacity(DEFAULT_CAPACITY))
346 }
347 (Codec::Float64, Some(Promotion::FloatToDouble)) => {
348 Self::Float32ToFloat64(Vec::with_capacity(DEFAULT_CAPACITY))
349 }
350 (Codec::Utf8 | Codec::Utf8View, Some(Promotion::BytesToString)) => Self::BytesToString(
351 OffsetBufferBuilder::new(DEFAULT_CAPACITY),
352 Vec::with_capacity(DEFAULT_CAPACITY),
353 ),
354 (Codec::Binary, Some(Promotion::StringToBytes)) => Self::StringToBytes(
355 OffsetBufferBuilder::new(DEFAULT_CAPACITY),
356 Vec::with_capacity(DEFAULT_CAPACITY),
357 ),
358 (Codec::Null, _) => Self::Null(0),
359 (Codec::Boolean, _) => Self::Boolean(BooleanBufferBuilder::new(DEFAULT_CAPACITY)),
360 (Codec::Int32, _) => Self::Int32(Vec::with_capacity(DEFAULT_CAPACITY)),
361 (Codec::Int64, _) => Self::Int64(Vec::with_capacity(DEFAULT_CAPACITY)),
362 (Codec::Float32, _) => Self::Float32(Vec::with_capacity(DEFAULT_CAPACITY)),
363 (Codec::Float64, _) => Self::Float64(Vec::with_capacity(DEFAULT_CAPACITY)),
364 (Codec::Binary, _) => Self::Binary(
365 OffsetBufferBuilder::new(DEFAULT_CAPACITY),
366 Vec::with_capacity(DEFAULT_CAPACITY),
367 ),
368 (Codec::Utf8, _) => Self::String(
369 OffsetBufferBuilder::new(DEFAULT_CAPACITY),
370 Vec::with_capacity(DEFAULT_CAPACITY),
371 ),
372 (Codec::Utf8View, _) => Self::StringView(
373 OffsetBufferBuilder::new(DEFAULT_CAPACITY),
374 Vec::with_capacity(DEFAULT_CAPACITY),
375 ),
376 (Codec::Date32, _) => Self::Date32(Vec::with_capacity(DEFAULT_CAPACITY)),
377 (Codec::TimeMillis, _) => Self::TimeMillis(Vec::with_capacity(DEFAULT_CAPACITY)),
378 (Codec::TimeMicros, _) => Self::TimeMicros(Vec::with_capacity(DEFAULT_CAPACITY)),
379 (Codec::TimestampMillis(tz), _) => {
380 Self::TimestampMillis(*tz, Vec::with_capacity(DEFAULT_CAPACITY))
381 }
382 (Codec::TimestampMicros(tz), _) => {
383 Self::TimestampMicros(*tz, Vec::with_capacity(DEFAULT_CAPACITY))
384 }
385 (Codec::TimestampNanos(tz), _) => {
386 Self::TimestampNanos(*tz, Vec::with_capacity(DEFAULT_CAPACITY))
387 }
388 #[cfg(feature = "avro_custom_types")]
389 (Codec::DurationNanos, _) => {
390 Self::DurationNanosecond(Vec::with_capacity(DEFAULT_CAPACITY))
391 }
392 #[cfg(feature = "avro_custom_types")]
393 (Codec::DurationMicros, _) => {
394 Self::DurationMicrosecond(Vec::with_capacity(DEFAULT_CAPACITY))
395 }
396 #[cfg(feature = "avro_custom_types")]
397 (Codec::DurationMillis, _) => {
398 Self::DurationMillisecond(Vec::with_capacity(DEFAULT_CAPACITY))
399 }
400 #[cfg(feature = "avro_custom_types")]
401 (Codec::DurationSeconds, _) => {
402 Self::DurationSecond(Vec::with_capacity(DEFAULT_CAPACITY))
403 }
404 #[cfg(feature = "avro_custom_types")]
405 (Codec::Int8, _) => Self::Int8(Vec::with_capacity(DEFAULT_CAPACITY)),
406 #[cfg(feature = "avro_custom_types")]
407 (Codec::Int16, _) => Self::Int16(Vec::with_capacity(DEFAULT_CAPACITY)),
408 #[cfg(feature = "avro_custom_types")]
409 (Codec::UInt8, _) => Self::UInt8(Vec::with_capacity(DEFAULT_CAPACITY)),
410 #[cfg(feature = "avro_custom_types")]
411 (Codec::UInt16, _) => Self::UInt16(Vec::with_capacity(DEFAULT_CAPACITY)),
412 #[cfg(feature = "avro_custom_types")]
413 (Codec::UInt32, _) => Self::UInt32(Vec::with_capacity(DEFAULT_CAPACITY)),
414 #[cfg(feature = "avro_custom_types")]
415 (Codec::UInt64, _) => Self::UInt64(Vec::with_capacity(DEFAULT_CAPACITY)),
416 #[cfg(feature = "avro_custom_types")]
417 (Codec::Float16, _) => Self::Float16(Vec::with_capacity(DEFAULT_CAPACITY)),
418 #[cfg(feature = "avro_custom_types")]
419 (Codec::Date64, _) => Self::Date64(Vec::with_capacity(DEFAULT_CAPACITY)),
420 #[cfg(feature = "avro_custom_types")]
421 (Codec::TimeNanos, _) => Self::TimeNanos(Vec::with_capacity(DEFAULT_CAPACITY)),
422 #[cfg(feature = "avro_custom_types")]
423 (Codec::Time32Secs, _) => Self::Time32Secs(Vec::with_capacity(DEFAULT_CAPACITY)),
424 #[cfg(feature = "avro_custom_types")]
425 (Codec::TimestampSecs(is_utc), _) => {
426 Self::TimestampSecs(*is_utc, Vec::with_capacity(DEFAULT_CAPACITY))
427 }
428 #[cfg(feature = "avro_custom_types")]
429 (Codec::IntervalYearMonth, _) => {
430 Self::IntervalYearMonth(Vec::with_capacity(DEFAULT_CAPACITY))
431 }
432 #[cfg(feature = "avro_custom_types")]
433 (Codec::IntervalMonthDayNano, _) => {
434 Self::IntervalMonthDayNano(Vec::with_capacity(DEFAULT_CAPACITY))
435 }
436 #[cfg(feature = "avro_custom_types")]
437 (Codec::IntervalDayTime, _) => {
438 Self::IntervalDayTime(Vec::with_capacity(DEFAULT_CAPACITY))
439 }
440 (Codec::Fixed(sz), _) => Self::Fixed(*sz, Vec::with_capacity(DEFAULT_CAPACITY)),
441 (Codec::Decimal(precision, scale, size), _) => {
442 let p = *precision;
443 let s = *scale;
444 let prec = p as u8;
445 let scl = s.unwrap_or(0) as i8;
446 #[cfg(feature = "small_decimals")]
447 {
448 if p <= DECIMAL32_MAX_PRECISION as usize {
449 let builder = Decimal32Builder::with_capacity(DEFAULT_CAPACITY)
450 .with_precision_and_scale(prec, scl)?;
451 Self::Decimal32(p, s, *size, builder)
452 } else if p <= DECIMAL64_MAX_PRECISION as usize {
453 let builder = Decimal64Builder::with_capacity(DEFAULT_CAPACITY)
454 .with_precision_and_scale(prec, scl)?;
455 Self::Decimal64(p, s, *size, builder)
456 } else if p <= DECIMAL128_MAX_PRECISION as usize {
457 let builder = Decimal128Builder::with_capacity(DEFAULT_CAPACITY)
458 .with_precision_and_scale(prec, scl)?;
459 Self::Decimal128(p, s, *size, builder)
460 } else if p <= DECIMAL256_MAX_PRECISION as usize {
461 let builder = Decimal256Builder::with_capacity(DEFAULT_CAPACITY)
462 .with_precision_and_scale(prec, scl)?;
463 Self::Decimal256(p, s, *size, builder)
464 } else {
465 return Err(AvroError::ParseError(format!(
466 "Decimal precision {p} exceeds maximum supported"
467 )));
468 }
469 }
470 #[cfg(not(feature = "small_decimals"))]
471 {
472 if p <= DECIMAL128_MAX_PRECISION as usize {
473 let builder = Decimal128Builder::with_capacity(DEFAULT_CAPACITY)
474 .with_precision_and_scale(prec, scl)?;
475 Self::Decimal128(p, s, *size, builder)
476 } else if p <= DECIMAL256_MAX_PRECISION as usize {
477 let builder = Decimal256Builder::with_capacity(DEFAULT_CAPACITY)
478 .with_precision_and_scale(prec, scl)?;
479 Self::Decimal256(p, s, *size, builder)
480 } else {
481 return Err(AvroError::ParseError(format!(
482 "Decimal precision {p} exceeds maximum supported"
483 )));
484 }
485 }
486 }
487 (Codec::Interval, _) => Self::Duration(IntervalMonthDayNanoBuilder::new()),
488 (Codec::List(item), _) => {
489 let decoder = Self::try_new(item)?;
490 Self::Array(
491 Arc::new(item.field_with_name("item")),
492 OffsetBufferBuilder::new(DEFAULT_CAPACITY),
493 Box::new(decoder),
494 )
495 }
496 (Codec::Enum(symbols), _) => {
497 let res = match data_type.resolution.as_ref() {
498 Some(ResolutionInfo::EnumMapping(mapping)) => {
499 Some(EnumResolution::new(mapping))
500 }
501 _ => None,
502 };
503 Self::Enum(Vec::with_capacity(DEFAULT_CAPACITY), symbols.clone(), res)
504 }
505 (Codec::Struct(fields), _) => {
506 let mut arrow_fields = Vec::with_capacity(fields.len());
507 let mut encodings = Vec::with_capacity(fields.len());
508 let mut field_defaults = Vec::with_capacity(fields.len());
509 for avro_field in fields.iter() {
510 let encoding = Self::try_new(avro_field.data_type())?;
511 arrow_fields.push(avro_field.field());
512 encodings.push(encoding);
513
514 if let Some(ResolutionInfo::DefaultValue(lit)) =
515 avro_field.data_type().resolution.as_ref()
516 {
517 field_defaults.push(Some(lit.clone()));
518 } else {
519 field_defaults.push(None);
520 }
521 }
522 let projector =
523 if let Some(ResolutionInfo::Record(rec)) = data_type.resolution.as_ref() {
524 Some(ProjectorBuilder::try_new(rec, &field_defaults).build()?)
525 } else {
526 None
527 };
528 Self::Record {
529 fields: arrow_fields.into(),
530 decoders: encodings,
531 defaults: field_defaults,
532 projector,
533 row_count: 0,
534 }
535 }
536 (Codec::Map(child), _) => {
537 let val_field = child.field_with_name(ArrowField::MAP_VALUE_FIELD_DEFAULT_NAME);
538 let map_field = Arc::new(ArrowField::new(
539 ArrowField::MAP_ENTRIES_FIELD_DEFAULT_NAME,
540 DataType::Struct(Fields::from(vec![
541 ArrowField::new(
542 ArrowField::MAP_KEY_FIELD_DEFAULT_NAME,
543 DataType::Utf8,
544 false,
545 ),
546 val_field,
547 ])),
548 false,
549 ));
550 let val_dec = Self::try_new(child)?;
551 Self::Map(
552 map_field,
553 OffsetBufferBuilder::new(DEFAULT_CAPACITY),
554 OffsetBufferBuilder::new(DEFAULT_CAPACITY),
555 Vec::with_capacity(DEFAULT_CAPACITY),
556 Box::new(val_dec),
557 )
558 }
559 (Codec::Uuid, _) => Self::Uuid(Vec::with_capacity(DEFAULT_CAPACITY)),
560 (Codec::Union(encodings, fields, UnionMode::Dense), _) => {
561 let decoders = encodings
562 .iter()
563 .map(Self::try_new_internal)
564 .collect::<Result<Vec<_>, _>>()?;
565 if fields.len() != decoders.len() {
566 return Err(AvroError::SchemaError(format!(
567 "Union has {} fields but {} decoders",
568 fields.len(),
569 decoders.len()
570 )));
571 }
572 let branch_count = decoders.len();
575 let max_addr = (i32::MAX as usize) + 1;
576 if branch_count > max_addr {
577 return Err(AvroError::SchemaError(format!(
578 "Union has {branch_count} branches, which exceeds the maximum addressable \
579 branches by an Avro int tag ({} + 1).",
580 i32::MAX
581 )));
582 }
583 let mut builder = UnionDecoderBuilder::new()
584 .with_fields(fields.clone())
585 .with_branches(decoders);
586 if let Some(ResolutionInfo::Union(info)) = data_type.resolution.as_ref()
587 && info.reader_is_union
588 {
589 builder = builder.with_resolved_union(info.clone());
590 }
591 Self::Union(builder.build()?)
592 }
593 (Codec::Union(_, _, _), _) => {
594 return Err(AvroError::NYI(
595 "Sparse Arrow unions are not yet supported".to_string(),
596 ));
597 }
598 #[cfg(feature = "avro_custom_types")]
599 (Codec::RunEndEncoded(values_dt, width_bits_or_bytes), _) => {
600 let inner = Self::try_new(values_dt)?;
601 let byte_width: u8 = match *width_bits_or_bytes {
602 2 | 4 | 8 => *width_bits_or_bytes,
603 16 => 2,
604 32 => 4,
605 64 => 8,
606 other => {
607 return Err(AvroError::InvalidArgument(format!(
608 "Unsupported run-end width {other} for RunEndEncoded; \
609 expected 16/32/64 bits or 2/4/8 bytes"
610 )));
611 }
612 };
613 Self::RunEndEncoded(byte_width, 0, Box::new(inner))
614 }
615 };
616 Ok(match data_type.nullability() {
617 Some(nullability) => {
618 let plan = match &data_type.resolution {
620 None => NullablePlan::ReadTag {
621 nullability,
622 resolution: ResolutionPlan::Promotion(Promotion::Direct),
623 },
624 Some(ResolutionInfo::Promotion(_)) => {
625 NullablePlan::FromSingle {
628 resolution: ResolutionPlan::Promotion(Promotion::Direct),
629 }
630 }
631 Some(ResolutionInfo::Union(info)) if !info.writer_is_union => {
632 let Some(Some((_, resolution))) = info.writer_to_reader.first() else {
633 return Err(AvroError::SchemaError(
634 "unexpected union resolution info for non-union writer and union reader type".into(),
635 ));
636 };
637 let resolution = ResolutionPlan::try_new(&decoder, resolution)?;
638 NullablePlan::FromSingle { resolution }
639 }
640 Some(ResolutionInfo::Union(info)) => {
641 let Some((_, resolution)) =
642 info.writer_to_reader[nullability.non_null_index()].as_ref()
643 else {
644 return Err(AvroError::SchemaError(
645 "unexpected union resolution info for nullable writer type".into(),
646 ));
647 };
648 NullablePlan::ReadTag {
649 nullability,
650 resolution: ResolutionPlan::try_new(&decoder, resolution)?,
651 }
652 }
653 Some(resolution) => NullablePlan::FromSingle {
654 resolution: ResolutionPlan::try_new(&decoder, resolution)?,
655 },
656 };
657 Self::Nullable(NullableDecoder::new(plan, decoder))
658 }
659 None => decoder,
660 })
661 }
662
663 fn append_null(&mut self) -> Result<(), AvroError> {
665 self.append_nulls(1)
666 }
667
668 fn append_nulls(&mut self, count: usize) -> Result<(), AvroError> {
671 if count == 0 {
672 return Ok(());
673 }
674 match self {
675 Self::Null(size) => *size += count,
676 Self::Boolean(values) => values.append_n(count, false),
677 Self::Int32(values) | Self::Date32(values) | Self::TimeMillis(values) => {
678 values.resize(values.len() + count, 0)
679 }
680 Self::Int64(values)
681 | Self::Int32ToInt64(values)
682 | Self::TimeMicros(values)
683 | Self::TimestampMillis(_, values)
684 | Self::TimestampMicros(_, values)
685 | Self::TimestampNanos(_, values) => values.resize(values.len() + count, 0),
686 #[cfg(feature = "avro_custom_types")]
687 Self::DurationSecond(values)
688 | Self::DurationMillisecond(values)
689 | Self::DurationMicrosecond(values)
690 | Self::DurationNanosecond(values)
691 | Self::Date64(values)
692 | Self::TimeNanos(values)
693 | Self::TimestampSecs(_, values) => values.resize(values.len() + count, 0),
694 #[cfg(feature = "avro_custom_types")]
695 Self::Int8(values) => values.resize(values.len() + count, 0),
696 #[cfg(feature = "avro_custom_types")]
697 Self::Int16(values) => values.resize(values.len() + count, 0),
698 #[cfg(feature = "avro_custom_types")]
699 Self::UInt8(values) => values.resize(values.len() + count, 0),
700 #[cfg(feature = "avro_custom_types")]
701 Self::UInt16(values) | Self::Float16(values) => values.resize(values.len() + count, 0),
702 #[cfg(feature = "avro_custom_types")]
703 Self::UInt32(values) => values.resize(values.len() + count, 0),
704 #[cfg(feature = "avro_custom_types")]
705 Self::UInt64(values) => values.resize(values.len() + count, 0),
706 #[cfg(feature = "avro_custom_types")]
707 Self::Time32Secs(values) | Self::IntervalYearMonth(values) => {
708 values.resize(values.len() + count, 0)
709 }
710 #[cfg(feature = "avro_custom_types")]
711 Self::IntervalDayTime(values) => {
712 values.resize(values.len() + count, IntervalDayTime::new(0, 0))
713 }
714 #[cfg(feature = "avro_custom_types")]
715 Self::IntervalMonthDayNano(values) => {
716 values.resize(values.len() + count, IntervalMonthDayNano::new(0, 0, 0))
717 }
718 Self::Float32(values) | Self::Int32ToFloat32(values) | Self::Int64ToFloat32(values) => {
719 values.resize(values.len() + count, 0.0)
720 }
721 Self::Float64(values)
722 | Self::Int32ToFloat64(values)
723 | Self::Int64ToFloat64(values)
724 | Self::Float32ToFloat64(values) => values.resize(values.len() + count, 0.0),
725 Self::Binary(offsets, _)
726 | Self::String(offsets, _)
727 | Self::StringView(offsets, _)
728 | Self::BytesToString(offsets, _)
729 | Self::StringToBytes(offsets, _)
730 | Self::Array(_, offsets, _)
731 | Self::Map(_, _, offsets, _, _) => {
732 offsets.reserve(count);
733 for _ in 0..count {
734 offsets.push_length(0);
735 }
736 }
737 Self::Record {
738 decoders,
739 row_count,
740 ..
741 } => {
742 for child in decoders.iter_mut() {
743 child.append_nulls(count)?;
744 }
745 *row_count += count;
746 }
747 Self::Fixed(width, values) => {
748 values.resize(values.len() + (*width as usize) * count, 0)
749 }
750 Self::Enum(values, _, _) => values.resize(values.len() + count, 0),
751 Self::Duration(builder) => builder.append_nulls(count),
752 Self::Uuid(values) => values.resize(values.len() + 16 * count, 0),
753 #[cfg(feature = "small_decimals")]
754 Self::Decimal32(_, _, _, builder) => builder.append_value_n(0, count),
755 #[cfg(feature = "small_decimals")]
756 Self::Decimal64(_, _, _, builder) => builder.append_value_n(0, count),
757 Self::Decimal128(_, _, _, builder) => builder.append_value_n(0, count),
758 Self::Decimal256(_, _, _, builder) => builder.append_value_n(i256::ZERO, count),
759 #[cfg(feature = "avro_custom_types")]
760 Self::RunEndEncoded(_, len, inner) => {
761 inner.append_nulls(count)?;
762 *len += count;
763 }
764 Self::Union(union) => {
765 for _ in 0..count {
766 union.append_null()?;
767 }
768 }
769 Self::Nullable(nullable) => nullable.pending_nulls += count,
770 }
771 Ok(())
772 }
773
774 fn append_default(&mut self, lit: &AvroLiteral) -> Result<(), AvroError> {
776 match self {
777 Self::Nullable(nullable) => {
778 if matches!(lit, AvroLiteral::Null) {
779 nullable.pending_nulls += 1;
780 } else {
781 nullable.materialize_pending()?;
782 nullable.values.append_default(lit)?;
783 nullable.validity.append_non_null();
784 }
785 Ok(())
786 }
787 Self::Null(count) => match lit {
788 AvroLiteral::Null => {
789 *count += 1;
790 Ok(())
791 }
792 _ => Err(AvroError::InvalidArgument(
793 "Non-null default for null type".to_string(),
794 )),
795 },
796 Self::Boolean(b) => match lit {
797 AvroLiteral::Boolean(v) => {
798 b.append(*v);
799 Ok(())
800 }
801 _ => Err(AvroError::InvalidArgument(
802 "Default for boolean must be boolean".to_string(),
803 )),
804 },
805 Self::Int32(v) | Self::Date32(v) | Self::TimeMillis(v) => match lit {
806 AvroLiteral::Int(i) => {
807 v.push(*i);
808 Ok(())
809 }
810 _ => Err(AvroError::InvalidArgument(
811 "Default for int32/date32/time-millis must be int".to_string(),
812 )),
813 },
814 #[cfg(feature = "avro_custom_types")]
815 Self::DurationSecond(v)
816 | Self::DurationMillisecond(v)
817 | Self::DurationMicrosecond(v)
818 | Self::DurationNanosecond(v) => match lit {
819 AvroLiteral::Long(i) => {
820 v.push(*i);
821 Ok(())
822 }
823 _ => Err(AvroError::InvalidArgument(
824 "Default for duration long must be long".to_string(),
825 )),
826 },
827 #[cfg(feature = "avro_custom_types")]
828 Self::Int8(v) => match lit {
829 AvroLiteral::Int(i) => {
830 let x = i8::try_from(*i).map_err(|_| {
831 AvroError::InvalidArgument(format!(
832 "Default for int8 out of range for i8: {i}"
833 ))
834 })?;
835 v.push(x);
836 Ok(())
837 }
838 _ => Err(AvroError::InvalidArgument(
839 "Default for int8 must be int".to_string(),
840 )),
841 },
842 #[cfg(feature = "avro_custom_types")]
843 Self::Int16(v) => match lit {
844 AvroLiteral::Int(i) => {
845 let x = i16::try_from(*i).map_err(|_| {
846 AvroError::InvalidArgument(format!(
847 "Default for int16 out of range for i16: {i}"
848 ))
849 })?;
850 v.push(x);
851 Ok(())
852 }
853 _ => Err(AvroError::InvalidArgument(
854 "Default for int16 must be int".to_string(),
855 )),
856 },
857 #[cfg(feature = "avro_custom_types")]
858 Self::UInt8(v) => match lit {
859 AvroLiteral::Int(i) => {
860 let x = u8::try_from(*i).map_err(|_| {
861 AvroError::InvalidArgument(format!(
862 "Default for uint8 out of range for u8: {i}"
863 ))
864 })?;
865 v.push(x);
866 Ok(())
867 }
868 _ => Err(AvroError::InvalidArgument(
869 "Default for uint8 must be int".to_string(),
870 )),
871 },
872 #[cfg(feature = "avro_custom_types")]
873 Self::UInt16(v) => match lit {
874 AvroLiteral::Int(i) => {
875 let x = u16::try_from(*i).map_err(|_| {
876 AvroError::InvalidArgument(format!(
877 "Default for uint16 out of range for u16: {i}"
878 ))
879 })?;
880 v.push(x);
881 Ok(())
882 }
883 _ => Err(AvroError::InvalidArgument(
884 "Default for uint16 must be int".to_string(),
885 )),
886 },
887 #[cfg(feature = "avro_custom_types")]
888 Self::UInt32(v) => match lit {
889 AvroLiteral::Long(i) => {
890 let x = u32::try_from(*i).map_err(|_| {
891 AvroError::InvalidArgument(format!(
892 "Default for uint32 out of range for u32: {i}"
893 ))
894 })?;
895 v.push(x);
896 Ok(())
897 }
898 _ => Err(AvroError::InvalidArgument(
899 "Default for uint32 must be long".to_string(),
900 )),
901 },
902 #[cfg(feature = "avro_custom_types")]
903 Self::UInt64(v) => match lit {
904 AvroLiteral::Bytes(b) => {
905 if b.len() != 8 {
906 return Err(AvroError::InvalidArgument(format!(
907 "uint64 default must be exactly 8 bytes, got {}",
908 b.len()
909 )));
910 }
911 v.push(u64::from_le_bytes([
912 b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7],
913 ]));
914 Ok(())
915 }
916 _ => Err(AvroError::InvalidArgument(
917 "Default for uint64 must be bytes (8-byte LE)".to_string(),
918 )),
919 },
920 #[cfg(feature = "avro_custom_types")]
921 Self::Float16(v) => match lit {
922 AvroLiteral::Bytes(b) => {
923 if b.len() != 2 {
924 return Err(AvroError::InvalidArgument(format!(
925 "float16 default must be exactly 2 bytes, got {}",
926 b.len()
927 )));
928 }
929 v.push(u16::from_le_bytes([b[0], b[1]]));
930 Ok(())
931 }
932 _ => Err(AvroError::InvalidArgument(
933 "Default for float16 must be bytes (2-byte LE IEEE-754)".to_string(),
934 )),
935 },
936 #[cfg(feature = "avro_custom_types")]
937 Self::Date64(v) | Self::TimeNanos(v) | Self::TimestampSecs(_, v) => match lit {
938 AvroLiteral::Long(i) => {
939 v.push(*i);
940 Ok(())
941 }
942 _ => Err(AvroError::InvalidArgument(
943 "Default for date64/time-nanos/timestamp-secs must be long".to_string(),
944 )),
945 },
946 #[cfg(feature = "avro_custom_types")]
947 Self::Time32Secs(v) => match lit {
948 AvroLiteral::Int(i) => {
949 v.push(*i);
950 Ok(())
951 }
952 _ => Err(AvroError::InvalidArgument(
953 "Default for time32-secs must be int".to_string(),
954 )),
955 },
956 #[cfg(feature = "avro_custom_types")]
957 Self::IntervalYearMonth(v) => match lit {
958 AvroLiteral::Bytes(b) => {
959 if b.len() != 4 {
960 return Err(AvroError::InvalidArgument(format!(
961 "interval-year-month default must be exactly 4 bytes, got {}",
962 b.len()
963 )));
964 }
965 v.push(i32::from_le_bytes([b[0], b[1], b[2], b[3]]));
966 Ok(())
967 }
968 _ => Err(AvroError::InvalidArgument(
969 "Default for interval-year-month must be bytes (4-byte LE)".to_string(),
970 )),
971 },
972 #[cfg(feature = "avro_custom_types")]
973 Self::IntervalMonthDayNano(v) => match lit {
974 AvroLiteral::Bytes(b) => {
975 if b.len() != 16 {
976 return Err(AvroError::InvalidArgument(format!(
977 "interval-month-day-nano default must be exactly 16 bytes, got {}",
978 b.len()
979 )));
980 }
981 let months = i32::from_le_bytes([b[0], b[1], b[2], b[3]]);
982 let days = i32::from_le_bytes([b[4], b[5], b[6], b[7]]);
983 let nanos =
984 i64::from_le_bytes([b[8], b[9], b[10], b[11], b[12], b[13], b[14], b[15]]);
985 v.push(IntervalMonthDayNano::new(months, days, nanos));
986 Ok(())
987 }
988 _ => Err(AvroError::InvalidArgument(
989 "Default for interval-month-day-nano must be bytes (16-byte LE)".to_string(),
990 )),
991 },
992 #[cfg(feature = "avro_custom_types")]
993 Self::IntervalDayTime(v) => match lit {
994 AvroLiteral::Bytes(b) => {
995 if b.len() != 8 {
996 return Err(AvroError::InvalidArgument(format!(
997 "interval-day-time default must be exactly 8 bytes, got {}",
998 b.len()
999 )));
1000 }
1001 let days = i32::from_le_bytes([b[0], b[1], b[2], b[3]]);
1002 let milliseconds = i32::from_le_bytes([b[4], b[5], b[6], b[7]]);
1003 v.push(IntervalDayTime::new(days, milliseconds));
1004 Ok(())
1005 }
1006 _ => Err(AvroError::InvalidArgument(
1007 "Default for interval-day-time must be bytes (8-byte LE)".to_string(),
1008 )),
1009 },
1010 Self::Int64(v)
1011 | Self::Int32ToInt64(v)
1012 | Self::TimeMicros(v)
1013 | Self::TimestampMillis(_, v)
1014 | Self::TimestampMicros(_, v)
1015 | Self::TimestampNanos(_, v) => match lit {
1016 AvroLiteral::Long(i) => {
1017 v.push(*i);
1018 Ok(())
1019 }
1020 AvroLiteral::Int(i) => {
1021 v.push(*i as i64);
1022 Ok(())
1023 }
1024 _ => Err(AvroError::InvalidArgument(
1025 "Default for long/time-micros/timestamp must be long or int".to_string(),
1026 )),
1027 },
1028 Self::Float32(v) | Self::Int32ToFloat32(v) | Self::Int64ToFloat32(v) => match lit {
1029 AvroLiteral::Float(f) => {
1030 v.push(*f);
1031 Ok(())
1032 }
1033 _ => Err(AvroError::InvalidArgument(
1034 "Default for float must be float".to_string(),
1035 )),
1036 },
1037 Self::Float64(v)
1038 | Self::Int32ToFloat64(v)
1039 | Self::Int64ToFloat64(v)
1040 | Self::Float32ToFloat64(v) => match lit {
1041 AvroLiteral::Double(f) => {
1042 v.push(*f);
1043 Ok(())
1044 }
1045 _ => Err(AvroError::InvalidArgument(
1046 "Default for double must be double".to_string(),
1047 )),
1048 },
1049 Self::Binary(offsets, values) | Self::StringToBytes(offsets, values) => match lit {
1050 AvroLiteral::Bytes(b) => {
1051 offsets.push_length(b.len());
1052 values.extend_from_slice(b);
1053 Ok(())
1054 }
1055 _ => Err(AvroError::InvalidArgument(
1056 "Default for bytes must be bytes".to_string(),
1057 )),
1058 },
1059 Self::BytesToString(offsets, values)
1060 | Self::String(offsets, values)
1061 | Self::StringView(offsets, values) => match lit {
1062 AvroLiteral::String(s) => {
1063 let b = s.as_bytes();
1064 offsets.push_length(b.len());
1065 values.extend_from_slice(b);
1066 Ok(())
1067 }
1068 _ => Err(AvroError::InvalidArgument(
1069 "Default for string must be string".to_string(),
1070 )),
1071 },
1072 Self::Uuid(values) => match lit {
1073 AvroLiteral::String(s) => {
1074 let uuid = Uuid::try_parse(s).map_err(|e| {
1075 AvroError::InvalidArgument(format!("Invalid UUID default: {s} ({e})"))
1076 })?;
1077 values.extend_from_slice(uuid.as_bytes());
1078 Ok(())
1079 }
1080 _ => Err(AvroError::InvalidArgument(
1081 "Default for uuid must be string".to_string(),
1082 )),
1083 },
1084 Self::Fixed(sz, accum) => match lit {
1085 AvroLiteral::Bytes(b) => {
1086 if b.len() != *sz as usize {
1087 return Err(AvroError::InvalidArgument(format!(
1088 "Fixed default length {} does not match size {sz}",
1089 b.len(),
1090 )));
1091 }
1092 accum.extend_from_slice(b);
1093 Ok(())
1094 }
1095 _ => Err(AvroError::InvalidArgument(
1096 "Default for fixed must be bytes".to_string(),
1097 )),
1098 },
1099 #[cfg(feature = "small_decimals")]
1100 Self::Decimal32(_, _, _, builder) => {
1101 append_decimal_default!(lit, builder, 4, i32, "decimal32")
1102 }
1103 #[cfg(feature = "small_decimals")]
1104 Self::Decimal64(_, _, _, builder) => {
1105 append_decimal_default!(lit, builder, 8, i64, "decimal64")
1106 }
1107 Self::Decimal128(_, _, _, builder) => {
1108 append_decimal_default!(lit, builder, 16, i128, "decimal128")
1109 }
1110 Self::Decimal256(_, _, _, builder) => {
1111 append_decimal_default!(lit, builder, 32, i256, "decimal256")
1112 }
1113 Self::Duration(builder) => match lit {
1114 AvroLiteral::Bytes(b) => {
1115 if b.len() != 12 {
1116 return Err(AvroError::InvalidArgument(format!(
1117 "Duration default must be exactly 12 bytes, got {}",
1118 b.len()
1119 )));
1120 }
1121 let months = u32::from_le_bytes([b[0], b[1], b[2], b[3]]);
1122 let days = u32::from_le_bytes([b[4], b[5], b[6], b[7]]);
1123 let millis = u32::from_le_bytes([b[8], b[9], b[10], b[11]]);
1124 let nanos = (millis as i64) * 1_000_000;
1125 builder.append_value(IntervalMonthDayNano::new(
1126 months as i32,
1127 days as i32,
1128 nanos,
1129 ));
1130 Ok(())
1131 }
1132 _ => Err(AvroError::InvalidArgument(
1133 "Default for duration must be 12-byte little-endian months/days/millis"
1134 .to_string(),
1135 )),
1136 },
1137 Self::Array(_, offsets, inner) => match lit {
1138 AvroLiteral::Array(items) => {
1139 offsets.push_length(items.len());
1140 for item in items {
1141 inner.append_default(item)?;
1142 }
1143 Ok(())
1144 }
1145 _ => Err(AvroError::InvalidArgument(
1146 "Default for array must be an array literal".to_string(),
1147 )),
1148 },
1149 Self::Map(_, koff, moff, kdata, valdec) => match lit {
1150 AvroLiteral::Map(entries) => {
1151 moff.push_length(entries.len());
1152 for (k, v) in entries {
1153 let kb = k.as_bytes();
1154 koff.push_length(kb.len());
1155 kdata.extend_from_slice(kb);
1156 valdec.append_default(v)?;
1157 }
1158 Ok(())
1159 }
1160 _ => Err(AvroError::InvalidArgument(
1161 "Default for map must be a map/object literal".to_string(),
1162 )),
1163 },
1164 Self::Enum(indices, symbols, _) => match lit {
1165 AvroLiteral::Enum(sym) => {
1166 let pos = symbols.iter().position(|s| s == sym).ok_or_else(|| {
1167 AvroError::InvalidArgument(format!(
1168 "Enum default symbol {sym:?} not in reader symbols"
1169 ))
1170 })?;
1171 indices.push(pos as i32);
1172 Ok(())
1173 }
1174 _ => Err(AvroError::InvalidArgument(
1175 "Default for enum must be a symbol".to_string(),
1176 )),
1177 },
1178 #[cfg(feature = "avro_custom_types")]
1179 Self::RunEndEncoded(_, len, inner) => {
1180 *len += 1;
1181 inner.append_default(lit)
1182 }
1183 Self::Union(u) => u.append_default(lit),
1184 Self::Record {
1185 fields: field_meta,
1186 decoders,
1187 defaults: field_defaults,
1188 row_count,
1189 ..
1190 } => match lit {
1191 AvroLiteral::Map(entries) => {
1192 for (i, dec) in decoders.iter_mut().enumerate() {
1193 let name = field_meta[i].name();
1194 if let Some(sub) = entries.get(name) {
1195 dec.append_default(sub)?;
1196 } else if let Some(default_literal) = field_defaults[i].as_ref() {
1197 dec.append_default(default_literal)?;
1198 } else {
1199 dec.append_null()?;
1200 }
1201 }
1202 *row_count += 1;
1203 Ok(())
1204 }
1205 AvroLiteral::Null => {
1206 for (i, dec) in decoders.iter_mut().enumerate() {
1207 if let Some(default_literal) = field_defaults[i].as_ref() {
1208 dec.append_default(default_literal)?;
1209 } else {
1210 dec.append_null()?;
1211 }
1212 }
1213 *row_count += 1;
1214 Ok(())
1215 }
1216 _ => Err(AvroError::InvalidArgument(
1217 "Default for record must be a map/object or null".to_string(),
1218 )),
1219 },
1220 }
1221 }
1222
1223 fn decode(&mut self, buf: &mut AvroCursor<'_>) -> Result<(), AvroError> {
1225 match self {
1226 Self::Null(x) => *x += 1,
1227 Self::Boolean(values) => values.append(buf.get_bool()?),
1228 Self::Int32(values) | Self::Date32(values) | Self::TimeMillis(values) => {
1229 values.push(buf.get_int()?)
1230 }
1231 Self::Int64(values)
1232 | Self::TimeMicros(values)
1233 | Self::TimestampMillis(_, values)
1234 | Self::TimestampMicros(_, values)
1235 | Self::TimestampNanos(_, values) => values.push(buf.get_long()?),
1236 #[cfg(feature = "avro_custom_types")]
1237 Self::DurationSecond(values)
1238 | Self::DurationMillisecond(values)
1239 | Self::DurationMicrosecond(values)
1240 | Self::DurationNanosecond(values) => values.push(buf.get_long()?),
1241 #[cfg(feature = "avro_custom_types")]
1242 Self::Int8(values) => {
1243 let raw = buf.get_int()?;
1244 let x = i8::try_from(raw).map_err(|_| {
1245 AvroError::ParseError(format!("int8 value {raw} out of range for i8"))
1246 })?;
1247 values.push(x);
1248 }
1249 #[cfg(feature = "avro_custom_types")]
1250 Self::Int16(values) => {
1251 let raw = buf.get_int()?;
1252 let x = i16::try_from(raw).map_err(|_| {
1253 AvroError::ParseError(format!("int16 value {raw} out of range for i16"))
1254 })?;
1255 values.push(x);
1256 }
1257 #[cfg(feature = "avro_custom_types")]
1258 Self::UInt8(values) => {
1259 let raw = buf.get_int()?;
1260 let x = u8::try_from(raw).map_err(|_| {
1261 AvroError::ParseError(format!("uint8 value {raw} out of range for u8"))
1262 })?;
1263 values.push(x);
1264 }
1265 #[cfg(feature = "avro_custom_types")]
1266 Self::UInt16(values) => {
1267 let raw = buf.get_int()?;
1268 let x = u16::try_from(raw).map_err(|_| {
1269 AvroError::ParseError(format!("uint16 value {raw} out of range for u16"))
1270 })?;
1271 values.push(x);
1272 }
1273 #[cfg(feature = "avro_custom_types")]
1274 Self::UInt32(values) => {
1275 let raw = buf.get_long()?;
1276 let x = u32::try_from(raw).map_err(|_| {
1277 AvroError::ParseError(format!("uint32 value {raw} out of range for u32"))
1278 })?;
1279 values.push(x);
1280 }
1281 #[cfg(feature = "avro_custom_types")]
1282 Self::UInt64(values) => {
1283 let b = buf.get_fixed(8)?;
1284 values.push(u64::from_le_bytes([
1285 b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7],
1286 ]));
1287 }
1288 #[cfg(feature = "avro_custom_types")]
1289 Self::Float16(values) => {
1290 let b = buf.get_fixed(2)?;
1291 values.push(u16::from_le_bytes([b[0], b[1]]));
1292 }
1293 #[cfg(feature = "avro_custom_types")]
1294 Self::Date64(values) | Self::TimeNanos(values) | Self::TimestampSecs(_, values) => {
1295 values.push(buf.get_long()?)
1296 }
1297 #[cfg(feature = "avro_custom_types")]
1298 Self::Time32Secs(values) => values.push(buf.get_int()?),
1299 #[cfg(feature = "avro_custom_types")]
1300 Self::IntervalYearMonth(values) => {
1301 let b = buf.get_fixed(4)?;
1302 values.push(i32::from_le_bytes([b[0], b[1], b[2], b[3]]));
1303 }
1304 #[cfg(feature = "avro_custom_types")]
1305 Self::IntervalMonthDayNano(values) => {
1306 let b = buf.get_fixed(16)?;
1307 let months = i32::from_le_bytes([b[0], b[1], b[2], b[3]]);
1308 let days = i32::from_le_bytes([b[4], b[5], b[6], b[7]]);
1309 let nanos =
1310 i64::from_le_bytes([b[8], b[9], b[10], b[11], b[12], b[13], b[14], b[15]]);
1311 values.push(IntervalMonthDayNano::new(months, days, nanos));
1312 }
1313 #[cfg(feature = "avro_custom_types")]
1314 Self::IntervalDayTime(values) => {
1315 let b = buf.get_fixed(8)?;
1316 let days = i32::from_le_bytes([b[0], b[1], b[2], b[3]]);
1318 let milliseconds = i32::from_le_bytes([b[4], b[5], b[6], b[7]]);
1319 values.push(IntervalDayTime::new(days, milliseconds));
1320 }
1321 Self::Float32(values) => values.push(buf.get_float()?),
1322 Self::Float64(values) => values.push(buf.get_double()?),
1323 Self::Int32ToInt64(values) => values.push(buf.get_int()? as i64),
1324 Self::Int32ToFloat32(values) => values.push(buf.get_int()? as f32),
1325 Self::Int32ToFloat64(values) => values.push(buf.get_int()? as f64),
1326 Self::Int64ToFloat32(values) => values.push(buf.get_long()? as f32),
1327 Self::Int64ToFloat64(values) => values.push(buf.get_long()? as f64),
1328 Self::Float32ToFloat64(values) => values.push(buf.get_float()? as f64),
1329 Self::StringToBytes(offsets, values)
1330 | Self::BytesToString(offsets, values)
1331 | Self::Binary(offsets, values)
1332 | Self::String(offsets, values)
1333 | Self::StringView(offsets, values) => {
1334 let data = buf.get_bytes()?;
1335 offsets.push_length(data.len());
1336 values.extend_from_slice(data);
1337 }
1338 Self::Uuid(values) => {
1339 let s_bytes = buf.get_bytes()?;
1340 let s = std::str::from_utf8(s_bytes).map_err(|e| {
1341 AvroError::ParseError(format!("UUID bytes are not valid UTF-8: {e}"))
1342 })?;
1343 let uuid = Uuid::try_parse(s)
1344 .map_err(|e| AvroError::ParseError(format!("Failed to parse uuid: {e}")))?;
1345 values.extend_from_slice(uuid.as_bytes());
1346 }
1347 Self::Array(_, off, encoding) => {
1348 let total_items = read_blocks(buf, |cursor| encoding.decode(cursor))?;
1349 off.push_length(total_items);
1350 }
1351 Self::Record {
1352 decoders,
1353 projector: None,
1354 row_count,
1355 ..
1356 } => {
1357 for encoding in decoders {
1358 encoding.decode(buf)?;
1359 }
1360 *row_count += 1;
1361 }
1362 Self::Record {
1363 decoders,
1364 projector: Some(proj),
1365 row_count,
1366 ..
1367 } => {
1368 proj.project_record(buf, decoders)?;
1369 *row_count += 1;
1370 }
1371 Self::Map(_, koff, moff, kdata, valdec) => {
1372 let newly_added = read_blocks(buf, |cur| {
1373 let kb = cur.get_bytes()?;
1374 koff.push_length(kb.len());
1375 kdata.extend_from_slice(kb);
1376 valdec.decode(cur)
1377 })?;
1378 moff.push_length(newly_added);
1379 }
1380 Self::Fixed(sz, accum) => {
1381 let fx = buf.get_fixed(*sz as usize)?;
1382 accum.extend_from_slice(fx);
1383 }
1384 #[cfg(feature = "small_decimals")]
1385 Self::Decimal32(_, _, size, builder) => {
1386 decode_decimal!(size, buf, builder, 4, i32);
1387 }
1388 #[cfg(feature = "small_decimals")]
1389 Self::Decimal64(_, _, size, builder) => {
1390 decode_decimal!(size, buf, builder, 8, i64);
1391 }
1392 Self::Decimal128(_, _, size, builder) => {
1393 decode_decimal!(size, buf, builder, 16, i128);
1394 }
1395 Self::Decimal256(_, _, size, builder) => {
1396 decode_decimal!(size, buf, builder, 32, i256);
1397 }
1398 Self::Enum(indices, _, None) => {
1399 indices.push(buf.get_int()?);
1400 }
1401 Self::Enum(indices, _, Some(res)) => {
1402 let raw = buf.get_int()?;
1403 let resolved = res.resolve(raw)?;
1404 indices.push(resolved);
1405 }
1406 Self::Duration(builder) => {
1407 let b = buf.get_fixed(12)?;
1408 let months = u32::from_le_bytes([b[0], b[1], b[2], b[3]]);
1409 let days = u32::from_le_bytes([b[4], b[5], b[6], b[7]]);
1410 let millis = u32::from_le_bytes([b[8], b[9], b[10], b[11]]);
1411 let nanos = (millis as i64) * 1_000_000;
1412 builder.append_value(IntervalMonthDayNano::new(months as i32, days as i32, nanos));
1413 }
1414 #[cfg(feature = "avro_custom_types")]
1415 Self::RunEndEncoded(_, len, inner) => {
1416 *len += 1;
1417 inner.decode(buf)?;
1418 }
1419 Self::Union(u) => u.decode(buf)?,
1420 Self::Nullable(nullable) => {
1421 let is_not_null = match &nullable.plan {
1422 NullablePlan::FromSingle { .. } => true,
1423 NullablePlan::ReadTag { nullability, .. } => {
1424 let branch = buf.read_vlq()?;
1425 match *nullability {
1426 Nullability::NullFirst => branch != 0,
1427 Nullability::NullSecond => branch == 0,
1428 }
1429 }
1430 };
1431
1432 if is_not_null {
1433 nullable.materialize_pending()?;
1434 let resolution = match &nullable.plan {
1435 NullablePlan::FromSingle { resolution }
1436 | NullablePlan::ReadTag { resolution, .. } => resolution,
1437 };
1438 nullable.values.decode_with_resolution(buf, resolution)?;
1440 nullable.validity.append_non_null();
1441 } else {
1442 nullable.pending_nulls += 1;
1443 }
1444 }
1445 }
1446 Ok(())
1447 }
1448
1449 fn decode_with_promotion(
1450 &mut self,
1451 buf: &mut AvroCursor<'_>,
1452 promotion: Promotion,
1453 ) -> Result<(), AvroError> {
1454 #[cfg(feature = "avro_custom_types")]
1455 if let Self::RunEndEncoded(_, len, inner) = self {
1456 *len += 1;
1457 return inner.decode_with_promotion(buf, promotion);
1458 }
1459
1460 macro_rules! promote_numeric_to {
1461 ($variant:ident, $getter:ident, $to:ty) => {{
1462 match self {
1463 Self::$variant(v) => {
1464 let x = buf.$getter()?;
1465 v.push(x as $to);
1466 Ok(())
1467 }
1468 other => Err(AvroError::ParseError(format!(
1469 "Promotion {promotion} target mismatch: expected {}, got {}",
1470 stringify!($variant),
1471 <Self as ::std::convert::AsRef<str>>::as_ref(other)
1472 ))),
1473 }
1474 }};
1475 }
1476 match promotion {
1477 Promotion::Direct => self.decode(buf),
1478 Promotion::IntToLong => promote_numeric_to!(Int64, get_int, i64),
1479 Promotion::IntToFloat => promote_numeric_to!(Float32, get_int, f32),
1480 Promotion::IntToDouble => promote_numeric_to!(Float64, get_int, f64),
1481 Promotion::LongToFloat => promote_numeric_to!(Float32, get_long, f32),
1482 Promotion::LongToDouble => promote_numeric_to!(Float64, get_long, f64),
1483 Promotion::FloatToDouble => promote_numeric_to!(Float64, get_float, f64),
1484 Promotion::StringToBytes => match self {
1485 Self::Binary(offsets, values) | Self::StringToBytes(offsets, values) => {
1486 let data = buf.get_bytes()?;
1487 offsets.push_length(data.len());
1488 values.extend_from_slice(data);
1489 Ok(())
1490 }
1491 other => Err(AvroError::ParseError(format!(
1492 "Promotion {promotion} target mismatch: expected bytes (Binary/StringToBytes), got {}",
1493 <Self as AsRef<str>>::as_ref(other)
1494 ))),
1495 },
1496 Promotion::BytesToString => match self {
1497 Self::String(offsets, values)
1498 | Self::StringView(offsets, values)
1499 | Self::BytesToString(offsets, values) => {
1500 let data = buf.get_bytes()?;
1501 offsets.push_length(data.len());
1502 values.extend_from_slice(data);
1503 Ok(())
1504 }
1505 other => Err(AvroError::ParseError(format!(
1506 "Promotion {promotion} target mismatch: expected string (String/StringView/BytesToString), got {}",
1507 <Self as AsRef<str>>::as_ref(other)
1508 ))),
1509 },
1510 }
1511 }
1512
1513 fn decode_with_resolution<'d>(
1514 &'d mut self,
1515 buf: &mut AvroCursor<'_>,
1516 resolution: &'d ResolutionPlan,
1517 ) -> Result<(), AvroError> {
1518 #[cfg(feature = "avro_custom_types")]
1519 if let Self::RunEndEncoded(_, len, inner) = self {
1520 *len += 1;
1521 return inner.decode_with_resolution(buf, resolution);
1522 }
1523
1524 match resolution {
1525 ResolutionPlan::Promotion(promotion) => {
1526 let promotion = *promotion;
1527 self.decode_with_promotion(buf, promotion)
1528 }
1529 ResolutionPlan::DefaultValue(lit) => self.append_default(lit),
1530 ResolutionPlan::EnumMapping(res) => {
1531 let Self::Enum(indices, _, _) = self else {
1532 return Err(AvroError::SchemaError(
1533 "enum mapping resolution provided for non-enum decoder".into(),
1534 ));
1535 };
1536 let raw = buf.get_int()?;
1537 let resolved = res.resolve(raw)?;
1538 indices.push(resolved);
1539 Ok(())
1540 }
1541 ResolutionPlan::Record(proj) => {
1542 let Self::Record {
1543 decoders,
1544 row_count,
1545 ..
1546 } = self
1547 else {
1548 return Err(AvroError::SchemaError(
1549 "record projection provided for non-record decoder".into(),
1550 ));
1551 };
1552 proj.project_record(buf, decoders)?;
1553 *row_count += 1;
1554 Ok(())
1555 }
1556 }
1557 }
1558
1559 fn flush(&mut self, nulls: Option<NullBuffer>) -> Result<ArrayRef, AvroError> {
1561 Ok(match self {
1562 Self::Nullable(nullable) => {
1563 nullable.materialize_pending()?;
1564 nullable.values.flush(nullable.validity.finish())?
1565 }
1566 Self::Null(size) => Arc::new(NullArray::new(std::mem::replace(size, 0))),
1567 Self::Boolean(b) => Arc::new(BooleanArray::new(b.finish(), nulls)),
1568 Self::Int32(values) => Arc::new(flush_primitive::<Int32Type>(values, nulls)),
1569 Self::Date32(values) => Arc::new(flush_primitive::<Date32Type>(values, nulls)),
1570 Self::Int64(values) => Arc::new(flush_primitive::<Int64Type>(values, nulls)),
1571 Self::TimeMillis(values) => {
1572 Arc::new(flush_primitive::<Time32MillisecondType>(values, nulls))
1573 }
1574 Self::TimeMicros(values) => {
1575 Arc::new(flush_primitive::<Time64MicrosecondType>(values, nulls))
1576 }
1577 Self::TimestampMillis(tz, values) => Arc::new(
1578 flush_primitive::<TimestampMillisecondType>(values, nulls)
1579 .with_timezone_opt(tz.as_ref().map(|tz| tz.to_string())),
1580 ),
1581 Self::TimestampMicros(tz, values) => Arc::new(
1582 flush_primitive::<TimestampMicrosecondType>(values, nulls)
1583 .with_timezone_opt(tz.as_ref().map(|tz| tz.to_string())),
1584 ),
1585 Self::TimestampNanos(tz, values) => Arc::new(
1586 flush_primitive::<TimestampNanosecondType>(values, nulls)
1587 .with_timezone_opt(tz.as_ref().map(|tz| tz.to_string())),
1588 ),
1589 #[cfg(feature = "avro_custom_types")]
1590 Self::DurationSecond(values) => {
1591 Arc::new(flush_primitive::<DurationSecondType>(values, nulls))
1592 }
1593 #[cfg(feature = "avro_custom_types")]
1594 Self::DurationMillisecond(values) => {
1595 Arc::new(flush_primitive::<DurationMillisecondType>(values, nulls))
1596 }
1597 #[cfg(feature = "avro_custom_types")]
1598 Self::DurationMicrosecond(values) => {
1599 Arc::new(flush_primitive::<DurationMicrosecondType>(values, nulls))
1600 }
1601 #[cfg(feature = "avro_custom_types")]
1602 Self::DurationNanosecond(values) => {
1603 Arc::new(flush_primitive::<DurationNanosecondType>(values, nulls))
1604 }
1605 #[cfg(feature = "avro_custom_types")]
1606 Self::Int8(values) => Arc::new(flush_primitive::<Int8Type>(values, nulls)),
1607 #[cfg(feature = "avro_custom_types")]
1608 Self::Int16(values) => Arc::new(flush_primitive::<Int16Type>(values, nulls)),
1609 #[cfg(feature = "avro_custom_types")]
1610 Self::UInt8(values) => Arc::new(flush_primitive::<UInt8Type>(values, nulls)),
1611 #[cfg(feature = "avro_custom_types")]
1612 Self::UInt16(values) => Arc::new(flush_primitive::<UInt16Type>(values, nulls)),
1613 #[cfg(feature = "avro_custom_types")]
1614 Self::UInt32(values) => Arc::new(flush_primitive::<UInt32Type>(values, nulls)),
1615 #[cfg(feature = "avro_custom_types")]
1616 Self::UInt64(values) => Arc::new(flush_primitive::<UInt64Type>(values, nulls)),
1617 #[cfg(feature = "avro_custom_types")]
1618 Self::Float16(values) => {
1619 let len = values.len();
1622 let buf: Buffer = std::mem::take(values).into();
1623 let scalar_buf = ScalarBuffer::new(buf, 0, len);
1624 Arc::new(Float16Array::new(scalar_buf, nulls))
1625 }
1626 #[cfg(feature = "avro_custom_types")]
1627 Self::Date64(values) => Arc::new(flush_primitive::<Date64Type>(values, nulls)),
1628 #[cfg(feature = "avro_custom_types")]
1629 Self::TimeNanos(values) => {
1630 Arc::new(flush_primitive::<Time64NanosecondType>(values, nulls))
1631 }
1632 #[cfg(feature = "avro_custom_types")]
1633 Self::Time32Secs(values) => {
1634 Arc::new(flush_primitive::<Time32SecondType>(values, nulls))
1635 }
1636 #[cfg(feature = "avro_custom_types")]
1637 Self::TimestampSecs(is_utc, values) => Arc::new(
1638 flush_primitive::<TimestampSecondType>(values, nulls)
1639 .with_timezone_opt(is_utc.then(|| "+00:00")),
1640 ),
1641 #[cfg(feature = "avro_custom_types")]
1642 Self::IntervalYearMonth(values) => {
1643 Arc::new(flush_primitive::<IntervalYearMonthType>(values, nulls))
1644 }
1645 #[cfg(feature = "avro_custom_types")]
1646 Self::IntervalMonthDayNano(values) => {
1647 Arc::new(flush_primitive::<IntervalMonthDayNanoType>(values, nulls))
1648 }
1649 #[cfg(feature = "avro_custom_types")]
1650 Self::IntervalDayTime(values) => {
1651 Arc::new(flush_primitive::<IntervalDayTimeType>(values, nulls))
1652 }
1653 Self::Float32(values) => Arc::new(flush_primitive::<Float32Type>(values, nulls)),
1654 Self::Float64(values) => Arc::new(flush_primitive::<Float64Type>(values, nulls)),
1655 Self::Int32ToInt64(values) => Arc::new(flush_primitive::<Int64Type>(values, nulls)),
1656 Self::Int32ToFloat32(values) | Self::Int64ToFloat32(values) => {
1657 Arc::new(flush_primitive::<Float32Type>(values, nulls))
1658 }
1659 Self::Int32ToFloat64(values)
1660 | Self::Int64ToFloat64(values)
1661 | Self::Float32ToFloat64(values) => {
1662 Arc::new(flush_primitive::<Float64Type>(values, nulls))
1663 }
1664 Self::StringToBytes(offsets, values) | Self::Binary(offsets, values) => {
1665 let offsets = flush_offsets(offsets);
1666 let values = flush_values(values).into();
1667 Arc::new(BinaryArray::try_new(offsets, values, nulls)?)
1668 }
1669 Self::BytesToString(offsets, values) | Self::String(offsets, values) => {
1670 let offsets = flush_offsets(offsets);
1671 let values = flush_values(values).into();
1672 Arc::new(StringArray::try_new(offsets, values, nulls)?)
1673 }
1674 Self::StringView(offsets, values) => {
1675 let offsets = flush_offsets(offsets);
1676 let values = flush_values(values);
1677 let array = StringArray::try_new(offsets, values.into(), nulls)?;
1678 Arc::new(StringViewArray::from(&array))
1679 }
1680 Self::Array(field, offsets, values) => {
1681 let values = values.flush(None)?;
1682 let offsets = flush_offsets(offsets);
1683 Arc::new(ListArray::try_new(field.clone(), offsets, values, nulls)?)
1684 }
1685 Self::Record {
1686 fields,
1687 decoders,
1688 row_count,
1689 ..
1690 } => {
1691 let arrays = decoders
1692 .iter_mut()
1693 .map(|x| x.flush(None))
1694 .collect::<Result<Vec<_>, _>>()?;
1695 Arc::new(StructArray::try_new_with_length(
1698 fields.clone(),
1699 arrays,
1700 nulls,
1701 std::mem::replace(row_count, 0),
1702 )?)
1703 }
1704 Self::Map(map_field, k_off, m_off, kdata, valdec) => {
1705 let moff = flush_offsets(m_off);
1706 let koff = flush_offsets(k_off);
1707 let kd = flush_values(kdata).into();
1708 let val_arr = valdec.flush(None)?;
1709 let key_arr = StringArray::try_new(koff, kd, None)?;
1710 if key_arr.len() != val_arr.len() {
1711 return Err(AvroError::InvalidArgument(format!(
1712 "Map keys length ({}) != map values length ({})",
1713 key_arr.len(),
1714 val_arr.len()
1715 )));
1716 }
1717 let final_len = moff.len() - 1;
1718 if let Some(n) = &nulls
1719 && n.len() != final_len
1720 {
1721 return Err(AvroError::InvalidArgument(format!(
1722 "Map array null buffer length {} != final map length {final_len}",
1723 n.len()
1724 )));
1725 }
1726 let entries_fields = match map_field.data_type() {
1727 DataType::Struct(fields) => fields.clone(),
1728 other => {
1729 return Err(AvroError::InvalidArgument(format!(
1730 "Map entries field must be a Struct, got {other:?}"
1731 )));
1732 }
1733 };
1734 let entries_struct =
1735 StructArray::try_new(entries_fields, vec![Arc::new(key_arr), val_arr], None)?;
1736 let map_arr =
1737 MapArray::try_new(map_field.clone(), moff, entries_struct, nulls, false)?;
1738 Arc::new(map_arr)
1739 }
1740 Self::Fixed(sz, accum) => {
1741 let b: Buffer = flush_values(accum).into();
1742 let arr = FixedSizeBinaryArray::try_new(*sz, b, nulls)
1743 .map_err(|e| AvroError::ParseError(e.to_string()))?;
1744 Arc::new(arr)
1745 }
1746 Self::Uuid(values) => {
1747 let arr = FixedSizeBinaryArray::try_new(16, std::mem::take(values).into(), nulls)
1748 .map_err(|e| AvroError::ParseError(e.to_string()))?;
1749 Arc::new(arr)
1750 }
1751 #[cfg(feature = "small_decimals")]
1752 Self::Decimal32(precision, scale, _, builder) => {
1753 flush_decimal!(builder, precision, scale, nulls, Decimal32Array)
1754 }
1755 #[cfg(feature = "small_decimals")]
1756 Self::Decimal64(precision, scale, _, builder) => {
1757 flush_decimal!(builder, precision, scale, nulls, Decimal64Array)
1758 }
1759 Self::Decimal128(precision, scale, _, builder) => {
1760 flush_decimal!(builder, precision, scale, nulls, Decimal128Array)
1761 }
1762 Self::Decimal256(precision, scale, _, builder) => {
1763 flush_decimal!(builder, precision, scale, nulls, Decimal256Array)
1764 }
1765 Self::Enum(indices, symbols, _) => flush_dict(indices, symbols, nulls)?,
1766 Self::Duration(builder) => {
1767 let (_, vals, _) = builder.finish().into_parts();
1768 let vals = IntervalMonthDayNanoArray::try_new(vals, nulls)
1769 .map_err(|e| AvroError::ParseError(e.to_string()))?;
1770 Arc::new(vals)
1771 }
1772 #[cfg(feature = "avro_custom_types")]
1773 Self::RunEndEncoded(width, len, inner) => {
1774 let values = inner.flush(nulls)?;
1775 let n = *len;
1776 let arr = values.as_ref();
1777 let mut run_starts: Vec<usize> = Vec::with_capacity(n);
1778 if n > 0 {
1779 run_starts.push(0);
1780 for i in 1..n {
1781 if !values_equal_at(arr, i - 1, i) {
1782 run_starts.push(i);
1783 }
1784 }
1785 }
1786 if n > (u32::MAX as usize) {
1787 return Err(AvroError::InvalidArgument(format!(
1788 "RunEndEncoded length {n} exceeds maximum supported by UInt32 indices for take",
1789 )));
1790 }
1791 let run_count = run_starts.len();
1792 let take_idx: PrimitiveArray<UInt32Type> =
1793 run_starts.iter().map(|&s| s as u32).collect();
1794 let per_run_values = if run_count == 0 {
1795 values.slice(0, 0)
1796 } else {
1797 take(arr, &take_idx, Option::from(TakeOptions::default())).map_err(|e| {
1798 AvroError::ParseError(format!("take() for REE values failed: {e}"))
1799 })?
1800 };
1801
1802 macro_rules! build_run_array {
1803 ($Native:ty, $ArrowTy:ty) => {{
1804 let mut ends: Vec<$Native> = Vec::with_capacity(run_count);
1805 for (idx, &_start) in run_starts.iter().enumerate() {
1806 let end = if idx + 1 < run_count {
1807 run_starts[idx + 1]
1808 } else {
1809 n
1810 };
1811 ends.push(end as $Native);
1812 }
1813 let ends: PrimitiveArray<$ArrowTy> = ends.into_iter().collect();
1814 let run_arr = RunArray::<$ArrowTy>::try_new(&ends, per_run_values.as_ref())
1815 .map_err(|e| AvroError::ParseError(e.to_string()))?;
1816 Arc::new(run_arr) as ArrayRef
1817 }};
1818 }
1819 match *width {
1820 2 => {
1821 if n > i16::MAX as usize {
1822 return Err(AvroError::InvalidArgument(format!(
1823 "RunEndEncoded length {n} exceeds i16::MAX for run end width 2"
1824 )));
1825 }
1826 build_run_array!(i16, Int16Type)
1827 }
1828 4 => build_run_array!(i32, Int32Type),
1829 8 => build_run_array!(i64, Int64Type),
1830 other => {
1831 return Err(AvroError::InvalidArgument(format!(
1832 "Unsupported run-end width {other} for RunEndEncoded"
1833 )));
1834 }
1835 }
1836 }
1837 Self::Union(u) => u.flush(nulls)?,
1838 })
1839 }
1840}
1841
1842#[derive(Debug)]
1844enum NullablePlan {
1845 ReadTag {
1847 nullability: Nullability,
1848 resolution: ResolutionPlan,
1849 },
1850 FromSingle { resolution: ResolutionPlan },
1853}
1854
1855#[derive(Debug)]
1857enum ResolutionPlan {
1858 Promotion(Promotion),
1860 DefaultValue(AvroLiteral),
1862 EnumMapping(EnumResolution),
1864 Record(Projector),
1866}
1867
1868impl ResolutionPlan {
1869 fn try_new(decoder: &Decoder, resolution: &ResolutionInfo) -> Result<Self, AvroError> {
1870 match (decoder, resolution) {
1871 (_, ResolutionInfo::Promotion(p)) => Ok(ResolutionPlan::Promotion(*p)),
1872 (_, ResolutionInfo::DefaultValue(lit)) => Ok(ResolutionPlan::DefaultValue(lit.clone())),
1873 (_, ResolutionInfo::EnumMapping(m)) => {
1874 Ok(ResolutionPlan::EnumMapping(EnumResolution::new(m)))
1875 }
1876 (
1877 Decoder::Record {
1878 defaults: field_defaults,
1879 ..
1880 },
1881 ResolutionInfo::Record(r),
1882 ) => Ok(ResolutionPlan::Record(
1883 ProjectorBuilder::try_new(r, field_defaults).build()?,
1884 )),
1885 (_, ResolutionInfo::Record(_)) => Err(AvroError::SchemaError(
1886 "record resolution on non-record decoder".into(),
1887 )),
1888 (_, ResolutionInfo::Union(_)) => Err(AvroError::SchemaError(
1889 "union variant cannot be resolved to a union type".into(),
1890 )),
1891 }
1892 }
1893}
1894
1895#[derive(Debug)]
1896struct EnumResolution {
1897 mapping: Arc<[i32]>,
1898 default_index: i32,
1899}
1900
1901impl EnumResolution {
1902 fn new(mapping: &EnumMapping) -> Self {
1903 EnumResolution {
1904 mapping: mapping.mapping.clone(),
1905 default_index: mapping.default_index,
1906 }
1907 }
1908
1909 fn resolve(&self, index: i32) -> Result<i32, AvroError> {
1910 let resolved = usize::try_from(index)
1911 .ok()
1912 .and_then(|idx| self.mapping.get(idx).copied())
1913 .filter(|&idx| idx >= 0)
1914 .unwrap_or(self.default_index);
1915 if resolved >= 0 {
1916 Ok(resolved)
1917 } else {
1918 Err(AvroError::ParseError(format!(
1919 "Enum symbol index {index} not resolvable and no default provided",
1920 )))
1921 }
1922 }
1923}
1924
1925#[derive(Debug)]
1927struct DispatchLookupTable {
1928 to_reader: Box<[i8]>,
1944 resolution: Box<[ResolutionPlan]>,
1949}
1950
1951const NO_SOURCE: i8 = -1;
1954
1955impl DispatchLookupTable {
1956 fn from_writer_to_reader(
1957 reader_branches: &[Decoder],
1958 resolution_map: &[Option<(usize, ResolutionInfo)>],
1959 ) -> Result<Self, AvroError> {
1960 let mut to_reader = Vec::with_capacity(resolution_map.len());
1961 let mut resolution = Vec::with_capacity(resolution_map.len());
1962 for map in resolution_map {
1963 match map {
1964 Some((idx, res)) => {
1965 let idx = *idx;
1966 let idx_i8 = i8::try_from(idx).map_err(|_| {
1967 AvroError::SchemaError(format!(
1968 "Reader branch index {idx} exceeds i8 range (max {})",
1969 i8::MAX
1970 ))
1971 })?;
1972 let plan = ResolutionPlan::try_new(&reader_branches[idx], res)?;
1973 to_reader.push(idx_i8);
1974 resolution.push(plan);
1975 }
1976 None => {
1977 to_reader.push(NO_SOURCE);
1978 resolution.push(ResolutionPlan::DefaultValue(AvroLiteral::Null));
1979 }
1980 }
1981 }
1982 Ok(Self {
1983 to_reader: to_reader.into_boxed_slice(),
1984 resolution: resolution.into_boxed_slice(),
1985 })
1986 }
1987
1988 #[inline]
1990 fn resolve(&self, writer_index: usize) -> Option<(usize, &ResolutionPlan)> {
1991 let reader_index = *self.to_reader.get(writer_index)?;
1992 (reader_index >= 0).then(|| (reader_index as usize, &self.resolution[writer_index]))
1993 }
1994}
1995
1996#[derive(Debug)]
1997struct UnionDecoder {
1998 fields: UnionFields,
1999 branches: UnionDecoderBranches,
2000 default_emit_idx: usize,
2001 null_emit_idx: usize,
2002 plan: UnionReadPlan,
2003}
2004
2005#[derive(Debug, Default)]
2006struct UnionDecoderBranches {
2007 decoders: Vec<Decoder>,
2008 reader_type_codes: Vec<i8>,
2009 type_ids: Vec<i8>,
2010 offsets: Vec<i32>,
2011 counts: Vec<i32>,
2012}
2013
2014impl UnionDecoderBranches {
2015 fn new(decoders: Vec<Decoder>, reader_type_codes: Vec<i8>) -> Self {
2016 let branch_len = decoders.len().max(reader_type_codes.len());
2017 Self {
2018 decoders,
2019 reader_type_codes,
2020 type_ids: Vec::with_capacity(DEFAULT_CAPACITY),
2021 offsets: Vec::with_capacity(DEFAULT_CAPACITY),
2022 counts: vec![0; branch_len],
2023 }
2024 }
2025
2026 fn emit_to(&mut self, reader_idx: usize) -> Result<&mut Decoder, AvroError> {
2027 let branches_len = self.decoders.len();
2028 let Some(reader_branch) = self.decoders.get_mut(reader_idx) else {
2029 return Err(AvroError::ParseError(format!(
2030 "Union branch index {reader_idx} out of range ({branches_len} branches)"
2031 )));
2032 };
2033 self.type_ids.push(self.reader_type_codes[reader_idx]);
2034 self.offsets.push(self.counts[reader_idx]);
2035 self.counts[reader_idx] += 1;
2036 Ok(reader_branch)
2037 }
2038}
2039
2040impl Default for UnionDecoder {
2041 fn default() -> Self {
2042 Self {
2043 fields: UnionFields::empty(),
2044 branches: Default::default(),
2045 default_emit_idx: 0,
2046 null_emit_idx: 0,
2047 plan: UnionReadPlan::Passthrough,
2048 }
2049 }
2050}
2051
2052#[derive(Debug)]
2053enum UnionReadPlan {
2054 ReaderUnion {
2055 lookup_table: DispatchLookupTable,
2056 },
2057 FromSingle {
2058 reader_idx: usize,
2059 resolution: ResolutionPlan,
2060 },
2061 ToSingle {
2062 target: Box<Decoder>,
2063 lookup_table: DispatchLookupTable,
2064 },
2065 Passthrough,
2066}
2067
2068impl UnionReadPlan {
2069 fn from_resolved(
2070 reader_branches: &[Decoder],
2071 resolved: Option<ResolvedUnion>,
2072 ) -> Result<Self, AvroError> {
2073 let Some(info) = resolved else {
2074 return Ok(Self::Passthrough);
2075 };
2076 match (info.writer_is_union, info.reader_is_union) {
2077 (true, true) => {
2078 let lookup_table =
2079 DispatchLookupTable::from_writer_to_reader(reader_branches, &info.writer_to_reader)?;
2080 Ok(Self::ReaderUnion { lookup_table })
2081 }
2082 (false, true) => {
2083 let Some((idx, resolution)) =
2084 info.writer_to_reader.first().and_then(Option::as_ref)
2085 else {
2086 return Err(AvroError::SchemaError(
2087 "Writer type does not match any reader union branch".to_string(),
2088 ));
2089 };
2090 let reader_idx = *idx;
2091 Ok(Self::FromSingle {
2092 reader_idx,
2093 resolution: ResolutionPlan::try_new(&reader_branches[reader_idx], resolution)?,
2094 })
2095 }
2096 (true, false) => Err(AvroError::InvalidArgument(
2097 "UnionDecoder::try_new cannot build writer-union to single; use UnionDecoderBuilder with a target"
2098 .to_string(),
2099 )),
2100 _ => Err(AvroError::SchemaError(
2102 "ResolvedUnion constructed for non-union sides; resolver should return None"
2103 .to_string(),
2104 )),
2105 }
2106 }
2107}
2108
2109impl UnionDecoder {
2110 fn try_new(
2111 fields: UnionFields,
2112 branches: Vec<Decoder>,
2113 resolved: Option<ResolvedUnion>,
2114 ) -> Result<Self, AvroError> {
2115 let reader_type_codes = fields.iter().map(|(tid, _)| tid).collect::<Vec<i8>>();
2116 let null_branch = branches.iter().position(|b| matches!(b, Decoder::Null(_)));
2117 let default_emit_idx = 0;
2118 let null_emit_idx = null_branch.unwrap_or(default_emit_idx);
2119 let max_addr = (i32::MAX as usize) + 1;
2121 if branches.len() > max_addr {
2122 return Err(AvroError::SchemaError(format!(
2123 "Reader union has {} branches, which exceeds the maximum addressable \
2124 branches by an Avro int tag ({} + 1).",
2125 branches.len(),
2126 i32::MAX
2127 )));
2128 }
2129 let plan = UnionReadPlan::from_resolved(&branches, resolved)?;
2130 Ok(Self {
2131 fields,
2132 branches: UnionDecoderBranches::new(branches, reader_type_codes),
2133 default_emit_idx,
2134 null_emit_idx,
2135 plan,
2136 })
2137 }
2138
2139 fn with_single_target(target: Decoder, info: ResolvedUnion) -> Result<Self, AvroError> {
2140 debug_assert!(info.writer_is_union && !info.reader_is_union);
2142 let mut reader_branches = [target];
2143 let lookup_table =
2144 DispatchLookupTable::from_writer_to_reader(&reader_branches, &info.writer_to_reader)?;
2145 let target = Box::new(mem::replace(&mut reader_branches[0], Decoder::Null(0)));
2146 Ok(Self {
2147 plan: UnionReadPlan::ToSingle {
2148 target,
2149 lookup_table,
2150 },
2151 ..Self::default()
2152 })
2153 }
2154
2155 #[inline]
2156 fn read_tag(buf: &mut AvroCursor<'_>) -> Result<usize, AvroError> {
2157 let raw = buf.get_long()?;
2162 if raw < 0 {
2163 return Err(AvroError::ParseError(format!(
2164 "Negative union branch index {raw}"
2165 )));
2166 }
2167 usize::try_from(raw).map_err(|_| {
2168 AvroError::ParseError(format!(
2169 "Union branch index {raw} does not fit into usize on this platform ({}-bit)",
2170 (usize::BITS as usize)
2171 ))
2172 })
2173 }
2174
2175 #[inline]
2176 fn on_decoder<F>(&mut self, fallback_idx: usize, action: F) -> Result<(), AvroError>
2177 where
2178 F: FnOnce(&mut Decoder) -> Result<(), AvroError>,
2179 {
2180 if let UnionReadPlan::ToSingle { target, .. } = &mut self.plan {
2181 return action(target);
2182 }
2183 let reader_idx = match &self.plan {
2184 UnionReadPlan::FromSingle { reader_idx, .. } => *reader_idx,
2185 _ => fallback_idx,
2186 };
2187 self.branches.emit_to(reader_idx).and_then(action)
2188 }
2189
2190 fn append_null(&mut self) -> Result<(), AvroError> {
2191 self.on_decoder(self.null_emit_idx, |decoder| decoder.append_null())
2192 }
2193
2194 fn append_default(&mut self, lit: &AvroLiteral) -> Result<(), AvroError> {
2195 self.on_decoder(self.default_emit_idx, |decoder| decoder.append_default(lit))
2196 }
2197
2198 fn decode(&mut self, buf: &mut AvroCursor<'_>) -> Result<(), AvroError> {
2199 match &mut self.plan {
2200 UnionReadPlan::Passthrough => {
2201 let reader_idx = Self::read_tag(buf)?;
2202 let decoder = self.branches.emit_to(reader_idx)?;
2203 decoder.decode(buf)
2204 }
2205 UnionReadPlan::ReaderUnion { lookup_table } => {
2206 let idx = Self::read_tag(buf)?;
2207 let Some((reader_idx, resolution)) = lookup_table.resolve(idx) else {
2208 return Err(AvroError::ParseError(format!(
2209 "Union branch index {idx} not resolvable by reader schema"
2210 )));
2211 };
2212 let decoder = self.branches.emit_to(reader_idx)?;
2213 decoder.decode_with_resolution(buf, resolution)
2214 }
2215 UnionReadPlan::FromSingle {
2216 reader_idx,
2217 resolution,
2218 } => {
2219 let decoder = self.branches.emit_to(*reader_idx)?;
2220 decoder.decode_with_resolution(buf, resolution)
2221 }
2222 UnionReadPlan::ToSingle {
2223 target,
2224 lookup_table,
2225 } => {
2226 let idx = Self::read_tag(buf)?;
2227 let Some((_, resolution)) = lookup_table.resolve(idx) else {
2228 return Err(AvroError::ParseError(format!(
2229 "Writer union branch index {idx} not resolvable by reader schema"
2230 )));
2231 };
2232 target.decode_with_resolution(buf, resolution)
2233 }
2234 }
2235 }
2236
2237 fn flush(&mut self, nulls: Option<NullBuffer>) -> Result<ArrayRef, AvroError> {
2238 if let UnionReadPlan::ToSingle { target, .. } = &mut self.plan {
2239 return target.flush(nulls);
2240 }
2241 debug_assert!(
2242 nulls.is_none(),
2243 "UnionArray does not accept a validity bitmap; \
2244 nulls should have been materialized as a Null child during decode"
2245 );
2246 let children = self
2247 .branches
2248 .decoders
2249 .iter_mut()
2250 .map(|d| d.flush(None))
2251 .collect::<Result<Vec<_>, _>>()?;
2252 let arr = UnionArray::try_new(
2253 self.fields.clone(),
2254 flush_values(&mut self.branches.type_ids)
2255 .into_iter()
2256 .collect(),
2257 Some(
2258 flush_values(&mut self.branches.offsets)
2259 .into_iter()
2260 .collect(),
2261 ),
2262 children,
2263 )
2264 .map_err(|e| AvroError::ParseError(e.to_string()))?;
2265 self.branches.counts.fill(0);
2266 Ok(Arc::new(arr))
2267 }
2268}
2269
2270#[derive(Debug, Default)]
2271struct UnionDecoderBuilder {
2272 fields: Option<UnionFields>,
2273 branches: Option<Vec<Decoder>>,
2274 resolved: Option<ResolvedUnion>,
2275 target: Option<Decoder>,
2276}
2277
2278impl UnionDecoderBuilder {
2279 fn new() -> Self {
2280 Self::default()
2281 }
2282
2283 fn with_fields(mut self, fields: UnionFields) -> Self {
2284 self.fields = Some(fields);
2285 self
2286 }
2287
2288 fn with_branches(mut self, branches: Vec<Decoder>) -> Self {
2289 self.branches = Some(branches);
2290 self
2291 }
2292
2293 fn with_resolved_union(mut self, resolved_union: ResolvedUnion) -> Self {
2294 self.resolved = Some(resolved_union);
2295 self
2296 }
2297
2298 fn with_target(mut self, target: Decoder) -> Self {
2299 self.target = Some(target);
2300 self
2301 }
2302
2303 fn build(self) -> Result<UnionDecoder, AvroError> {
2304 match (self.resolved, self.fields, self.branches, self.target) {
2305 (resolved, Some(fields), Some(branches), None) => {
2306 UnionDecoder::try_new(fields, branches, resolved)
2307 }
2308 (Some(info), None, None, Some(target))
2309 if info.writer_is_union && !info.reader_is_union =>
2310 {
2311 UnionDecoder::with_single_target(target, info)
2312 }
2313 _ => Err(AvroError::InvalidArgument(
2314 "Invalid UnionDecoderBuilder configuration: expected either \
2315 (fields + branches + resolved) with no target for reader-unions, or \
2316 (resolved + target) with no fields/branches for writer-union to single."
2317 .to_string(),
2318 )),
2319 }
2320 }
2321}
2322
2323#[derive(Debug, Copy, Clone)]
2324enum NegativeBlockBehavior {
2325 ProcessItems,
2326 SkipBySize,
2327}
2328
2329#[inline]
2330fn skip_blocks(
2331 buf: &mut AvroCursor,
2332 mut skip_item: impl FnMut(&mut AvroCursor) -> Result<(), AvroError>,
2333) -> Result<usize, AvroError> {
2334 process_blockwise(
2335 buf,
2336 move |c| skip_item(c),
2337 NegativeBlockBehavior::SkipBySize,
2338 )
2339}
2340
2341#[inline]
2342fn flush_dict(
2343 indices: &mut Vec<i32>,
2344 symbols: &[String],
2345 nulls: Option<NullBuffer>,
2346) -> Result<ArrayRef, AvroError> {
2347 let keys = flush_primitive::<Int32Type>(indices, nulls);
2348 let values = Arc::new(StringArray::from_iter_values(
2349 symbols.iter().map(|s| s.as_str()),
2350 ));
2351 DictionaryArray::try_new(keys, values)
2352 .map_err(Into::into)
2353 .map(|arr| Arc::new(arr) as ArrayRef)
2354}
2355
2356#[inline]
2357fn read_blocks(
2358 buf: &mut AvroCursor,
2359 decode_entry: impl FnMut(&mut AvroCursor) -> Result<(), AvroError>,
2360) -> Result<usize, AvroError> {
2361 process_blockwise(buf, decode_entry, NegativeBlockBehavior::ProcessItems)
2362}
2363
2364#[inline]
2365fn process_blockwise(
2366 buf: &mut AvroCursor,
2367 mut on_item: impl FnMut(&mut AvroCursor) -> Result<(), AvroError>,
2368 negative_behavior: NegativeBlockBehavior,
2369) -> Result<usize, AvroError> {
2370 let mut total = 0usize;
2371 loop {
2372 let block_count = buf.get_long()?;
2377 match block_count.cmp(&0) {
2378 Ordering::Equal => break,
2379 Ordering::Less => {
2380 let count = block_count.unsigned_abs() as usize;
2382 let raw_size = buf.get_long()?;
2384 let size_in_bytes = usize::try_from(raw_size).map_err(|_| {
2385 AvroError::ParseError(format!("Block size cannot be negative, got {raw_size}"))
2386 })?;
2387 match negative_behavior {
2388 NegativeBlockBehavior::ProcessItems => {
2389 total = process_block_items(buf, count, total, &mut on_item)?;
2391 }
2392 NegativeBlockBehavior::SkipBySize => {
2393 let _ = buf.get_fixed(size_in_bytes)?;
2395 total = total.saturating_add(count);
2396 }
2397 }
2398 }
2399 Ordering::Greater => {
2400 let count = block_count as usize;
2401 total = process_block_items(buf, count, total, &mut on_item)?;
2402 }
2403 }
2404 }
2405 Ok(total)
2406}
2407
2408#[inline]
2413fn process_block_items(
2414 buf: &mut AvroCursor,
2415 count: usize,
2416 total: usize,
2417 on_item: &mut impl FnMut(&mut AvroCursor) -> Result<(), AvroError>,
2418) -> Result<usize, AvroError> {
2419 let Some(new_total) = total
2420 .checked_add(count)
2421 .filter(|&t| i32::try_from(t).is_ok())
2422 else {
2423 return Err(AvroError::ParseError(
2424 "Capacity overflow when decoding array/map item blocks".to_string(),
2425 ));
2426 };
2427 for _ in 0..count {
2428 on_item(buf)?;
2429 }
2430 Ok(new_total)
2431}
2432
2433#[inline]
2434fn flush_values<T>(values: &mut Vec<T>) -> Vec<T> {
2435 std::mem::replace(values, Vec::with_capacity(DEFAULT_CAPACITY))
2436}
2437
2438#[inline]
2439fn flush_offsets(offsets: &mut OffsetBufferBuilder<i32>) -> OffsetBuffer<i32> {
2440 std::mem::replace(offsets, OffsetBufferBuilder::new(DEFAULT_CAPACITY)).finish()
2441}
2442
2443#[inline]
2444fn flush_primitive<T: ArrowPrimitiveType>(
2445 values: &mut Vec<T::Native>,
2446 nulls: Option<NullBuffer>,
2447) -> PrimitiveArray<T> {
2448 PrimitiveArray::new(flush_values(values).into(), nulls)
2449}
2450
2451#[inline]
2452fn read_decimal_bytes_be<const N: usize>(
2453 buf: &mut AvroCursor<'_>,
2454 size: Option<usize>,
2455) -> Result<[u8; N], AvroError> {
2456 match size {
2457 Some(n) if n == N => {
2458 let raw = buf.get_fixed(N)?;
2459 let mut arr = [0u8; N];
2460 arr.copy_from_slice(raw);
2461 Ok(arr)
2462 }
2463 Some(n) => {
2464 let raw = buf.get_fixed(n)?;
2465 sign_cast_to::<N>(raw)
2466 }
2467 None => {
2468 let raw = buf.get_bytes()?;
2469 sign_cast_to::<N>(raw)
2470 }
2471 }
2472}
2473
2474#[inline]
2483fn sign_cast_to<const N: usize>(raw: &[u8]) -> Result<[u8; N], AvroError> {
2484 let len = raw.len();
2485 if len == N {
2487 let mut out = [0u8; N];
2488 out.copy_from_slice(raw);
2489 return Ok(out);
2490 }
2491 let first = raw.first().copied().unwrap_or(0u8);
2493 let sign_byte = if (first & 0x80) == 0 { 0x00 } else { 0xFF };
2494 let mut out = [sign_byte; N];
2496 if len > N {
2497 let extra = len - N;
2500 if raw[..extra].iter().any(|&b| b != sign_byte) {
2502 return Err(AvroError::ParseError(format!(
2503 "Decimal value with {len} bytes cannot be represented in {N} bytes without overflow"
2504 )));
2505 }
2506 if N > 0 {
2507 let first_kept = raw[extra];
2508 let sign_bit_mismatch = ((first_kept ^ sign_byte) & 0x80) != 0;
2509 if sign_bit_mismatch {
2510 return Err(AvroError::ParseError(format!(
2511 "Decimal value with {len} bytes cannot be represented in {N} bytes without overflow"
2512 )));
2513 }
2514 }
2515 out.copy_from_slice(&raw[extra..]);
2516 return Ok(out);
2517 }
2518 out[N - len..].copy_from_slice(raw);
2519 Ok(out)
2520}
2521
2522#[cfg(feature = "avro_custom_types")]
2523#[inline]
2524fn values_equal_at(arr: &dyn Array, i: usize, j: usize) -> bool {
2525 match (arr.is_null(i), arr.is_null(j)) {
2526 (true, true) => true,
2527 (true, false) | (false, true) => false,
2528 (false, false) => {
2529 let a = arr.slice(i, 1);
2530 let b = arr.slice(j, 1);
2531 a == b
2532 }
2533 }
2534}
2535
2536#[derive(Debug)]
2537struct Projector {
2538 writer_projections: Vec<FieldProjection>,
2539 default_injections: Arc<[(usize, AvroLiteral)]>,
2540}
2541
2542#[derive(Debug)]
2543enum FieldProjection {
2544 ToReader(usize),
2545 Skip(Skipper),
2546}
2547
2548#[derive(Debug)]
2549struct ProjectorBuilder<'a> {
2550 rec: &'a ResolvedRecord,
2551 field_defaults: &'a [Option<AvroLiteral>],
2552}
2553
2554impl<'a> ProjectorBuilder<'a> {
2555 #[inline]
2556 fn try_new(rec: &'a ResolvedRecord, field_defaults: &'a [Option<AvroLiteral>]) -> Self {
2557 Self {
2558 rec,
2559 field_defaults,
2560 }
2561 }
2562
2563 #[inline]
2564 fn build(self) -> Result<Projector, AvroError> {
2565 let mut default_injections: Vec<(usize, AvroLiteral)> =
2566 Vec::with_capacity(self.rec.default_fields.len());
2567 for &idx in self.rec.default_fields.as_ref() {
2568 let lit = self
2569 .field_defaults
2570 .get(idx)
2571 .and_then(|lit| lit.clone())
2572 .unwrap_or(AvroLiteral::Null);
2573 default_injections.push((idx, lit));
2574 }
2575 let writer_projections = self
2576 .rec
2577 .writer_fields
2578 .iter()
2579 .map(|field| match field {
2580 ResolvedField::ToReader(index, _) => Ok(FieldProjection::ToReader(*index)),
2581 ResolvedField::Skip(datatype) => {
2582 let skipper = Skipper::from_avro(datatype)?;
2583 Ok(FieldProjection::Skip(skipper))
2584 }
2585 })
2586 .collect::<Result<_, AvroError>>()?;
2587 Ok(Projector {
2588 writer_projections,
2589 default_injections: default_injections.into(),
2590 })
2591 }
2592}
2593
2594impl Projector {
2595 #[inline]
2596 fn project_record(
2597 &self,
2598 buf: &mut AvroCursor<'_>,
2599 encodings: &mut [Decoder],
2600 ) -> Result<(), AvroError> {
2601 for field_proj in &self.writer_projections {
2602 match field_proj {
2603 FieldProjection::ToReader(index) => encodings[*index].decode(buf)?,
2604 FieldProjection::Skip(skipper) => skipper.skip(buf)?,
2605 }
2606 }
2607 for (reader_index, lit) in self.default_injections.as_ref() {
2608 encodings[*reader_index].append_default(lit)?;
2609 }
2610 Ok(())
2611 }
2612}
2613
2614#[derive(Debug)]
2620enum Skipper {
2621 Null,
2622 Boolean,
2623 Int32,
2624 Int64,
2625 Float32,
2626 Float64,
2627 Bytes,
2628 String,
2629 TimeMicros,
2630 TimestampMillis,
2631 TimestampMicros,
2632 TimestampNanos,
2633 Fixed(usize),
2634 Decimal(Option<usize>),
2635 UuidString,
2636 Enum,
2637 DurationFixed12,
2638 List(Box<Skipper>),
2639 Map(Box<Skipper>),
2640 Struct(Vec<Skipper>),
2641 Union(Vec<Skipper>),
2642 Nullable(Nullability, Box<Skipper>),
2643 #[cfg(feature = "avro_custom_types")]
2644 RunEndEncoded(Box<Skipper>),
2645}
2646
2647impl Skipper {
2648 fn from_avro(dt: &AvroDataType) -> Result<Self, AvroError> {
2649 let mut base = match dt.codec() {
2650 Codec::Null => Self::Null,
2651 Codec::Boolean => Self::Boolean,
2652 Codec::Int32 | Codec::Date32 | Codec::TimeMillis => Self::Int32,
2653 Codec::Int64 => Self::Int64,
2654 Codec::TimeMicros => Self::TimeMicros,
2655 Codec::TimestampMillis(_) => Self::TimestampMillis,
2656 Codec::TimestampMicros(_) => Self::TimestampMicros,
2657 Codec::TimestampNanos(_) => Self::TimestampNanos,
2658 #[cfg(feature = "avro_custom_types")]
2659 Codec::DurationNanos
2660 | Codec::DurationMicros
2661 | Codec::DurationMillis
2662 | Codec::DurationSeconds => Self::Int64,
2663 #[cfg(feature = "avro_custom_types")]
2664 Codec::Int8 | Codec::Int16 | Codec::UInt8 | Codec::UInt16 | Codec::Time32Secs => {
2665 Self::Int32
2666 }
2667 #[cfg(feature = "avro_custom_types")]
2668 Codec::UInt32 | Codec::Date64 | Codec::TimeNanos | Codec::TimestampSecs(_) => {
2669 Self::Int64
2670 }
2671 #[cfg(feature = "avro_custom_types")]
2672 Codec::UInt64 => Self::Fixed(8),
2673 #[cfg(feature = "avro_custom_types")]
2674 Codec::Float16 => Self::Fixed(2),
2675 #[cfg(feature = "avro_custom_types")]
2676 Codec::IntervalYearMonth => Self::Fixed(4),
2677 #[cfg(feature = "avro_custom_types")]
2678 Codec::IntervalMonthDayNano => Self::Fixed(16),
2679 #[cfg(feature = "avro_custom_types")]
2680 Codec::IntervalDayTime => Self::Fixed(8),
2681 Codec::Float32 => Self::Float32,
2682 Codec::Float64 => Self::Float64,
2683 Codec::Binary => Self::Bytes,
2684 Codec::Utf8 | Codec::Utf8View => Self::String,
2685 Codec::Fixed(sz) => Self::Fixed(*sz as usize),
2686 Codec::Decimal(_, _, size) => Self::Decimal(*size),
2687 Codec::Uuid => Self::UuidString, Codec::Enum(_) => Self::Enum,
2689 Codec::List(item) => Self::List(Box::new(Skipper::from_avro(item)?)),
2690 Codec::Struct(fields) => {
2691 if let Some(ResolutionInfo::Record(rec)) = dt.resolution.as_ref() {
2692 Self::Struct(
2693 rec.writer_fields
2694 .iter()
2695 .map(|wf| match wf {
2696 ResolvedField::ToReader(_, wdt) | ResolvedField::Skip(wdt) => {
2697 Skipper::from_avro(wdt)
2698 }
2699 })
2700 .collect::<Result<_, _>>()?,
2701 )
2702 } else {
2703 Self::Struct(
2704 fields
2705 .iter()
2706 .map(|f| Skipper::from_avro(f.data_type()))
2707 .collect::<Result<_, _>>()?,
2708 )
2709 }
2710 }
2711 Codec::Map(values) => Self::Map(Box::new(Skipper::from_avro(values)?)),
2712 Codec::Interval => Self::DurationFixed12,
2713 Codec::Union(encodings, _, _) => {
2714 let max_addr = (i32::MAX as usize) + 1;
2715 if encodings.len() > max_addr {
2716 return Err(AvroError::SchemaError(format!(
2717 "Writer union has {} branches, which exceeds the maximum addressable \
2718 branches by an Avro int tag ({} + 1).",
2719 encodings.len(),
2720 i32::MAX
2721 )));
2722 }
2723 Self::Union(
2724 encodings
2725 .iter()
2726 .map(Skipper::from_avro)
2727 .collect::<Result<_, _>>()?,
2728 )
2729 }
2730 #[cfg(feature = "avro_custom_types")]
2731 Codec::RunEndEncoded(inner, _w) => {
2732 Self::RunEndEncoded(Box::new(Skipper::from_avro(inner)?))
2733 }
2734 };
2735 if let Some(n) = dt.nullability() {
2736 base = Self::Nullable(n, Box::new(base));
2737 }
2738 Ok(base)
2739 }
2740
2741 fn skip(&self, buf: &mut AvroCursor<'_>) -> Result<(), AvroError> {
2742 match self {
2743 Self::Null => Ok(()),
2744 Self::Boolean => {
2745 buf.get_bool()?;
2746 Ok(())
2747 }
2748 Self::Int32 => {
2749 buf.skip_int()?;
2750 Ok(())
2751 }
2752 Self::Int64
2753 | Self::TimeMicros
2754 | Self::TimestampMillis
2755 | Self::TimestampMicros
2756 | Self::TimestampNanos => {
2757 buf.skip_long()?;
2758 Ok(())
2759 }
2760 Self::Float32 => {
2761 buf.get_float()?;
2762 Ok(())
2763 }
2764 Self::Float64 => {
2765 buf.get_double()?;
2766 Ok(())
2767 }
2768 Self::Bytes | Self::String | Self::UuidString => {
2769 buf.get_bytes()?;
2770 Ok(())
2771 }
2772 Self::Fixed(sz) => {
2773 buf.get_fixed(*sz)?;
2774 Ok(())
2775 }
2776 Self::Decimal(size) => {
2777 if let Some(s) = size {
2778 buf.get_fixed(*s)
2779 } else {
2780 buf.get_bytes()
2781 }?;
2782 Ok(())
2783 }
2784 Self::Enum => {
2785 buf.skip_int()?;
2786 Ok(())
2787 }
2788 Self::DurationFixed12 => {
2789 buf.get_fixed(12)?;
2790 Ok(())
2791 }
2792 Self::List(item) => {
2793 skip_blocks(buf, |c| item.skip(c))?;
2794 Ok(())
2795 }
2796 Self::Map(value) => {
2797 skip_blocks(buf, |c| {
2798 c.get_bytes()?; value.skip(c)
2800 })?;
2801 Ok(())
2802 }
2803 Self::Struct(fields) => {
2804 for f in fields {
2805 f.skip(buf)?
2806 }
2807 Ok(())
2808 }
2809 Self::Union(encodings) => {
2810 let raw = buf.get_long()?;
2812 if raw < 0 {
2813 return Err(AvroError::ParseError(format!(
2814 "Negative union branch index {raw}"
2815 )));
2816 }
2817 let idx: usize = usize::try_from(raw).map_err(|_| {
2818 AvroError::ParseError(format!(
2819 "Union branch index {raw} does not fit into usize on this platform ({}-bit)",
2820 (usize::BITS as usize)
2821 ))
2822 })?;
2823 let Some(encoding) = encodings.get(idx) else {
2824 return Err(AvroError::ParseError(format!(
2825 "Union branch index {idx} out of range for skipper ({} branches)",
2826 encodings.len()
2827 )));
2828 };
2829 encoding.skip(buf)
2830 }
2831 Self::Nullable(order, inner) => {
2832 let branch = buf.read_vlq()?;
2833 let is_not_null = match *order {
2834 Nullability::NullFirst => branch != 0,
2835 Nullability::NullSecond => branch == 0,
2836 };
2837 if is_not_null {
2838 inner.skip(buf)?;
2839 }
2840 Ok(())
2841 }
2842 #[cfg(feature = "avro_custom_types")]
2843 Self::RunEndEncoded(inner) => inner.skip(buf),
2844 }
2845 }
2846}
2847
2848#[cfg(test)]
2849mod tests {
2850 use super::*;
2851 use crate::codec::AvroFieldBuilder;
2852 use crate::schema::{Attributes, ComplexType, Field, PrimitiveType, Record, Schema, TypeName};
2853 use arrow_array::cast::AsArray;
2854 use indexmap::IndexMap;
2855 use std::collections::HashMap;
2856
2857 fn encode_avro_int(value: i32) -> Vec<u8> {
2858 let mut buf = Vec::new();
2859 let mut v = (value << 1) ^ (value >> 31);
2860 while v & !0x7F != 0 {
2861 buf.push(((v & 0x7F) | 0x80) as u8);
2862 v >>= 7;
2863 }
2864 buf.push(v as u8);
2865 buf
2866 }
2867
2868 fn encode_avro_long(value: i64) -> Vec<u8> {
2869 let mut buf = Vec::new();
2870 let mut v = (value << 1) ^ (value >> 63);
2871 while v & !0x7F != 0 {
2872 buf.push(((v & 0x7F) | 0x80) as u8);
2873 v >>= 7;
2874 }
2875 buf.push(v as u8);
2876 buf
2877 }
2878
2879 fn encode_avro_bytes(bytes: &[u8]) -> Vec<u8> {
2880 let mut buf = encode_avro_long(bytes.len() as i64);
2881 buf.extend_from_slice(bytes);
2882 buf
2883 }
2884
2885 fn avro_from_codec(codec: Codec) -> AvroDataType {
2886 AvroDataType::new(codec, Default::default(), None)
2887 }
2888
2889 fn resolved_root_datatype(
2890 writer: Schema<'static>,
2891 reader: Schema<'static>,
2892 use_utf8view: bool,
2893 strict_mode: bool,
2894 ) -> AvroDataType {
2895 let writer_record = Schema::Complex(ComplexType::Record(Record {
2897 name: "Root",
2898 namespace: None,
2899 doc: None,
2900 aliases: vec![],
2901 fields: vec![Field {
2902 name: "v",
2903 r#type: writer,
2904 default: None,
2905 doc: None,
2906 aliases: vec![],
2907 }],
2908 attributes: Attributes::default(),
2909 }));
2910
2911 let reader_record = Schema::Complex(ComplexType::Record(Record {
2913 name: "Root",
2914 namespace: None,
2915 doc: None,
2916 aliases: vec![],
2917 fields: vec![Field {
2918 name: "v",
2919 r#type: reader,
2920 default: None,
2921 doc: None,
2922 aliases: vec![],
2923 }],
2924 attributes: Attributes::default(),
2925 }));
2926
2927 let field = AvroFieldBuilder::new(&writer_record)
2929 .with_reader_schema(&reader_record)
2930 .with_utf8view(use_utf8view)
2931 .with_strict_mode(strict_mode)
2932 .build()
2933 .expect("schema resolution should succeed");
2934
2935 match field.data_type().codec() {
2936 Codec::Struct(fields) => fields[0].data_type().clone(),
2937 other => panic!("expected wrapper struct, got {other:?}"),
2938 }
2939 }
2940
2941 fn decoder_for_promotion(
2942 writer: PrimitiveType,
2943 reader: PrimitiveType,
2944 use_utf8view: bool,
2945 ) -> Decoder {
2946 let ws = Schema::TypeName(TypeName::Primitive(writer));
2947 let rs = Schema::TypeName(TypeName::Primitive(reader));
2948 let dt = resolved_root_datatype(ws, rs, use_utf8view, false);
2949 Decoder::try_new(&dt).unwrap()
2950 }
2951
2952 fn make_avro_dt(codec: Codec, nullability: Option<Nullability>) -> AvroDataType {
2953 AvroDataType::new(codec, HashMap::new(), nullability)
2954 }
2955
2956 #[cfg(feature = "avro_custom_types")]
2957 fn encode_vlq_u64(mut x: u64) -> Vec<u8> {
2958 let mut out = Vec::with_capacity(10);
2959 while x >= 0x80 {
2960 out.push((x as u8) | 0x80);
2961 x >>= 7;
2962 }
2963 out.push(x as u8);
2964 out
2965 }
2966
2967 #[test]
2968 fn test_union_resolution_writer_union_reader_union_reorder_and_promotion_dense() {
2969 let ws = Schema::Union(vec![
2970 Schema::TypeName(TypeName::Primitive(PrimitiveType::Int)),
2971 Schema::TypeName(TypeName::Primitive(PrimitiveType::String)),
2972 ]);
2973 let rs = Schema::Union(vec![
2974 Schema::TypeName(TypeName::Primitive(PrimitiveType::String)),
2975 Schema::TypeName(TypeName::Primitive(PrimitiveType::Long)),
2976 ]);
2977
2978 let dt = resolved_root_datatype(ws, rs, false, false);
2979 let mut dec = Decoder::try_new(&dt).unwrap();
2980
2981 let mut rec1 = encode_avro_long(0);
2982 rec1.extend(encode_avro_int(7));
2983 let mut cur1 = AvroCursor::new(&rec1);
2984 dec.decode(&mut cur1).unwrap();
2985
2986 let mut rec2 = encode_avro_long(1);
2987 rec2.extend(encode_avro_bytes(b"abc"));
2988 let mut cur2 = AvroCursor::new(&rec2);
2989 dec.decode(&mut cur2).unwrap();
2990
2991 let arr = dec.flush(None).unwrap();
2992 let ua = arr
2993 .as_any()
2994 .downcast_ref::<UnionArray>()
2995 .expect("dense union output");
2996
2997 assert_eq!(
2998 ua.type_id(0),
2999 1,
3000 "first value must select reader 'long' branch"
3001 );
3002 assert_eq!(ua.value_offset(0), 0);
3003
3004 assert_eq!(
3005 ua.type_id(1),
3006 0,
3007 "second value must select reader 'string' branch"
3008 );
3009 assert_eq!(ua.value_offset(1), 0);
3010
3011 let long_child = ua.child(1).as_any().downcast_ref::<Int64Array>().unwrap();
3012 assert_eq!(long_child.len(), 1);
3013 assert_eq!(long_child.value(0), 7);
3014
3015 let str_child = ua.child(0).as_any().downcast_ref::<StringArray>().unwrap();
3016 assert_eq!(str_child.len(), 1);
3017 assert_eq!(str_child.value(0), "abc");
3018 }
3019
3020 #[test]
3021 fn test_union_resolution_writer_union_reader_nonunion_promotion_int_to_long() {
3022 let ws = Schema::Union(vec![
3023 Schema::TypeName(TypeName::Primitive(PrimitiveType::Int)),
3024 Schema::TypeName(TypeName::Primitive(PrimitiveType::String)),
3025 ]);
3026 let rs = Schema::TypeName(TypeName::Primitive(PrimitiveType::Long));
3027
3028 let dt = resolved_root_datatype(ws, rs, false, false);
3029 let mut dec = Decoder::try_new(&dt).unwrap();
3030
3031 let mut data = encode_avro_long(0);
3032 data.extend(encode_avro_int(5));
3033 let mut cur = AvroCursor::new(&data);
3034 dec.decode(&mut cur).unwrap();
3035
3036 let arr = dec.flush(None).unwrap();
3037 let out = arr.as_any().downcast_ref::<Int64Array>().unwrap();
3038 assert_eq!(out.len(), 1);
3039 assert_eq!(out.value(0), 5);
3040 }
3041
3042 #[test]
3043 fn test_union_resolution_writer_union_reader_nonunion_mismatch_errors() {
3044 let ws = Schema::Union(vec![
3045 Schema::TypeName(TypeName::Primitive(PrimitiveType::Int)),
3046 Schema::TypeName(TypeName::Primitive(PrimitiveType::String)),
3047 ]);
3048 let rs = Schema::TypeName(TypeName::Primitive(PrimitiveType::Long));
3049
3050 let dt = resolved_root_datatype(ws, rs, false, false);
3051 let mut dec = Decoder::try_new(&dt).unwrap();
3052
3053 let mut data = encode_avro_long(1);
3054 data.extend(encode_avro_bytes(b"z"));
3055 let mut cur = AvroCursor::new(&data);
3056 let res = dec.decode(&mut cur);
3057 assert!(
3058 res.is_err(),
3059 "expected error when writer union branch does not resolve to reader non-union type"
3060 );
3061 }
3062
3063 #[test]
3064 fn test_union_resolution_writer_nonunion_reader_union_selects_matching_branch() {
3065 let ws = Schema::TypeName(TypeName::Primitive(PrimitiveType::Int));
3066 let rs = Schema::Union(vec![
3067 Schema::TypeName(TypeName::Primitive(PrimitiveType::String)),
3068 Schema::TypeName(TypeName::Primitive(PrimitiveType::Long)),
3069 ]);
3070
3071 let dt = resolved_root_datatype(ws, rs, false, false);
3072 let mut dec = Decoder::try_new(&dt).unwrap();
3073
3074 let data = encode_avro_int(6);
3075 let mut cur = AvroCursor::new(&data);
3076 dec.decode(&mut cur).unwrap();
3077
3078 let arr = dec.flush(None).unwrap();
3079 let ua = arr
3080 .as_any()
3081 .downcast_ref::<UnionArray>()
3082 .expect("dense union output");
3083 assert_eq!(ua.len(), 1);
3084 assert_eq!(
3085 ua.type_id(0),
3086 1,
3087 "must resolve to reader 'long' branch (type_id 1)"
3088 );
3089 assert_eq!(ua.value_offset(0), 0);
3090
3091 let long_child = ua.child(1).as_any().downcast_ref::<Int64Array>().unwrap();
3092 assert_eq!(long_child.len(), 1);
3093 assert_eq!(long_child.value(0), 6);
3094
3095 let str_child = ua.child(0).as_any().downcast_ref::<StringArray>().unwrap();
3096 assert_eq!(str_child.len(), 0, "string branch must be empty");
3097 }
3098
3099 #[test]
3100 fn test_union_resolution_writer_union_reader_union_unmapped_branch_errors() {
3101 let ws = Schema::Union(vec![
3102 Schema::TypeName(TypeName::Primitive(PrimitiveType::Int)),
3103 Schema::TypeName(TypeName::Primitive(PrimitiveType::Boolean)),
3104 ]);
3105 let rs = Schema::Union(vec![
3106 Schema::TypeName(TypeName::Primitive(PrimitiveType::String)),
3107 Schema::TypeName(TypeName::Primitive(PrimitiveType::Long)),
3108 ]);
3109
3110 let dt = resolved_root_datatype(ws, rs, false, false);
3111 let mut dec = Decoder::try_new(&dt).unwrap();
3112
3113 let mut data = encode_avro_long(1);
3114 data.push(1);
3115 let mut cur = AvroCursor::new(&data);
3116 let res = dec.decode(&mut cur);
3117 assert!(
3118 res.is_err(),
3119 "expected error for unmapped writer 'boolean' branch"
3120 );
3121 }
3122
3123 #[test]
3124 fn test_schema_resolution_promotion_int_to_long() {
3125 let mut dec = decoder_for_promotion(PrimitiveType::Int, PrimitiveType::Long, false);
3126 assert!(matches!(dec, Decoder::Int32ToInt64(_)));
3127 for v in [0, 1, -2, 123456] {
3128 let data = encode_avro_int(v);
3129 let mut cur = AvroCursor::new(&data);
3130 dec.decode(&mut cur).unwrap();
3131 }
3132 let arr = dec.flush(None).unwrap();
3133 let a = arr.as_any().downcast_ref::<Int64Array>().unwrap();
3134 assert_eq!(a.value(0), 0);
3135 assert_eq!(a.value(1), 1);
3136 assert_eq!(a.value(2), -2);
3137 assert_eq!(a.value(3), 123456);
3138 }
3139
3140 #[test]
3141 fn test_schema_resolution_promotion_int_to_float() {
3142 let mut dec = decoder_for_promotion(PrimitiveType::Int, PrimitiveType::Float, false);
3143 assert!(matches!(dec, Decoder::Int32ToFloat32(_)));
3144 for v in [0, 42, -7] {
3145 let data = encode_avro_int(v);
3146 let mut cur = AvroCursor::new(&data);
3147 dec.decode(&mut cur).unwrap();
3148 }
3149 let arr = dec.flush(None).unwrap();
3150 let a = arr.as_any().downcast_ref::<Float32Array>().unwrap();
3151 assert_eq!(a.value(0), 0.0);
3152 assert_eq!(a.value(1), 42.0);
3153 assert_eq!(a.value(2), -7.0);
3154 }
3155
3156 #[test]
3157 fn test_schema_resolution_promotion_int_to_double() {
3158 let mut dec = decoder_for_promotion(PrimitiveType::Int, PrimitiveType::Double, false);
3159 assert!(matches!(dec, Decoder::Int32ToFloat64(_)));
3160 for v in [1, -1, 10_000] {
3161 let data = encode_avro_int(v);
3162 let mut cur = AvroCursor::new(&data);
3163 dec.decode(&mut cur).unwrap();
3164 }
3165 let arr = dec.flush(None).unwrap();
3166 let a = arr.as_any().downcast_ref::<Float64Array>().unwrap();
3167 assert_eq!(a.value(0), 1.0);
3168 assert_eq!(a.value(1), -1.0);
3169 assert_eq!(a.value(2), 10_000.0);
3170 }
3171
3172 #[test]
3173 fn test_schema_resolution_promotion_long_to_float() {
3174 let mut dec = decoder_for_promotion(PrimitiveType::Long, PrimitiveType::Float, false);
3175 assert!(matches!(dec, Decoder::Int64ToFloat32(_)));
3176 for v in [0_i64, 1_000_000_i64, -123_i64] {
3177 let data = encode_avro_long(v);
3178 let mut cur = AvroCursor::new(&data);
3179 dec.decode(&mut cur).unwrap();
3180 }
3181 let arr = dec.flush(None).unwrap();
3182 let a = arr.as_any().downcast_ref::<Float32Array>().unwrap();
3183 assert_eq!(a.value(0), 0.0);
3184 assert_eq!(a.value(1), 1_000_000.0);
3185 assert_eq!(a.value(2), -123.0);
3186 }
3187
3188 #[test]
3189 fn test_schema_resolution_promotion_long_to_double() {
3190 let mut dec = decoder_for_promotion(PrimitiveType::Long, PrimitiveType::Double, false);
3191 assert!(matches!(dec, Decoder::Int64ToFloat64(_)));
3192 for v in [2_i64, -2_i64, 9_223_372_i64] {
3193 let data = encode_avro_long(v);
3194 let mut cur = AvroCursor::new(&data);
3195 dec.decode(&mut cur).unwrap();
3196 }
3197 let arr = dec.flush(None).unwrap();
3198 let a = arr.as_any().downcast_ref::<Float64Array>().unwrap();
3199 assert_eq!(a.value(0), 2.0);
3200 assert_eq!(a.value(1), -2.0);
3201 assert_eq!(a.value(2), 9_223_372.0);
3202 }
3203
3204 #[test]
3205 fn test_schema_resolution_promotion_float_to_double() {
3206 let mut dec = decoder_for_promotion(PrimitiveType::Float, PrimitiveType::Double, false);
3207 assert!(matches!(dec, Decoder::Float32ToFloat64(_)));
3208 for v in [0.5_f32, -3.25_f32, 1.0e6_f32] {
3209 let data = v.to_le_bytes().to_vec();
3210 let mut cur = AvroCursor::new(&data);
3211 dec.decode(&mut cur).unwrap();
3212 }
3213 let arr = dec.flush(None).unwrap();
3214 let a = arr.as_any().downcast_ref::<Float64Array>().unwrap();
3215 assert_eq!(a.value(0), 0.5_f64);
3216 assert_eq!(a.value(1), -3.25_f64);
3217 assert_eq!(a.value(2), 1.0e6_f64);
3218 }
3219
3220 #[test]
3221 fn test_schema_resolution_promotion_bytes_to_string_utf8() {
3222 let mut dec = decoder_for_promotion(PrimitiveType::Bytes, PrimitiveType::String, false);
3223 assert!(matches!(dec, Decoder::BytesToString(_, _)));
3224 for s in ["hello", "world", "héllo"] {
3225 let data = encode_avro_bytes(s.as_bytes());
3226 let mut cur = AvroCursor::new(&data);
3227 dec.decode(&mut cur).unwrap();
3228 }
3229 let arr = dec.flush(None).unwrap();
3230 let a = arr.as_any().downcast_ref::<StringArray>().unwrap();
3231 assert_eq!(a.value(0), "hello");
3232 assert_eq!(a.value(1), "world");
3233 assert_eq!(a.value(2), "héllo");
3234 }
3235
3236 #[test]
3237 fn test_schema_resolution_promotion_bytes_to_string_utf8view_enabled() {
3238 let mut dec = decoder_for_promotion(PrimitiveType::Bytes, PrimitiveType::String, true);
3239 assert!(matches!(dec, Decoder::BytesToString(_, _)));
3240 let data = encode_avro_bytes(b"abc");
3241 let mut cur = AvroCursor::new(&data);
3242 dec.decode(&mut cur).unwrap();
3243 let arr = dec.flush(None).unwrap();
3244 let a = arr.as_any().downcast_ref::<StringArray>().unwrap();
3245 assert_eq!(a.value(0), "abc");
3246 }
3247
3248 #[test]
3249 fn test_schema_resolution_promotion_string_to_bytes() {
3250 let mut dec = decoder_for_promotion(PrimitiveType::String, PrimitiveType::Bytes, false);
3251 assert!(matches!(dec, Decoder::StringToBytes(_, _)));
3252 for s in ["", "abc", "data"] {
3253 let data = encode_avro_bytes(s.as_bytes());
3254 let mut cur = AvroCursor::new(&data);
3255 dec.decode(&mut cur).unwrap();
3256 }
3257 let arr = dec.flush(None).unwrap();
3258 let a = arr.as_any().downcast_ref::<BinaryArray>().unwrap();
3259 assert_eq!(a.value(0), b"");
3260 assert_eq!(a.value(1), b"abc");
3261 assert_eq!(a.value(2), b"data");
3262 }
3263
3264 #[test]
3265 fn test_schema_resolution_no_promotion_passthrough_int() {
3266 let ws = Schema::TypeName(TypeName::Primitive(PrimitiveType::Int));
3267 let rs = Schema::TypeName(TypeName::Primitive(PrimitiveType::Int));
3268 let writer_record = Schema::Complex(ComplexType::Record(Record {
3270 name: "Root",
3271 namespace: None,
3272 doc: None,
3273 aliases: vec![],
3274 fields: vec![Field {
3275 name: "v",
3276 r#type: ws,
3277 default: None,
3278 doc: None,
3279 aliases: vec![],
3280 }],
3281 attributes: Attributes::default(),
3282 }));
3283 let reader_record = Schema::Complex(ComplexType::Record(Record {
3284 name: "Root",
3285 namespace: None,
3286 doc: None,
3287 aliases: vec![],
3288 fields: vec![Field {
3289 name: "v",
3290 r#type: rs,
3291 default: None,
3292 doc: None,
3293 aliases: vec![],
3294 }],
3295 attributes: Attributes::default(),
3296 }));
3297 let field = AvroFieldBuilder::new(&writer_record)
3298 .with_reader_schema(&reader_record)
3299 .with_utf8view(false)
3300 .with_strict_mode(false)
3301 .build()
3302 .unwrap();
3303 let dt = match field.data_type().codec() {
3305 Codec::Struct(fields) => fields[0].data_type().clone(),
3306 other => panic!("expected wrapper struct, got {other:?}"),
3307 };
3308 let mut dec = Decoder::try_new(&dt).unwrap();
3309 assert!(matches!(dec, Decoder::Int32(_)));
3310 for v in [7, -9] {
3311 let data = encode_avro_int(v);
3312 let mut cur = AvroCursor::new(&data);
3313 dec.decode(&mut cur).unwrap();
3314 }
3315 let arr = dec.flush(None).unwrap();
3316 let a = arr.as_any().downcast_ref::<Int32Array>().unwrap();
3317 assert_eq!(a.value(0), 7);
3318 assert_eq!(a.value(1), -9);
3319 }
3320
3321 #[test]
3322 fn test_schema_resolution_illegal_promotion_int_to_boolean_errors() {
3323 let ws = Schema::TypeName(TypeName::Primitive(PrimitiveType::Int));
3324 let rs = Schema::TypeName(TypeName::Primitive(PrimitiveType::Boolean));
3325 let writer_record = Schema::Complex(ComplexType::Record(Record {
3326 name: "Root",
3327 namespace: None,
3328 doc: None,
3329 aliases: vec![],
3330 fields: vec![Field {
3331 name: "v",
3332 r#type: ws,
3333 default: None,
3334 doc: None,
3335 aliases: vec![],
3336 }],
3337 attributes: Attributes::default(),
3338 }));
3339 let reader_record = Schema::Complex(ComplexType::Record(Record {
3340 name: "Root",
3341 namespace: None,
3342 doc: None,
3343 aliases: vec![],
3344 fields: vec![Field {
3345 name: "v",
3346 r#type: rs,
3347 default: None,
3348 doc: None,
3349 aliases: vec![],
3350 }],
3351 attributes: Attributes::default(),
3352 }));
3353 let res = AvroFieldBuilder::new(&writer_record)
3354 .with_reader_schema(&reader_record)
3355 .with_utf8view(false)
3356 .with_strict_mode(false)
3357 .build();
3358 assert!(res.is_err(), "expected error for illegal promotion");
3359 }
3360
3361 #[test]
3362 fn test_map_decoding_one_entry() {
3363 let value_type = avro_from_codec(Codec::Utf8);
3364 let map_type = avro_from_codec(Codec::Map(Arc::new(value_type)));
3365 let mut decoder = Decoder::try_new(&map_type).unwrap();
3366 let mut data = Vec::new();
3368 data.extend_from_slice(&encode_avro_long(1));
3369 data.extend_from_slice(&encode_avro_bytes(b"hello")); data.extend_from_slice(&encode_avro_bytes(b"world")); data.extend_from_slice(&encode_avro_long(0));
3372 let mut cursor = AvroCursor::new(&data);
3373 decoder.decode(&mut cursor).unwrap();
3374 let array = decoder.flush(None).unwrap();
3375 let map_arr = array.as_any().downcast_ref::<MapArray>().unwrap();
3376 assert_eq!(map_arr.len(), 1); assert_eq!(map_arr.value_length(0), 1);
3378 let entries = map_arr.value(0);
3379 let struct_entries = entries.as_any().downcast_ref::<StructArray>().unwrap();
3380 assert_eq!(struct_entries.len(), 1);
3381 let key_arr = struct_entries
3382 .column_by_name("key")
3383 .unwrap()
3384 .as_any()
3385 .downcast_ref::<StringArray>()
3386 .unwrap();
3387 let val_arr = struct_entries
3388 .column_by_name("value")
3389 .unwrap()
3390 .as_any()
3391 .downcast_ref::<StringArray>()
3392 .unwrap();
3393 assert_eq!(key_arr.value(0), "hello");
3394 assert_eq!(val_arr.value(0), "world");
3395 }
3396
3397 #[test]
3398 fn test_map_decoding_empty() {
3399 let value_type = avro_from_codec(Codec::Utf8);
3400 let map_type = avro_from_codec(Codec::Map(Arc::new(value_type)));
3401 let mut decoder = Decoder::try_new(&map_type).unwrap();
3402 let data = encode_avro_long(0);
3403 decoder.decode(&mut AvroCursor::new(&data)).unwrap();
3404 let array = decoder.flush(None).unwrap();
3405 let map_arr = array.as_any().downcast_ref::<MapArray>().unwrap();
3406 assert_eq!(map_arr.len(), 1);
3407 assert_eq!(map_arr.value_length(0), 0);
3408 }
3409
3410 #[test]
3411 fn test_fixed_decoding() {
3412 let avro_type = avro_from_codec(Codec::Fixed(3));
3413 let mut decoder = Decoder::try_new(&avro_type).expect("Failed to create decoder");
3414
3415 let data1 = [1u8, 2, 3];
3416 let mut cursor1 = AvroCursor::new(&data1);
3417 decoder
3418 .decode(&mut cursor1)
3419 .expect("Failed to decode data1");
3420 assert_eq!(cursor1.position(), 3, "Cursor should advance by fixed size");
3421 let data2 = [4u8, 5, 6];
3422 let mut cursor2 = AvroCursor::new(&data2);
3423 decoder
3424 .decode(&mut cursor2)
3425 .expect("Failed to decode data2");
3426 assert_eq!(cursor2.position(), 3, "Cursor should advance by fixed size");
3427 let array = decoder.flush(None).expect("Failed to flush decoder");
3428 assert_eq!(array.len(), 2, "Array should contain two items");
3429 let fixed_size_binary_array = array
3430 .as_any()
3431 .downcast_ref::<FixedSizeBinaryArray>()
3432 .expect("Failed to downcast to FixedSizeBinaryArray");
3433 assert_eq!(
3434 fixed_size_binary_array.value_length(),
3435 3,
3436 "Fixed size of binary values should be 3"
3437 );
3438 assert_eq!(
3439 fixed_size_binary_array.value(0),
3440 &[1, 2, 3],
3441 "First item mismatch"
3442 );
3443 assert_eq!(
3444 fixed_size_binary_array.value(1),
3445 &[4, 5, 6],
3446 "Second item mismatch"
3447 );
3448 }
3449
3450 #[test]
3451 fn test_fixed_decoding_empty() {
3452 let avro_type = avro_from_codec(Codec::Fixed(5));
3453 let mut decoder = Decoder::try_new(&avro_type).expect("Failed to create decoder");
3454
3455 let array = decoder
3456 .flush(None)
3457 .expect("Failed to flush decoder for empty input");
3458
3459 assert_eq!(array.len(), 0, "Array should be empty");
3460 let fixed_size_binary_array = array
3461 .as_any()
3462 .downcast_ref::<FixedSizeBinaryArray>()
3463 .expect("Failed to downcast to FixedSizeBinaryArray for empty array");
3464
3465 assert_eq!(
3466 fixed_size_binary_array.value_length(),
3467 5,
3468 "Fixed size of binary values should be 5 as per type"
3469 );
3470 }
3471
3472 #[test]
3473 fn test_uuid_decoding() {
3474 let avro_type = avro_from_codec(Codec::Uuid);
3475 let mut decoder = Decoder::try_new(&avro_type).expect("Failed to create decoder");
3476 let uuid_str = "f81d4fae-7dec-11d0-a765-00a0c91e6bf6";
3477 let data = encode_avro_bytes(uuid_str.as_bytes());
3478 let mut cursor = AvroCursor::new(&data);
3479 decoder.decode(&mut cursor).expect("Failed to decode data");
3480 assert_eq!(
3481 cursor.position(),
3482 data.len(),
3483 "Cursor should advance by varint size + data size"
3484 );
3485 let array = decoder.flush(None).expect("Failed to flush decoder");
3486 let fixed_size_binary_array = array
3487 .as_any()
3488 .downcast_ref::<FixedSizeBinaryArray>()
3489 .expect("Array should be a FixedSizeBinaryArray");
3490 assert_eq!(fixed_size_binary_array.len(), 1);
3491 assert_eq!(fixed_size_binary_array.value_length(), 16);
3492 let expected_bytes = [
3493 0xf8, 0x1d, 0x4f, 0xae, 0x7d, 0xec, 0x11, 0xd0, 0xa7, 0x65, 0x00, 0xa0, 0xc9, 0x1e,
3494 0x6b, 0xf6,
3495 ];
3496 assert_eq!(fixed_size_binary_array.value(0), &expected_bytes);
3497 }
3498
3499 #[test]
3500 fn test_array_decoding() {
3501 let item_dt = avro_from_codec(Codec::Int32);
3502 let list_dt = avro_from_codec(Codec::List(Arc::new(item_dt)));
3503 let mut decoder = Decoder::try_new(&list_dt).unwrap();
3504 let mut row1 = Vec::new();
3505 row1.extend_from_slice(&encode_avro_long(2));
3506 row1.extend_from_slice(&encode_avro_int(10));
3507 row1.extend_from_slice(&encode_avro_int(20));
3508 row1.extend_from_slice(&encode_avro_long(0));
3509 let row2 = encode_avro_long(0);
3510 let mut cursor = AvroCursor::new(&row1);
3511 decoder.decode(&mut cursor).unwrap();
3512 let mut cursor2 = AvroCursor::new(&row2);
3513 decoder.decode(&mut cursor2).unwrap();
3514 let array = decoder.flush(None).unwrap();
3515 let list_arr = array.as_any().downcast_ref::<ListArray>().unwrap();
3516 assert_eq!(list_arr.len(), 2);
3517 let offsets = list_arr.value_offsets();
3518 assert_eq!(offsets, &[0, 2, 2]);
3519 let values = list_arr.values();
3520 let int_arr = values.as_primitive::<Int32Type>();
3521 assert_eq!(int_arr.len(), 2);
3522 assert_eq!(int_arr.value(0), 10);
3523 assert_eq!(int_arr.value(1), 20);
3524 }
3525
3526 #[test]
3527 fn test_array_decoding_with_negative_block_count() {
3528 let item_dt = avro_from_codec(Codec::Int32);
3529 let list_dt = avro_from_codec(Codec::List(Arc::new(item_dt)));
3530 let mut decoder = Decoder::try_new(&list_dt).unwrap();
3531 let mut data = encode_avro_long(-3);
3532 data.extend_from_slice(&encode_avro_long(12));
3533 data.extend_from_slice(&encode_avro_int(1));
3534 data.extend_from_slice(&encode_avro_int(2));
3535 data.extend_from_slice(&encode_avro_int(3));
3536 data.extend_from_slice(&encode_avro_long(0));
3537 let mut cursor = AvroCursor::new(&data);
3538 decoder.decode(&mut cursor).unwrap();
3539 let array = decoder.flush(None).unwrap();
3540 let list_arr = array.as_any().downcast_ref::<ListArray>().unwrap();
3541 assert_eq!(list_arr.len(), 1);
3542 assert_eq!(list_arr.value_length(0), 3);
3543 let values = list_arr.values().as_primitive::<Int32Type>();
3544 assert_eq!(values.len(), 3);
3545 assert_eq!(values.value(0), 1);
3546 assert_eq!(values.value(1), 2);
3547 assert_eq!(values.value(2), 3);
3548 }
3549
3550 fn encode_avro_long_extreme(value: i64) -> Vec<u8> {
3553 let mut n = ((value << 1) ^ (value >> 63)) as u64;
3554 let mut out = Vec::new();
3555 while n >= 0x80 {
3556 out.push((n as u8) | 0x80);
3557 n >>= 7;
3558 }
3559 out.push(n as u8);
3560 out
3561 }
3562
3563 fn array_of_null_decoder() -> Decoder {
3566 let list_dt = avro_from_codec(Codec::List(Arc::new(avro_from_codec(Codec::Null))));
3567 Decoder::try_new(&list_dt).unwrap()
3568 }
3569
3570 #[test]
3571 fn test_array_of_null_decodes() {
3572 let mut decoder = array_of_null_decoder();
3573 let mut data = encode_avro_long(3); data.extend_from_slice(&encode_avro_long(0)); decoder.decode(&mut AvroCursor::new(&data)).unwrap();
3576 }
3577
3578 #[test]
3579 fn test_array_block_count_i64_max_errors() {
3580 let mut decoder = array_of_null_decoder();
3582 let mut data = encode_avro_long_extreme(i64::MAX); data.extend_from_slice(&encode_avro_long(0)); let err = decoder.decode(&mut AvroCursor::new(&data)).unwrap_err();
3585 assert!(
3586 err.to_string().contains("Capacity overflow"),
3587 "unexpected error: {err}",
3588 );
3589 }
3590
3591 #[test]
3592 fn test_array_block_count_i64_min_errors() {
3593 let mut decoder = array_of_null_decoder();
3595 let mut data = encode_avro_long_extreme(i64::MIN); data.extend_from_slice(&encode_avro_long(0)); let err = decoder.decode(&mut AvroCursor::new(&data)).unwrap_err();
3598 assert!(
3599 err.to_string().contains("Capacity overflow"),
3600 "unexpected error: {err}",
3601 );
3602 }
3603
3604 #[test]
3605 fn test_nested_array_decoding() {
3606 let inner_ty = avro_from_codec(Codec::List(Arc::new(avro_from_codec(Codec::Int32))));
3607 let nested_ty = avro_from_codec(Codec::List(Arc::new(inner_ty.clone())));
3608 let mut decoder = Decoder::try_new(&nested_ty).unwrap();
3609 let mut buf = Vec::new();
3610 buf.extend(encode_avro_long(1));
3611 buf.extend(encode_avro_long(2));
3612 buf.extend(encode_avro_int(5));
3613 buf.extend(encode_avro_int(6));
3614 buf.extend(encode_avro_long(0));
3615 buf.extend(encode_avro_long(0));
3616 let mut cursor = AvroCursor::new(&buf);
3617 decoder.decode(&mut cursor).unwrap();
3618 let arr = decoder.flush(None).unwrap();
3619 let outer = arr.as_any().downcast_ref::<ListArray>().unwrap();
3620 assert_eq!(outer.len(), 1);
3621 assert_eq!(outer.value_length(0), 1);
3622 let inner = outer.values().as_any().downcast_ref::<ListArray>().unwrap();
3623 assert_eq!(inner.len(), 1);
3624 assert_eq!(inner.value_length(0), 2);
3625 let values = inner
3626 .values()
3627 .as_any()
3628 .downcast_ref::<Int32Array>()
3629 .unwrap();
3630 assert_eq!(values.values(), &[5, 6]);
3631 }
3632
3633 #[test]
3634 fn test_array_decoding_empty_array() {
3635 let value_type = avro_from_codec(Codec::Utf8);
3636 let map_type = avro_from_codec(Codec::List(Arc::new(value_type)));
3637 let mut decoder = Decoder::try_new(&map_type).unwrap();
3638 let data = encode_avro_long(0);
3639 decoder.decode(&mut AvroCursor::new(&data)).unwrap();
3640 let array = decoder.flush(None).unwrap();
3641 let list_arr = array.as_any().downcast_ref::<ListArray>().unwrap();
3642 assert_eq!(list_arr.len(), 1);
3643 assert_eq!(list_arr.value_length(0), 0);
3644 }
3645
3646 #[test]
3647 fn test_array_decoding_writer_nonunion_items_reader_nullable_items() {
3648 use crate::schema::Array;
3649 let writer_schema = Schema::Complex(ComplexType::Array(Array {
3650 items: Box::new(Schema::TypeName(TypeName::Primitive(PrimitiveType::Int))),
3651 attributes: Attributes::default(),
3652 }));
3653 let reader_schema = Schema::Complex(ComplexType::Array(Array {
3654 items: Box::new(Schema::Union(vec![
3655 Schema::TypeName(TypeName::Primitive(PrimitiveType::Null)),
3656 Schema::TypeName(TypeName::Primitive(PrimitiveType::Int)),
3657 ])),
3658 attributes: Attributes::default(),
3659 }));
3660 let dt = resolved_root_datatype(writer_schema, reader_schema, false, false);
3661 if let Codec::List(inner) = dt.codec() {
3662 assert_eq!(
3663 inner.nullability(),
3664 Some(Nullability::NullFirst),
3665 "items should be nullable"
3666 );
3667 } else {
3668 panic!("expected List codec");
3669 }
3670 let mut decoder = Decoder::try_new(&dt).unwrap();
3671 let mut data = encode_avro_long(2);
3672 data.extend(encode_avro_int(10));
3673 data.extend(encode_avro_int(20));
3674 data.extend(encode_avro_long(0));
3675 let mut cursor = AvroCursor::new(&data);
3676 decoder.decode(&mut cursor).unwrap();
3677 assert_eq!(
3678 cursor.position(),
3679 data.len(),
3680 "all bytes should be consumed"
3681 );
3682 let array = decoder.flush(None).unwrap();
3683 let list_arr = array.as_any().downcast_ref::<ListArray>().unwrap();
3684 assert_eq!(list_arr.len(), 1, "one list/row");
3685 assert_eq!(list_arr.value_length(0), 2, "two items in the list");
3686 let values = list_arr.values().as_primitive::<Int32Type>();
3687 assert_eq!(values.len(), 2);
3688 assert_eq!(values.value(0), 10);
3689 assert_eq!(values.value(1), 20);
3690 assert!(!values.is_null(0));
3691 assert!(!values.is_null(1));
3692 }
3693
3694 #[test]
3695 fn test_decimal_decoding_fixed256() {
3696 let dt = avro_from_codec(Codec::Decimal(50, Some(2), Some(32)));
3697 let mut decoder = Decoder::try_new(&dt).unwrap();
3698 let row1 = [
3699 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
3700 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
3701 0x00, 0x00, 0x30, 0x39,
3702 ];
3703 let row2 = [
3704 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
3705 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
3706 0xFF, 0xFF, 0xFF, 0x85,
3707 ];
3708 let mut data = Vec::new();
3709 data.extend_from_slice(&row1);
3710 data.extend_from_slice(&row2);
3711 let mut cursor = AvroCursor::new(&data);
3712 decoder.decode(&mut cursor).unwrap();
3713 decoder.decode(&mut cursor).unwrap();
3714 let arr = decoder.flush(None).unwrap();
3715 let dec = arr.as_any().downcast_ref::<Decimal256Array>().unwrap();
3716 assert_eq!(dec.len(), 2);
3717 assert_eq!(dec.value_as_string(0), "123.45");
3718 assert_eq!(dec.value_as_string(1), "-1.23");
3719 }
3720
3721 #[test]
3722 fn test_decimal_decoding_fixed128() {
3723 let dt = avro_from_codec(Codec::Decimal(28, Some(2), Some(16)));
3724 let mut decoder = Decoder::try_new(&dt).unwrap();
3725 let row1 = [
3726 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
3727 0x30, 0x39,
3728 ];
3729 let row2 = [
3730 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
3731 0xFF, 0x85,
3732 ];
3733 let mut data = Vec::new();
3734 data.extend_from_slice(&row1);
3735 data.extend_from_slice(&row2);
3736 let mut cursor = AvroCursor::new(&data);
3737 decoder.decode(&mut cursor).unwrap();
3738 decoder.decode(&mut cursor).unwrap();
3739 let arr = decoder.flush(None).unwrap();
3740 let dec = arr.as_any().downcast_ref::<Decimal128Array>().unwrap();
3741 assert_eq!(dec.len(), 2);
3742 assert_eq!(dec.value_as_string(0), "123.45");
3743 assert_eq!(dec.value_as_string(1), "-1.23");
3744 }
3745
3746 #[test]
3747 fn test_decimal_decoding_fixed32_from_32byte_fixed_storage() {
3748 let dt = avro_from_codec(Codec::Decimal(5, Some(2), Some(32)));
3749 let mut decoder = Decoder::try_new(&dt).unwrap();
3750 let row1 = [
3751 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
3752 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
3753 0x00, 0x00, 0x30, 0x39,
3754 ];
3755 let row2 = [
3756 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
3757 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
3758 0xFF, 0xFF, 0xFF, 0x85,
3759 ];
3760 let mut data = Vec::new();
3761 data.extend_from_slice(&row1);
3762 data.extend_from_slice(&row2);
3763 let mut cursor = AvroCursor::new(&data);
3764 decoder.decode(&mut cursor).unwrap();
3765 decoder.decode(&mut cursor).unwrap();
3766 let arr = decoder.flush(None).unwrap();
3767 #[cfg(feature = "small_decimals")]
3768 {
3769 let dec = arr.as_any().downcast_ref::<Decimal32Array>().unwrap();
3770 assert_eq!(dec.len(), 2);
3771 assert_eq!(dec.value_as_string(0), "123.45");
3772 assert_eq!(dec.value_as_string(1), "-1.23");
3773 }
3774 #[cfg(not(feature = "small_decimals"))]
3775 {
3776 let dec = arr.as_any().downcast_ref::<Decimal128Array>().unwrap();
3777 assert_eq!(dec.len(), 2);
3778 assert_eq!(dec.value_as_string(0), "123.45");
3779 assert_eq!(dec.value_as_string(1), "-1.23");
3780 }
3781 }
3782
3783 #[test]
3784 fn test_decimal_decoding_fixed32_from_16byte_fixed_storage() {
3785 let dt = avro_from_codec(Codec::Decimal(5, Some(2), Some(16)));
3786 let mut decoder = Decoder::try_new(&dt).unwrap();
3787 let row1 = [
3788 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
3789 0x30, 0x39,
3790 ];
3791 let row2 = [
3792 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
3793 0xFF, 0x85,
3794 ];
3795 let mut data = Vec::new();
3796 data.extend_from_slice(&row1);
3797 data.extend_from_slice(&row2);
3798 let mut cursor = AvroCursor::new(&data);
3799 decoder.decode(&mut cursor).unwrap();
3800 decoder.decode(&mut cursor).unwrap();
3801
3802 let arr = decoder.flush(None).unwrap();
3803 #[cfg(feature = "small_decimals")]
3804 {
3805 let dec = arr.as_any().downcast_ref::<Decimal32Array>().unwrap();
3806 assert_eq!(dec.len(), 2);
3807 assert_eq!(dec.value_as_string(0), "123.45");
3808 assert_eq!(dec.value_as_string(1), "-1.23");
3809 }
3810 #[cfg(not(feature = "small_decimals"))]
3811 {
3812 let dec = arr.as_any().downcast_ref::<Decimal128Array>().unwrap();
3813 assert_eq!(dec.len(), 2);
3814 assert_eq!(dec.value_as_string(0), "123.45");
3815 assert_eq!(dec.value_as_string(1), "-1.23");
3816 }
3817 }
3818
3819 #[test]
3820 fn test_decimal_decoding_bytes_with_nulls() {
3821 let dt = avro_from_codec(Codec::Decimal(4, Some(1), None));
3822 let inner = Decoder::try_new(&dt).unwrap();
3823 let mut decoder = Decoder::Nullable(NullableDecoder::new(
3824 NullablePlan::ReadTag {
3825 nullability: Nullability::NullSecond,
3826 resolution: ResolutionPlan::Promotion(Promotion::Direct),
3827 },
3828 inner,
3829 ));
3830 let mut data = Vec::new();
3831 data.extend_from_slice(&encode_avro_int(0));
3832 data.extend_from_slice(&encode_avro_bytes(&[0x04, 0xD2]));
3833 data.extend_from_slice(&encode_avro_int(1));
3834 data.extend_from_slice(&encode_avro_int(0));
3835 data.extend_from_slice(&encode_avro_bytes(&[0xFB, 0x2E]));
3836 let mut cursor = AvroCursor::new(&data);
3837 decoder.decode(&mut cursor).unwrap();
3838 decoder.decode(&mut cursor).unwrap();
3839 decoder.decode(&mut cursor).unwrap();
3840 let arr = decoder.flush(None).unwrap();
3841 #[cfg(feature = "small_decimals")]
3842 {
3843 let dec_arr = arr.as_any().downcast_ref::<Decimal32Array>().unwrap();
3844 assert_eq!(dec_arr.len(), 3);
3845 assert!(dec_arr.is_valid(0));
3846 assert!(!dec_arr.is_valid(1));
3847 assert!(dec_arr.is_valid(2));
3848 assert_eq!(dec_arr.value_as_string(0), "123.4");
3849 assert_eq!(dec_arr.value_as_string(2), "-123.4");
3850 }
3851 #[cfg(not(feature = "small_decimals"))]
3852 {
3853 let dec_arr = arr.as_any().downcast_ref::<Decimal128Array>().unwrap();
3854 assert_eq!(dec_arr.len(), 3);
3855 assert!(dec_arr.is_valid(0));
3856 assert!(!dec_arr.is_valid(1));
3857 assert!(dec_arr.is_valid(2));
3858 assert_eq!(dec_arr.value_as_string(0), "123.4");
3859 assert_eq!(dec_arr.value_as_string(2), "-123.4");
3860 }
3861 }
3862
3863 #[test]
3864 fn test_decimal_decoding_bytes_with_nulls_fixed_size_narrow_result() {
3865 let dt = avro_from_codec(Codec::Decimal(6, Some(2), Some(16)));
3866 let inner = Decoder::try_new(&dt).unwrap();
3867 let mut decoder = Decoder::Nullable(NullableDecoder::new(
3868 NullablePlan::ReadTag {
3869 nullability: Nullability::NullSecond,
3870 resolution: ResolutionPlan::Promotion(Promotion::Direct),
3871 },
3872 inner,
3873 ));
3874 let row1 = [
3875 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
3876 0xE2, 0x40,
3877 ];
3878 let row3 = [
3879 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE,
3880 0x1D, 0xC0,
3881 ];
3882 let mut data = Vec::new();
3883 data.extend_from_slice(&encode_avro_int(0));
3884 data.extend_from_slice(&row1);
3885 data.extend_from_slice(&encode_avro_int(1));
3886 data.extend_from_slice(&encode_avro_int(0));
3887 data.extend_from_slice(&row3);
3888 let mut cursor = AvroCursor::new(&data);
3889 decoder.decode(&mut cursor).unwrap();
3890 decoder.decode(&mut cursor).unwrap();
3891 decoder.decode(&mut cursor).unwrap();
3892 let arr = decoder.flush(None).unwrap();
3893 #[cfg(feature = "small_decimals")]
3894 {
3895 let dec_arr = arr.as_any().downcast_ref::<Decimal32Array>().unwrap();
3896 assert_eq!(dec_arr.len(), 3);
3897 assert!(dec_arr.is_valid(0));
3898 assert!(!dec_arr.is_valid(1));
3899 assert!(dec_arr.is_valid(2));
3900 assert_eq!(dec_arr.value_as_string(0), "1234.56");
3901 assert_eq!(dec_arr.value_as_string(2), "-1234.56");
3902 }
3903 #[cfg(not(feature = "small_decimals"))]
3904 {
3905 let dec_arr = arr.as_any().downcast_ref::<Decimal128Array>().unwrap();
3906 assert_eq!(dec_arr.len(), 3);
3907 assert!(dec_arr.is_valid(0));
3908 assert!(!dec_arr.is_valid(1));
3909 assert!(dec_arr.is_valid(2));
3910 assert_eq!(dec_arr.value_as_string(0), "1234.56");
3911 assert_eq!(dec_arr.value_as_string(2), "-1234.56");
3912 }
3913 }
3914
3915 #[test]
3916 fn test_enum_decoding() {
3917 let symbols: Arc<[String]> = vec!["A", "B", "C"].into_iter().map(String::from).collect();
3918 let avro_type = avro_from_codec(Codec::Enum(symbols.clone()));
3919 let mut decoder = Decoder::try_new(&avro_type).unwrap();
3920 let mut data = Vec::new();
3921 data.extend_from_slice(&encode_avro_int(2));
3922 data.extend_from_slice(&encode_avro_int(0));
3923 data.extend_from_slice(&encode_avro_int(1));
3924 let mut cursor = AvroCursor::new(&data);
3925 decoder.decode(&mut cursor).unwrap();
3926 decoder.decode(&mut cursor).unwrap();
3927 decoder.decode(&mut cursor).unwrap();
3928 let array = decoder.flush(None).unwrap();
3929 let dict_array = array
3930 .as_any()
3931 .downcast_ref::<DictionaryArray<Int32Type>>()
3932 .unwrap();
3933 assert_eq!(dict_array.len(), 3);
3934 let values = dict_array
3935 .values()
3936 .as_any()
3937 .downcast_ref::<StringArray>()
3938 .unwrap();
3939 assert_eq!(values.value(0), "A");
3940 assert_eq!(values.value(1), "B");
3941 assert_eq!(values.value(2), "C");
3942 assert_eq!(dict_array.keys().values(), &[2, 0, 1]);
3943 }
3944
3945 #[test]
3946 fn test_enum_decoding_with_nulls() {
3947 let symbols: Arc<[String]> = vec!["X", "Y"].into_iter().map(String::from).collect();
3948 let enum_codec = Codec::Enum(symbols.clone());
3949 let avro_type =
3950 AvroDataType::new(enum_codec, Default::default(), Some(Nullability::NullFirst));
3951 let mut decoder = Decoder::try_new(&avro_type).unwrap();
3952 let mut data = Vec::new();
3953 data.extend_from_slice(&encode_avro_long(1));
3954 data.extend_from_slice(&encode_avro_int(1));
3955 data.extend_from_slice(&encode_avro_long(0));
3956 data.extend_from_slice(&encode_avro_long(1));
3957 data.extend_from_slice(&encode_avro_int(0));
3958 let mut cursor = AvroCursor::new(&data);
3959 decoder.decode(&mut cursor).unwrap();
3960 decoder.decode(&mut cursor).unwrap();
3961 decoder.decode(&mut cursor).unwrap();
3962 let array = decoder.flush(None).unwrap();
3963 let dict_array = array
3964 .as_any()
3965 .downcast_ref::<DictionaryArray<Int32Type>>()
3966 .unwrap();
3967 assert_eq!(dict_array.len(), 3);
3968 assert!(dict_array.is_valid(0));
3969 assert!(dict_array.is_null(1));
3970 assert!(dict_array.is_valid(2));
3971 let expected_keys = Int32Array::from(vec![Some(1), None, Some(0)]);
3972 assert_eq!(dict_array.keys(), &expected_keys);
3973 let values = dict_array
3974 .values()
3975 .as_any()
3976 .downcast_ref::<StringArray>()
3977 .unwrap();
3978 assert_eq!(values.value(0), "X");
3979 assert_eq!(values.value(1), "Y");
3980 }
3981
3982 #[test]
3983 fn test_duration_decoding_with_nulls() {
3984 let duration_codec = Codec::Interval;
3985 let avro_type = AvroDataType::new(
3986 duration_codec,
3987 Default::default(),
3988 Some(Nullability::NullFirst),
3989 );
3990 let mut decoder = Decoder::try_new(&avro_type).unwrap();
3991 let mut data = Vec::new();
3992 data.extend_from_slice(&encode_avro_long(1)); let mut duration1 = Vec::new();
3995 duration1.extend_from_slice(&1u32.to_le_bytes());
3996 duration1.extend_from_slice(&2u32.to_le_bytes());
3997 duration1.extend_from_slice(&3u32.to_le_bytes());
3998 data.extend_from_slice(&duration1);
3999 data.extend_from_slice(&encode_avro_long(0)); data.extend_from_slice(&encode_avro_long(1)); let mut duration2 = Vec::new();
4003 duration2.extend_from_slice(&4u32.to_le_bytes());
4004 duration2.extend_from_slice(&5u32.to_le_bytes());
4005 duration2.extend_from_slice(&6u32.to_le_bytes());
4006 data.extend_from_slice(&duration2);
4007 let mut cursor = AvroCursor::new(&data);
4008 decoder.decode(&mut cursor).unwrap();
4009 decoder.decode(&mut cursor).unwrap();
4010 decoder.decode(&mut cursor).unwrap();
4011 let array = decoder.flush(None).unwrap();
4012 let interval_array = array
4013 .as_any()
4014 .downcast_ref::<IntervalMonthDayNanoArray>()
4015 .unwrap();
4016 assert_eq!(interval_array.len(), 3);
4017 assert!(interval_array.is_valid(0));
4018 assert!(interval_array.is_null(1));
4019 assert!(interval_array.is_valid(2));
4020 let expected = IntervalMonthDayNanoArray::from(vec![
4021 Some(IntervalMonthDayNano {
4022 months: 1,
4023 days: 2,
4024 nanoseconds: 3_000_000,
4025 }),
4026 None,
4027 Some(IntervalMonthDayNano {
4028 months: 4,
4029 days: 5,
4030 nanoseconds: 6_000_000,
4031 }),
4032 ]);
4033 assert_eq!(interval_array, &expected);
4034 }
4035
4036 #[cfg(feature = "avro_custom_types")]
4037 #[test]
4038 fn test_interval_month_day_nano_custom_decoding_with_nulls() {
4039 let avro_type = AvroDataType::new(
4040 Codec::IntervalMonthDayNano,
4041 Default::default(),
4042 Some(Nullability::NullFirst),
4043 );
4044 let mut decoder = Decoder::try_new(&avro_type).unwrap();
4045 let mut data = Vec::new();
4046 data.extend_from_slice(&encode_avro_long(1));
4048 data.extend_from_slice(&1i32.to_le_bytes());
4049 data.extend_from_slice(&(-2i32).to_le_bytes());
4050 data.extend_from_slice(&3i64.to_le_bytes());
4051 data.extend_from_slice(&encode_avro_long(0));
4053 data.extend_from_slice(&encode_avro_long(1));
4055 data.extend_from_slice(&(-4i32).to_le_bytes());
4056 data.extend_from_slice(&5i32.to_le_bytes());
4057 data.extend_from_slice(&(-6i64).to_le_bytes());
4058 let mut cursor = AvroCursor::new(&data);
4059 decoder.decode(&mut cursor).unwrap();
4060 decoder.decode(&mut cursor).unwrap();
4061 decoder.decode(&mut cursor).unwrap();
4062 let array = decoder.flush(None).unwrap();
4063 let interval_array = array
4064 .as_any()
4065 .downcast_ref::<IntervalMonthDayNanoArray>()
4066 .unwrap();
4067 assert_eq!(interval_array.len(), 3);
4068 let expected = IntervalMonthDayNanoArray::from(vec![
4069 Some(IntervalMonthDayNano::new(1, -2, 3)),
4070 None,
4071 Some(IntervalMonthDayNano::new(-4, 5, -6)),
4072 ]);
4073 assert_eq!(interval_array, &expected);
4074 }
4075
4076 #[test]
4077 fn test_duration_decoding_empty() {
4078 let duration_codec = Codec::Interval;
4079 let avro_type = AvroDataType::new(duration_codec, Default::default(), None);
4080 let mut decoder = Decoder::try_new(&avro_type).unwrap();
4081 let array = decoder.flush(None).unwrap();
4082 assert_eq!(array.len(), 0);
4083 }
4084
4085 #[test]
4086 #[cfg(feature = "avro_custom_types")]
4087 fn test_duration_seconds_decoding() {
4088 let avro_type = AvroDataType::new(Codec::DurationSeconds, Default::default(), None);
4089 let mut decoder = Decoder::try_new(&avro_type).unwrap();
4090 let mut data = Vec::new();
4091 data.extend_from_slice(&encode_avro_long(0));
4093 data.extend_from_slice(&encode_avro_long(-1));
4094 data.extend_from_slice(&encode_avro_long(2));
4095 let mut cursor = AvroCursor::new(&data);
4096 decoder.decode(&mut cursor).unwrap();
4097 decoder.decode(&mut cursor).unwrap();
4098 decoder.decode(&mut cursor).unwrap();
4099 let array = decoder.flush(None).unwrap();
4100 let dur = array
4101 .as_any()
4102 .downcast_ref::<DurationSecondArray>()
4103 .unwrap();
4104 assert_eq!(dur.values(), &[0, -1, 2]);
4105 }
4106
4107 #[test]
4108 #[cfg(feature = "avro_custom_types")]
4109 fn test_duration_milliseconds_decoding() {
4110 let avro_type = AvroDataType::new(Codec::DurationMillis, Default::default(), None);
4111 let mut decoder = Decoder::try_new(&avro_type).unwrap();
4112 let mut data = Vec::new();
4113 for v in [1i64, 0, -2] {
4114 data.extend_from_slice(&encode_avro_long(v));
4115 }
4116 let mut cursor = AvroCursor::new(&data);
4117 for _ in 0..3 {
4118 decoder.decode(&mut cursor).unwrap();
4119 }
4120 let array = decoder.flush(None).unwrap();
4121 let dur = array
4122 .as_any()
4123 .downcast_ref::<DurationMillisecondArray>()
4124 .unwrap();
4125 assert_eq!(dur.values(), &[1, 0, -2]);
4126 }
4127
4128 #[test]
4129 #[cfg(feature = "avro_custom_types")]
4130 fn test_duration_microseconds_decoding() {
4131 let avro_type = AvroDataType::new(Codec::DurationMicros, Default::default(), None);
4132 let mut decoder = Decoder::try_new(&avro_type).unwrap();
4133 let mut data = Vec::new();
4134 for v in [5i64, -6, 7] {
4135 data.extend_from_slice(&encode_avro_long(v));
4136 }
4137 let mut cursor = AvroCursor::new(&data);
4138 for _ in 0..3 {
4139 decoder.decode(&mut cursor).unwrap();
4140 }
4141 let array = decoder.flush(None).unwrap();
4142 let dur = array
4143 .as_any()
4144 .downcast_ref::<DurationMicrosecondArray>()
4145 .unwrap();
4146 assert_eq!(dur.values(), &[5, -6, 7]);
4147 }
4148
4149 #[test]
4150 #[cfg(feature = "avro_custom_types")]
4151 fn test_duration_nanoseconds_decoding() {
4152 let avro_type = AvroDataType::new(Codec::DurationNanos, Default::default(), None);
4153 let mut decoder = Decoder::try_new(&avro_type).unwrap();
4154 let mut data = Vec::new();
4155 for v in [8i64, 9, -10] {
4156 data.extend_from_slice(&encode_avro_long(v));
4157 }
4158 let mut cursor = AvroCursor::new(&data);
4159 for _ in 0..3 {
4160 decoder.decode(&mut cursor).unwrap();
4161 }
4162 let array = decoder.flush(None).unwrap();
4163 let dur = array
4164 .as_any()
4165 .downcast_ref::<DurationNanosecondArray>()
4166 .unwrap();
4167 assert_eq!(dur.values(), &[8, 9, -10]);
4168 }
4169
4170 #[test]
4171 fn test_nullable_decode_error_bitmap_corruption() {
4172 let avro_type = AvroDataType::new(
4174 Codec::Int32,
4175 Default::default(),
4176 Some(Nullability::NullSecond),
4177 );
4178 let mut decoder = Decoder::try_new(&avro_type).unwrap();
4179
4180 let mut row1 = Vec::new();
4182 row1.extend_from_slice(&encode_avro_int(1));
4183
4184 let mut row2 = Vec::new();
4186 row2.extend_from_slice(&encode_avro_int(0)); let mut row3 = Vec::new();
4190 row3.extend_from_slice(&encode_avro_int(0)); row3.extend_from_slice(&encode_avro_int(42)); decoder.decode(&mut AvroCursor::new(&row1)).unwrap();
4194 assert!(decoder.decode(&mut AvroCursor::new(&row2)).is_err()); decoder.decode(&mut AvroCursor::new(&row3)).unwrap();
4196
4197 let array = decoder.flush(None).unwrap();
4198
4199 assert_eq!(array.len(), 2);
4201 let int_array = array.as_any().downcast_ref::<Int32Array>().unwrap();
4202 assert!(int_array.is_null(0)); assert_eq!(int_array.value(1), 42); }
4205
4206 #[test]
4207 fn test_enum_mapping_reordered_symbols() {
4208 let reader_symbols: Arc<[String]> =
4209 vec!["B".to_string(), "C".to_string(), "A".to_string()].into();
4210 let mapping: Arc<[i32]> = Arc::from(vec![2, 0, 1]);
4211 let default_index: i32 = -1;
4212 let mut dec = Decoder::Enum(
4213 Vec::with_capacity(DEFAULT_CAPACITY),
4214 reader_symbols.clone(),
4215 Some(EnumResolution {
4216 mapping,
4217 default_index,
4218 }),
4219 );
4220 let mut data = Vec::new();
4221 data.extend_from_slice(&encode_avro_int(0));
4222 data.extend_from_slice(&encode_avro_int(1));
4223 data.extend_from_slice(&encode_avro_int(2));
4224 let mut cur = AvroCursor::new(&data);
4225 dec.decode(&mut cur).unwrap();
4226 dec.decode(&mut cur).unwrap();
4227 dec.decode(&mut cur).unwrap();
4228 let arr = dec.flush(None).unwrap();
4229 let dict = arr
4230 .as_any()
4231 .downcast_ref::<DictionaryArray<Int32Type>>()
4232 .unwrap();
4233 let expected_keys = Int32Array::from(vec![2, 0, 1]);
4234 assert_eq!(dict.keys(), &expected_keys);
4235 let values = dict
4236 .values()
4237 .as_any()
4238 .downcast_ref::<StringArray>()
4239 .unwrap();
4240 assert_eq!(values.value(0), "B");
4241 assert_eq!(values.value(1), "C");
4242 assert_eq!(values.value(2), "A");
4243 }
4244
4245 #[test]
4246 fn test_enum_mapping_unknown_symbol_and_out_of_range_fall_back_to_default() {
4247 let reader_symbols: Arc<[String]> = vec!["A".to_string(), "B".to_string()].into();
4248 let default_index: i32 = 1;
4249 let mapping: Arc<[i32]> = Arc::from(vec![0, 1]);
4250 let mut dec = Decoder::Enum(
4251 Vec::with_capacity(DEFAULT_CAPACITY),
4252 reader_symbols.clone(),
4253 Some(EnumResolution {
4254 mapping,
4255 default_index,
4256 }),
4257 );
4258 let mut data = Vec::new();
4259 data.extend_from_slice(&encode_avro_int(0));
4260 data.extend_from_slice(&encode_avro_int(1));
4261 data.extend_from_slice(&encode_avro_int(99));
4262 let mut cur = AvroCursor::new(&data);
4263 dec.decode(&mut cur).unwrap();
4264 dec.decode(&mut cur).unwrap();
4265 dec.decode(&mut cur).unwrap();
4266 let arr = dec.flush(None).unwrap();
4267 let dict = arr
4268 .as_any()
4269 .downcast_ref::<DictionaryArray<Int32Type>>()
4270 .unwrap();
4271 let expected_keys = Int32Array::from(vec![0, 1, 1]);
4272 assert_eq!(dict.keys(), &expected_keys);
4273 let values = dict
4274 .values()
4275 .as_any()
4276 .downcast_ref::<StringArray>()
4277 .unwrap();
4278 assert_eq!(values.value(0), "A");
4279 assert_eq!(values.value(1), "B");
4280 }
4281
4282 #[test]
4283 fn test_enum_mapping_unknown_symbol_without_default_errors() {
4284 let reader_symbols: Arc<[String]> = vec!["A".to_string()].into();
4285 let default_index: i32 = -1; let mapping: Arc<[i32]> = Arc::from(vec![-1]);
4287 let mut dec = Decoder::Enum(
4288 Vec::with_capacity(DEFAULT_CAPACITY),
4289 reader_symbols,
4290 Some(EnumResolution {
4291 mapping,
4292 default_index,
4293 }),
4294 );
4295 let data = encode_avro_int(0);
4296 let mut cur = AvroCursor::new(&data);
4297 let err = dec
4298 .decode(&mut cur)
4299 .expect_err("expected decode error for unresolved enum without default");
4300 let msg = err.to_string();
4301 assert!(
4302 msg.contains("not resolvable") && msg.contains("no default"),
4303 "unexpected error message: {msg}"
4304 );
4305 }
4306
4307 fn make_record_resolved_decoder(
4308 reader_fields: &[(&str, DataType, bool)],
4309 writer_projections: Vec<FieldProjection>,
4310 ) -> Decoder {
4311 let mut field_refs: Vec<FieldRef> = Vec::with_capacity(reader_fields.len());
4312 let mut encodings: Vec<Decoder> = Vec::with_capacity(reader_fields.len());
4313 for (name, dt, nullable) in reader_fields {
4314 field_refs.push(Arc::new(ArrowField::new(*name, dt.clone(), *nullable)));
4315 let enc = match dt {
4316 DataType::Int32 => Decoder::Int32(Vec::new()),
4317 DataType::Int64 => Decoder::Int64(Vec::new()),
4318 DataType::Utf8 => {
4319 Decoder::String(OffsetBufferBuilder::new(DEFAULT_CAPACITY), Vec::new())
4320 }
4321 other => panic!("Unsupported test reader field type: {other:?}"),
4322 };
4323 encodings.push(enc);
4324 }
4325 let fields: Fields = field_refs.into();
4326 Decoder::Record {
4327 fields,
4328 decoders: encodings,
4329 defaults: vec![None; reader_fields.len()],
4330 projector: Some(Projector {
4331 writer_projections,
4332 default_injections: Arc::from(Vec::<(usize, AvroLiteral)>::new()),
4333 }),
4334 row_count: 0,
4335 }
4336 }
4337
4338 #[test]
4339 fn test_skip_writer_trailing_field_int32() {
4340 let mut dec = make_record_resolved_decoder(
4341 &[("id", arrow_schema::DataType::Int32, false)],
4342 vec![
4343 FieldProjection::ToReader(0),
4344 FieldProjection::Skip(super::Skipper::Int32),
4345 ],
4346 );
4347 let mut data = Vec::new();
4348 data.extend_from_slice(&encode_avro_int(7));
4349 data.extend_from_slice(&encode_avro_int(999));
4350 let mut cur = AvroCursor::new(&data);
4351 dec.decode(&mut cur).unwrap();
4352 assert_eq!(cur.position(), data.len());
4353 let arr = dec.flush(None).unwrap();
4354 let struct_arr = arr.as_any().downcast_ref::<StructArray>().unwrap();
4355 assert_eq!(struct_arr.len(), 1);
4356 let id = struct_arr
4357 .column_by_name("id")
4358 .unwrap()
4359 .as_any()
4360 .downcast_ref::<Int32Array>()
4361 .unwrap();
4362 assert_eq!(id.value(0), 7);
4363 }
4364
4365 #[test]
4366 fn test_skip_writer_middle_field_string() {
4367 let mut dec = make_record_resolved_decoder(
4368 &[
4369 ("id", DataType::Int32, false),
4370 ("score", DataType::Int64, false),
4371 ],
4372 vec![
4373 FieldProjection::ToReader(0),
4374 FieldProjection::Skip(Skipper::String),
4375 FieldProjection::ToReader(1),
4376 ],
4377 );
4378 let mut data = Vec::new();
4379 data.extend_from_slice(&encode_avro_int(42));
4380 data.extend_from_slice(&encode_avro_bytes(b"abcdef"));
4381 data.extend_from_slice(&encode_avro_long(1000));
4382 let mut cur = AvroCursor::new(&data);
4383 dec.decode(&mut cur).unwrap();
4384 assert_eq!(cur.position(), data.len());
4385 let arr = dec.flush(None).unwrap();
4386 let s = arr.as_any().downcast_ref::<StructArray>().unwrap();
4387 let id = s
4388 .column_by_name("id")
4389 .unwrap()
4390 .as_any()
4391 .downcast_ref::<Int32Array>()
4392 .unwrap();
4393 let score = s
4394 .column_by_name("score")
4395 .unwrap()
4396 .as_any()
4397 .downcast_ref::<Int64Array>()
4398 .unwrap();
4399 assert_eq!(id.value(0), 42);
4400 assert_eq!(score.value(0), 1000);
4401 }
4402
4403 #[test]
4404 fn test_skip_writer_array_with_negative_block_count_fast() {
4405 let mut dec = make_record_resolved_decoder(
4406 &[("id", DataType::Int32, false)],
4407 vec![
4408 FieldProjection::Skip(super::Skipper::List(Box::new(Skipper::Int32))),
4409 FieldProjection::ToReader(0),
4410 ],
4411 );
4412 let mut array_payload = Vec::new();
4413 array_payload.extend_from_slice(&encode_avro_int(1));
4414 array_payload.extend_from_slice(&encode_avro_int(2));
4415 array_payload.extend_from_slice(&encode_avro_int(3));
4416 let mut data = Vec::new();
4417 data.extend_from_slice(&encode_avro_long(-3));
4418 data.extend_from_slice(&encode_avro_long(array_payload.len() as i64));
4419 data.extend_from_slice(&array_payload);
4420 data.extend_from_slice(&encode_avro_long(0));
4421 data.extend_from_slice(&encode_avro_int(5));
4422 let mut cur = AvroCursor::new(&data);
4423 dec.decode(&mut cur).unwrap();
4424 assert_eq!(cur.position(), data.len());
4425 let arr = dec.flush(None).unwrap();
4426 let s = arr.as_any().downcast_ref::<StructArray>().unwrap();
4427 let id = s
4428 .column_by_name("id")
4429 .unwrap()
4430 .as_any()
4431 .downcast_ref::<Int32Array>()
4432 .unwrap();
4433 assert_eq!(id.len(), 1);
4434 assert_eq!(id.value(0), 5);
4435 }
4436
4437 #[test]
4438 fn test_skip_writer_map_with_negative_block_count_fast() {
4439 let mut dec = make_record_resolved_decoder(
4440 &[("id", DataType::Int32, false)],
4441 vec![
4442 FieldProjection::Skip(Skipper::Map(Box::new(Skipper::Int32))),
4443 FieldProjection::ToReader(0),
4444 ],
4445 );
4446 let mut entries = Vec::new();
4447 entries.extend_from_slice(&encode_avro_bytes(b"k1"));
4448 entries.extend_from_slice(&encode_avro_int(10));
4449 entries.extend_from_slice(&encode_avro_bytes(b"k2"));
4450 entries.extend_from_slice(&encode_avro_int(20));
4451 let mut data = Vec::new();
4452 data.extend_from_slice(&encode_avro_long(-2));
4453 data.extend_from_slice(&encode_avro_long(entries.len() as i64));
4454 data.extend_from_slice(&entries);
4455 data.extend_from_slice(&encode_avro_long(0));
4456 data.extend_from_slice(&encode_avro_int(123));
4457 let mut cur = AvroCursor::new(&data);
4458 dec.decode(&mut cur).unwrap();
4459 assert_eq!(cur.position(), data.len());
4460 let arr = dec.flush(None).unwrap();
4461 let s = arr.as_any().downcast_ref::<StructArray>().unwrap();
4462 let id = s
4463 .column_by_name("id")
4464 .unwrap()
4465 .as_any()
4466 .downcast_ref::<Int32Array>()
4467 .unwrap();
4468 assert_eq!(id.len(), 1);
4469 assert_eq!(id.value(0), 123);
4470 }
4471
4472 #[test]
4473 fn test_skip_writer_nullable_field_union_nullfirst() {
4474 let mut dec = make_record_resolved_decoder(
4475 &[("id", DataType::Int32, false)],
4476 vec![
4477 FieldProjection::Skip(super::Skipper::Nullable(
4478 Nullability::NullFirst,
4479 Box::new(super::Skipper::Int32),
4480 )),
4481 FieldProjection::ToReader(0),
4482 ],
4483 );
4484 let mut row1 = Vec::new();
4485 row1.extend_from_slice(&encode_avro_long(0));
4486 row1.extend_from_slice(&encode_avro_int(5));
4487 let mut row2 = Vec::new();
4488 row2.extend_from_slice(&encode_avro_long(1));
4489 row2.extend_from_slice(&encode_avro_int(123));
4490 row2.extend_from_slice(&encode_avro_int(7));
4491 let mut cur1 = AvroCursor::new(&row1);
4492 let mut cur2 = AvroCursor::new(&row2);
4493 dec.decode(&mut cur1).unwrap();
4494 dec.decode(&mut cur2).unwrap();
4495 assert_eq!(cur1.position(), row1.len());
4496 assert_eq!(cur2.position(), row2.len());
4497 let arr = dec.flush(None).unwrap();
4498 let s = arr.as_any().downcast_ref::<StructArray>().unwrap();
4499 let id = s
4500 .column_by_name("id")
4501 .unwrap()
4502 .as_any()
4503 .downcast_ref::<Int32Array>()
4504 .unwrap();
4505 assert_eq!(id.len(), 2);
4506 assert_eq!(id.value(0), 5);
4507 assert_eq!(id.value(1), 7);
4508 }
4509
4510 fn make_dense_union_avro(
4511 children: Vec<(Codec, &'_ str, DataType)>,
4512 type_ids: Vec<i8>,
4513 ) -> AvroDataType {
4514 let mut avro_children: Vec<AvroDataType> = Vec::with_capacity(children.len());
4515 let mut fields: Vec<arrow_schema::Field> = Vec::with_capacity(children.len());
4516 for (codec, name, dt) in children {
4517 avro_children.push(AvroDataType::new(codec, Default::default(), None));
4518 fields.push(arrow_schema::Field::new(name, dt, true));
4519 }
4520 let union_fields = UnionFields::try_new(type_ids, fields).unwrap();
4521 let union_codec = Codec::Union(avro_children.into(), union_fields, UnionMode::Dense);
4522 AvroDataType::new(union_codec, Default::default(), None)
4523 }
4524
4525 #[test]
4526 fn test_union_dense_two_children_custom_type_ids() {
4527 let union_dt = make_dense_union_avro(
4528 vec![
4529 (Codec::Int32, "i", DataType::Int32),
4530 (Codec::Utf8, "s", DataType::Utf8),
4531 ],
4532 vec![2, 5],
4533 );
4534 let mut dec = Decoder::try_new(&union_dt).unwrap();
4535 let mut r1 = Vec::new();
4536 r1.extend_from_slice(&encode_avro_long(0));
4537 r1.extend_from_slice(&encode_avro_int(7));
4538 let mut r2 = Vec::new();
4539 r2.extend_from_slice(&encode_avro_long(1));
4540 r2.extend_from_slice(&encode_avro_bytes(b"x"));
4541 let mut r3 = Vec::new();
4542 r3.extend_from_slice(&encode_avro_long(0));
4543 r3.extend_from_slice(&encode_avro_int(-1));
4544 dec.decode(&mut AvroCursor::new(&r1)).unwrap();
4545 dec.decode(&mut AvroCursor::new(&r2)).unwrap();
4546 dec.decode(&mut AvroCursor::new(&r3)).unwrap();
4547 let array = dec.flush(None).unwrap();
4548 let ua = array
4549 .as_any()
4550 .downcast_ref::<UnionArray>()
4551 .expect("expected UnionArray");
4552 assert_eq!(ua.len(), 3);
4553 assert_eq!(ua.type_id(0), 2);
4554 assert_eq!(ua.type_id(1), 5);
4555 assert_eq!(ua.type_id(2), 2);
4556 assert_eq!(ua.value_offset(0), 0);
4557 assert_eq!(ua.value_offset(1), 0);
4558 assert_eq!(ua.value_offset(2), 1);
4559 let int_child = ua
4560 .child(2)
4561 .as_any()
4562 .downcast_ref::<Int32Array>()
4563 .expect("int child");
4564 assert_eq!(int_child.len(), 2);
4565 assert_eq!(int_child.value(0), 7);
4566 assert_eq!(int_child.value(1), -1);
4567 let str_child = ua
4568 .child(5)
4569 .as_any()
4570 .downcast_ref::<StringArray>()
4571 .expect("string child");
4572 assert_eq!(str_child.len(), 1);
4573 assert_eq!(str_child.value(0), "x");
4574 }
4575
4576 #[test]
4577 fn test_union_dense_with_null_and_string_children() {
4578 let union_dt = make_dense_union_avro(
4579 vec![
4580 (Codec::Null, "n", DataType::Null),
4581 (Codec::Utf8, "s", DataType::Utf8),
4582 ],
4583 vec![42, 7],
4584 );
4585 let mut dec = Decoder::try_new(&union_dt).unwrap();
4586 let r1 = encode_avro_long(0);
4587 let mut r2 = Vec::new();
4588 r2.extend_from_slice(&encode_avro_long(1));
4589 r2.extend_from_slice(&encode_avro_bytes(b"abc"));
4590 let r3 = encode_avro_long(0);
4591 dec.decode(&mut AvroCursor::new(&r1)).unwrap();
4592 dec.decode(&mut AvroCursor::new(&r2)).unwrap();
4593 dec.decode(&mut AvroCursor::new(&r3)).unwrap();
4594 let array = dec.flush(None).unwrap();
4595 let ua = array
4596 .as_any()
4597 .downcast_ref::<UnionArray>()
4598 .expect("expected UnionArray");
4599 assert_eq!(ua.len(), 3);
4600 assert_eq!(ua.type_id(0), 42);
4601 assert_eq!(ua.type_id(1), 7);
4602 assert_eq!(ua.type_id(2), 42);
4603 assert_eq!(ua.value_offset(0), 0);
4604 assert_eq!(ua.value_offset(1), 0);
4605 assert_eq!(ua.value_offset(2), 1);
4606 let null_child = ua
4607 .child(42)
4608 .as_any()
4609 .downcast_ref::<NullArray>()
4610 .expect("null child");
4611 assert_eq!(null_child.len(), 2);
4612 let str_child = ua
4613 .child(7)
4614 .as_any()
4615 .downcast_ref::<StringArray>()
4616 .expect("string child");
4617 assert_eq!(str_child.len(), 1);
4618 assert_eq!(str_child.value(0), "abc");
4619 }
4620
4621 #[test]
4622 fn test_union_dense_offsets_reset_after_flush() {
4623 let union_dt = make_dense_union_avro(
4624 vec![
4625 (Codec::Null, "n", DataType::Null),
4626 (Codec::Utf8, "s", DataType::Utf8),
4627 ],
4628 vec![0, 1],
4629 );
4630 let mut dec = Decoder::try_new(&union_dt).unwrap();
4631
4632 for _ in 0..2 {
4633 dec.decode(&mut AvroCursor::new(&encode_avro_long(0)))
4634 .unwrap();
4635 let array = dec.flush(None).unwrap();
4636 let union = array.as_union();
4637 assert_eq!(union.value_offset(0), 0);
4638 }
4639 }
4640
4641 #[test]
4642 fn test_union_decode_negative_branch_index_errors() {
4643 let union_dt = make_dense_union_avro(
4644 vec![
4645 (Codec::Int32, "i", DataType::Int32),
4646 (Codec::Utf8, "s", DataType::Utf8),
4647 ],
4648 vec![0, 1],
4649 );
4650 let mut dec = Decoder::try_new(&union_dt).unwrap();
4651 let row = encode_avro_long(-1); let err = dec
4653 .decode(&mut AvroCursor::new(&row))
4654 .expect_err("expected error for negative branch index");
4655 let msg = err.to_string();
4656 assert!(
4657 msg.contains("Negative union branch index"),
4658 "unexpected error message: {msg}"
4659 );
4660 }
4661
4662 #[test]
4663 fn test_union_decode_out_of_range_branch_index_errors() {
4664 let union_dt = make_dense_union_avro(
4665 vec![
4666 (Codec::Int32, "i", DataType::Int32),
4667 (Codec::Utf8, "s", DataType::Utf8),
4668 ],
4669 vec![10, 11],
4670 );
4671 let mut dec = Decoder::try_new(&union_dt).unwrap();
4672 let row = encode_avro_long(2);
4673 let err = dec
4674 .decode(&mut AvroCursor::new(&row))
4675 .expect_err("expected error for out-of-range branch index");
4676 let msg = err.to_string();
4677 assert!(
4678 msg.contains("out of range"),
4679 "unexpected error message: {msg}"
4680 );
4681 }
4682
4683 #[test]
4684 fn test_union_sparse_mode_not_supported() {
4685 let children: Vec<AvroDataType> = vec![
4686 AvroDataType::new(Codec::Int32, Default::default(), None),
4687 AvroDataType::new(Codec::Utf8, Default::default(), None),
4688 ];
4689 let uf = UnionFields::try_new(
4690 vec![1, 3],
4691 vec![
4692 arrow_schema::Field::new("i", DataType::Int32, true),
4693 arrow_schema::Field::new("s", DataType::Utf8, true),
4694 ],
4695 )
4696 .unwrap();
4697 let codec = Codec::Union(children.into(), uf, UnionMode::Sparse);
4698 let dt = AvroDataType::new(codec, Default::default(), None);
4699 let err = Decoder::try_new(&dt).expect_err("sparse union should not be supported");
4700 let msg = err.to_string();
4701 assert!(
4702 msg.contains("Sparse Arrow unions are not yet supported"),
4703 "unexpected error message: {msg}"
4704 );
4705 }
4706
4707 fn make_record_decoder_with_projector_defaults(
4708 reader_fields: &[(&str, DataType, bool)],
4709 field_defaults: Vec<Option<AvroLiteral>>,
4710 default_injections: Vec<(usize, AvroLiteral)>,
4711 ) -> Decoder {
4712 assert_eq!(
4713 field_defaults.len(),
4714 reader_fields.len(),
4715 "field_defaults must have one entry per reader field"
4716 );
4717 let mut field_refs: Vec<FieldRef> = Vec::with_capacity(reader_fields.len());
4718 let mut encodings: Vec<Decoder> = Vec::with_capacity(reader_fields.len());
4719 for (name, dt, nullable) in reader_fields {
4720 field_refs.push(Arc::new(ArrowField::new(*name, dt.clone(), *nullable)));
4721 let enc = match dt {
4722 DataType::Int32 => Decoder::Int32(Vec::with_capacity(DEFAULT_CAPACITY)),
4723 DataType::Int64 => Decoder::Int64(Vec::with_capacity(DEFAULT_CAPACITY)),
4724 DataType::Utf8 => Decoder::String(
4725 OffsetBufferBuilder::new(DEFAULT_CAPACITY),
4726 Vec::with_capacity(DEFAULT_CAPACITY),
4727 ),
4728 other => panic!("Unsupported test field type in helper: {other:?}"),
4729 };
4730 encodings.push(enc);
4731 }
4732 let fields: Fields = field_refs.into();
4733 let projector = Projector {
4734 writer_projections: vec![],
4735 default_injections: Arc::from(default_injections),
4736 };
4737 Decoder::Record {
4738 fields,
4739 decoders: encodings,
4740 defaults: field_defaults,
4741 projector: Some(projector),
4742 row_count: 0,
4743 }
4744 }
4745
4746 #[cfg(feature = "avro_custom_types")]
4747 #[test]
4748 fn test_default_append_custom_integer_range_validation() {
4749 let mut d_i8 = Decoder::Int8(Vec::with_capacity(DEFAULT_CAPACITY));
4750 d_i8.append_default(&AvroLiteral::Int(i8::MIN as i32))
4751 .unwrap();
4752 d_i8.append_default(&AvroLiteral::Int(i8::MAX as i32))
4753 .unwrap();
4754 let err_i8_high = d_i8
4755 .append_default(&AvroLiteral::Int(i8::MAX as i32 + 1))
4756 .unwrap_err();
4757 assert!(err_i8_high.to_string().contains("out of range for i8"));
4758 let err_i8_low = d_i8
4759 .append_default(&AvroLiteral::Int(i8::MIN as i32 - 1))
4760 .unwrap_err();
4761 assert!(err_i8_low.to_string().contains("out of range for i8"));
4762 let arr_i8 = d_i8.flush(None).unwrap();
4763 let values_i8 = arr_i8.as_any().downcast_ref::<Int8Array>().unwrap();
4764 assert_eq!(values_i8.values(), &[i8::MIN, i8::MAX]);
4765
4766 let mut d_i16 = Decoder::Int16(Vec::with_capacity(DEFAULT_CAPACITY));
4767 d_i16
4768 .append_default(&AvroLiteral::Int(i16::MIN as i32))
4769 .unwrap();
4770 d_i16
4771 .append_default(&AvroLiteral::Int(i16::MAX as i32))
4772 .unwrap();
4773 let err_i16_high = d_i16
4774 .append_default(&AvroLiteral::Int(i16::MAX as i32 + 1))
4775 .unwrap_err();
4776 assert!(err_i16_high.to_string().contains("out of range for i16"));
4777 let err_i16_low = d_i16
4778 .append_default(&AvroLiteral::Int(i16::MIN as i32 - 1))
4779 .unwrap_err();
4780 assert!(err_i16_low.to_string().contains("out of range for i16"));
4781 let arr_i16 = d_i16.flush(None).unwrap();
4782 let values_i16 = arr_i16.as_any().downcast_ref::<Int16Array>().unwrap();
4783 assert_eq!(values_i16.values(), &[i16::MIN, i16::MAX]);
4784
4785 let mut d_u8 = Decoder::UInt8(Vec::with_capacity(DEFAULT_CAPACITY));
4786 d_u8.append_default(&AvroLiteral::Int(0)).unwrap();
4787 d_u8.append_default(&AvroLiteral::Int(u8::MAX as i32))
4788 .unwrap();
4789 let err_u8_neg = d_u8.append_default(&AvroLiteral::Int(-1)).unwrap_err();
4790 assert!(err_u8_neg.to_string().contains("out of range for u8"));
4791 let err_u8_high = d_u8
4792 .append_default(&AvroLiteral::Int(u8::MAX as i32 + 1))
4793 .unwrap_err();
4794 assert!(err_u8_high.to_string().contains("out of range for u8"));
4795 let arr_u8 = d_u8.flush(None).unwrap();
4796 let values_u8 = arr_u8.as_any().downcast_ref::<UInt8Array>().unwrap();
4797 assert_eq!(values_u8.values(), &[0, u8::MAX]);
4798
4799 let mut d_u16 = Decoder::UInt16(Vec::with_capacity(DEFAULT_CAPACITY));
4800 d_u16.append_default(&AvroLiteral::Int(0)).unwrap();
4801 d_u16
4802 .append_default(&AvroLiteral::Int(u16::MAX as i32))
4803 .unwrap();
4804 let err_u16_neg = d_u16.append_default(&AvroLiteral::Int(-1)).unwrap_err();
4805 assert!(err_u16_neg.to_string().contains("out of range for u16"));
4806 let err_u16_high = d_u16
4807 .append_default(&AvroLiteral::Int(u16::MAX as i32 + 1))
4808 .unwrap_err();
4809 assert!(err_u16_high.to_string().contains("out of range for u16"));
4810 let arr_u16 = d_u16.flush(None).unwrap();
4811 let values_u16 = arr_u16.as_any().downcast_ref::<UInt16Array>().unwrap();
4812 assert_eq!(values_u16.values(), &[0, u16::MAX]);
4813
4814 let mut d_u32 = Decoder::UInt32(Vec::with_capacity(DEFAULT_CAPACITY));
4815 d_u32.append_default(&AvroLiteral::Long(0)).unwrap();
4816 d_u32
4817 .append_default(&AvroLiteral::Long(u32::MAX as i64))
4818 .unwrap();
4819 let err_u32_neg = d_u32.append_default(&AvroLiteral::Long(-1)).unwrap_err();
4820 assert!(err_u32_neg.to_string().contains("out of range for u32"));
4821 let err_u32_high = d_u32
4822 .append_default(&AvroLiteral::Long(u32::MAX as i64 + 1))
4823 .unwrap_err();
4824 assert!(err_u32_high.to_string().contains("out of range for u32"));
4825 let arr_u32 = d_u32.flush(None).unwrap();
4826 let values_u32 = arr_u32.as_any().downcast_ref::<UInt32Array>().unwrap();
4827 assert_eq!(values_u32.values(), &[0, u32::MAX]);
4828 }
4829
4830 #[cfg(feature = "avro_custom_types")]
4831 #[test]
4832 fn test_decode_custom_integer_range_validation() {
4833 let mut d_i8 = Decoder::try_new(&avro_from_codec(Codec::Int8)).unwrap();
4834 d_i8.decode(&mut AvroCursor::new(&encode_avro_int(i8::MIN as i32)))
4835 .unwrap();
4836 d_i8.decode(&mut AvroCursor::new(&encode_avro_int(i8::MAX as i32)))
4837 .unwrap();
4838 let err_i8_high = d_i8
4839 .decode(&mut AvroCursor::new(&encode_avro_int(i8::MAX as i32 + 1)))
4840 .unwrap_err();
4841 assert!(err_i8_high.to_string().contains("out of range for i8"));
4842 let err_i8_low = d_i8
4843 .decode(&mut AvroCursor::new(&encode_avro_int(i8::MIN as i32 - 1)))
4844 .unwrap_err();
4845 assert!(err_i8_low.to_string().contains("out of range for i8"));
4846 let arr_i8 = d_i8.flush(None).unwrap();
4847 let values_i8 = arr_i8.as_any().downcast_ref::<Int8Array>().unwrap();
4848 assert_eq!(values_i8.values(), &[i8::MIN, i8::MAX]);
4849
4850 let mut d_i16 = Decoder::try_new(&avro_from_codec(Codec::Int16)).unwrap();
4851 d_i16
4852 .decode(&mut AvroCursor::new(&encode_avro_int(i16::MIN as i32)))
4853 .unwrap();
4854 d_i16
4855 .decode(&mut AvroCursor::new(&encode_avro_int(i16::MAX as i32)))
4856 .unwrap();
4857 let err_i16_high = d_i16
4858 .decode(&mut AvroCursor::new(&encode_avro_int(i16::MAX as i32 + 1)))
4859 .unwrap_err();
4860 assert!(err_i16_high.to_string().contains("out of range for i16"));
4861 let err_i16_low = d_i16
4862 .decode(&mut AvroCursor::new(&encode_avro_int(i16::MIN as i32 - 1)))
4863 .unwrap_err();
4864 assert!(err_i16_low.to_string().contains("out of range for i16"));
4865 let arr_i16 = d_i16.flush(None).unwrap();
4866 let values_i16 = arr_i16.as_any().downcast_ref::<Int16Array>().unwrap();
4867 assert_eq!(values_i16.values(), &[i16::MIN, i16::MAX]);
4868
4869 let mut d_u8 = Decoder::try_new(&avro_from_codec(Codec::UInt8)).unwrap();
4870 d_u8.decode(&mut AvroCursor::new(&encode_avro_int(0)))
4871 .unwrap();
4872 d_u8.decode(&mut AvroCursor::new(&encode_avro_int(u8::MAX as i32)))
4873 .unwrap();
4874 let err_u8_neg = d_u8
4875 .decode(&mut AvroCursor::new(&encode_avro_int(-1)))
4876 .unwrap_err();
4877 assert!(err_u8_neg.to_string().contains("out of range for u8"));
4878 let err_u8_high = d_u8
4879 .decode(&mut AvroCursor::new(&encode_avro_int(u8::MAX as i32 + 1)))
4880 .unwrap_err();
4881 assert!(err_u8_high.to_string().contains("out of range for u8"));
4882 let arr_u8 = d_u8.flush(None).unwrap();
4883 let values_u8 = arr_u8.as_any().downcast_ref::<UInt8Array>().unwrap();
4884 assert_eq!(values_u8.values(), &[0, u8::MAX]);
4885
4886 let mut d_u16 = Decoder::try_new(&avro_from_codec(Codec::UInt16)).unwrap();
4887 d_u16
4888 .decode(&mut AvroCursor::new(&encode_avro_int(0)))
4889 .unwrap();
4890 d_u16
4891 .decode(&mut AvroCursor::new(&encode_avro_int(u16::MAX as i32)))
4892 .unwrap();
4893 let err_u16_neg = d_u16
4894 .decode(&mut AvroCursor::new(&encode_avro_int(-1)))
4895 .unwrap_err();
4896 assert!(err_u16_neg.to_string().contains("out of range for u16"));
4897 let err_u16_high = d_u16
4898 .decode(&mut AvroCursor::new(&encode_avro_int(u16::MAX as i32 + 1)))
4899 .unwrap_err();
4900 assert!(err_u16_high.to_string().contains("out of range for u16"));
4901 let arr_u16 = d_u16.flush(None).unwrap();
4902 let values_u16 = arr_u16.as_any().downcast_ref::<UInt16Array>().unwrap();
4903 assert_eq!(values_u16.values(), &[0, u16::MAX]);
4904
4905 let mut d_u32 = Decoder::try_new(&avro_from_codec(Codec::UInt32)).unwrap();
4906 d_u32
4907 .decode(&mut AvroCursor::new(&encode_avro_long(0)))
4908 .unwrap();
4909 d_u32
4910 .decode(&mut AvroCursor::new(&encode_avro_long(u32::MAX as i64)))
4911 .unwrap();
4912 let err_u32_neg = d_u32
4913 .decode(&mut AvroCursor::new(&encode_avro_long(-1)))
4914 .unwrap_err();
4915 assert!(err_u32_neg.to_string().contains("out of range for u32"));
4916 let err_u32_high = d_u32
4917 .decode(&mut AvroCursor::new(&encode_avro_long(u32::MAX as i64 + 1)))
4918 .unwrap_err();
4919 assert!(err_u32_high.to_string().contains("out of range for u32"));
4920 let arr_u32 = d_u32.flush(None).unwrap();
4921 let values_u32 = arr_u32.as_any().downcast_ref::<UInt32Array>().unwrap();
4922 assert_eq!(values_u32.values(), &[0, u32::MAX]);
4923 }
4924
4925 #[test]
4926 fn test_default_append_int32_and_int64_from_int_and_long() {
4927 let mut d_i32 = Decoder::Int32(Vec::with_capacity(DEFAULT_CAPACITY));
4928 d_i32.append_default(&AvroLiteral::Int(42)).unwrap();
4929 let arr = d_i32.flush(None).unwrap();
4930 let a = arr.as_any().downcast_ref::<Int32Array>().unwrap();
4931 assert_eq!(a.len(), 1);
4932 assert_eq!(a.value(0), 42);
4933 let mut d_i64 = Decoder::Int64(Vec::with_capacity(DEFAULT_CAPACITY));
4934 d_i64.append_default(&AvroLiteral::Int(5)).unwrap();
4935 d_i64.append_default(&AvroLiteral::Long(7)).unwrap();
4936 let arr64 = d_i64.flush(None).unwrap();
4937 let a64 = arr64.as_any().downcast_ref::<Int64Array>().unwrap();
4938 assert_eq!(a64.len(), 2);
4939 assert_eq!(a64.value(0), 5);
4940 assert_eq!(a64.value(1), 7);
4941 }
4942
4943 #[test]
4944 fn test_default_append_floats_and_doubles() {
4945 let mut d_f32 = Decoder::Float32(Vec::with_capacity(DEFAULT_CAPACITY));
4946 d_f32.append_default(&AvroLiteral::Float(1.5)).unwrap();
4947 let arr32 = d_f32.flush(None).unwrap();
4948 let a = arr32.as_any().downcast_ref::<Float32Array>().unwrap();
4949 assert_eq!(a.value(0), 1.5);
4950 let mut d_f64 = Decoder::Float64(Vec::with_capacity(DEFAULT_CAPACITY));
4951 d_f64.append_default(&AvroLiteral::Double(2.25)).unwrap();
4952 let arr64 = d_f64.flush(None).unwrap();
4953 let b = arr64.as_any().downcast_ref::<Float64Array>().unwrap();
4954 assert_eq!(b.value(0), 2.25);
4955 }
4956
4957 #[test]
4958 fn test_default_append_string_and_bytes() {
4959 let mut d_str = Decoder::String(
4960 OffsetBufferBuilder::new(DEFAULT_CAPACITY),
4961 Vec::with_capacity(DEFAULT_CAPACITY),
4962 );
4963 d_str
4964 .append_default(&AvroLiteral::String("hi".into()))
4965 .unwrap();
4966 let s_arr = d_str.flush(None).unwrap();
4967 let arr = s_arr.as_any().downcast_ref::<StringArray>().unwrap();
4968 assert_eq!(arr.value(0), "hi");
4969 let mut d_bytes = Decoder::Binary(
4970 OffsetBufferBuilder::new(DEFAULT_CAPACITY),
4971 Vec::with_capacity(DEFAULT_CAPACITY),
4972 );
4973 d_bytes
4974 .append_default(&AvroLiteral::Bytes(vec![1, 2, 3]))
4975 .unwrap();
4976 let b_arr = d_bytes.flush(None).unwrap();
4977 let barr = b_arr.as_any().downcast_ref::<BinaryArray>().unwrap();
4978 assert_eq!(barr.value(0), &[1, 2, 3]);
4979 let mut d_str_err = Decoder::String(
4980 OffsetBufferBuilder::new(DEFAULT_CAPACITY),
4981 Vec::with_capacity(DEFAULT_CAPACITY),
4982 );
4983 let err = d_str_err
4984 .append_default(&AvroLiteral::Bytes(vec![0x61, 0x62]))
4985 .unwrap_err();
4986 assert!(
4987 err.to_string()
4988 .contains("Default for string must be string"),
4989 "unexpected error: {err:?}"
4990 );
4991 }
4992
4993 #[test]
4994 fn test_nullable_null_runs_defer_validity_and_values_together() {
4995 let mut decoder = Decoder::Nullable(NullableDecoder::new(
4996 NullablePlan::ReadTag {
4997 nullability: Nullability::NullFirst,
4998 resolution: ResolutionPlan::Promotion(Promotion::Direct),
4999 },
5000 Decoder::Int32(Vec::new()),
5001 ));
5002
5003 decoder.append_nulls(64).unwrap();
5004 let Decoder::Nullable(nullable) = &decoder else {
5005 unreachable!();
5006 };
5007 assert_eq!(nullable.pending_nulls, 64);
5008 assert_eq!(nullable.validity.len(), 0);
5009 let Decoder::Int32(values) = nullable.values.as_ref() else {
5010 unreachable!();
5011 };
5012 assert!(values.is_empty());
5013
5014 decoder.append_default(&AvroLiteral::Int(7)).unwrap();
5015 let Decoder::Nullable(nullable) = &decoder else {
5016 unreachable!();
5017 };
5018 assert_eq!(nullable.pending_nulls, 0);
5019 assert_eq!(nullable.validity.len(), 65);
5020 let Decoder::Int32(values) = nullable.values.as_ref() else {
5021 unreachable!();
5022 };
5023 assert_eq!(values.len(), 65);
5024 assert_eq!(values[64], 7);
5025
5026 decoder.append_nulls(32).unwrap();
5027 let values = decoder.flush(None).unwrap();
5028 assert_eq!(values.len(), 97);
5029 assert_eq!(values.null_count(), 96);
5030 assert_eq!(values.as_primitive::<Int32Type>().value(64), 7);
5031 }
5032
5033 #[test]
5034 fn test_default_append_nullable_int32_null_and_value() {
5035 let inner = Decoder::Int32(Vec::with_capacity(DEFAULT_CAPACITY));
5036 let mut dec = Decoder::Nullable(NullableDecoder::new(
5037 NullablePlan::ReadTag {
5038 nullability: Nullability::NullFirst,
5039 resolution: ResolutionPlan::Promotion(Promotion::Direct),
5040 },
5041 inner,
5042 ));
5043 dec.append_default(&AvroLiteral::Null).unwrap();
5044 dec.append_default(&AvroLiteral::Null).unwrap();
5045 dec.append_default(&AvroLiteral::Int(11)).unwrap();
5046 dec.append_default(&AvroLiteral::Null).unwrap();
5047 dec.append_default(&AvroLiteral::Null).unwrap();
5048 dec.append_default(&AvroLiteral::Int(12)).unwrap();
5049 let arr = dec.flush(None).unwrap();
5050 let a = arr.as_any().downcast_ref::<Int32Array>().unwrap();
5051 assert_eq!(a.len(), 6);
5052 assert!(a.is_null(0));
5053 assert!(a.is_null(1));
5054 assert_eq!(a.value(2), 11);
5055 assert!(a.is_null(3));
5056 assert!(a.is_null(4));
5057 assert_eq!(a.value(5), 12);
5058 }
5059
5060 #[test]
5061 fn test_default_append_array_of_ints() {
5062 let list_dt = avro_from_codec(Codec::List(Arc::new(avro_from_codec(Codec::Int32))));
5063 let mut d = Decoder::try_new(&list_dt).unwrap();
5064 let items = vec![
5065 AvroLiteral::Int(1),
5066 AvroLiteral::Int(2),
5067 AvroLiteral::Int(3),
5068 ];
5069 d.append_default(&AvroLiteral::Array(items)).unwrap();
5070 let arr = d.flush(None).unwrap();
5071 let list = arr.as_any().downcast_ref::<ListArray>().unwrap();
5072 assert_eq!(list.len(), 1);
5073 assert_eq!(list.value_length(0), 3);
5074 let vals = list.values().as_any().downcast_ref::<Int32Array>().unwrap();
5075 assert_eq!(vals.values(), &[1, 2, 3]);
5076 }
5077
5078 #[test]
5079 fn test_default_append_map_string_to_int() {
5080 let map_dt = avro_from_codec(Codec::Map(Arc::new(avro_from_codec(Codec::Int32))));
5081 let mut d = Decoder::try_new(&map_dt).unwrap();
5082 let mut m: IndexMap<String, AvroLiteral> = IndexMap::new();
5083 m.insert("k1".to_string(), AvroLiteral::Int(10));
5084 m.insert("k2".to_string(), AvroLiteral::Int(20));
5085 d.append_default(&AvroLiteral::Map(m)).unwrap();
5086 let arr = d.flush(None).unwrap();
5087 let map = arr.as_any().downcast_ref::<MapArray>().unwrap();
5088 assert_eq!(map.len(), 1);
5089 assert_eq!(map.value_length(0), 2);
5090 let binding = map.value(0);
5091 let entries = binding.as_any().downcast_ref::<StructArray>().unwrap();
5092 let k = entries
5093 .column_by_name("key")
5094 .unwrap()
5095 .as_any()
5096 .downcast_ref::<StringArray>()
5097 .unwrap();
5098 let v = entries
5099 .column_by_name("value")
5100 .unwrap()
5101 .as_any()
5102 .downcast_ref::<Int32Array>()
5103 .unwrap();
5104 let keys: std::collections::HashSet<&str> = (0..k.len()).map(|i| k.value(i)).collect();
5105 assert_eq!(keys, ["k1", "k2"].into_iter().collect());
5106 let vals: std::collections::HashSet<i32> = (0..v.len()).map(|i| v.value(i)).collect();
5107 assert_eq!(vals, [10, 20].into_iter().collect());
5108 }
5109
5110 #[test]
5111 fn test_default_append_enum_by_symbol() {
5112 let symbols: Arc<[String]> = vec!["A".into(), "B".into(), "C".into()].into();
5113 let mut d = Decoder::Enum(Vec::with_capacity(DEFAULT_CAPACITY), symbols.clone(), None);
5114 d.append_default(&AvroLiteral::Enum("B".into())).unwrap();
5115 let arr = d.flush(None).unwrap();
5116 let dict = arr
5117 .as_any()
5118 .downcast_ref::<DictionaryArray<Int32Type>>()
5119 .unwrap();
5120 assert_eq!(dict.len(), 1);
5121 let expected = Int32Array::from(vec![1]);
5122 assert_eq!(dict.keys(), &expected);
5123 let values = dict
5124 .values()
5125 .as_any()
5126 .downcast_ref::<StringArray>()
5127 .unwrap();
5128 assert_eq!(values.value(1), "B");
5129 }
5130
5131 #[test]
5132 fn test_default_append_uuid_and_type_error() {
5133 let mut d = Decoder::Uuid(Vec::with_capacity(DEFAULT_CAPACITY));
5134 let uuid_str = "123e4567-e89b-12d3-a456-426614174000";
5135 d.append_default(&AvroLiteral::String(uuid_str.into()))
5136 .unwrap();
5137 let arr_ref = d.flush(None).unwrap();
5138 let arr = arr_ref
5139 .as_any()
5140 .downcast_ref::<FixedSizeBinaryArray>()
5141 .unwrap();
5142 assert_eq!(arr.value_length(), 16);
5143 assert_eq!(arr.len(), 1);
5144 let mut d2 = Decoder::Uuid(Vec::with_capacity(DEFAULT_CAPACITY));
5145 let err = d2
5146 .append_default(&AvroLiteral::Bytes(vec![0u8; 16]))
5147 .unwrap_err();
5148 assert!(
5149 err.to_string().contains("Default for uuid must be string"),
5150 "unexpected error: {err:?}"
5151 );
5152 }
5153
5154 #[test]
5155 fn test_default_append_fixed_and_length_mismatch() {
5156 let mut d = Decoder::Fixed(4, Vec::with_capacity(DEFAULT_CAPACITY));
5157 d.append_default(&AvroLiteral::Bytes(vec![1, 2, 3, 4]))
5158 .unwrap();
5159 let arr_ref = d.flush(None).unwrap();
5160 let arr = arr_ref
5161 .as_any()
5162 .downcast_ref::<FixedSizeBinaryArray>()
5163 .unwrap();
5164 assert_eq!(arr.value_length(), 4);
5165 assert_eq!(arr.value(0), &[1, 2, 3, 4]);
5166 let mut d_err = Decoder::Fixed(4, Vec::with_capacity(DEFAULT_CAPACITY));
5167 let err = d_err
5168 .append_default(&AvroLiteral::Bytes(vec![1, 2, 3]))
5169 .unwrap_err();
5170 assert!(
5171 err.to_string().contains("Fixed default length"),
5172 "unexpected error: {err:?}"
5173 );
5174 }
5175
5176 #[test]
5177 fn test_default_append_duration_and_length_validation() {
5178 let dt = avro_from_codec(Codec::Interval);
5179 let mut d = Decoder::try_new(&dt).unwrap();
5180 let mut bytes = Vec::with_capacity(12);
5181 bytes.extend_from_slice(&1u32.to_le_bytes());
5182 bytes.extend_from_slice(&2u32.to_le_bytes());
5183 bytes.extend_from_slice(&3u32.to_le_bytes());
5184 d.append_default(&AvroLiteral::Bytes(bytes)).unwrap();
5185 let arr_ref = d.flush(None).unwrap();
5186 let arr = arr_ref
5187 .as_any()
5188 .downcast_ref::<IntervalMonthDayNanoArray>()
5189 .unwrap();
5190 assert_eq!(arr.len(), 1);
5191 let v = arr.value(0);
5192 assert_eq!(v.months, 1);
5193 assert_eq!(v.days, 2);
5194 assert_eq!(v.nanoseconds, 3_000_000);
5195 let mut d_err = Decoder::try_new(&avro_from_codec(Codec::Interval)).unwrap();
5196 let err = d_err
5197 .append_default(&AvroLiteral::Bytes(vec![0u8; 11]))
5198 .unwrap_err();
5199 assert!(
5200 err.to_string()
5201 .contains("Duration default must be exactly 12 bytes"),
5202 "unexpected error: {err:?}"
5203 );
5204 }
5205
5206 #[test]
5207 fn test_default_append_decimal256_from_bytes() {
5208 let dt = avro_from_codec(Codec::Decimal(50, Some(2), Some(32)));
5209 let mut d = Decoder::try_new(&dt).unwrap();
5210 let pos: [u8; 32] = [
5211 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
5212 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
5213 0x00, 0x00, 0x30, 0x39,
5214 ];
5215 d.append_default(&AvroLiteral::Bytes(pos.to_vec())).unwrap();
5216 let neg: [u8; 32] = [
5217 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
5218 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
5219 0xFF, 0xFF, 0xFF, 0x85,
5220 ];
5221 d.append_default(&AvroLiteral::Bytes(neg.to_vec())).unwrap();
5222 let arr = d.flush(None).unwrap();
5223 let dec = arr.as_any().downcast_ref::<Decimal256Array>().unwrap();
5224 assert_eq!(dec.len(), 2);
5225 assert_eq!(dec.value_as_string(0), "123.45");
5226 assert_eq!(dec.value_as_string(1), "-1.23");
5227 }
5228
5229 #[test]
5230 fn test_record_append_default_map_missing_fields_uses_projector_field_defaults() {
5231 let field_defaults = vec![None, Some(AvroLiteral::String("hi".into()))];
5232 let mut rec = make_record_decoder_with_projector_defaults(
5233 &[("a", DataType::Int32, false), ("b", DataType::Utf8, false)],
5234 field_defaults,
5235 vec![],
5236 );
5237 let mut map: IndexMap<String, AvroLiteral> = IndexMap::new();
5238 map.insert("a".to_string(), AvroLiteral::Int(7));
5239 rec.append_default(&AvroLiteral::Map(map)).unwrap();
5240 let arr = rec.flush(None).unwrap();
5241 let s = arr.as_any().downcast_ref::<StructArray>().unwrap();
5242 let a = s
5243 .column_by_name("a")
5244 .unwrap()
5245 .as_any()
5246 .downcast_ref::<Int32Array>()
5247 .unwrap();
5248 let b = s
5249 .column_by_name("b")
5250 .unwrap()
5251 .as_any()
5252 .downcast_ref::<StringArray>()
5253 .unwrap();
5254 assert_eq!(a.value(0), 7);
5255 assert_eq!(b.value(0), "hi");
5256 }
5257
5258 #[test]
5259 fn test_record_append_default_null_uses_projector_field_defaults() {
5260 let field_defaults = vec![
5261 Some(AvroLiteral::Int(5)),
5262 Some(AvroLiteral::String("x".into())),
5263 ];
5264 let mut rec = make_record_decoder_with_projector_defaults(
5265 &[("a", DataType::Int32, false), ("b", DataType::Utf8, false)],
5266 field_defaults,
5267 vec![],
5268 );
5269 rec.append_default(&AvroLiteral::Null).unwrap();
5270 let arr = rec.flush(None).unwrap();
5271 let s = arr.as_any().downcast_ref::<StructArray>().unwrap();
5272 let a = s
5273 .column_by_name("a")
5274 .unwrap()
5275 .as_any()
5276 .downcast_ref::<Int32Array>()
5277 .unwrap();
5278 let b = s
5279 .column_by_name("b")
5280 .unwrap()
5281 .as_any()
5282 .downcast_ref::<StringArray>()
5283 .unwrap();
5284 assert_eq!(a.value(0), 5);
5285 assert_eq!(b.value(0), "x");
5286 }
5287
5288 #[test]
5289 fn test_record_append_default_missing_fields_without_projector_defaults_yields_type_nulls_or_empties()
5290 {
5291 let fields = vec![("a", DataType::Int32, true), ("b", DataType::Utf8, true)];
5292 let mut field_refs: Vec<FieldRef> = Vec::new();
5293 let mut encoders: Vec<Decoder> = Vec::new();
5294 for (name, dt, nullable) in &fields {
5295 field_refs.push(Arc::new(ArrowField::new(*name, dt.clone(), *nullable)));
5296 }
5297 let enc_a = Decoder::Nullable(NullableDecoder::new(
5298 NullablePlan::ReadTag {
5299 nullability: Nullability::NullSecond,
5300 resolution: ResolutionPlan::Promotion(Promotion::Direct),
5301 },
5302 Decoder::Int32(Vec::with_capacity(DEFAULT_CAPACITY)),
5303 ));
5304 let enc_b = Decoder::Nullable(NullableDecoder::new(
5305 NullablePlan::ReadTag {
5306 nullability: Nullability::NullSecond,
5307 resolution: ResolutionPlan::Promotion(Promotion::Direct),
5308 },
5309 Decoder::String(
5310 OffsetBufferBuilder::new(DEFAULT_CAPACITY),
5311 Vec::with_capacity(DEFAULT_CAPACITY),
5312 ),
5313 ));
5314 encoders.push(enc_a);
5315 encoders.push(enc_b);
5316 let field_defaults = vec![None, None]; let projector = Projector {
5318 writer_projections: vec![],
5319 default_injections: Arc::from(Vec::<(usize, AvroLiteral)>::new()),
5320 };
5321 let mut rec = Decoder::Record {
5322 fields: field_refs.into(),
5323 decoders: encoders,
5324 defaults: field_defaults,
5325 projector: Some(projector),
5326 row_count: 0,
5327 };
5328 let mut map: IndexMap<String, AvroLiteral> = IndexMap::new();
5329 map.insert("a".to_string(), AvroLiteral::Int(9));
5330 rec.append_default(&AvroLiteral::Map(map)).unwrap();
5331 let arr = rec.flush(None).unwrap();
5332 let s = arr.as_any().downcast_ref::<StructArray>().unwrap();
5333 let a = s
5334 .column_by_name("a")
5335 .unwrap()
5336 .as_any()
5337 .downcast_ref::<Int32Array>()
5338 .unwrap();
5339 let b = s
5340 .column_by_name("b")
5341 .unwrap()
5342 .as_any()
5343 .downcast_ref::<StringArray>()
5344 .unwrap();
5345 assert!(a.is_valid(0));
5346 assert_eq!(a.value(0), 9);
5347 assert!(b.is_null(0));
5348 }
5349
5350 #[test]
5351 fn test_projector_default_injection_when_writer_lacks_fields() {
5352 let defaults = vec![None, None];
5353 let injections = vec![
5354 (0, AvroLiteral::Int(99)),
5355 (1, AvroLiteral::String("alice".into())),
5356 ];
5357 let mut rec = make_record_decoder_with_projector_defaults(
5358 &[
5359 ("id", DataType::Int32, false),
5360 ("name", DataType::Utf8, false),
5361 ],
5362 defaults,
5363 injections,
5364 );
5365 rec.decode(&mut AvroCursor::new(&[])).unwrap();
5366 let arr = rec.flush(None).unwrap();
5367 let s = arr.as_any().downcast_ref::<StructArray>().unwrap();
5368 let id = s
5369 .column_by_name("id")
5370 .unwrap()
5371 .as_any()
5372 .downcast_ref::<Int32Array>()
5373 .unwrap();
5374 let name = s
5375 .column_by_name("name")
5376 .unwrap()
5377 .as_any()
5378 .downcast_ref::<StringArray>()
5379 .unwrap();
5380 assert_eq!(id.value(0), 99);
5381 assert_eq!(name.value(0), "alice");
5382 }
5383
5384 #[test]
5385 fn union_type_ids_are_not_child_indexes() {
5386 let encodings: Vec<AvroDataType> =
5387 vec![avro_from_codec(Codec::Int32), avro_from_codec(Codec::Utf8)];
5388 let fields: UnionFields = [
5389 (42_i8, Arc::new(ArrowField::new("a", DataType::Int32, true))),
5390 (7_i8, Arc::new(ArrowField::new("b", DataType::Utf8, true))),
5391 ]
5392 .into_iter()
5393 .collect();
5394 let dt = avro_from_codec(Codec::Union(
5395 encodings.into(),
5396 fields.clone(),
5397 UnionMode::Dense,
5398 ));
5399 let mut dec = Decoder::try_new(&dt).expect("decoder");
5400 let mut b1 = encode_avro_long(1);
5401 b1.extend(encode_avro_bytes(b"hi"));
5402 dec.decode(&mut AvroCursor::new(&b1)).expect("decode b1");
5403 let mut b0 = encode_avro_long(0);
5404 b0.extend(encode_avro_int(5));
5405 dec.decode(&mut AvroCursor::new(&b0)).expect("decode b0");
5406 let arr = dec.flush(None).expect("flush");
5407 let ua = arr.as_any().downcast_ref::<UnionArray>().expect("union");
5408 assert_eq!(ua.len(), 2);
5409 assert_eq!(ua.type_id(0), 7, "type id must come from UnionFields");
5410 assert_eq!(ua.type_id(1), 42, "type id must come from UnionFields");
5411 assert_eq!(ua.value_offset(0), 0);
5412 assert_eq!(ua.value_offset(1), 0);
5413 let utf8_child = ua.child(7).as_any().downcast_ref::<StringArray>().unwrap();
5414 assert_eq!(utf8_child.len(), 1);
5415 assert_eq!(utf8_child.value(0), "hi");
5416 let int_child = ua.child(42).as_any().downcast_ref::<Int32Array>().unwrap();
5417 assert_eq!(int_child.len(), 1);
5418 assert_eq!(int_child.value(0), 5);
5419 let type_ids: Vec<i8> = fields.iter().map(|(tid, _)| tid).collect();
5420 assert_eq!(type_ids, vec![42_i8, 7_i8]);
5421 }
5422
5423 #[cfg(feature = "avro_custom_types")]
5424 #[test]
5425 fn skipper_from_avro_maps_custom_duration_variants_to_int64() -> Result<(), AvroError> {
5426 for codec in [
5427 Codec::DurationNanos,
5428 Codec::DurationMicros,
5429 Codec::DurationMillis,
5430 Codec::DurationSeconds,
5431 ] {
5432 let dt = make_avro_dt(codec.clone(), None);
5433 let s = Skipper::from_avro(&dt)?;
5434 match s {
5435 Skipper::Int64 => {}
5436 other => panic!("expected Int64 skipper for {codec:?}, got {other:?}"),
5437 }
5438 }
5439 Ok(())
5440 }
5441
5442 #[cfg(feature = "avro_custom_types")]
5443 #[test]
5444 fn skipper_skip_consumes_one_long_for_custom_durations() -> Result<(), AvroError> {
5445 let values: [i64; 7] = [0, 1, -1, 150, -150, i64::MAX / 3, i64::MIN / 3];
5446 for codec in [
5447 Codec::DurationNanos,
5448 Codec::DurationMicros,
5449 Codec::DurationMillis,
5450 Codec::DurationSeconds,
5451 ] {
5452 let dt = make_avro_dt(codec.clone(), None);
5453 let s = Skipper::from_avro(&dt)?;
5454 for &v in &values {
5455 let bytes = encode_avro_long(v);
5456 let mut cursor = AvroCursor::new(&bytes);
5457 s.skip(&mut cursor)?;
5458 assert_eq!(
5459 cursor.position(),
5460 bytes.len(),
5461 "did not consume all bytes for {codec:?} value {v}"
5462 );
5463 }
5464 }
5465 Ok(())
5466 }
5467
5468 #[cfg(feature = "avro_custom_types")]
5469 #[test]
5470 fn skipper_nullable_custom_duration_respects_null_first() -> Result<(), AvroError> {
5471 let dt = make_avro_dt(Codec::DurationNanos, Some(Nullability::NullFirst));
5472 let s = Skipper::from_avro(&dt)?;
5473 match &s {
5474 Skipper::Nullable(Nullability::NullFirst, inner) => match **inner {
5475 Skipper::Int64 => {}
5476 ref other => panic!("expected inner Int64, got {other:?}"),
5477 },
5478 other => panic!("expected Nullable(NullFirst, Int64), got {other:?}"),
5479 }
5480 {
5481 let buf = encode_vlq_u64(0);
5482 let mut cursor = AvroCursor::new(&buf);
5483 s.skip(&mut cursor)?;
5484 assert_eq!(cursor.position(), 1, "expected to consume only tag=0");
5485 }
5486 {
5487 let mut buf = encode_vlq_u64(1);
5488 buf.extend(encode_avro_long(0));
5489 let mut cursor = AvroCursor::new(&buf);
5490 s.skip(&mut cursor)?;
5491 assert_eq!(cursor.position(), 2, "expected to consume tag=1 + long(0)");
5492 }
5493
5494 Ok(())
5495 }
5496
5497 #[cfg(feature = "avro_custom_types")]
5498 #[test]
5499 fn skipper_nullable_custom_duration_respects_null_second() -> Result<(), AvroError> {
5500 let dt = make_avro_dt(Codec::DurationMicros, Some(Nullability::NullSecond));
5501 let s = Skipper::from_avro(&dt)?;
5502 match &s {
5503 Skipper::Nullable(Nullability::NullSecond, inner) => match **inner {
5504 Skipper::Int64 => {}
5505 ref other => panic!("expected inner Int64, got {other:?}"),
5506 },
5507 other => panic!("expected Nullable(NullSecond, Int64), got {other:?}"),
5508 }
5509 {
5510 let buf = encode_vlq_u64(1);
5511 let mut cursor = AvroCursor::new(&buf);
5512 s.skip(&mut cursor)?;
5513 assert_eq!(cursor.position(), 1, "expected to consume only tag=1");
5514 }
5515 {
5516 let mut buf = encode_vlq_u64(0);
5517 buf.extend(encode_avro_long(-1));
5518 let mut cursor = AvroCursor::new(&buf);
5519 s.skip(&mut cursor)?;
5520 assert_eq!(
5521 cursor.position(),
5522 1 + encode_avro_long(-1).len(),
5523 "expected to consume tag=0 + long(-1)"
5524 );
5525 }
5526 Ok(())
5527 }
5528
5529 #[test]
5530 fn skipper_interval_is_fixed12_and_skips_12_bytes() -> Result<(), AvroError> {
5531 let dt = make_avro_dt(Codec::Interval, None);
5532 let s = Skipper::from_avro(&dt)?;
5533 match s {
5534 Skipper::DurationFixed12 => {}
5535 other => panic!("expected DurationFixed12, got {other:?}"),
5536 }
5537 let payload = vec![0u8; 12];
5538 let mut cursor = AvroCursor::new(&payload);
5539 s.skip(&mut cursor)?;
5540 assert_eq!(cursor.position(), 12, "expected to consume 12 fixed bytes");
5541 Ok(())
5542 }
5543
5544 #[cfg(feature = "avro_custom_types")]
5545 #[test]
5546 fn test_run_end_encoded_width16_int32_basic_grouping() {
5547 use arrow_array::RunArray;
5548 use std::sync::Arc;
5549 let inner = avro_from_codec(Codec::Int32);
5550 let ree = AvroDataType::new(
5551 Codec::RunEndEncoded(Arc::new(inner), 16),
5552 Default::default(),
5553 None,
5554 );
5555 let mut dec = Decoder::try_new(&ree).expect("create REE decoder");
5556 for v in [1, 1, 1, 2, 2, 3, 3, 3, 3] {
5557 let bytes = encode_avro_int(v);
5558 dec.decode(&mut AvroCursor::new(&bytes)).expect("decode");
5559 }
5560 let arr = dec.flush(None).expect("flush");
5561 let ra = arr
5562 .as_any()
5563 .downcast_ref::<RunArray<Int16Type>>()
5564 .expect("RunArray<Int16Type>");
5565 assert_eq!(ra.len(), 9);
5566 assert_eq!(ra.run_ends().values(), &[3, 5, 9]);
5567 let vals = ra
5568 .values()
5569 .as_ref()
5570 .as_any()
5571 .downcast_ref::<Int32Array>()
5572 .expect("values Int32");
5573 assert_eq!(vals.values(), &[1, 2, 3]);
5574 }
5575
5576 #[cfg(feature = "avro_custom_types")]
5577 #[test]
5578 fn test_run_end_encoded_width32_nullable_values_group_nulls() {
5579 use arrow_array::RunArray;
5580 use std::sync::Arc;
5581 let inner = AvroDataType::new(
5582 Codec::Int32,
5583 Default::default(),
5584 Some(Nullability::NullSecond),
5585 );
5586 let ree = AvroDataType::new(
5587 Codec::RunEndEncoded(Arc::new(inner), 32),
5588 Default::default(),
5589 None,
5590 );
5591 let mut dec = Decoder::try_new(&ree).expect("create REE decoder");
5592 let seq: [Option<i32>; 8] = [
5593 None,
5594 None,
5595 Some(7),
5596 Some(7),
5597 Some(7),
5598 None,
5599 Some(5),
5600 Some(5),
5601 ];
5602 for item in seq {
5603 let mut bytes = Vec::new();
5604 match item {
5605 None => bytes.extend_from_slice(&encode_vlq_u64(1)),
5606 Some(v) => {
5607 bytes.extend_from_slice(&encode_vlq_u64(0));
5608 bytes.extend_from_slice(&encode_avro_int(v));
5609 }
5610 }
5611 dec.decode(&mut AvroCursor::new(&bytes)).expect("decode");
5612 }
5613 let arr = dec.flush(None).expect("flush");
5614 let ra = arr
5615 .as_any()
5616 .downcast_ref::<RunArray<Int32Type>>()
5617 .expect("RunArray<Int32Type>");
5618 assert_eq!(ra.len(), 8);
5619 assert_eq!(ra.run_ends().values(), &[2, 5, 6, 8]);
5620 let vals = ra
5621 .values()
5622 .as_ref()
5623 .as_any()
5624 .downcast_ref::<Int32Array>()
5625 .expect("values Int32 (nullable)");
5626 assert_eq!(vals.len(), 4);
5627 assert!(vals.is_null(0));
5628 assert_eq!(vals.value(1), 7);
5629 assert!(vals.is_null(2));
5630 assert_eq!(vals.value(3), 5);
5631 }
5632
5633 #[cfg(feature = "avro_custom_types")]
5634 #[test]
5635 fn test_run_end_encoded_decode_with_promotion_int_to_double_via_nullable_from_single() {
5636 use arrow_array::RunArray;
5637 let inner_values = Decoder::Float64(Vec::with_capacity(DEFAULT_CAPACITY));
5638 let ree = Decoder::RunEndEncoded(
5639 8, 0,
5641 Box::new(inner_values),
5642 );
5643 let mut dec = Decoder::Nullable(NullableDecoder::new(
5644 NullablePlan::FromSingle {
5645 resolution: ResolutionPlan::Promotion(Promotion::IntToDouble),
5646 },
5647 ree,
5648 ));
5649 for v in [1, 1, 2, 2, 2] {
5650 let bytes = encode_avro_int(v);
5651 dec.decode(&mut AvroCursor::new(&bytes)).expect("decode");
5652 }
5653 let arr = dec.flush(None).expect("flush");
5654 let ra = arr
5655 .as_any()
5656 .downcast_ref::<RunArray<Int64Type>>()
5657 .expect("RunArray<Int64Type>");
5658 assert_eq!(ra.len(), 5);
5659 assert_eq!(ra.run_ends().values(), &[2, 5]);
5660 let vals = ra
5661 .values()
5662 .as_ref()
5663 .as_any()
5664 .downcast_ref::<Float64Array>()
5665 .expect("values Float64");
5666 assert_eq!(vals.values(), &[1.0, 2.0]);
5667 }
5668
5669 #[cfg(feature = "avro_custom_types")]
5670 #[test]
5671 fn test_run_end_encoded_unsupported_run_end_width_errors() {
5672 use std::sync::Arc;
5673 let inner = avro_from_codec(Codec::Int32);
5674 let dt = AvroDataType::new(
5675 Codec::RunEndEncoded(Arc::new(inner), 3),
5676 Default::default(),
5677 None,
5678 );
5679 let err = Decoder::try_new(&dt).expect_err("must reject unsupported width");
5680 let msg = err.to_string();
5681 assert!(
5682 msg.contains("Unsupported run-end width")
5683 && msg.contains("16/32/64 bits or 2/4/8 bytes"),
5684 "unexpected error message: {msg}"
5685 );
5686 }
5687
5688 #[cfg(feature = "avro_custom_types")]
5689 #[test]
5690 fn test_run_end_encoded_empty_input_is_empty_runarray() {
5691 use arrow_array::RunArray;
5692 use std::sync::Arc;
5693 let inner = avro_from_codec(Codec::Utf8);
5694 let dt = AvroDataType::new(
5695 Codec::RunEndEncoded(Arc::new(inner), 4),
5696 Default::default(),
5697 None,
5698 );
5699 let mut dec = Decoder::try_new(&dt).expect("create REE decoder");
5700 let arr = dec.flush(None).expect("flush");
5701 let ra = arr
5702 .as_any()
5703 .downcast_ref::<RunArray<Int32Type>>()
5704 .expect("RunArray<Int32Type>");
5705 assert_eq!(ra.len(), 0);
5706 assert_eq!(ra.run_ends().len(), 0);
5707 assert_eq!(ra.values().len(), 0);
5708 }
5709
5710 #[cfg(feature = "avro_custom_types")]
5711 #[test]
5712 fn test_run_end_encoded_strings_grouping_width32_bits() {
5713 use arrow_array::RunArray;
5714 use std::sync::Arc;
5715 let inner = avro_from_codec(Codec::Utf8);
5716 let dt = AvroDataType::new(
5717 Codec::RunEndEncoded(Arc::new(inner), 32),
5718 Default::default(),
5719 None,
5720 );
5721 let mut dec = Decoder::try_new(&dt).expect("create REE decoder");
5722 for s in ["a", "a", "bb", "bb", "bb", "a"] {
5723 let bytes = encode_avro_bytes(s.as_bytes());
5724 dec.decode(&mut AvroCursor::new(&bytes)).expect("decode");
5725 }
5726 let arr = dec.flush(None).expect("flush");
5727 let ra = arr
5728 .as_any()
5729 .downcast_ref::<RunArray<Int32Type>>()
5730 .expect("RunArray<Int32Type>");
5731 assert_eq!(ra.run_ends().values(), &[2, 5, 6]);
5732 let vals = ra
5733 .values()
5734 .as_ref()
5735 .as_any()
5736 .downcast_ref::<StringArray>()
5737 .expect("values String");
5738 assert_eq!(vals.len(), 3);
5739 assert_eq!(vals.value(0), "a");
5740 assert_eq!(vals.value(1), "bb");
5741 assert_eq!(vals.value(2), "a");
5742 }
5743
5744 #[cfg(not(feature = "avro_custom_types"))]
5745 #[test]
5746 fn test_no_custom_types_feature_smoke_decodes_plain_int32() {
5747 let dt = avro_from_codec(Codec::Int32);
5748 let mut dec = Decoder::try_new(&dt).expect("create Int32 decoder");
5749 for v in [1, 2, 3] {
5750 let bytes = encode_avro_int(v);
5751 dec.decode(&mut AvroCursor::new(&bytes)).expect("decode");
5752 }
5753 let arr = dec.flush(None).expect("flush");
5754 let a = arr
5755 .as_any()
5756 .downcast_ref::<Int32Array>()
5757 .expect("Int32Array");
5758 assert_eq!(a.values(), &[1, 2, 3]);
5759 }
5760
5761 #[test]
5762 fn test_timestamp_nanos_decoding_offset_zero() {
5763 let avro_type = avro_from_codec(Codec::TimestampNanos(Some(Tz::OffsetZero)));
5764 let mut decoder = Decoder::try_new(&avro_type).expect("create TimestampNanos decoder");
5765 let mut data = Vec::new();
5766 for v in [0_i64, 1_i64, -1_i64, 1_234_567_890_i64] {
5767 data.extend_from_slice(&encode_avro_long(v));
5768 }
5769 let mut cur = AvroCursor::new(&data);
5770 for _ in 0..4 {
5771 decoder.decode(&mut cur).expect("decode nanos ts");
5772 }
5773 let array = decoder.flush(None).expect("flush nanos ts");
5774 let ts = array
5775 .as_any()
5776 .downcast_ref::<TimestampNanosecondArray>()
5777 .expect("TimestampNanosecondArray");
5778 assert_eq!(ts.values(), &[0, 1, -1, 1_234_567_890]);
5779 match ts.data_type() {
5780 DataType::Timestamp(arrow_schema::TimeUnit::Nanosecond, tz) => {
5781 assert_eq!(tz.as_deref(), Some("+00:00"));
5782 }
5783 other => panic!("expected Timestamp(Nanosecond, Some(\"+00:00\")), got {other:?}"),
5784 }
5785 }
5786
5787 #[test]
5788 fn test_timestamp_nanos_decoding_utc() {
5789 let avro_type = avro_from_codec(Codec::TimestampNanos(Some(Tz::Utc)));
5790 let mut decoder = Decoder::try_new(&avro_type).expect("create TimestampNanos decoder");
5791 let mut data = Vec::new();
5792 for v in [0_i64, 1_i64, -1_i64, 1_234_567_890_i64] {
5793 data.extend_from_slice(&encode_avro_long(v));
5794 }
5795 let mut cur = AvroCursor::new(&data);
5796 for _ in 0..4 {
5797 decoder.decode(&mut cur).expect("decode nanos ts");
5798 }
5799 let array = decoder.flush(None).expect("flush nanos ts");
5800 let ts = array
5801 .as_any()
5802 .downcast_ref::<TimestampNanosecondArray>()
5803 .expect("TimestampNanosecondArray");
5804 assert_eq!(ts.values(), &[0, 1, -1, 1_234_567_890]);
5805 match ts.data_type() {
5806 DataType::Timestamp(arrow_schema::TimeUnit::Nanosecond, tz) => {
5807 assert_eq!(tz.as_deref(), Some("UTC"));
5808 }
5809 other => panic!("expected Timestamp(Nanosecond, Some(\"UTC\")), got {other:?}"),
5810 }
5811 }
5812
5813 #[test]
5814 fn test_timestamp_nanos_decoding_local() {
5815 let avro_type = avro_from_codec(Codec::TimestampNanos(None));
5816 let mut decoder = Decoder::try_new(&avro_type).expect("create TimestampNanos decoder");
5817 let mut data = Vec::new();
5818 for v in [10_i64, 20_i64, -30_i64] {
5819 data.extend_from_slice(&encode_avro_long(v));
5820 }
5821 let mut cur = AvroCursor::new(&data);
5822 for _ in 0..3 {
5823 decoder.decode(&mut cur).expect("decode nanos ts");
5824 }
5825 let array = decoder.flush(None).expect("flush nanos ts");
5826 let ts = array
5827 .as_any()
5828 .downcast_ref::<TimestampNanosecondArray>()
5829 .expect("TimestampNanosecondArray");
5830 assert_eq!(ts.values(), &[10, 20, -30]);
5831 match ts.data_type() {
5832 DataType::Timestamp(arrow_schema::TimeUnit::Nanosecond, tz) => {
5833 assert_eq!(tz.as_deref(), None);
5834 }
5835 other => panic!("expected Timestamp(Nanosecond, None), got {other:?}"),
5836 }
5837 }
5838
5839 #[test]
5840 fn test_timestamp_nanos_decoding_with_nulls() {
5841 let avro_type = AvroDataType::new(
5842 Codec::TimestampNanos(None),
5843 Default::default(),
5844 Some(Nullability::NullFirst),
5845 );
5846 let mut decoder = Decoder::try_new(&avro_type).expect("create nullable TimestampNanos");
5847 let mut data = Vec::new();
5848 data.extend_from_slice(&encode_avro_long(1));
5849 data.extend_from_slice(&encode_avro_long(42));
5850 data.extend_from_slice(&encode_avro_long(0));
5851 data.extend_from_slice(&encode_avro_long(1));
5852 data.extend_from_slice(&encode_avro_long(-7));
5853 let mut cur = AvroCursor::new(&data);
5854 for _ in 0..3 {
5855 decoder.decode(&mut cur).expect("decode nullable nanos ts");
5856 }
5857 let array = decoder.flush(None).expect("flush nullable nanos ts");
5858 let ts = array
5859 .as_any()
5860 .downcast_ref::<TimestampNanosecondArray>()
5861 .expect("TimestampNanosecondArray");
5862 assert_eq!(ts.len(), 3);
5863 assert!(ts.is_valid(0));
5864 assert!(ts.is_null(1));
5865 assert!(ts.is_valid(2));
5866 assert_eq!(ts.value(0), 42);
5867 assert_eq!(ts.value(2), -7);
5868 match ts.data_type() {
5869 DataType::Timestamp(arrow_schema::TimeUnit::Nanosecond, tz) => {
5870 assert_eq!(tz.as_deref(), None);
5871 }
5872 other => panic!("expected Timestamp(Nanosecond, None), got {other:?}"),
5873 }
5874 }
5875}