1use std::marker::PhantomData;
6use std::sync::Arc;
7
8use arrow_array::{
9 ArrayRef, FixedSizeListArray, GenericListArray, GenericListViewArray, OffsetSizeTrait,
10};
11use arrow_buffer::{NullBufferBuilder, OffsetBuffer, ScalarBuffer};
12use arrow_schema::{ArrowError, DataType, FieldRef};
13
14use crate::reader::tape::{Tape, TapeElement};
15use crate::reader::{ArrayDecoder, DecoderContext};
16
17pub type ListArrayDecoder<O> = ListLikeArrayDecoder<O, false>;
18pub type ListViewArrayDecoder<O> = ListLikeArrayDecoder<O, true>;
19
20pub struct ListLikeArrayDecoder<O, const IS_VIEW: bool> {
21 field: FieldRef,
22 decoder: Box<dyn ArrayDecoder>,
23 phantom: PhantomData<O>,
24 ignore_type_conflicts: bool,
25 is_nullable: bool,
26}
27
28impl<O: OffsetSizeTrait, const IS_VIEW: bool> ListLikeArrayDecoder<O, IS_VIEW> {
29 pub fn new(
30 ctx: &DecoderContext,
31 data_type: &DataType,
32 is_nullable: bool,
33 ) -> Result<Self, ArrowError> {
34 let field = match (IS_VIEW, data_type) {
35 (false, DataType::List(f)) if !O::IS_LARGE => f,
36 (false, DataType::LargeList(f)) if O::IS_LARGE => f,
37 (true, DataType::ListView(f)) if !O::IS_LARGE => f,
38 (true, DataType::LargeListView(f)) if O::IS_LARGE => f,
39 _ => unreachable!(),
40 };
41 let decoder = ctx.make_decoder(field, field.is_nullable())?;
42
43 Ok(Self {
44 field: field.clone(),
45 decoder,
46 phantom: Default::default(),
47 ignore_type_conflicts: ctx.ignore_type_conflicts(),
48 is_nullable,
49 })
50 }
51}
52
53impl<O: OffsetSizeTrait, const IS_VIEW: bool> ArrayDecoder for ListLikeArrayDecoder<O, IS_VIEW> {
54 fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef, ArrowError> {
55 let mut child_pos = Vec::with_capacity(pos.len());
56 let mut offsets = Vec::with_capacity(pos.len() + 1);
57 offsets.push(O::from_usize(0).unwrap());
58
59 let mut nulls = self.is_nullable.then(|| NullBufferBuilder::new(pos.len()));
60
61 for p in pos {
62 let end_idx = match (tape.get(*p), nulls.as_mut()) {
63 (TapeElement::StartList(end_idx), None) => end_idx,
64 (TapeElement::StartList(end_idx), Some(nulls)) => {
65 nulls.append_non_null();
66 end_idx
67 }
68 (TapeElement::Null, Some(nulls)) => {
69 nulls.append_null();
70 *p + 1
71 }
72 (_, Some(nulls)) if self.ignore_type_conflicts => {
73 nulls.append_null();
74 *p + 1
75 }
76 _ => return Err(tape.error(*p, "[")),
77 };
78
79 let mut cur_idx = *p + 1;
80 while cur_idx < end_idx {
81 child_pos.push(cur_idx);
82
83 cur_idx = tape.next(cur_idx, "list value")?;
85 }
86
87 let offset = O::from_usize(child_pos.len()).ok_or_else(|| {
88 ArrowError::JsonError(format!("offset overflow decoding {}ListArray", O::PREFIX))
89 })?;
90 offsets.push(offset);
91 }
92
93 let values = self.decoder.decode(tape, &child_pos)?;
94 let nulls = nulls.as_mut().and_then(|x| x.finish());
95
96 if IS_VIEW {
97 let mut sizes = Vec::with_capacity(offsets.len() - 1);
98 for i in 1..offsets.len() {
99 sizes.push(offsets[i] - offsets[i - 1]);
100 }
101 offsets.pop();
102 let array = GenericListViewArray::<O>::try_new(
103 self.field.clone(),
104 ScalarBuffer::from(offsets),
105 ScalarBuffer::from(sizes),
106 values,
107 nulls,
108 )?;
109 Ok(Arc::new(array))
110 } else {
111 let offsets = unsafe { OffsetBuffer::<O>::new_unchecked(ScalarBuffer::from(offsets)) };
113
114 let array = GenericListArray::<O>::try_new(self.field.clone(), offsets, values, nulls)?;
115 Ok(Arc::new(array))
116 }
117 }
118}
119
120pub struct FixedSizeListArrayDecoder {
121 field: FieldRef,
122 size: i32,
123 decoder: Box<dyn ArrayDecoder>,
124 ignore_type_conflicts: bool,
125 is_nullable: bool,
126}
127
128impl FixedSizeListArrayDecoder {
129 pub fn new(
130 ctx: &DecoderContext,
131 data_type: &DataType,
132 is_nullable: bool,
133 ) -> Result<Self, ArrowError> {
134 let (field, size) = match data_type {
135 DataType::FixedSizeList(f, s) => (f, *s),
136 _ => unreachable!(),
137 };
138 let decoder = ctx.make_decoder(field, field.is_nullable())?;
139
140 Ok(Self {
141 field: field.clone(),
142 size,
143 decoder,
144 ignore_type_conflicts: ctx.ignore_type_conflicts(),
145 is_nullable,
146 })
147 }
148}
149
150impl ArrayDecoder for FixedSizeListArrayDecoder {
151 fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef, ArrowError> {
152 let expected = self.size as usize;
153 let mut child_pos = Vec::with_capacity(pos.len() * expected);
154
155 let mut nulls = self.is_nullable.then(|| NullBufferBuilder::new(pos.len()));
156
157 for p in pos {
158 let end_idx = match (tape.get(*p), nulls.as_mut()) {
159 (TapeElement::StartList(end_idx), None) => end_idx,
160 (TapeElement::StartList(end_idx), Some(nulls)) => {
161 nulls.append_non_null();
162 end_idx
163 }
164 (TapeElement::Null, Some(nulls)) => {
165 nulls.append_null();
166 child_pos.resize(child_pos.len() + expected, 0);
167 continue;
168 }
169 (_, Some(nulls)) if self.ignore_type_conflicts => {
170 nulls.append_null();
171 child_pos.resize(child_pos.len() + expected, 0);
172 continue;
173 }
174 _ => return Err(tape.error(*p, "[")),
175 };
176
177 let child_start = child_pos.len();
178 let mut cur_idx = *p + 1;
179 while cur_idx < end_idx {
180 child_pos.push(cur_idx);
181 cur_idx = tape.next(cur_idx, "fixed-size list value")?;
182 }
183
184 let actual = child_pos.len() - child_start;
185 if actual != expected {
186 return Err(ArrowError::JsonError(format!(
187 "Incorrect number of elements for FixedSizeList, \
188 expected {expected} but got {actual}"
189 )));
190 }
191 }
192
193 let values = self.decoder.decode(tape, &child_pos)?;
194 let nulls = nulls.as_mut().and_then(|x| x.finish());
195
196 let array = FixedSizeListArray::try_new_with_length(
197 self.field.clone(),
198 self.size,
199 values,
200 nulls,
201 pos.len(),
202 )?;
203 Ok(Arc::new(array))
204 }
205}