Skip to main content

arrow_json/reader/
list_array.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use std::marker::PhantomData;
6use std::sync::Arc;
7
8use arrow_array::{
9    ArrayRef, FixedSizeListArray, GenericListArray, GenericListViewArray, OffsetSizeTrait,
10};
11use arrow_buffer::{NullBufferBuilder, OffsetBuffer, ScalarBuffer};
12use arrow_schema::{ArrowError, DataType, FieldRef};
13
14use crate::reader::tape::{Tape, TapeElement};
15use crate::reader::{ArrayDecoder, DecoderContext};
16
17pub type ListArrayDecoder<O> = ListLikeArrayDecoder<O, false>;
18pub type ListViewArrayDecoder<O> = ListLikeArrayDecoder<O, true>;
19
20pub struct ListLikeArrayDecoder<O, const IS_VIEW: bool> {
21    field: FieldRef,
22    decoder: Box<dyn ArrayDecoder>,
23    phantom: PhantomData<O>,
24    ignore_type_conflicts: bool,
25    is_nullable: bool,
26}
27
28impl<O: OffsetSizeTrait, const IS_VIEW: bool> ListLikeArrayDecoder<O, IS_VIEW> {
29    pub fn new(
30        ctx: &DecoderContext,
31        data_type: &DataType,
32        is_nullable: bool,
33    ) -> Result<Self, ArrowError> {
34        let field = match (IS_VIEW, data_type) {
35            (false, DataType::List(f)) if !O::IS_LARGE => f,
36            (false, DataType::LargeList(f)) if O::IS_LARGE => f,
37            (true, DataType::ListView(f)) if !O::IS_LARGE => f,
38            (true, DataType::LargeListView(f)) if O::IS_LARGE => f,
39            _ => unreachable!(),
40        };
41        let decoder = ctx.make_decoder(field, field.is_nullable())?;
42
43        Ok(Self {
44            field: field.clone(),
45            decoder,
46            phantom: Default::default(),
47            ignore_type_conflicts: ctx.ignore_type_conflicts(),
48            is_nullable,
49        })
50    }
51}
52
53impl<O: OffsetSizeTrait, const IS_VIEW: bool> ArrayDecoder for ListLikeArrayDecoder<O, IS_VIEW> {
54    fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef, ArrowError> {
55        let mut child_pos = Vec::with_capacity(pos.len());
56        let mut offsets = Vec::with_capacity(pos.len() + 1);
57        offsets.push(O::from_usize(0).unwrap());
58
59        let mut nulls = self.is_nullable.then(|| NullBufferBuilder::new(pos.len()));
60
61        for p in pos {
62            let end_idx = match (tape.get(*p), nulls.as_mut()) {
63                (TapeElement::StartList(end_idx), None) => end_idx,
64                (TapeElement::StartList(end_idx), Some(nulls)) => {
65                    nulls.append_non_null();
66                    end_idx
67                }
68                (TapeElement::Null, Some(nulls)) => {
69                    nulls.append_null();
70                    *p + 1
71                }
72                (_, Some(nulls)) if self.ignore_type_conflicts => {
73                    nulls.append_null();
74                    *p + 1
75                }
76                _ => return Err(tape.error(*p, "[")),
77            };
78
79            let mut cur_idx = *p + 1;
80            while cur_idx < end_idx {
81                child_pos.push(cur_idx);
82
83                // Advance to next field
84                cur_idx = tape.next(cur_idx, "list value")?;
85            }
86
87            let offset = O::from_usize(child_pos.len()).ok_or_else(|| {
88                ArrowError::JsonError(format!("offset overflow decoding {}ListArray", O::PREFIX))
89            })?;
90            offsets.push(offset);
91        }
92
93        let values = self.decoder.decode(tape, &child_pos)?;
94        let nulls = nulls.as_mut().and_then(|x| x.finish());
95
96        if IS_VIEW {
97            let mut sizes = Vec::with_capacity(offsets.len() - 1);
98            for i in 1..offsets.len() {
99                sizes.push(offsets[i] - offsets[i - 1]);
100            }
101            offsets.pop();
102            let array = GenericListViewArray::<O>::try_new(
103                self.field.clone(),
104                ScalarBuffer::from(offsets),
105                ScalarBuffer::from(sizes),
106                values,
107                nulls,
108            )?;
109            Ok(Arc::new(array))
110        } else {
111            // SAFETY: offsets are built monotonically starting from 0
112            let offsets = unsafe { OffsetBuffer::<O>::new_unchecked(ScalarBuffer::from(offsets)) };
113
114            let array = GenericListArray::<O>::try_new(self.field.clone(), offsets, values, nulls)?;
115            Ok(Arc::new(array))
116        }
117    }
118}
119
120pub struct FixedSizeListArrayDecoder {
121    field: FieldRef,
122    size: i32,
123    decoder: Box<dyn ArrayDecoder>,
124    ignore_type_conflicts: bool,
125    is_nullable: bool,
126}
127
128impl FixedSizeListArrayDecoder {
129    pub fn new(
130        ctx: &DecoderContext,
131        data_type: &DataType,
132        is_nullable: bool,
133    ) -> Result<Self, ArrowError> {
134        let (field, size) = match data_type {
135            DataType::FixedSizeList(f, s) => (f, *s),
136            _ => unreachable!(),
137        };
138        let decoder = ctx.make_decoder(field, field.is_nullable())?;
139
140        Ok(Self {
141            field: field.clone(),
142            size,
143            decoder,
144            ignore_type_conflicts: ctx.ignore_type_conflicts(),
145            is_nullable,
146        })
147    }
148}
149
150impl ArrayDecoder for FixedSizeListArrayDecoder {
151    fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef, ArrowError> {
152        let expected = self.size as usize;
153        let mut child_pos = Vec::with_capacity(pos.len() * expected);
154
155        let mut nulls = self.is_nullable.then(|| NullBufferBuilder::new(pos.len()));
156
157        for p in pos {
158            let end_idx = match (tape.get(*p), nulls.as_mut()) {
159                (TapeElement::StartList(end_idx), None) => end_idx,
160                (TapeElement::StartList(end_idx), Some(nulls)) => {
161                    nulls.append_non_null();
162                    end_idx
163                }
164                (TapeElement::Null, Some(nulls)) => {
165                    nulls.append_null();
166                    child_pos.resize(child_pos.len() + expected, 0);
167                    continue;
168                }
169                (_, Some(nulls)) if self.ignore_type_conflicts => {
170                    nulls.append_null();
171                    child_pos.resize(child_pos.len() + expected, 0);
172                    continue;
173                }
174                _ => return Err(tape.error(*p, "[")),
175            };
176
177            let child_start = child_pos.len();
178            let mut cur_idx = *p + 1;
179            while cur_idx < end_idx {
180                child_pos.push(cur_idx);
181                cur_idx = tape.next(cur_idx, "fixed-size list value")?;
182            }
183
184            let actual = child_pos.len() - child_start;
185            if actual != expected {
186                return Err(ArrowError::JsonError(format!(
187                    "Incorrect number of elements for FixedSizeList, \
188                     expected {expected} but got {actual}"
189                )));
190            }
191        }
192
193        let values = self.decoder.decode(tape, &child_pos)?;
194        let nulls = nulls.as_mut().and_then(|x| x.finish());
195
196        let array = FixedSizeListArray::try_new_with_length(
197            self.field.clone(),
198            self.size,
199            values,
200            nulls,
201            pos.len(),
202        )?;
203        Ok(Arc::new(array))
204    }
205}