arrow_integration_test/
field.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use crate::{data_type_from_json, data_type_to_json};
19use arrow::datatypes::{DataType, Field};
20use arrow::error::{ArrowError, Result};
21use std::collections::HashMap;
22use std::sync::Arc;
23
24/// Parse a `Field` definition from a JSON representation.
25pub fn field_from_json(json: &serde_json::Value) -> Result<Field> {
26    use serde_json::Value;
27    match *json {
28        Value::Object(ref map) => {
29            let name = match map.get("name") {
30                Some(Value::String(name)) => name.clone(),
31                _ => {
32                    return Err(ArrowError::ParseError(
33                        "Field missing 'name' attribute".to_string(),
34                    ));
35                }
36            };
37            let Some(&Value::Bool(nullable)) = map.get("nullable") else {
38                return Err(ArrowError::ParseError(
39                    "Field missing 'nullable' attribute".to_string(),
40                ));
41            };
42            let data_type = match map.get("type") {
43                Some(t) => data_type_from_json(t)?,
44                _ => {
45                    return Err(ArrowError::ParseError(
46                        "Field missing 'type' attribute".to_string(),
47                    ));
48                }
49            };
50
51            // Referenced example file: testing/data/arrow-ipc-stream/integration/1.0.0-littleendian/generated_custom_metadata.json.gz
52            let metadata = match map.get("metadata") {
53                Some(Value::Array(values)) => {
54                    let mut res: HashMap<String, String> = HashMap::default();
55                    for value in values {
56                        match value.as_object() {
57                            Some(map) => {
58                                if map.len() != 2 {
59                                    return Err(ArrowError::ParseError(
60                                        "Field 'metadata' must have exact two entries for each key-value map".to_string(),
61                                    ));
62                                }
63                                if let (Some(k), Some(v)) = (map.get("key"), map.get("value")) {
64                                    if let (Some(k_str), Some(v_str)) = (k.as_str(), v.as_str()) {
65                                        res.insert(
66                                            k_str.to_string().clone(),
67                                            v_str.to_string().clone(),
68                                        );
69                                    } else {
70                                        return Err(ArrowError::ParseError(
71                                            "Field 'metadata' must have map value of string type"
72                                                .to_string(),
73                                        ));
74                                    }
75                                } else {
76                                    return Err(ArrowError::ParseError("Field 'metadata' lacks map keys named \"key\" or \"value\"".to_string()));
77                                }
78                            }
79                            _ => {
80                                return Err(ArrowError::ParseError(
81                                    "Field 'metadata' contains non-object key-value pair"
82                                        .to_string(),
83                                ));
84                            }
85                        }
86                    }
87                    res
88                }
89                // We also support map format, because Schema's metadata supports this.
90                // See https://github.com/apache/arrow/pull/5907
91                Some(Value::Object(values)) => {
92                    let mut res: HashMap<String, String> = HashMap::default();
93                    for (k, v) in values {
94                        if let Some(str_value) = v.as_str() {
95                            res.insert(k.clone(), str_value.to_string().clone());
96                        } else {
97                            return Err(ArrowError::ParseError(format!(
98                                "Field 'metadata' contains non-string value for key {k}"
99                            )));
100                        }
101                    }
102                    res
103                }
104                Some(_) => {
105                    return Err(ArrowError::ParseError(
106                        "Field `metadata` is not json array".to_string(),
107                    ));
108                }
109                _ => HashMap::default(),
110            };
111
112            // if data_type is a struct or list, get its children
113            let data_type = match data_type {
114                DataType::List(_)
115                | DataType::LargeList(_)
116                | DataType::ListView(_)
117                | DataType::LargeListView(_)
118                | DataType::FixedSizeList(_, _) => match map.get("children") {
119                    Some(Value::Array(values)) => {
120                        if values.len() != 1 {
121                            return Err(ArrowError::ParseError(
122                                "Field 'children' must have one element for a list data type"
123                                    .to_string(),
124                            ));
125                        }
126                        match data_type {
127                            DataType::List(_) => {
128                                DataType::List(Arc::new(field_from_json(&values[0])?))
129                            }
130                            DataType::LargeList(_) => {
131                                DataType::LargeList(Arc::new(field_from_json(&values[0])?))
132                            }
133                            DataType::ListView(_) => {
134                                DataType::ListView(Arc::new(field_from_json(&values[0])?))
135                            }
136                            DataType::LargeListView(_) => {
137                                DataType::LargeListView(Arc::new(field_from_json(&values[0])?))
138                            }
139                            DataType::FixedSizeList(_, int) => {
140                                DataType::FixedSizeList(Arc::new(field_from_json(&values[0])?), int)
141                            }
142                            _ => unreachable!(
143                                "Data type should be a list, largelist, listview, largelistview or fixedsizelist"
144                            ),
145                        }
146                    }
147                    Some(_) => {
148                        return Err(ArrowError::ParseError(
149                            "Field 'children' must be an array".to_string(),
150                        ));
151                    }
152                    None => {
153                        return Err(ArrowError::ParseError(
154                            "Field missing 'children' attribute".to_string(),
155                        ));
156                    }
157                },
158                DataType::Struct(_) => match map.get("children") {
159                    Some(Value::Array(values)) => {
160                        DataType::Struct(values.iter().map(field_from_json).collect::<Result<_>>()?)
161                    }
162                    Some(_) => {
163                        return Err(ArrowError::ParseError(
164                            "Field 'children' must be an array".to_string(),
165                        ));
166                    }
167                    None => {
168                        return Err(ArrowError::ParseError(
169                            "Field missing 'children' attribute".to_string(),
170                        ));
171                    }
172                },
173                DataType::Map(_, keys_sorted) => {
174                    match map.get("children") {
175                        Some(Value::Array(values)) if values.len() == 1 => {
176                            let child = field_from_json(&values[0])?;
177                            // child must be a struct
178                            match child.data_type() {
179                                DataType::Struct(map_fields) if map_fields.len() == 2 => {
180                                    DataType::Map(Arc::new(child), keys_sorted)
181                                }
182                                t => {
183                                    return Err(ArrowError::ParseError(format!(
184                                        "Map children should be a struct with 2 fields, found {t:?}"
185                                    )));
186                                }
187                            }
188                        }
189                        Some(_) => {
190                            return Err(ArrowError::ParseError(
191                                "Field 'children' must be an array with 1 element".to_string(),
192                            ));
193                        }
194                        None => {
195                            return Err(ArrowError::ParseError(
196                                "Field missing 'children' attribute".to_string(),
197                            ));
198                        }
199                    }
200                }
201                DataType::Union(fields, mode) => match map.get("children") {
202                    Some(Value::Array(values)) => {
203                        let fields = fields
204                            .iter()
205                            .zip(values)
206                            .map(|((id, _), value)| Ok((id, Arc::new(field_from_json(value)?))))
207                            .collect::<Result<_>>()?;
208
209                        DataType::Union(fields, mode)
210                    }
211                    Some(_) => {
212                        return Err(ArrowError::ParseError(
213                            "Field 'children' must be an array".to_string(),
214                        ));
215                    }
216                    None => {
217                        return Err(ArrowError::ParseError(
218                            "Field missing 'children' attribute".to_string(),
219                        ));
220                    }
221                },
222                DataType::RunEndEncoded(_, _) => match map.get("children") {
223                    Some(Value::Array(values)) => {
224                        if values.len() != 2 {
225                            return Err(ArrowError::ParseError(
226                                "Field 'children' must have exactly 2 elements for RunEndEncoded"
227                                    .to_string(),
228                            ));
229                        }
230                        let run_ends = Arc::new(field_from_json(&values[0])?);
231                        let values_field = Arc::new(field_from_json(&values[1])?);
232                        DataType::RunEndEncoded(run_ends, values_field)
233                    }
234                    Some(_) => {
235                        return Err(ArrowError::ParseError(
236                            "Field 'children' must be an array".to_string(),
237                        ));
238                    }
239                    None => {
240                        return Err(ArrowError::ParseError(
241                            "Field missing 'children' attribute".to_string(),
242                        ));
243                    }
244                },
245                _ => data_type,
246            };
247
248            let mut dict_id = 0;
249            let mut dict_is_ordered = false;
250
251            let data_type = match map.get("dictionary") {
252                Some(dictionary) => {
253                    let index_type = match dictionary.get("indexType") {
254                        Some(t) => data_type_from_json(t)?,
255                        _ => {
256                            return Err(ArrowError::ParseError(
257                                "Field missing 'indexType' attribute".to_string(),
258                            ));
259                        }
260                    };
261                    dict_id = match dictionary.get("id") {
262                        Some(Value::Number(n)) => n.as_i64().unwrap(),
263                        _ => {
264                            return Err(ArrowError::ParseError(
265                                "Field missing 'id' attribute".to_string(),
266                            ));
267                        }
268                    };
269                    dict_is_ordered = match dictionary.get("isOrdered") {
270                        Some(&Value::Bool(n)) => n,
271                        _ => {
272                            return Err(ArrowError::ParseError(
273                                "Field missing 'isOrdered' attribute".to_string(),
274                            ));
275                        }
276                    };
277                    DataType::Dictionary(Box::new(index_type), Box::new(data_type))
278                }
279                _ => data_type,
280            };
281
282            #[expect(deprecated)]
283            let mut field = Field::new_dict(name, data_type, nullable, dict_id, dict_is_ordered);
284            field.set_metadata(metadata);
285            Ok(field)
286        }
287        _ => Err(ArrowError::ParseError(
288            "Invalid json value type for field".to_string(),
289        )),
290    }
291}
292
293/// Generate a JSON representation of the `Field`.
294pub fn field_to_json(field: &Field) -> serde_json::Value {
295    let children: Vec<serde_json::Value> = match field.data_type() {
296        DataType::Struct(fields) => fields.iter().map(|x| field_to_json(x.as_ref())).collect(),
297        DataType::List(field)
298        | DataType::LargeList(field)
299        | DataType::ListView(field)
300        | DataType::LargeListView(field)
301        | DataType::FixedSizeList(field, _)
302        | DataType::Map(field, _) => vec![field_to_json(field)],
303        DataType::RunEndEncoded(run_ends, values) => {
304            vec![field_to_json(run_ends), field_to_json(values)]
305        }
306        _ => vec![],
307    };
308
309    match field.data_type() {
310        DataType::Dictionary(index_type, value_type) => {
311            #[expect(deprecated)]
312            let dict_id = field.dict_id().unwrap();
313            serde_json::json!({
314                "name": field.name(),
315                "nullable": field.is_nullable(),
316                "type": data_type_to_json(value_type),
317                "children": children,
318                "dictionary": {
319                    "id": dict_id,
320                    "indexType": data_type_to_json(index_type),
321                    "isOrdered": field.dict_is_ordered().unwrap(),
322                }
323            })
324        }
325        _ => serde_json::json!({
326            "name": field.name(),
327            "nullable": field.is_nullable(),
328            "type": data_type_to_json(field.data_type()),
329            "children": children
330        }),
331    }
332}
333
334#[cfg(test)]
335mod tests {
336    use super::*;
337    use arrow::datatypes::UnionMode;
338    use serde_json::Value;
339
340    #[test]
341    fn struct_field_to_json() {
342        let f = Field::new_struct(
343            "address",
344            vec![
345                Field::new("street", DataType::Utf8, false),
346                Field::new("zip", DataType::UInt16, false),
347            ],
348            false,
349        );
350        let value: Value = serde_json::from_str(
351            r#"{
352                "name": "address",
353                "nullable": false,
354                "type": {
355                    "name": "struct"
356                },
357                "children": [
358                    {
359                        "name": "street",
360                        "nullable": false,
361                        "type": {
362                            "name": "utf8"
363                        },
364                        "children": []
365                    },
366                    {
367                        "name": "zip",
368                        "nullable": false,
369                        "type": {
370                            "name": "int",
371                            "bitWidth": 16,
372                            "isSigned": false
373                        },
374                        "children": []
375                    }
376                ]
377            }"#,
378        )
379        .unwrap();
380        assert_eq!(value, field_to_json(&f));
381    }
382
383    #[test]
384    fn map_field_to_json() {
385        let f = Field::new_map(
386            "my_map",
387            "my_entries",
388            Field::new("my_keys", DataType::Utf8, false),
389            Field::new("my_values", DataType::UInt16, true),
390            true,
391            false,
392        );
393        let value: Value = serde_json::from_str(
394            r#"{
395                "name": "my_map",
396                "nullable": false,
397                "type": {
398                    "name": "map",
399                    "keysSorted": true
400                },
401                "children": [
402                    {
403                        "name": "my_entries",
404                        "nullable": false,
405                        "type": {
406                            "name": "struct"
407                        },
408                        "children": [
409                            {
410                                "name": "my_keys",
411                                "nullable": false,
412                                "type": {
413                                    "name": "utf8"
414                                },
415                                "children": []
416                            },
417                            {
418                                "name": "my_values",
419                                "nullable": true,
420                                "type": {
421                                    "name": "int",
422                                    "bitWidth": 16,
423                                    "isSigned": false
424                                },
425                                "children": []
426                            }
427                        ]
428                    }
429                ]
430            }"#,
431        )
432        .unwrap();
433        assert_eq!(value, field_to_json(&f));
434    }
435
436    #[test]
437    fn primitive_field_to_json() {
438        let f = Field::new("first_name", DataType::Utf8, false);
439        let value: Value = serde_json::from_str(
440            r#"{
441                "name": "first_name",
442                "nullable": false,
443                "type": {
444                    "name": "utf8"
445                },
446                "children": []
447            }"#,
448        )
449        .unwrap();
450        assert_eq!(value, field_to_json(&f));
451    }
452    #[test]
453    fn parse_struct_from_json() {
454        let json = r#"
455        {
456            "name": "address",
457            "type": {
458                "name": "struct"
459            },
460            "nullable": false,
461            "children": [
462                {
463                    "name": "street",
464                    "type": {
465                    "name": "utf8"
466                    },
467                    "nullable": false,
468                    "children": []
469                },
470                {
471                    "name": "zip",
472                    "type": {
473                    "name": "int",
474                    "isSigned": false,
475                    "bitWidth": 16
476                    },
477                    "nullable": false,
478                    "children": []
479                }
480            ]
481        }
482        "#;
483        let value: Value = serde_json::from_str(json).unwrap();
484        let dt = field_from_json(&value).unwrap();
485
486        let expected = Field::new_struct(
487            "address",
488            vec![
489                Field::new("street", DataType::Utf8, false),
490                Field::new("zip", DataType::UInt16, false),
491            ],
492            false,
493        );
494
495        assert_eq!(expected, dt);
496    }
497
498    #[test]
499    fn parse_map_from_json() {
500        let json = r#"
501        {
502            "name": "my_map",
503            "nullable": false,
504            "type": {
505                "name": "map",
506                "keysSorted": true
507            },
508            "children": [
509                {
510                    "name": "my_entries",
511                    "nullable": false,
512                    "type": {
513                        "name": "struct"
514                    },
515                    "children": [
516                        {
517                            "name": "my_keys",
518                            "nullable": false,
519                            "type": {
520                                "name": "utf8"
521                            },
522                            "children": []
523                        },
524                        {
525                            "name": "my_values",
526                            "nullable": true,
527                            "type": {
528                                "name": "int",
529                                "bitWidth": 16,
530                                "isSigned": false
531                            },
532                            "children": []
533                        }
534                    ]
535                }
536            ]
537        }
538        "#;
539        let value: Value = serde_json::from_str(json).unwrap();
540        let dt = field_from_json(&value).unwrap();
541
542        let expected = Field::new_map(
543            "my_map",
544            "my_entries",
545            Field::new("my_keys", DataType::Utf8, false),
546            Field::new("my_values", DataType::UInt16, true),
547            true,
548            false,
549        );
550
551        assert_eq!(expected, dt);
552    }
553
554    #[test]
555    fn parse_union_from_json() {
556        let json = r#"
557        {
558            "name": "my_union",
559            "nullable": false,
560            "type": {
561                "name": "union",
562                "mode": "SPARSE",
563                "typeIds": [
564                    5,
565                    7
566                ]
567            },
568            "children": [
569                {
570                    "name": "f1",
571                    "type": {
572                        "name": "int",
573                        "isSigned": true,
574                        "bitWidth": 32
575                    },
576                    "nullable": true,
577                    "children": []
578                },
579                {
580                    "name": "f2",
581                    "type": {
582                        "name": "utf8"
583                    },
584                    "nullable": true,
585                    "children": []
586                }
587            ]
588        }
589        "#;
590        let value: Value = serde_json::from_str(json).unwrap();
591        let dt = field_from_json(&value).unwrap();
592
593        let expected = Field::new_union(
594            "my_union",
595            vec![5, 7],
596            vec![
597                Field::new("f1", DataType::Int32, true),
598                Field::new("f2", DataType::Utf8, true),
599            ],
600            UnionMode::Sparse,
601        );
602
603        assert_eq!(expected, dt);
604    }
605}
arrow_integration_test/field.rs

arrow_integration_test/
field.rs
