arrow_integration_test/
field.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use crate::{data_type_from_json, data_type_to_json};
19use arrow::datatypes::{DataType, Field};
20use arrow::error::{ArrowError, Result};
21use std::collections::HashMap;
22use std::sync::Arc;
23
24/// Parse a `Field` definition from a JSON representation.
25pub fn field_from_json(json: &serde_json::Value) -> Result<Field> {
26    use serde_json::Value;
27    match *json {
28        Value::Object(ref map) => {
29            let name = match map.get("name") {
30                Some(Value::String(name)) => name.clone(),
31                _ => {
32                    return Err(ArrowError::ParseError(
33                        "Field missing 'name' attribute".to_string(),
34                    ));
35                }
36            };
37            let Some(&Value::Bool(nullable)) = map.get("nullable") else {
38                return Err(ArrowError::ParseError(
39                    "Field missing 'nullable' attribute".to_string(),
40                ));
41            };
42            let data_type = match map.get("type") {
43                Some(t) => data_type_from_json(t)?,
44                _ => {
45                    return Err(ArrowError::ParseError(
46                        "Field missing 'type' attribute".to_string(),
47                    ));
48                }
49            };
50
51            // Referenced example file: testing/data/arrow-ipc-stream/integration/1.0.0-littleendian/generated_custom_metadata.json.gz
52            let metadata = match map.get("metadata") {
53                Some(Value::Array(values)) => {
54                    let mut res: HashMap<String, String> = HashMap::default();
55                    for value in values {
56                        match value.as_object() {
57                            Some(map) => {
58                                if map.len() != 2 {
59                                    return Err(ArrowError::ParseError(
60                                        "Field 'metadata' must have exact two entries for each key-value map".to_string(),
61                                    ));
62                                }
63                                if let (Some(k), Some(v)) = (map.get("key"), map.get("value")) {
64                                    if let (Some(k_str), Some(v_str)) = (k.as_str(), v.as_str()) {
65                                        res.insert(
66                                            k_str.to_string().clone(),
67                                            v_str.to_string().clone(),
68                                        );
69                                    } else {
70                                        return Err(ArrowError::ParseError(
71                                            "Field 'metadata' must have map value of string type"
72                                                .to_string(),
73                                        ));
74                                    }
75                                } else {
76                                    return Err(ArrowError::ParseError("Field 'metadata' lacks map keys named \"key\" or \"value\"".to_string()));
77                                }
78                            }
79                            _ => {
80                                return Err(ArrowError::ParseError(
81                                    "Field 'metadata' contains non-object key-value pair"
82                                        .to_string(),
83                                ));
84                            }
85                        }
86                    }
87                    res
88                }
89                // We also support map format, because Schema's metadata supports this.
90                // See https://github.com/apache/arrow/pull/5907
91                Some(Value::Object(values)) => {
92                    let mut res: HashMap<String, String> = HashMap::default();
93                    for (k, v) in values {
94                        if let Some(str_value) = v.as_str() {
95                            res.insert(k.clone(), str_value.to_string().clone());
96                        } else {
97                            return Err(ArrowError::ParseError(format!(
98                                "Field 'metadata' contains non-string value for key {k}"
99                            )));
100                        }
101                    }
102                    res
103                }
104                Some(_) => {
105                    return Err(ArrowError::ParseError(
106                        "Field `metadata` is not json array".to_string(),
107                    ));
108                }
109                _ => HashMap::default(),
110            };
111
112            // if data_type is a struct or list, get its children
113            let data_type = match data_type {
114                DataType::List(_)
115                | DataType::LargeList(_)
116                | DataType::ListView(_)
117                | DataType::LargeListView(_)
118                | DataType::FixedSizeList(_, _) => match map.get("children") {
119                    Some(Value::Array(values)) => {
120                        if values.len() != 1 {
121                            return Err(ArrowError::ParseError(
122                                "Field 'children' must have one element for a list data type"
123                                    .to_string(),
124                            ));
125                        }
126                        match data_type {
127                            DataType::List(_) => {
128                                DataType::List(Arc::new(field_from_json(&values[0])?))
129                            }
130                            DataType::LargeList(_) => {
131                                DataType::LargeList(Arc::new(field_from_json(&values[0])?))
132                            }
133                            DataType::ListView(_) => {
134                                DataType::ListView(Arc::new(field_from_json(&values[0])?))
135                            }
136                            DataType::LargeListView(_) => {
137                                DataType::LargeListView(Arc::new(field_from_json(&values[0])?))
138                            }
139                            DataType::FixedSizeList(_, int) => {
140                                DataType::FixedSizeList(Arc::new(field_from_json(&values[0])?), int)
141                            }
142                            _ => unreachable!(
143                                "Data type should be a list, largelist, listview, largelistview or fixedsizelist"
144                            ),
145                        }
146                    }
147                    Some(_) => {
148                        return Err(ArrowError::ParseError(
149                            "Field 'children' must be an array".to_string(),
150                        ));
151                    }
152                    None => {
153                        return Err(ArrowError::ParseError(
154                            "Field missing 'children' attribute".to_string(),
155                        ));
156                    }
157                },
158                DataType::Struct(_) => match map.get("children") {
159                    Some(Value::Array(values)) => {
160                        DataType::Struct(values.iter().map(field_from_json).collect::<Result<_>>()?)
161                    }
162                    Some(_) => {
163                        return Err(ArrowError::ParseError(
164                            "Field 'children' must be an array".to_string(),
165                        ));
166                    }
167                    None => {
168                        return Err(ArrowError::ParseError(
169                            "Field missing 'children' attribute".to_string(),
170                        ));
171                    }
172                },
173                DataType::Map(_, keys_sorted) => {
174                    match map.get("children") {
175                        Some(Value::Array(values)) if values.len() == 1 => {
176                            let child = field_from_json(&values[0])?;
177                            // child must be a struct
178                            match child.data_type() {
179                                DataType::Struct(map_fields) if map_fields.len() == 2 => {
180                                    DataType::Map(Arc::new(child), keys_sorted)
181                                }
182                                t => {
183                                    return Err(ArrowError::ParseError(format!(
184                                        "Map children should be a struct with 2 fields, found {t:?}"
185                                    )));
186                                }
187                            }
188                        }
189                        Some(_) => {
190                            return Err(ArrowError::ParseError(
191                                "Field 'children' must be an array with 1 element".to_string(),
192                            ));
193                        }
194                        None => {
195                            return Err(ArrowError::ParseError(
196                                "Field missing 'children' attribute".to_string(),
197                            ));
198                        }
199                    }
200                }
201                DataType::Union(fields, mode) => match map.get("children") {
202                    Some(Value::Array(values)) => {
203                        let fields = fields
204                            .iter()
205                            .zip(values)
206                            .map(|((id, _), value)| Ok((id, Arc::new(field_from_json(value)?))))
207                            .collect::<Result<_>>()?;
208
209                        DataType::Union(fields, mode)
210                    }
211                    Some(_) => {
212                        return Err(ArrowError::ParseError(
213                            "Field 'children' must be an array".to_string(),
214                        ));
215                    }
216                    None => {
217                        return Err(ArrowError::ParseError(
218                            "Field missing 'children' attribute".to_string(),
219                        ));
220                    }
221                },
222                DataType::RunEndEncoded(_, _) => match map.get("children") {
223                    Some(Value::Array(values)) => {
224                        if values.len() != 2 {
225                            return Err(ArrowError::ParseError(
226                                "Field 'children' must have exactly 2 elements for RunEndEncoded"
227                                    .to_string(),
228                            ));
229                        }
230                        let run_ends = Arc::new(field_from_json(&values[0])?);
231                        let values_field = Arc::new(field_from_json(&values[1])?);
232                        DataType::RunEndEncoded(run_ends, values_field)
233                    }
234                    Some(_) => {
235                        return Err(ArrowError::ParseError(
236                            "Field 'children' must be an array".to_string(),
237                        ));
238                    }
239                    None => {
240                        return Err(ArrowError::ParseError(
241                            "Field missing 'children' attribute".to_string(),
242                        ));
243                    }
244                },
245                _ => data_type,
246            };
247
248            let mut dict_id = 0;
249            let mut dict_is_ordered = false;
250
251            let data_type = match map.get("dictionary") {
252                Some(dictionary) => {
253                    let index_type = match dictionary.get("indexType") {
254                        Some(t) => data_type_from_json(t)?,
255                        _ => {
256                            return Err(ArrowError::ParseError(
257                                "Field missing 'indexType' attribute".to_string(),
258                            ));
259                        }
260                    };
261                    dict_id = match dictionary.get("id") {
262                        Some(Value::Number(n)) => n.as_i64().ok_or_else(|| {
263                            ArrowError::ParseError(
264                                "Field 'id' attribute is not an integer".to_string(),
265                            )
266                        })?,
267                        _ => {
268                            return Err(ArrowError::ParseError(
269                                "Field missing 'id' attribute".to_string(),
270                            ));
271                        }
272                    };
273                    dict_is_ordered = match dictionary.get("isOrdered") {
274                        Some(&Value::Bool(n)) => n,
275                        _ => {
276                            return Err(ArrowError::ParseError(
277                                "Field missing 'isOrdered' attribute".to_string(),
278                            ));
279                        }
280                    };
281                    DataType::Dictionary(Box::new(index_type), Box::new(data_type))
282                }
283                _ => data_type,
284            };
285
286            #[expect(deprecated)]
287            let mut field = Field::new_dict(name, data_type, nullable, dict_id, dict_is_ordered);
288            field.set_metadata(metadata);
289            Ok(field)
290        }
291        _ => Err(ArrowError::ParseError(
292            "Invalid json value type for field".to_string(),
293        )),
294    }
295}
296
297/// Generate a JSON representation of the `Field`.
298pub fn field_to_json(field: &Field) -> serde_json::Value {
299    let children: Vec<serde_json::Value> = match field.data_type() {
300        DataType::Struct(fields) => fields.iter().map(|x| field_to_json(x.as_ref())).collect(),
301        DataType::List(field)
302        | DataType::LargeList(field)
303        | DataType::ListView(field)
304        | DataType::LargeListView(field)
305        | DataType::FixedSizeList(field, _)
306        | DataType::Map(field, _) => vec![field_to_json(field)],
307        DataType::RunEndEncoded(run_ends, values) => {
308            vec![field_to_json(run_ends), field_to_json(values)]
309        }
310        _ => vec![],
311    };
312
313    let mut json = match field.data_type() {
314        DataType::Dictionary(index_type, value_type) => {
315            #[expect(deprecated)]
316            let dict_id = field.dict_id().unwrap();
317            serde_json::json!({
318                "name": field.name(),
319                "nullable": field.is_nullable(),
320                "type": data_type_to_json(value_type),
321                "children": children,
322                "dictionary": {
323                    "id": dict_id,
324                    "indexType": data_type_to_json(index_type),
325                    "isOrdered": field.dict_is_ordered().unwrap(),
326                }
327            })
328        }
329        _ => serde_json::json!({
330            "name": field.name(),
331            "nullable": field.is_nullable(),
332            "type": data_type_to_json(field.data_type()),
333            "children": children
334        }),
335    };
336
337    if !field.metadata().is_empty() {
338        json["metadata"] = field
339            .metadata()
340            .iter()
341            .map(|(key, value)| (key.clone(), value.clone()))
342            .collect();
343    }
344
345    json
346}
347
348#[cfg(test)]
349mod tests {
350    use super::*;
351    use arrow::datatypes::UnionMode;
352    use serde_json::Value;
353
354    #[test]
355    fn struct_field_to_json() {
356        let f = Field::new_struct(
357            "address",
358            vec![
359                Field::new("street", DataType::Utf8, false),
360                Field::new("zip", DataType::UInt16, false),
361            ],
362            false,
363        );
364        let value: Value = serde_json::from_str(
365            r#"{
366                "name": "address",
367                "nullable": false,
368                "type": {
369                    "name": "struct"
370                },
371                "children": [
372                    {
373                        "name": "street",
374                        "nullable": false,
375                        "type": {
376                            "name": "utf8"
377                        },
378                        "children": []
379                    },
380                    {
381                        "name": "zip",
382                        "nullable": false,
383                        "type": {
384                            "name": "int",
385                            "bitWidth": 16,
386                            "isSigned": false
387                        },
388                        "children": []
389                    }
390                ]
391            }"#,
392        )
393        .unwrap();
394        assert_eq!(value, field_to_json(&f));
395    }
396
397    #[test]
398    fn map_field_to_json() {
399        let f = Field::new_map(
400            "my_map",
401            "my_entries",
402            Field::new("my_keys", DataType::Utf8, false),
403            Field::new("my_values", DataType::UInt16, true),
404            true,
405            false,
406        );
407        let value: Value = serde_json::from_str(
408            r#"{
409                "name": "my_map",
410                "nullable": false,
411                "type": {
412                    "name": "map",
413                    "keysSorted": true
414                },
415                "children": [
416                    {
417                        "name": "my_entries",
418                        "nullable": false,
419                        "type": {
420                            "name": "struct"
421                        },
422                        "children": [
423                            {
424                                "name": "my_keys",
425                                "nullable": false,
426                                "type": {
427                                    "name": "utf8"
428                                },
429                                "children": []
430                            },
431                            {
432                                "name": "my_values",
433                                "nullable": true,
434                                "type": {
435                                    "name": "int",
436                                    "bitWidth": 16,
437                                    "isSigned": false
438                                },
439                                "children": []
440                            }
441                        ]
442                    }
443                ]
444            }"#,
445        )
446        .unwrap();
447        assert_eq!(value, field_to_json(&f));
448    }
449
450    #[test]
451    fn primitive_field_to_json() {
452        let f = Field::new("first_name", DataType::Utf8, false);
453        let value: Value = serde_json::from_str(
454            r#"{
455                "name": "first_name",
456                "nullable": false,
457                "type": {
458                    "name": "utf8"
459                },
460                "children": []
461            }"#,
462        )
463        .unwrap();
464        assert_eq!(value, field_to_json(&f));
465    }
466    #[test]
467    fn parse_struct_from_json() {
468        let json = r#"
469        {
470            "name": "address",
471            "type": {
472                "name": "struct"
473            },
474            "nullable": false,
475            "children": [
476                {
477                    "name": "street",
478                    "type": {
479                    "name": "utf8"
480                    },
481                    "nullable": false,
482                    "children": []
483                },
484                {
485                    "name": "zip",
486                    "type": {
487                    "name": "int",
488                    "isSigned": false,
489                    "bitWidth": 16
490                    },
491                    "nullable": false,
492                    "children": []
493                }
494            ]
495        }
496        "#;
497        let value: Value = serde_json::from_str(json).unwrap();
498        let dt = field_from_json(&value).unwrap();
499
500        let expected = Field::new_struct(
501            "address",
502            vec![
503                Field::new("street", DataType::Utf8, false),
504                Field::new("zip", DataType::UInt16, false),
505            ],
506            false,
507        );
508
509        assert_eq!(expected, dt);
510    }
511
512    #[test]
513    fn parse_map_from_json() {
514        let json = r#"
515        {
516            "name": "my_map",
517            "nullable": false,
518            "type": {
519                "name": "map",
520                "keysSorted": true
521            },
522            "children": [
523                {
524                    "name": "my_entries",
525                    "nullable": false,
526                    "type": {
527                        "name": "struct"
528                    },
529                    "children": [
530                        {
531                            "name": "my_keys",
532                            "nullable": false,
533                            "type": {
534                                "name": "utf8"
535                            },
536                            "children": []
537                        },
538                        {
539                            "name": "my_values",
540                            "nullable": true,
541                            "type": {
542                                "name": "int",
543                                "bitWidth": 16,
544                                "isSigned": false
545                            },
546                            "children": []
547                        }
548                    ]
549                }
550            ]
551        }
552        "#;
553        let value: Value = serde_json::from_str(json).unwrap();
554        let dt = field_from_json(&value).unwrap();
555
556        let expected = Field::new_map(
557            "my_map",
558            "my_entries",
559            Field::new("my_keys", DataType::Utf8, false),
560            Field::new("my_values", DataType::UInt16, true),
561            true,
562            false,
563        );
564
565        assert_eq!(expected, dt);
566    }
567
568    #[test]
569    fn parse_union_from_json() {
570        let json = r#"
571        {
572            "name": "my_union",
573            "nullable": false,
574            "type": {
575                "name": "union",
576                "mode": "SPARSE",
577                "typeIds": [
578                    5,
579                    7
580                ]
581            },
582            "children": [
583                {
584                    "name": "f1",
585                    "type": {
586                        "name": "int",
587                        "isSigned": true,
588                        "bitWidth": 32
589                    },
590                    "nullable": true,
591                    "children": []
592                },
593                {
594                    "name": "f2",
595                    "type": {
596                        "name": "utf8"
597                    },
598                    "nullable": true,
599                    "children": []
600                }
601            ]
602        }
603        "#;
604        let value: Value = serde_json::from_str(json).unwrap();
605        let dt = field_from_json(&value).unwrap();
606
607        let expected = Field::new_union(
608            "my_union",
609            vec![5, 7],
610            vec![
611                Field::new("f1", DataType::Int32, true),
612                Field::new("f2", DataType::Utf8, true),
613            ],
614            UnionMode::Sparse,
615        );
616
617        assert_eq!(expected, dt);
618    }
619}
arrow_integration_test/field.rs

arrow_integration_test/
field.rs
