parquet_variant/builder/
metadata.rs1use std::collections::HashMap;
6
7use arrow_schema::ArrowError;
8use indexmap::IndexSet;
9
10use crate::{VARIANT_SPEC_VERSION, VariantMetadata, int_size};
11
12fn write_offset(buf: &mut Vec<u8>, value: usize, nbytes: u8) {
14 let bytes = value.to_le_bytes();
15 buf.extend_from_slice(&bytes[..nbytes as usize]);
16}
17
18pub trait MetadataBuilder: std::fmt::Debug {
24 fn try_upsert_field_name(&mut self, field_name: &str) -> Result<u32, ArrowError>;
28
29 fn field_name(&self, field_id: usize) -> &str;
32
33 fn num_field_names(&self) -> usize;
37
38 fn truncate_field_names(&mut self, new_size: usize);
40
41 fn finish(&mut self) -> usize;
43}
44
45impl MetadataBuilder for WritableMetadataBuilder {
46 fn try_upsert_field_name(&mut self, field_name: &str) -> Result<u32, ArrowError> {
47 Ok(self.upsert_field_name(field_name))
48 }
49 fn field_name(&self, field_id: usize) -> &str {
50 self.field_name(field_id)
51 }
52 fn num_field_names(&self) -> usize {
53 self.num_field_names()
54 }
55 fn truncate_field_names(&mut self, new_size: usize) {
56 self.field_names.truncate(new_size)
57 }
58 fn finish(&mut self) -> usize {
59 self.finish()
60 }
61}
62
63#[derive(Debug)]
72pub struct ReadOnlyMetadataBuilder<'m> {
73 metadata: &'m VariantMetadata<'m>,
74 known_field_names: HashMap<&'m str, u32>,
79}
80
81impl<'m> ReadOnlyMetadataBuilder<'m> {
82 pub fn new(metadata: &'m VariantMetadata<'m>) -> Self {
84 Self {
85 metadata,
86 known_field_names: HashMap::new(),
87 }
88 }
89}
90
91impl MetadataBuilder for ReadOnlyMetadataBuilder<'_> {
92 fn try_upsert_field_name(&mut self, field_name: &str) -> Result<u32, ArrowError> {
93 if let Some(field_id) = self.metadata.borrowed_field_id(field_name) {
101 return Ok(field_id);
102 }
103
104 if let Some(field_id) = self.known_field_names.get(field_name) {
105 return Ok(*field_id);
106 }
107
108 let Some((field_id, field_name)) = self.metadata.get_entry(field_name) else {
109 return Err(ArrowError::InvalidArgumentError(format!(
110 "Field name '{field_name}' not found in metadata dictionary"
111 )));
112 };
113
114 self.known_field_names.insert(field_name, field_id);
115 Ok(field_id)
116 }
117 fn field_name(&self, field_id: usize) -> &str {
118 &self.metadata[field_id]
119 }
120 fn num_field_names(&self) -> usize {
121 self.metadata.len()
122 }
123 fn truncate_field_names(&mut self, new_size: usize) {
124 debug_assert_eq!(self.metadata.len(), new_size);
125 }
126 fn finish(&mut self) -> usize {
127 self.metadata.bytes.len()
128 }
129}
130
131#[derive(Default, Debug)]
140pub struct WritableMetadataBuilder {
141 pub(crate) field_names: IndexSet<String>,
142
143 pub(crate) is_sorted: bool,
144
145 metadata_buffer: Vec<u8>,
147}
148
149impl WritableMetadataBuilder {
150 pub fn upsert_field_name(&mut self, field_name: &str) -> u32 {
152 let (id, new_entry) = self.field_names.insert_full(field_name.to_string());
153
154 if new_entry {
155 let n = self.num_field_names();
156
157 self.is_sorted =
162 n == 1 || self.is_sorted && (self.field_names[n - 2] < self.field_names[n - 1]);
163 }
164
165 id as u32
166 }
167
168 pub fn offset(&self) -> usize {
170 self.metadata_buffer.len()
171 }
172
173 fn num_field_names(&self) -> usize {
180 let n = self.field_names.len();
181 assert!(u32::try_from(n).is_ok());
182
183 n
184 }
185
186 fn field_name(&self, i: usize) -> &str {
187 &self.field_names[i]
188 }
189
190 fn metadata_size(&self) -> usize {
191 self.field_names.iter().map(|k| k.len()).sum()
192 }
193
194 pub fn finish(&mut self) -> usize {
198 let nkeys = self.num_field_names();
199
200 let total_dict_size = self.metadata_size();
202
203 let metadata_buffer = &mut self.metadata_buffer;
204 let is_sorted = std::mem::take(&mut self.is_sorted);
205 let field_names = std::mem::take(&mut self.field_names);
206
207 let max_offset = std::cmp::max(total_dict_size, nkeys);
209 let offset_size = int_size(max_offset) as u8;
210
211 let offset_start = 1 + offset_size as usize;
212 let string_start = offset_start + (nkeys + 1) * offset_size as usize;
213 let metadata_size = string_start + total_dict_size;
214
215 metadata_buffer.reserve(metadata_size);
216
217 metadata_buffer
219 .push(VARIANT_SPEC_VERSION | ((is_sorted as u8) << 4) | ((offset_size - 1) << 6));
220
221 write_offset(metadata_buffer, nkeys, offset_size);
223
224 let mut cur_offset = 0;
226 for key in &field_names {
227 write_offset(metadata_buffer, cur_offset, offset_size);
228 cur_offset += key.len();
229 }
230 write_offset(metadata_buffer, cur_offset, offset_size);
232
233 for key in field_names {
235 metadata_buffer.extend_from_slice(key.as_bytes());
236 }
237
238 metadata_buffer.len()
239 }
240
241 pub fn into_inner(self) -> Vec<u8> {
243 self.metadata_buffer
244 }
245}
246
247impl<S: AsRef<str>> FromIterator<S> for WritableMetadataBuilder {
248 fn from_iter<T: IntoIterator<Item = S>>(iter: T) -> Self {
249 let mut this = Self::default();
250 this.extend(iter);
251
252 this
253 }
254}
255
256impl<S: AsRef<str>> Extend<S> for WritableMetadataBuilder {
257 fn extend<T: IntoIterator<Item = S>>(&mut self, iter: T) {
258 let iter = iter.into_iter();
259 let (min, _) = iter.size_hint();
260
261 self.field_names.reserve(min);
262
263 for field_name in iter {
264 self.upsert_field_name(field_name.as_ref());
265 }
266 }
267}
268
269#[cfg(test)]
270mod test {
271 use crate::{
272 ParentState, ValueBuilder, VariantMetadata,
273 builder::{
274 metadata::{ReadOnlyMetadataBuilder, WritableMetadataBuilder},
275 object::ObjectBuilder,
276 },
277 };
278
279 #[test]
280 fn test_metadata_builder_from_iter() {
281 let metadata = WritableMetadataBuilder::from_iter(vec!["apple", "banana", "cherry"]);
282 assert_eq!(metadata.num_field_names(), 3);
283 assert_eq!(metadata.field_name(0), "apple");
284 assert_eq!(metadata.field_name(1), "banana");
285 assert_eq!(metadata.field_name(2), "cherry");
286 assert!(metadata.is_sorted);
287
288 let metadata = WritableMetadataBuilder::from_iter(["zebra", "apple", "banana"]);
289 assert_eq!(metadata.num_field_names(), 3);
290 assert_eq!(metadata.field_name(0), "zebra");
291 assert_eq!(metadata.field_name(1), "apple");
292 assert_eq!(metadata.field_name(2), "banana");
293 assert!(!metadata.is_sorted);
294
295 let metadata = WritableMetadataBuilder::from_iter(Vec::<&str>::new());
296 assert_eq!(metadata.num_field_names(), 0);
297 assert!(!metadata.is_sorted);
298 }
299
300 #[test]
301 fn test_metadata_builder_extend() {
302 let mut metadata = WritableMetadataBuilder::default();
303 assert_eq!(metadata.num_field_names(), 0);
304 assert!(!metadata.is_sorted);
305
306 metadata.extend(["apple", "cherry"]);
307 assert_eq!(metadata.num_field_names(), 2);
308 assert_eq!(metadata.field_name(0), "apple");
309 assert_eq!(metadata.field_name(1), "cherry");
310 assert!(metadata.is_sorted);
311
312 metadata.extend(vec!["dinosaur", "monkey"]);
314 assert_eq!(metadata.num_field_names(), 4);
315 assert_eq!(metadata.field_name(2), "dinosaur");
316 assert_eq!(metadata.field_name(3), "monkey");
317 assert!(metadata.is_sorted);
318
319 let initial_count = metadata.num_field_names();
321 metadata.extend(["apple", "monkey"]);
322 assert_eq!(metadata.num_field_names(), initial_count); }
324
325 #[test]
326 fn test_metadata_builder_extend_sort_order() {
327 let mut metadata = WritableMetadataBuilder::default();
328
329 metadata.extend(["middle"]);
330 assert!(metadata.is_sorted);
331
332 metadata.extend(["zebra"]);
333 assert!(metadata.is_sorted);
334
335 metadata.extend(["apple"]);
337 assert!(!metadata.is_sorted);
338 }
339
340 #[test]
341 fn test_metadata_builder_from_iter_with_string_types() {
342 let metadata = WritableMetadataBuilder::from_iter(["a", "b", "c"]);
344 assert_eq!(metadata.num_field_names(), 3);
345
346 let metadata = WritableMetadataBuilder::from_iter(vec![
348 "a".to_string(),
349 "b".to_string(),
350 "c".to_string(),
351 ]);
352 assert_eq!(metadata.num_field_names(), 3);
353
354 let field_names: Vec<Box<str>> = vec!["a".into(), "b".into(), "c".into()];
356 let metadata = WritableMetadataBuilder::from_iter(field_names);
357 assert_eq!(metadata.num_field_names(), 3);
358 }
359
360 #[test]
361 fn test_read_only_metadata_builder_fails_on_unknown_field() {
362 let mut default_builder = WritableMetadataBuilder::default();
364 default_builder.upsert_field_name("known_field");
365 default_builder.finish();
366 let metadata_bytes = default_builder.into_inner();
367
368 let metadata = VariantMetadata::try_new(&metadata_bytes).unwrap();
370 let mut metadata_builder = ReadOnlyMetadataBuilder::new(&metadata);
371 let mut value_builder = ValueBuilder::new();
372
373 {
374 let state = ParentState::variant(&mut value_builder, &mut metadata_builder);
375 let mut obj = ObjectBuilder::new(state, false);
376
377 obj.insert("known_field", "value");
379
380 let result = obj.try_insert("unknown_field", "value");
382 assert!(result.is_err());
383 assert!(
384 result
385 .unwrap_err()
386 .to_string()
387 .contains("Field name 'unknown_field' not found")
388 );
389 }
390 }
391}