parquet/file/page_index/
column_index.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! [`ColumnIndexMetaData`] structures holding decoded [`ColumnIndex`] information
19//!
20//! [`ColumnIndex`]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
21//!
22
23use crate::{
24    data_type::{ByteArray, FixedLenByteArray},
25    errors::{ParquetError, Result},
26    parquet_thrift::{
27        ElementType, FieldType, ThriftCompactOutputProtocol, WriteThrift, WriteThriftField,
28    },
29};
30use std::ops::Deref;
31
32use crate::{
33    basic::BoundaryOrder,
34    data_type::{Int96, private::ParquetValueType},
35    file::page_index::index_reader::ThriftColumnIndex,
36};
37
38/// Common bits of the column index
39#[derive(Debug, Clone, PartialEq)]
40pub struct ColumnIndex {
41    pub(crate) null_pages: Vec<bool>,
42    pub(crate) boundary_order: BoundaryOrder,
43    pub(crate) null_counts: Option<Vec<i64>>,
44    pub(crate) repetition_level_histograms: Option<Vec<i64>>,
45    pub(crate) definition_level_histograms: Option<Vec<i64>>,
46}
47
48impl ColumnIndex {
49    /// Returns the number of pages
50    pub fn num_pages(&self) -> u64 {
51        self.null_pages.len() as u64
52    }
53
54    /// Returns the number of null values in the page indexed by `idx`
55    ///
56    /// Returns `None` if no null counts have been set in the index
57    pub fn null_count(&self, idx: usize) -> Option<i64> {
58        self.null_counts.as_ref().map(|nc| nc[idx])
59    }
60
61    /// Returns the repetition level histogram for the page indexed by `idx`
62    pub fn repetition_level_histogram(&self, idx: usize) -> Option<&[i64]> {
63        if let Some(rep_hists) = self.repetition_level_histograms.as_ref() {
64            let num_lvls = rep_hists.len() / self.num_pages() as usize;
65            let start = num_lvls * idx;
66            Some(&rep_hists[start..start + num_lvls])
67        } else {
68            None
69        }
70    }
71
72    /// Returns the definition level histogram for the page indexed by `idx`
73    pub fn definition_level_histogram(&self, idx: usize) -> Option<&[i64]> {
74        if let Some(def_hists) = self.definition_level_histograms.as_ref() {
75            let num_lvls = def_hists.len() / self.num_pages() as usize;
76            let start = num_lvls * idx;
77            Some(&def_hists[start..start + num_lvls])
78        } else {
79            None
80        }
81    }
82
83    /// Returns whether the page indexed by `idx` consists of all null values
84    pub fn is_null_page(&self, idx: usize) -> bool {
85        self.null_pages[idx]
86    }
87}
88
89/// Column index for primitive types
90#[derive(Debug, Clone, PartialEq)]
91pub struct PrimitiveColumnIndex<T> {
92    pub(crate) column_index: ColumnIndex,
93    pub(crate) min_values: Vec<T>,
94    pub(crate) max_values: Vec<T>,
95}
96
97impl<T: ParquetValueType> PrimitiveColumnIndex<T> {
98    pub(crate) fn try_new(
99        null_pages: Vec<bool>,
100        boundary_order: BoundaryOrder,
101        null_counts: Option<Vec<i64>>,
102        repetition_level_histograms: Option<Vec<i64>>,
103        definition_level_histograms: Option<Vec<i64>>,
104        min_bytes: Vec<&[u8]>,
105        max_bytes: Vec<&[u8]>,
106    ) -> Result<Self> {
107        let len = null_pages.len();
108
109        if min_bytes.len() != len || max_bytes.len() != len {
110            return Err(ParquetError::General(format!(
111                "ColumnIndex min/max length mismatch: expected {len}, got min={} max={}",
112                min_bytes.len(),
113                max_bytes.len()
114            )));
115        }
116        if let Some(ref nc) = null_counts {
117            if nc.len() != len {
118                return Err(ParquetError::General(format!(
119                    "ColumnIndex null_counts length mismatch: expected {len}, got {}",
120                    nc.len()
121                )));
122            }
123        }
124        if let Some(ref rep) = repetition_level_histograms {
125            if len != 0 && rep.len() % len != 0 {
126                return Err(ParquetError::General(
127                    "Invalid repetition_level_histograms length".to_string(),
128                ));
129            }
130        }
131        if let Some(ref def) = definition_level_histograms {
132            if len != 0 && def.len() % len != 0 {
133                return Err(ParquetError::General(
134                    "Invalid definition_level_histograms length".to_string(),
135                ));
136            }
137        }
138
139        let mut min_values = Vec::with_capacity(len);
140        let mut max_values = Vec::with_capacity(len);
141
142        for (i, is_null) in null_pages.iter().enumerate().take(len) {
143            if !is_null {
144                let min = min_bytes[i];
145                min_values.push(T::try_from_le_slice(min)?);
146
147                let max = max_bytes[i];
148                max_values.push(T::try_from_le_slice(max)?);
149            } else {
150                // need placeholders
151                min_values.push(Default::default());
152                max_values.push(Default::default());
153            }
154        }
155
156        Ok(Self {
157            column_index: ColumnIndex {
158                null_pages,
159                boundary_order,
160                null_counts,
161                repetition_level_histograms,
162                definition_level_histograms,
163            },
164            min_values,
165            max_values,
166        })
167    }
168
169    pub(super) fn try_from_thrift(index: ThriftColumnIndex) -> Result<Self> {
170        Self::try_new(
171            index.null_pages,
172            index.boundary_order,
173            index.null_counts,
174            index.repetition_level_histograms,
175            index.definition_level_histograms,
176            index.min_values,
177            index.max_values,
178        )
179    }
180}
181
182impl<T> PrimitiveColumnIndex<T> {
183    /// Returns an array containing the min values for each page.
184    ///
185    /// Values in the returned slice are only valid if [`ColumnIndex::is_null_page()`]
186    /// is `false` for the same index.
187    pub fn min_values(&self) -> &[T] {
188        &self.min_values
189    }
190
191    /// Returns an array containing the max values for each page.
192    ///
193    /// Values in the returned slice are only valid if [`ColumnIndex::is_null_page()`]
194    /// is `false` for the same index.
195    pub fn max_values(&self) -> &[T] {
196        &self.max_values
197    }
198
199    /// Returns an iterator over the min values.
200    ///
201    /// Values may be `None` when [`ColumnIndex::is_null_page()`] is `true`.
202    pub fn min_values_iter(&self) -> impl Iterator<Item = Option<&T>> {
203        self.min_values.iter().enumerate().map(|(i, min)| {
204            if self.is_null_page(i) {
205                None
206            } else {
207                Some(min)
208            }
209        })
210    }
211
212    /// Returns an iterator over the max values.
213    ///
214    /// Values may be `None` when [`ColumnIndex::is_null_page()`] is `true`.
215    pub fn max_values_iter(&self) -> impl Iterator<Item = Option<&T>> {
216        self.max_values.iter().enumerate().map(|(i, min)| {
217            if self.is_null_page(i) {
218                None
219            } else {
220                Some(min)
221            }
222        })
223    }
224
225    /// Returns the min value for the page indexed by `idx`
226    ///
227    /// It is `None` when all values are null
228    #[inline]
229    pub fn min_value(&self, idx: usize) -> Option<&T> {
230        if self.null_pages[idx] {
231            None
232        } else {
233            Some(&self.min_values[idx])
234        }
235    }
236
237    /// Returns the max value for the page indexed by `idx`
238    ///
239    /// It is `None` when all values are null
240    #[inline]
241    pub fn max_value(&self, idx: usize) -> Option<&T> {
242        if self.null_pages[idx] {
243            None
244        } else {
245            Some(&self.max_values[idx])
246        }
247    }
248}
249
250impl<T> Deref for PrimitiveColumnIndex<T> {
251    type Target = ColumnIndex;
252
253    fn deref(&self) -> &Self::Target {
254        &self.column_index
255    }
256}
257
258impl<T: ParquetValueType> WriteThrift for PrimitiveColumnIndex<T> {
259    const ELEMENT_TYPE: ElementType = ElementType::Struct;
260    fn write_thrift<W: std::io::Write>(
261        &self,
262        writer: &mut ThriftCompactOutputProtocol<W>,
263    ) -> Result<()> {
264        self.null_pages.write_thrift_field(writer, 1, 0)?;
265
266        // need to handle min/max manually
267        let len = self.null_pages.len();
268        writer.write_field_begin(FieldType::List, 2, 1)?;
269        writer.write_list_begin(ElementType::Binary, len)?;
270        for i in 0..len {
271            let min = self.min_value(i).map(|m| m.as_bytes()).unwrap_or(&[]);
272            min.write_thrift(writer)?;
273        }
274        writer.write_field_begin(FieldType::List, 3, 2)?;
275        writer.write_list_begin(ElementType::Binary, len)?;
276        for i in 0..len {
277            let max = self.max_value(i).map(|m| m.as_bytes()).unwrap_or(&[]);
278            max.write_thrift(writer)?;
279        }
280        let mut last_field_id = self.boundary_order.write_thrift_field(writer, 4, 3)?;
281        if let Some(null_counts) = &self.null_counts {
282            last_field_id = null_counts.write_thrift_field(writer, 5, last_field_id)?;
283        }
284        if let Some(repetition_level_histograms) = &self.repetition_level_histograms {
285            last_field_id =
286                repetition_level_histograms.write_thrift_field(writer, 6, last_field_id)?;
287        }
288        if let Some(definition_level_histograms) = &self.definition_level_histograms {
289            definition_level_histograms.write_thrift_field(writer, 7, last_field_id)?;
290        }
291        writer.write_struct_end()
292    }
293}
294
295/// Column index for byte arrays (fixed length and variable)
296#[derive(Debug, Clone, PartialEq)]
297pub struct ByteArrayColumnIndex {
298    pub(crate) column_index: ColumnIndex,
299    // raw bytes for min and max values
300    pub(crate) min_bytes: Vec<u8>,
301    pub(crate) min_offsets: Vec<usize>,
302    pub(crate) max_bytes: Vec<u8>,
303    pub(crate) max_offsets: Vec<usize>,
304}
305
306impl ByteArrayColumnIndex {
307    pub(crate) fn try_new(
308        null_pages: Vec<bool>,
309        boundary_order: BoundaryOrder,
310        null_counts: Option<Vec<i64>>,
311        repetition_level_histograms: Option<Vec<i64>>,
312        definition_level_histograms: Option<Vec<i64>>,
313        min_values: Vec<&[u8]>,
314        max_values: Vec<&[u8]>,
315    ) -> Result<Self> {
316        let len = null_pages.len();
317
318        if min_values.len() != len || max_values.len() != len {
319            return Err(ParquetError::General(format!(
320                "ColumnIndex min/max length mismatch: expected {len}, got min={} max={}",
321                min_values.len(),
322                max_values.len()
323            )));
324        }
325        if let Some(ref nc) = null_counts {
326            if nc.len() != len {
327                return Err(ParquetError::General(format!(
328                    "ColumnIndex null_counts length mismatch: expected {len}, got {}",
329                    nc.len()
330                )));
331            }
332        }
333        if let Some(ref rep) = repetition_level_histograms {
334            if len != 0 && rep.len() % len != 0 {
335                return Err(ParquetError::General(
336                    "Invalid repetition_level_histograms length".to_string(),
337                ));
338            }
339        }
340        if let Some(ref def) = definition_level_histograms {
341            if len != 0 && def.len() % len != 0 {
342                return Err(ParquetError::General(
343                    "Invalid definition_level_histograms length".to_string(),
344                ));
345            }
346        }
347
348        let min_len = min_values.iter().map(|&v| v.len()).sum();
349        let max_len = max_values.iter().map(|&v| v.len()).sum();
350        let mut min_bytes = vec![0u8; min_len];
351        let mut max_bytes = vec![0u8; max_len];
352
353        let mut min_offsets = vec![0usize; len + 1];
354        let mut max_offsets = vec![0usize; len + 1];
355
356        let mut min_pos = 0;
357        let mut max_pos = 0;
358
359        for (i, is_null) in null_pages.iter().enumerate().take(len) {
360            if !is_null {
361                let min = min_values[i];
362                let dst = &mut min_bytes[min_pos..min_pos + min.len()];
363                dst.copy_from_slice(min);
364                min_offsets[i] = min_pos;
365                min_pos += min.len();
366
367                let max = max_values[i];
368                let dst = &mut max_bytes[max_pos..max_pos + max.len()];
369                dst.copy_from_slice(max);
370                max_offsets[i] = max_pos;
371                max_pos += max.len();
372            } else {
373                min_offsets[i] = min_pos;
374                max_offsets[i] = max_pos;
375            }
376        }
377
378        min_offsets[len] = min_pos;
379        max_offsets[len] = max_pos;
380
381        Ok(Self {
382            column_index: ColumnIndex {
383                null_pages,
384                boundary_order,
385                null_counts,
386                repetition_level_histograms,
387                definition_level_histograms,
388            },
389            min_bytes,
390            min_offsets,
391            max_bytes,
392            max_offsets,
393        })
394    }
395
396    pub(super) fn try_from_thrift(index: ThriftColumnIndex) -> Result<Self> {
397        Self::try_new(
398            index.null_pages,
399            index.boundary_order,
400            index.null_counts,
401            index.repetition_level_histograms,
402            index.definition_level_histograms,
403            index.min_values,
404            index.max_values,
405        )
406    }
407
408    /// Returns the min value for the page indexed by `idx`
409    ///
410    /// It is `None` when all values are null
411    pub fn min_value(&self, idx: usize) -> Option<&[u8]> {
412        if self.null_pages[idx] {
413            None
414        } else {
415            let start = self.min_offsets[idx];
416            let end = self.min_offsets[idx + 1];
417            Some(&self.min_bytes[start..end])
418        }
419    }
420
421    /// Returns the max value for the page indexed by `idx`
422    ///
423    /// It is `None` when all values are null
424    pub fn max_value(&self, idx: usize) -> Option<&[u8]> {
425        if self.null_pages[idx] {
426            None
427        } else {
428            let start = self.max_offsets[idx];
429            let end = self.max_offsets[idx + 1];
430            Some(&self.max_bytes[start..end])
431        }
432    }
433
434    /// Returns an iterator over the min values.
435    ///
436    /// Values may be `None` when [`ColumnIndex::is_null_page()`] is `true`.
437    pub fn min_values_iter(&self) -> impl Iterator<Item = Option<&[u8]>> {
438        (0..self.num_pages() as usize).map(|i| self.min_value(i))
439    }
440
441    /// Returns an iterator over the max values.
442    ///
443    /// Values may be `None` when [`ColumnIndex::is_null_page()`] is `true`.
444    pub fn max_values_iter(&self) -> impl Iterator<Item = Option<&[u8]>> {
445        (0..self.num_pages() as usize).map(|i| self.max_value(i))
446    }
447}
448
449impl Deref for ByteArrayColumnIndex {
450    type Target = ColumnIndex;
451
452    fn deref(&self) -> &Self::Target {
453        &self.column_index
454    }
455}
456
457impl WriteThrift for ByteArrayColumnIndex {
458    const ELEMENT_TYPE: ElementType = ElementType::Struct;
459    fn write_thrift<W: std::io::Write>(
460        &self,
461        writer: &mut ThriftCompactOutputProtocol<W>,
462    ) -> Result<()> {
463        self.null_pages.write_thrift_field(writer, 1, 0)?;
464
465        // need to handle min/max manually
466        let len = self.null_pages.len();
467        writer.write_field_begin(FieldType::List, 2, 1)?;
468        writer.write_list_begin(ElementType::Binary, len)?;
469        for i in 0..len {
470            let min = self.min_value(i).unwrap_or(&[]);
471            min.write_thrift(writer)?;
472        }
473        writer.write_field_begin(FieldType::List, 3, 2)?;
474        writer.write_list_begin(ElementType::Binary, len)?;
475        for i in 0..len {
476            let max = self.max_value(i).unwrap_or(&[]);
477            max.write_thrift(writer)?;
478        }
479        let mut last_field_id = self.boundary_order.write_thrift_field(writer, 4, 3)?;
480        if let Some(null_counts) = &self.null_counts {
481            last_field_id = null_counts.write_thrift_field(writer, 5, last_field_id)?;
482        }
483        if let Some(repetition_level_histograms) = &self.repetition_level_histograms {
484            last_field_id =
485                repetition_level_histograms.write_thrift_field(writer, 6, last_field_id)?;
486        }
487        if let Some(definition_level_histograms) = &self.definition_level_histograms {
488            definition_level_histograms.write_thrift_field(writer, 7, last_field_id)?;
489        }
490        writer.write_struct_end()
491    }
492}
493
494// Macro to generate getter functions for ColumnIndexMetaData.
495macro_rules! colidx_enum_func {
496    ($self:ident, $func:ident, $arg:ident) => {{
497        match *$self {
498            Self::BOOLEAN(ref typed) => typed.$func($arg),
499            Self::INT32(ref typed) => typed.$func($arg),
500            Self::INT64(ref typed) => typed.$func($arg),
501            Self::INT96(ref typed) => typed.$func($arg),
502            Self::FLOAT(ref typed) => typed.$func($arg),
503            Self::DOUBLE(ref typed) => typed.$func($arg),
504            Self::BYTE_ARRAY(ref typed) => typed.$func($arg),
505            Self::FIXED_LEN_BYTE_ARRAY(ref typed) => typed.$func($arg),
506            _ => panic!(concat!(
507                "Cannot call ",
508                stringify!($func),
509                " on ColumnIndexMetaData::NONE"
510            )),
511        }
512    }};
513    ($self:ident, $func:ident) => {{
514        match *$self {
515            Self::BOOLEAN(ref typed) => typed.$func(),
516            Self::INT32(ref typed) => typed.$func(),
517            Self::INT64(ref typed) => typed.$func(),
518            Self::INT96(ref typed) => typed.$func(),
519            Self::FLOAT(ref typed) => typed.$func(),
520            Self::DOUBLE(ref typed) => typed.$func(),
521            Self::BYTE_ARRAY(ref typed) => typed.$func(),
522            Self::FIXED_LEN_BYTE_ARRAY(ref typed) => typed.$func(),
523            _ => panic!(concat!(
524                "Cannot call ",
525                stringify!($func),
526                " on ColumnIndexMetaData::NONE"
527            )),
528        }
529    }};
530}
531
532/// Parsed [`ColumnIndex`] information for a Parquet file.
533///
534/// See [`ParquetColumnIndex`] for more information.
535///
536/// [`ParquetColumnIndex`]: crate::file::metadata::ParquetColumnIndex
537/// [`ColumnIndex`]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
538#[derive(Debug, Clone, PartialEq)]
539#[allow(non_camel_case_types)]
540pub enum ColumnIndexMetaData {
541    /// Sometimes reading page index from parquet file
542    /// will only return pageLocations without min_max index,
543    /// `NONE` represents this lack of index information
544    NONE,
545    /// Boolean type index
546    BOOLEAN(PrimitiveColumnIndex<bool>),
547    /// 32-bit integer type index
548    INT32(PrimitiveColumnIndex<i32>),
549    /// 64-bit integer type index
550    INT64(PrimitiveColumnIndex<i64>),
551    /// 96-bit integer type (timestamp) index
552    INT96(PrimitiveColumnIndex<Int96>),
553    /// 32-bit floating point type index
554    FLOAT(PrimitiveColumnIndex<f32>),
555    /// 64-bit floating point type index
556    DOUBLE(PrimitiveColumnIndex<f64>),
557    /// Byte array type index
558    BYTE_ARRAY(ByteArrayColumnIndex),
559    /// Fixed length byte array type index
560    FIXED_LEN_BYTE_ARRAY(ByteArrayColumnIndex),
561}
562
563impl ColumnIndexMetaData {
564    /// Return min/max elements inside ColumnIndex are ordered or not.
565    pub fn is_sorted(&self) -> bool {
566        // 0:UNORDERED, 1:ASCENDING ,2:DESCENDING,
567        if let Some(order) = self.get_boundary_order() {
568            order != BoundaryOrder::UNORDERED
569        } else {
570            false
571        }
572    }
573
574    /// Get boundary_order of this page index.
575    pub fn get_boundary_order(&self) -> Option<BoundaryOrder> {
576        match self {
577            Self::NONE => None,
578            Self::BOOLEAN(index) => Some(index.boundary_order),
579            Self::INT32(index) => Some(index.boundary_order),
580            Self::INT64(index) => Some(index.boundary_order),
581            Self::INT96(index) => Some(index.boundary_order),
582            Self::FLOAT(index) => Some(index.boundary_order),
583            Self::DOUBLE(index) => Some(index.boundary_order),
584            Self::BYTE_ARRAY(index) => Some(index.boundary_order),
585            Self::FIXED_LEN_BYTE_ARRAY(index) => Some(index.boundary_order),
586        }
587    }
588
589    /// Returns array of null counts, one per page.
590    ///
591    /// Returns `None` if now null counts have been set in the index
592    pub fn null_counts(&self) -> Option<&Vec<i64>> {
593        match self {
594            Self::NONE => None,
595            Self::BOOLEAN(index) => index.null_counts.as_ref(),
596            Self::INT32(index) => index.null_counts.as_ref(),
597            Self::INT64(index) => index.null_counts.as_ref(),
598            Self::INT96(index) => index.null_counts.as_ref(),
599            Self::FLOAT(index) => index.null_counts.as_ref(),
600            Self::DOUBLE(index) => index.null_counts.as_ref(),
601            Self::BYTE_ARRAY(index) => index.null_counts.as_ref(),
602            Self::FIXED_LEN_BYTE_ARRAY(index) => index.null_counts.as_ref(),
603        }
604    }
605
606    /// Returns the number of pages
607    pub fn num_pages(&self) -> u64 {
608        colidx_enum_func!(self, num_pages)
609    }
610
611    /// Returns the number of null values in the page indexed by `idx`
612    ///
613    /// Returns `None` if no null counts have been set in the index
614    pub fn null_count(&self, idx: usize) -> Option<i64> {
615        colidx_enum_func!(self, null_count, idx)
616    }
617
618    /// Returns the repetition level histogram for the page indexed by `idx`
619    pub fn repetition_level_histogram(&self, idx: usize) -> Option<&[i64]> {
620        colidx_enum_func!(self, repetition_level_histogram, idx)
621    }
622
623    /// Returns the definition level histogram for the page indexed by `idx`
624    pub fn definition_level_histogram(&self, idx: usize) -> Option<&[i64]> {
625        colidx_enum_func!(self, definition_level_histogram, idx)
626    }
627
628    /// Returns whether the page indexed by `idx` consists of all null values
629    #[inline]
630    pub fn is_null_page(&self, idx: usize) -> bool {
631        colidx_enum_func!(self, is_null_page, idx)
632    }
633}
634
635/// Provides iterators over min and max values of a [`ColumnIndexMetaData`]
636pub trait ColumnIndexIterators {
637    /// Can be one of `bool`, `i32`, `i64`, `Int96`, `f32`, `f64`, [`ByteArray`],
638    /// or [`FixedLenByteArray`]
639    type Item;
640
641    /// Return iterator over the min values for the index
642    fn min_values_iter(colidx: &ColumnIndexMetaData) -> impl Iterator<Item = Option<Self::Item>>;
643
644    /// Return iterator over the max values for the index
645    fn max_values_iter(colidx: &ColumnIndexMetaData) -> impl Iterator<Item = Option<Self::Item>>;
646}
647
648macro_rules! column_index_iters {
649    ($item: ident, $variant: ident, $conv:expr) => {
650        impl ColumnIndexIterators for $item {
651            type Item = $item;
652
653            fn min_values_iter(
654                colidx: &ColumnIndexMetaData,
655            ) -> impl Iterator<Item = Option<Self::Item>> {
656                if let ColumnIndexMetaData::$variant(index) = colidx {
657                    index.min_values_iter().map($conv)
658                } else {
659                    panic!(concat!("Wrong type for ", stringify!($item), " iterator"))
660                }
661            }
662
663            fn max_values_iter(
664                colidx: &ColumnIndexMetaData,
665            ) -> impl Iterator<Item = Option<Self::Item>> {
666                if let ColumnIndexMetaData::$variant(index) = colidx {
667                    index.max_values_iter().map($conv)
668                } else {
669                    panic!(concat!("Wrong type for ", stringify!($item), " iterator"))
670                }
671            }
672        }
673    };
674}
675
676column_index_iters!(bool, BOOLEAN, |v| v.copied());
677column_index_iters!(i32, INT32, |v| v.copied());
678column_index_iters!(i64, INT64, |v| v.copied());
679column_index_iters!(Int96, INT96, |v| v.copied());
680column_index_iters!(f32, FLOAT, |v| v.copied());
681column_index_iters!(f64, DOUBLE, |v| v.copied());
682column_index_iters!(ByteArray, BYTE_ARRAY, |v| v
683    .map(|v| ByteArray::from(v.to_owned())));
684column_index_iters!(FixedLenByteArray, FIXED_LEN_BYTE_ARRAY, |v| v
685    .map(|v| FixedLenByteArray::from(v.to_owned())));
686
687impl WriteThrift for ColumnIndexMetaData {
688    const ELEMENT_TYPE: ElementType = ElementType::Struct;
689
690    fn write_thrift<W: std::io::Write>(
691        &self,
692        writer: &mut ThriftCompactOutputProtocol<W>,
693    ) -> Result<()> {
694        match self {
695            ColumnIndexMetaData::BOOLEAN(index) => index.write_thrift(writer),
696            ColumnIndexMetaData::INT32(index) => index.write_thrift(writer),
697            ColumnIndexMetaData::INT64(index) => index.write_thrift(writer),
698            ColumnIndexMetaData::INT96(index) => index.write_thrift(writer),
699            ColumnIndexMetaData::FLOAT(index) => index.write_thrift(writer),
700            ColumnIndexMetaData::DOUBLE(index) => index.write_thrift(writer),
701            ColumnIndexMetaData::BYTE_ARRAY(index) => index.write_thrift(writer),
702            ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(index) => index.write_thrift(writer),
703            _ => Err(general_err!("Cannot serialize NONE index")),
704        }
705    }
706}
707
708#[cfg(test)]
709mod tests {
710    use super::*;
711
712    #[test]
713    fn test_page_index_min_max_null() {
714        let column_index = PrimitiveColumnIndex {
715            column_index: ColumnIndex {
716                null_pages: vec![false],
717                boundary_order: BoundaryOrder::ASCENDING,
718                null_counts: Some(vec![0]),
719                repetition_level_histograms: Some(vec![1, 2]),
720                definition_level_histograms: Some(vec![1, 2, 3]),
721            },
722            min_values: vec![-123],
723            max_values: vec![234],
724        };
725
726        assert_eq!(column_index.min_value(0), Some(&-123));
727        assert_eq!(column_index.max_value(0), Some(&234));
728        assert_eq!(column_index.null_count(0), Some(0));
729        assert_eq!(column_index.repetition_level_histogram(0).unwrap(), &[1, 2]);
730        assert_eq!(
731            column_index.definition_level_histogram(0).unwrap(),
732            &[1, 2, 3]
733        );
734    }
735
736    #[test]
737    fn test_page_index_min_max_null_none() {
738        let column_index: PrimitiveColumnIndex<i32> = PrimitiveColumnIndex::<i32> {
739            column_index: ColumnIndex {
740                null_pages: vec![true],
741                boundary_order: BoundaryOrder::ASCENDING,
742                null_counts: Some(vec![1]),
743                repetition_level_histograms: None,
744                definition_level_histograms: Some(vec![1, 0]),
745            },
746            min_values: vec![Default::default()],
747            max_values: vec![Default::default()],
748        };
749
750        assert_eq!(column_index.min_value(0), None);
751        assert_eq!(column_index.max_value(0), None);
752        assert_eq!(column_index.null_count(0), Some(1));
753        assert_eq!(column_index.repetition_level_histogram(0), None);
754        assert_eq!(column_index.definition_level_histogram(0).unwrap(), &[1, 0]);
755    }
756
757    #[test]
758    fn test_invalid_column_index() {
759        let column_index = ThriftColumnIndex {
760            null_pages: vec![true, false],
761            min_values: vec![
762                &[],
763                &[], // this shouldn't be empty as null_pages[1] is false
764            ],
765            max_values: vec![
766                &[],
767                &[], // this shouldn't be empty as null_pages[1] is false
768            ],
769            null_counts: None,
770            repetition_level_histograms: None,
771            definition_level_histograms: None,
772            boundary_order: BoundaryOrder::UNORDERED,
773        };
774
775        let err = PrimitiveColumnIndex::<i32>::try_from_thrift(column_index).unwrap_err();
776        assert_eq!(
777            err.to_string(),
778            "Parquet error: error converting value, expected 4 bytes got 0"
779        );
780    }
781
782    #[test]
783    fn test_column_index_rejects_mismatched_min_max_lengths() {
784        // Two pages, but only one min/max entry. The entry itself is valid i32 bytes,
785        // so this specifically checks that lengths must match the number of pages.
786        let column_index = ThriftColumnIndex {
787            null_pages: vec![false, false],
788            min_values: vec![&[1u8, 0, 0, 0]],
789            max_values: vec![&[10u8, 0, 0, 0]],
790            null_counts: None,
791            repetition_level_histograms: None,
792            definition_level_histograms: None,
793            boundary_order: BoundaryOrder::UNORDERED,
794        };
795
796        // ColumnIndex arrays must align with the number of pages (null_pages.len()).
797        let err = PrimitiveColumnIndex::<i32>::try_from_thrift(column_index).unwrap_err();
798        // Should fail because min/max lengths don’t match null_pages
799        assert!(err.to_string().contains("length mismatch"));
800    }
801}
parquet/file/page_index/column_index.rs

parquet/file/page_index/
column_index.rs
