Skip to main content

parquet/file/page_index/
column_index.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! [`ColumnIndexMetaData`] structures holding decoded [`ColumnIndex`] information
6//!
7//! [`ColumnIndex`]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
8//!
9
10use crate::{
11    data_type::{ByteArray, FixedLenByteArray},
12    errors::{ParquetError, Result},
13    parquet_thrift::{
14        ElementType, FieldType, ThriftCompactOutputProtocol, WriteThrift, WriteThriftField,
15    },
16};
17use std::ops::Deref;
18
19use crate::{
20    basic::BoundaryOrder,
21    data_type::{Int96, private::ParquetValueType},
22    file::page_index::index_reader::ThriftColumnIndex,
23};
24
25/// Common bits of the column index
26#[derive(Debug, Clone, PartialEq)]
27pub struct ColumnIndex {
28    pub(crate) null_pages: Vec<bool>,
29    pub(crate) boundary_order: BoundaryOrder,
30    pub(crate) null_counts: Option<Vec<i64>>,
31    pub(crate) repetition_level_histograms: Option<Vec<i64>>,
32    pub(crate) definition_level_histograms: Option<Vec<i64>>,
33    pub(crate) nan_counts: Option<Vec<i64>>,
34}
35
36impl ColumnIndex {
37    /// Returns the number of pages
38    pub fn num_pages(&self) -> u64 {
39        self.null_pages.len() as u64
40    }
41
42    /// Returns the number of null values in the page indexed by `idx`
43    ///
44    /// Returns `None` if no null counts have been set in the index
45    pub fn null_count(&self, idx: usize) -> Option<i64> {
46        self.null_counts.as_ref().map(|nc| nc[idx])
47    }
48
49    /// Returns the number of NaN values in the page indexed by `idx`
50    ///
51    /// Returns `None` if no NaN counts have been set in the index
52    pub fn nan_count(&self, idx: usize) -> Option<i64> {
53        self.nan_counts.as_ref().map(|nc| nc[idx])
54    }
55
56    /// Returns the repetition level histogram for the page indexed by `idx`
57    pub fn repetition_level_histogram(&self, idx: usize) -> Option<&[i64]> {
58        if let Some(rep_hists) = self.repetition_level_histograms.as_ref() {
59            let num_lvls = rep_hists.len() / self.num_pages() as usize;
60            let start = num_lvls * idx;
61            Some(&rep_hists[start..start + num_lvls])
62        } else {
63            None
64        }
65    }
66
67    /// Returns the definition level histogram for the page indexed by `idx`
68    pub fn definition_level_histogram(&self, idx: usize) -> Option<&[i64]> {
69        if let Some(def_hists) = self.definition_level_histograms.as_ref() {
70            let num_lvls = def_hists.len() / self.num_pages() as usize;
71            let start = num_lvls * idx;
72            Some(&def_hists[start..start + num_lvls])
73        } else {
74            None
75        }
76    }
77
78    /// Returns whether the page indexed by `idx` consists of all null values
79    pub fn is_null_page(&self, idx: usize) -> bool {
80        self.null_pages[idx]
81    }
82}
83
84/// Column index for primitive types
85#[derive(Debug, Clone, PartialEq)]
86pub struct PrimitiveColumnIndex<T> {
87    pub(crate) column_index: ColumnIndex,
88    pub(crate) min_values: Vec<T>,
89    pub(crate) max_values: Vec<T>,
90}
91
92impl<T: ParquetValueType> PrimitiveColumnIndex<T> {
93    #[expect(clippy::too_many_arguments)]
94    pub(crate) fn try_new(
95        null_pages: Vec<bool>,
96        boundary_order: BoundaryOrder,
97        null_counts: Option<Vec<i64>>,
98        nan_counts: Option<Vec<i64>>,
99        repetition_level_histograms: Option<Vec<i64>>,
100        definition_level_histograms: Option<Vec<i64>>,
101        min_bytes: Vec<&[u8]>,
102        max_bytes: Vec<&[u8]>,
103    ) -> Result<Self> {
104        let len = null_pages.len();
105
106        if min_bytes.len() != len || max_bytes.len() != len {
107            return Err(ParquetError::General(format!(
108                "ColumnIndex min/max length mismatch: expected {len}, got min={} max={}",
109                min_bytes.len(),
110                max_bytes.len()
111            )));
112        }
113        if let Some(ref nc) = null_counts
114            && nc.len() != len
115        {
116            return Err(ParquetError::General(format!(
117                "ColumnIndex null_counts length mismatch: expected {len}, got {}",
118                nc.len()
119            )));
120        }
121        if let Some(ref rep) = repetition_level_histograms
122            && len != 0
123            && rep.len() % len != 0
124        {
125            return Err(ParquetError::General(
126                "Invalid repetition_level_histograms length".to_string(),
127            ));
128        }
129        if let Some(ref def) = definition_level_histograms
130            && len != 0
131            && def.len() % len != 0
132        {
133            return Err(ParquetError::General(
134                "Invalid definition_level_histograms length".to_string(),
135            ));
136        }
137
138        let mut min_values = Vec::with_capacity(len);
139        let mut max_values = Vec::with_capacity(len);
140
141        for (i, is_null) in null_pages.iter().enumerate().take(len) {
142            if !is_null {
143                let min = min_bytes[i];
144                min_values.push(T::try_from_le_slice(min)?);
145
146                let max = max_bytes[i];
147                max_values.push(T::try_from_le_slice(max)?);
148            } else {
149                // need placeholders
150                min_values.push(Default::default());
151                max_values.push(Default::default());
152            }
153        }
154
155        Ok(Self {
156            column_index: ColumnIndex {
157                null_pages,
158                boundary_order,
159                null_counts,
160                repetition_level_histograms,
161                definition_level_histograms,
162                nan_counts,
163            },
164            min_values,
165            max_values,
166        })
167    }
168
169    pub(super) fn try_from_thrift(index: ThriftColumnIndex) -> Result<Self> {
170        Self::try_new(
171            index.null_pages,
172            index.boundary_order,
173            index.null_counts,
174            index.nan_counts,
175            index.repetition_level_histograms,
176            index.definition_level_histograms,
177            index.min_values,
178            index.max_values,
179        )
180    }
181}
182
183impl<T> PrimitiveColumnIndex<T> {
184    /// Returns an array containing the min values for each page.
185    ///
186    /// Values in the returned slice are only valid if [`ColumnIndex::is_null_page()`]
187    /// is `false` for the same index.
188    pub fn min_values(&self) -> &[T] {
189        &self.min_values
190    }
191
192    /// Returns an array containing the max values for each page.
193    ///
194    /// Values in the returned slice are only valid if [`ColumnIndex::is_null_page()`]
195    /// is `false` for the same index.
196    pub fn max_values(&self) -> &[T] {
197        &self.max_values
198    }
199
200    /// Returns an iterator over the min values.
201    ///
202    /// Values may be `None` when [`ColumnIndex::is_null_page()`] is `true`.
203    pub fn min_values_iter(&self) -> impl Iterator<Item = Option<&T>> {
204        self.min_values.iter().enumerate().map(|(i, min)| {
205            if self.is_null_page(i) {
206                None
207            } else {
208                Some(min)
209            }
210        })
211    }
212
213    /// Returns an iterator over the max values.
214    ///
215    /// Values may be `None` when [`ColumnIndex::is_null_page()`] is `true`.
216    pub fn max_values_iter(&self) -> impl Iterator<Item = Option<&T>> {
217        self.max_values.iter().enumerate().map(|(i, min)| {
218            if self.is_null_page(i) {
219                None
220            } else {
221                Some(min)
222            }
223        })
224    }
225
226    /// Returns the min value for the page indexed by `idx`
227    ///
228    /// It is `None` when all values are null
229    #[inline]
230    pub fn min_value(&self, idx: usize) -> Option<&T> {
231        if self.null_pages[idx] {
232            None
233        } else {
234            Some(&self.min_values[idx])
235        }
236    }
237
238    /// Returns the max value for the page indexed by `idx`
239    ///
240    /// It is `None` when all values are null
241    #[inline]
242    pub fn max_value(&self, idx: usize) -> Option<&T> {
243        if self.null_pages[idx] {
244            None
245        } else {
246            Some(&self.max_values[idx])
247        }
248    }
249}
250
251impl<T> Deref for PrimitiveColumnIndex<T> {
252    type Target = ColumnIndex;
253
254    fn deref(&self) -> &Self::Target {
255        &self.column_index
256    }
257}
258
259impl<T: ParquetValueType> WriteThrift for PrimitiveColumnIndex<T> {
260    const ELEMENT_TYPE: ElementType = ElementType::Struct;
261    fn write_thrift<W: std::io::Write>(
262        &self,
263        writer: &mut ThriftCompactOutputProtocol<W>,
264    ) -> Result<()> {
265        self.null_pages.write_thrift_field(writer, 1, 0)?;
266
267        // need to handle min/max manually
268        let len = self.null_pages.len();
269        writer.write_field_begin(FieldType::List, 2, 1)?;
270        writer.write_list_begin(ElementType::Binary, len)?;
271        for i in 0..len {
272            let min = self.min_value(i).map(|m| m.as_bytes()).unwrap_or(&[]);
273            min.write_thrift(writer)?;
274        }
275        writer.write_field_begin(FieldType::List, 3, 2)?;
276        writer.write_list_begin(ElementType::Binary, len)?;
277        for i in 0..len {
278            let max = self.max_value(i).map(|m| m.as_bytes()).unwrap_or(&[]);
279            max.write_thrift(writer)?;
280        }
281        let mut last_field_id = self.boundary_order.write_thrift_field(writer, 4, 3)?;
282        if let Some(null_counts) = &self.null_counts {
283            last_field_id = null_counts.write_thrift_field(writer, 5, last_field_id)?;
284        }
285        if let Some(repetition_level_histograms) = &self.repetition_level_histograms {
286            last_field_id =
287                repetition_level_histograms.write_thrift_field(writer, 6, last_field_id)?;
288        }
289        if let Some(definition_level_histograms) = &self.definition_level_histograms {
290            last_field_id =
291                definition_level_histograms.write_thrift_field(writer, 7, last_field_id)?;
292        }
293        if let Some(nan_counts) = &self.nan_counts {
294            nan_counts.write_thrift_field(writer, 8, last_field_id)?;
295        }
296        writer.write_struct_end()
297    }
298}
299
300/// Column index for byte arrays (fixed length and variable)
301#[derive(Debug, Clone, PartialEq)]
302pub struct ByteArrayColumnIndex {
303    pub(crate) column_index: ColumnIndex,
304    // raw bytes for min and max values
305    pub(crate) min_bytes: Vec<u8>,
306    pub(crate) min_offsets: Vec<usize>,
307    pub(crate) max_bytes: Vec<u8>,
308    pub(crate) max_offsets: Vec<usize>,
309}
310
311impl ByteArrayColumnIndex {
312    #[expect(clippy::too_many_arguments)]
313    pub(crate) fn try_new(
314        null_pages: Vec<bool>,
315        boundary_order: BoundaryOrder,
316        null_counts: Option<Vec<i64>>,
317        nan_counts: Option<Vec<i64>>,
318        repetition_level_histograms: Option<Vec<i64>>,
319        definition_level_histograms: Option<Vec<i64>>,
320        min_values: Vec<&[u8]>,
321        max_values: Vec<&[u8]>,
322    ) -> Result<Self> {
323        let len = null_pages.len();
324
325        if min_values.len() != len || max_values.len() != len {
326            return Err(ParquetError::General(format!(
327                "ColumnIndex min/max length mismatch: expected {len}, got min={} max={}",
328                min_values.len(),
329                max_values.len()
330            )));
331        }
332        if let Some(ref nc) = null_counts
333            && nc.len() != len
334        {
335            return Err(ParquetError::General(format!(
336                "ColumnIndex null_counts length mismatch: expected {len}, got {}",
337                nc.len()
338            )));
339        }
340        if let Some(ref rep) = repetition_level_histograms
341            && len != 0
342            && rep.len() % len != 0
343        {
344            return Err(ParquetError::General(
345                "Invalid repetition_level_histograms length".to_string(),
346            ));
347        }
348        if let Some(ref def) = definition_level_histograms
349            && len != 0
350            && def.len() % len != 0
351        {
352            return Err(ParquetError::General(
353                "Invalid definition_level_histograms length".to_string(),
354            ));
355        }
356
357        let min_len = min_values.iter().map(|&v| v.len()).sum();
358        let max_len = max_values.iter().map(|&v| v.len()).sum();
359        let mut min_bytes = vec![0u8; min_len];
360        let mut max_bytes = vec![0u8; max_len];
361
362        let mut min_offsets = vec![0usize; len + 1];
363        let mut max_offsets = vec![0usize; len + 1];
364
365        let mut min_pos = 0;
366        let mut max_pos = 0;
367
368        for (i, is_null) in null_pages.iter().enumerate().take(len) {
369            if !is_null {
370                let min = min_values[i];
371                let dst = &mut min_bytes[min_pos..min_pos + min.len()];
372                dst.copy_from_slice(min);
373                min_offsets[i] = min_pos;
374                min_pos += min.len();
375
376                let max = max_values[i];
377                let dst = &mut max_bytes[max_pos..max_pos + max.len()];
378                dst.copy_from_slice(max);
379                max_offsets[i] = max_pos;
380                max_pos += max.len();
381            } else {
382                min_offsets[i] = min_pos;
383                max_offsets[i] = max_pos;
384            }
385        }
386
387        min_offsets[len] = min_pos;
388        max_offsets[len] = max_pos;
389
390        Ok(Self {
391            column_index: ColumnIndex {
392                null_pages,
393                boundary_order,
394                null_counts,
395                repetition_level_histograms,
396                definition_level_histograms,
397                nan_counts,
398            },
399            min_bytes,
400            min_offsets,
401            max_bytes,
402            max_offsets,
403        })
404    }
405
406    pub(super) fn try_from_thrift(index: ThriftColumnIndex) -> Result<Self> {
407        Self::try_new(
408            index.null_pages,
409            index.boundary_order,
410            index.null_counts,
411            index.nan_counts,
412            index.repetition_level_histograms,
413            index.definition_level_histograms,
414            index.min_values,
415            index.max_values,
416        )
417    }
418
419    /// Returns the min value for the page indexed by `idx`
420    ///
421    /// It is `None` when all values are null
422    pub fn min_value(&self, idx: usize) -> Option<&[u8]> {
423        if self.null_pages[idx] {
424            None
425        } else {
426            let start = self.min_offsets[idx];
427            let end = self.min_offsets[idx + 1];
428            Some(&self.min_bytes[start..end])
429        }
430    }
431
432    /// Returns the max value for the page indexed by `idx`
433    ///
434    /// It is `None` when all values are null
435    pub fn max_value(&self, idx: usize) -> Option<&[u8]> {
436        if self.null_pages[idx] {
437            None
438        } else {
439            let start = self.max_offsets[idx];
440            let end = self.max_offsets[idx + 1];
441            Some(&self.max_bytes[start..end])
442        }
443    }
444
445    /// Returns an iterator over the min values.
446    ///
447    /// Values may be `None` when [`ColumnIndex::is_null_page()`] is `true`.
448    pub fn min_values_iter(&self) -> impl Iterator<Item = Option<&[u8]>> {
449        (0..self.num_pages() as usize).map(|i| self.min_value(i))
450    }
451
452    /// Returns an iterator over the max values.
453    ///
454    /// Values may be `None` when [`ColumnIndex::is_null_page()`] is `true`.
455    pub fn max_values_iter(&self) -> impl Iterator<Item = Option<&[u8]>> {
456        (0..self.num_pages() as usize).map(|i| self.max_value(i))
457    }
458}
459
460impl Deref for ByteArrayColumnIndex {
461    type Target = ColumnIndex;
462
463    fn deref(&self) -> &Self::Target {
464        &self.column_index
465    }
466}
467
468impl WriteThrift for ByteArrayColumnIndex {
469    const ELEMENT_TYPE: ElementType = ElementType::Struct;
470    fn write_thrift<W: std::io::Write>(
471        &self,
472        writer: &mut ThriftCompactOutputProtocol<W>,
473    ) -> Result<()> {
474        self.null_pages.write_thrift_field(writer, 1, 0)?;
475
476        // need to handle min/max manually
477        let len = self.null_pages.len();
478        writer.write_field_begin(FieldType::List, 2, 1)?;
479        writer.write_list_begin(ElementType::Binary, len)?;
480        for i in 0..len {
481            let min = self.min_value(i).unwrap_or(&[]);
482            min.write_thrift(writer)?;
483        }
484        writer.write_field_begin(FieldType::List, 3, 2)?;
485        writer.write_list_begin(ElementType::Binary, len)?;
486        for i in 0..len {
487            let max = self.max_value(i).unwrap_or(&[]);
488            max.write_thrift(writer)?;
489        }
490        let mut last_field_id = self.boundary_order.write_thrift_field(writer, 4, 3)?;
491        if let Some(null_counts) = &self.null_counts {
492            last_field_id = null_counts.write_thrift_field(writer, 5, last_field_id)?;
493        }
494        if let Some(repetition_level_histograms) = &self.repetition_level_histograms {
495            last_field_id =
496                repetition_level_histograms.write_thrift_field(writer, 6, last_field_id)?;
497        }
498        if let Some(definition_level_histograms) = &self.definition_level_histograms {
499            last_field_id =
500                definition_level_histograms.write_thrift_field(writer, 7, last_field_id)?;
501        }
502        if let Some(nan_counts) = &self.nan_counts {
503            nan_counts.write_thrift_field(writer, 8, last_field_id)?;
504        }
505        writer.write_struct_end()
506    }
507}
508
509// Macro to generate getter functions for ColumnIndexMetaData.
510macro_rules! colidx_enum_func {
511    ($self:ident, $func:ident, $arg:ident) => {{
512        match *$self {
513            Self::BOOLEAN(ref typed) => typed.$func($arg),
514            Self::INT32(ref typed) => typed.$func($arg),
515            Self::INT64(ref typed) => typed.$func($arg),
516            Self::INT96(ref typed) => typed.$func($arg),
517            Self::FLOAT(ref typed) => typed.$func($arg),
518            Self::DOUBLE(ref typed) => typed.$func($arg),
519            Self::BYTE_ARRAY(ref typed) => typed.$func($arg),
520            Self::FIXED_LEN_BYTE_ARRAY(ref typed) => typed.$func($arg),
521        }
522    }};
523    ($self:ident, $func:ident) => {{
524        match *$self {
525            Self::BOOLEAN(ref typed) => typed.$func(),
526            Self::INT32(ref typed) => typed.$func(),
527            Self::INT64(ref typed) => typed.$func(),
528            Self::INT96(ref typed) => typed.$func(),
529            Self::FLOAT(ref typed) => typed.$func(),
530            Self::DOUBLE(ref typed) => typed.$func(),
531            Self::BYTE_ARRAY(ref typed) => typed.$func(),
532            Self::FIXED_LEN_BYTE_ARRAY(ref typed) => typed.$func(),
533        }
534    }};
535}
536
537/// Parsed [`ColumnIndex`] information for a Parquet file.
538///
539/// See [`PageIndex`] for more information.
540///
541/// [`PageIndex`]: crate::file::metadata::page_index::PageIndex
542/// [`ColumnIndex`]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
543#[derive(Debug, Clone, PartialEq)]
544#[expect(non_camel_case_types)]
545pub enum ColumnIndexMetaData {
546    /// Boolean type index
547    BOOLEAN(PrimitiveColumnIndex<bool>),
548    /// 32-bit integer type index
549    INT32(PrimitiveColumnIndex<i32>),
550    /// 64-bit integer type index
551    INT64(PrimitiveColumnIndex<i64>),
552    /// 96-bit integer type (timestamp) index
553    INT96(PrimitiveColumnIndex<Int96>),
554    /// 32-bit floating point type index
555    FLOAT(PrimitiveColumnIndex<f32>),
556    /// 64-bit floating point type index
557    DOUBLE(PrimitiveColumnIndex<f64>),
558    /// Byte array type index
559    BYTE_ARRAY(ByteArrayColumnIndex),
560    /// Fixed length byte array type index
561    FIXED_LEN_BYTE_ARRAY(ByteArrayColumnIndex),
562}
563
564impl ColumnIndexMetaData {
565    /// Return min/max elements inside ColumnIndex are ordered or not.
566    pub fn is_sorted(&self) -> bool {
567        // 0:UNORDERED, 1:ASCENDING ,2:DESCENDING,
568        if let Some(order) = self.get_boundary_order() {
569            order != BoundaryOrder::UNORDERED
570        } else {
571            false
572        }
573    }
574
575    /// Get boundary_order of this page index.
576    pub fn get_boundary_order(&self) -> Option<BoundaryOrder> {
577        match self {
578            Self::BOOLEAN(index) => Some(index.boundary_order),
579            Self::INT32(index) => Some(index.boundary_order),
580            Self::INT64(index) => Some(index.boundary_order),
581            Self::INT96(index) => Some(index.boundary_order),
582            Self::FLOAT(index) => Some(index.boundary_order),
583            Self::DOUBLE(index) => Some(index.boundary_order),
584            Self::BYTE_ARRAY(index) => Some(index.boundary_order),
585            Self::FIXED_LEN_BYTE_ARRAY(index) => Some(index.boundary_order),
586        }
587    }
588
589    /// Returns array of null counts, one per page.
590    ///
591    /// Returns `None` if no null counts have been set in the index
592    pub fn null_counts(&self) -> Option<&Vec<i64>> {
593        match self {
594            Self::BOOLEAN(index) => index.null_counts.as_ref(),
595            Self::INT32(index) => index.null_counts.as_ref(),
596            Self::INT64(index) => index.null_counts.as_ref(),
597            Self::INT96(index) => index.null_counts.as_ref(),
598            Self::FLOAT(index) => index.null_counts.as_ref(),
599            Self::DOUBLE(index) => index.null_counts.as_ref(),
600            Self::BYTE_ARRAY(index) => index.null_counts.as_ref(),
601            Self::FIXED_LEN_BYTE_ARRAY(index) => index.null_counts.as_ref(),
602        }
603    }
604
605    /// Returns array of NaN counts, one per page.
606    ///
607    /// Returns `None` if no NaN counts have been set in the index
608    pub fn nan_counts(&self) -> Option<&Vec<i64>> {
609        match self {
610            Self::BOOLEAN(index) => index.nan_counts.as_ref(),
611            Self::INT32(index) => index.nan_counts.as_ref(),
612            Self::INT64(index) => index.nan_counts.as_ref(),
613            Self::INT96(index) => index.nan_counts.as_ref(),
614            Self::FLOAT(index) => index.nan_counts.as_ref(),
615            Self::DOUBLE(index) => index.nan_counts.as_ref(),
616            Self::BYTE_ARRAY(index) => index.nan_counts.as_ref(),
617            Self::FIXED_LEN_BYTE_ARRAY(index) => index.nan_counts.as_ref(),
618        }
619    }
620
621    /// Returns the number of pages
622    pub fn num_pages(&self) -> u64 {
623        colidx_enum_func!(self, num_pages)
624    }
625
626    /// Returns the number of null values in the page indexed by `idx`
627    ///
628    /// Returns `None` if no null counts have been set in the index
629    pub fn null_count(&self, idx: usize) -> Option<i64> {
630        colidx_enum_func!(self, null_count, idx)
631    }
632
633    /// Returns the number of NaN values in the page indexed by `idx`
634    ///
635    /// Returns `None` if no NaN counts have been set in the index
636    pub fn nan_count(&self, idx: usize) -> Option<i64> {
637        colidx_enum_func!(self, nan_count, idx)
638    }
639
640    /// Returns the repetition level histogram for the page indexed by `idx`
641    pub fn repetition_level_histogram(&self, idx: usize) -> Option<&[i64]> {
642        colidx_enum_func!(self, repetition_level_histogram, idx)
643    }
644
645    /// Returns the definition level histogram for the page indexed by `idx`
646    pub fn definition_level_histogram(&self, idx: usize) -> Option<&[i64]> {
647        colidx_enum_func!(self, definition_level_histogram, idx)
648    }
649
650    /// Returns whether the page indexed by `idx` consists of all null values
651    #[inline]
652    pub fn is_null_page(&self, idx: usize) -> bool {
653        colidx_enum_func!(self, is_null_page, idx)
654    }
655}
656
657/// Provides iterators over min and max values of a [`ColumnIndexMetaData`]
658pub trait ColumnIndexIterators {
659    /// Can be one of `bool`, `i32`, `i64`, `Int96`, `f32`, `f64`, [`ByteArray`],
660    /// or [`FixedLenByteArray`]
661    type Item;
662
663    /// Return iterator over the min values for the index
664    fn min_values_iter(colidx: &ColumnIndexMetaData) -> impl Iterator<Item = Option<Self::Item>>;
665
666    /// Return iterator over the max values for the index
667    fn max_values_iter(colidx: &ColumnIndexMetaData) -> impl Iterator<Item = Option<Self::Item>>;
668}
669
670macro_rules! column_index_iters {
671    ($item: ident, $variant: ident, $conv:expr) => {
672        impl ColumnIndexIterators for $item {
673            type Item = $item;
674
675            fn min_values_iter(
676                colidx: &ColumnIndexMetaData,
677            ) -> impl Iterator<Item = Option<Self::Item>> {
678                if let ColumnIndexMetaData::$variant(index) = colidx {
679                    index.min_values_iter().map($conv)
680                } else {
681                    panic!(concat!("Wrong type for ", stringify!($item), " iterator"))
682                }
683            }
684
685            fn max_values_iter(
686                colidx: &ColumnIndexMetaData,
687            ) -> impl Iterator<Item = Option<Self::Item>> {
688                if let ColumnIndexMetaData::$variant(index) = colidx {
689                    index.max_values_iter().map($conv)
690                } else {
691                    panic!(concat!("Wrong type for ", stringify!($item), " iterator"))
692                }
693            }
694        }
695    };
696}
697
698column_index_iters!(bool, BOOLEAN, |v| v.copied());
699column_index_iters!(i32, INT32, |v| v.copied());
700column_index_iters!(i64, INT64, |v| v.copied());
701column_index_iters!(Int96, INT96, |v| v.copied());
702column_index_iters!(f32, FLOAT, |v| v.copied());
703column_index_iters!(f64, DOUBLE, |v| v.copied());
704column_index_iters!(ByteArray, BYTE_ARRAY, |v| v
705    .map(|v| ByteArray::from(v.to_owned())));
706column_index_iters!(FixedLenByteArray, FIXED_LEN_BYTE_ARRAY, |v| v
707    .map(|v| FixedLenByteArray::from(v.to_owned())));
708
709impl WriteThrift for ColumnIndexMetaData {
710    const ELEMENT_TYPE: ElementType = ElementType::Struct;
711
712    fn write_thrift<W: std::io::Write>(
713        &self,
714        writer: &mut ThriftCompactOutputProtocol<W>,
715    ) -> Result<()> {
716        match self {
717            ColumnIndexMetaData::BOOLEAN(index) => index.write_thrift(writer),
718            ColumnIndexMetaData::INT32(index) => index.write_thrift(writer),
719            ColumnIndexMetaData::INT64(index) => index.write_thrift(writer),
720            ColumnIndexMetaData::INT96(index) => index.write_thrift(writer),
721            ColumnIndexMetaData::FLOAT(index) => index.write_thrift(writer),
722            ColumnIndexMetaData::DOUBLE(index) => index.write_thrift(writer),
723            ColumnIndexMetaData::BYTE_ARRAY(index) => index.write_thrift(writer),
724            ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(index) => index.write_thrift(writer),
725        }
726    }
727}
728
729#[cfg(test)]
730mod tests {
731    use super::*;
732
733    #[test]
734    fn test_page_index_min_max_null() {
735        let column_index = PrimitiveColumnIndex {
736            column_index: ColumnIndex {
737                null_pages: vec![false],
738                boundary_order: BoundaryOrder::ASCENDING,
739                null_counts: Some(vec![0]),
740                nan_counts: None,
741                repetition_level_histograms: Some(vec![1, 2]),
742                definition_level_histograms: Some(vec![1, 2, 3]),
743            },
744            min_values: vec![-123],
745            max_values: vec![234],
746        };
747
748        assert_eq!(column_index.min_value(0), Some(&-123));
749        assert_eq!(column_index.max_value(0), Some(&234));
750        assert_eq!(column_index.null_count(0), Some(0));
751        assert_eq!(column_index.repetition_level_histogram(0).unwrap(), &[1, 2]);
752        assert_eq!(
753            column_index.definition_level_histogram(0).unwrap(),
754            &[1, 2, 3]
755        );
756    }
757
758    #[test]
759    fn test_page_index_min_max_null_none() {
760        let column_index: PrimitiveColumnIndex<i32> = PrimitiveColumnIndex::<i32> {
761            column_index: ColumnIndex {
762                null_pages: vec![true],
763                boundary_order: BoundaryOrder::ASCENDING,
764                null_counts: Some(vec![1]),
765                nan_counts: None,
766                repetition_level_histograms: None,
767                definition_level_histograms: Some(vec![1, 0]),
768            },
769            min_values: vec![Default::default()],
770            max_values: vec![Default::default()],
771        };
772
773        assert_eq!(column_index.min_value(0), None);
774        assert_eq!(column_index.max_value(0), None);
775        assert_eq!(column_index.null_count(0), Some(1));
776        assert_eq!(column_index.repetition_level_histogram(0), None);
777        assert_eq!(column_index.definition_level_histogram(0).unwrap(), &[1, 0]);
778    }
779
780    #[test]
781    fn test_invalid_column_index() {
782        let column_index = ThriftColumnIndex {
783            null_pages: vec![true, false],
784            min_values: vec![
785                &[],
786                &[], // this shouldn't be empty as null_pages[1] is false
787            ],
788            max_values: vec![
789                &[],
790                &[], // this shouldn't be empty as null_pages[1] is false
791            ],
792            null_counts: None,
793            nan_counts: None,
794            repetition_level_histograms: None,
795            definition_level_histograms: None,
796            boundary_order: BoundaryOrder::UNORDERED,
797        };
798
799        let err = PrimitiveColumnIndex::<i32>::try_from_thrift(column_index).unwrap_err();
800        assert_eq!(
801            err.to_string(),
802            "Parquet error: error converting value, expected 4 bytes got 0"
803        );
804    }
805
806    #[test]
807    fn test_column_index_rejects_mismatched_min_max_lengths() {
808        // Two pages, but only one min/max entry. The entry itself is valid i32 bytes,
809        // so this specifically checks that lengths must match the number of pages.
810        let column_index = ThriftColumnIndex {
811            null_pages: vec![false, false],
812            min_values: vec![&[1u8, 0, 0, 0]],
813            max_values: vec![&[10u8, 0, 0, 0]],
814            null_counts: None,
815            repetition_level_histograms: None,
816            definition_level_histograms: None,
817            boundary_order: BoundaryOrder::UNORDERED,
818            nan_counts: None,
819        };
820
821        // ColumnIndex arrays must align with the number of pages (null_pages.len()).
822        let err = PrimitiveColumnIndex::<i32>::try_from_thrift(column_index).unwrap_err();
823        // Should fail because min/max lengths don’t match null_pages
824        assert!(err.to_string().contains("length mismatch"));
825    }
826}