parquet/file/page_index/
column_index.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! [`ColumnIndexMetaData`] structures holding decoded [`ColumnIndex`] information
19//!
20//! [`ColumnIndex`]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
21//!
22
23use crate::{
24    data_type::{ByteArray, FixedLenByteArray},
25    errors::{ParquetError, Result},
26    parquet_thrift::{
27        ElementType, FieldType, ThriftCompactOutputProtocol, WriteThrift, WriteThriftField,
28    },
29};
30use std::ops::Deref;
31
32use crate::{
33    basic::BoundaryOrder,
34    data_type::{Int96, private::ParquetValueType},
35    file::page_index::index_reader::ThriftColumnIndex,
36};
37
38/// Common bits of the column index
39#[derive(Debug, Clone, PartialEq)]
40pub struct ColumnIndex {
41    pub(crate) null_pages: Vec<bool>,
42    pub(crate) boundary_order: BoundaryOrder,
43    pub(crate) null_counts: Option<Vec<i64>>,
44    pub(crate) repetition_level_histograms: Option<Vec<i64>>,
45    pub(crate) definition_level_histograms: Option<Vec<i64>>,
46    pub(crate) nan_counts: Option<Vec<i64>>,
47}
48
49impl ColumnIndex {
50    /// Returns the number of pages
51    pub fn num_pages(&self) -> u64 {
52        self.null_pages.len() as u64
53    }
54
55    /// Returns the number of null values in the page indexed by `idx`
56    ///
57    /// Returns `None` if no null counts have been set in the index
58    pub fn null_count(&self, idx: usize) -> Option<i64> {
59        self.null_counts.as_ref().map(|nc| nc[idx])
60    }
61
62    /// Returns the number of NaN values in the page indexed by `idx`
63    ///
64    /// Returns `None` if no NaN counts have been set in the index
65    pub fn nan_count(&self, idx: usize) -> Option<i64> {
66        self.nan_counts.as_ref().map(|nc| nc[idx])
67    }
68
69    /// Returns the repetition level histogram for the page indexed by `idx`
70    pub fn repetition_level_histogram(&self, idx: usize) -> Option<&[i64]> {
71        if let Some(rep_hists) = self.repetition_level_histograms.as_ref() {
72            let num_lvls = rep_hists.len() / self.num_pages() as usize;
73            let start = num_lvls * idx;
74            Some(&rep_hists[start..start + num_lvls])
75        } else {
76            None
77        }
78    }
79
80    /// Returns the definition level histogram for the page indexed by `idx`
81    pub fn definition_level_histogram(&self, idx: usize) -> Option<&[i64]> {
82        if let Some(def_hists) = self.definition_level_histograms.as_ref() {
83            let num_lvls = def_hists.len() / self.num_pages() as usize;
84            let start = num_lvls * idx;
85            Some(&def_hists[start..start + num_lvls])
86        } else {
87            None
88        }
89    }
90
91    /// Returns whether the page indexed by `idx` consists of all null values
92    pub fn is_null_page(&self, idx: usize) -> bool {
93        self.null_pages[idx]
94    }
95}
96
97/// Column index for primitive types
98#[derive(Debug, Clone, PartialEq)]
99pub struct PrimitiveColumnIndex<T> {
100    pub(crate) column_index: ColumnIndex,
101    pub(crate) min_values: Vec<T>,
102    pub(crate) max_values: Vec<T>,
103}
104
105impl<T: ParquetValueType> PrimitiveColumnIndex<T> {
106    #[expect(clippy::too_many_arguments)]
107    pub(crate) fn try_new(
108        null_pages: Vec<bool>,
109        boundary_order: BoundaryOrder,
110        null_counts: Option<Vec<i64>>,
111        nan_counts: Option<Vec<i64>>,
112        repetition_level_histograms: Option<Vec<i64>>,
113        definition_level_histograms: Option<Vec<i64>>,
114        min_bytes: Vec<&[u8]>,
115        max_bytes: Vec<&[u8]>,
116    ) -> Result<Self> {
117        let len = null_pages.len();
118
119        if min_bytes.len() != len || max_bytes.len() != len {
120            return Err(ParquetError::General(format!(
121                "ColumnIndex min/max length mismatch: expected {len}, got min={} max={}",
122                min_bytes.len(),
123                max_bytes.len()
124            )));
125        }
126        if let Some(ref nc) = null_counts
127            && nc.len() != len
128        {
129            return Err(ParquetError::General(format!(
130                "ColumnIndex null_counts length mismatch: expected {len}, got {}",
131                nc.len()
132            )));
133        }
134        if let Some(ref rep) = repetition_level_histograms
135            && len != 0
136            && rep.len() % len != 0
137        {
138            return Err(ParquetError::General(
139                "Invalid repetition_level_histograms length".to_string(),
140            ));
141        }
142        if let Some(ref def) = definition_level_histograms
143            && len != 0
144            && def.len() % len != 0
145        {
146            return Err(ParquetError::General(
147                "Invalid definition_level_histograms length".to_string(),
148            ));
149        }
150
151        let mut min_values = Vec::with_capacity(len);
152        let mut max_values = Vec::with_capacity(len);
153
154        for (i, is_null) in null_pages.iter().enumerate().take(len) {
155            if !is_null {
156                let min = min_bytes[i];
157                min_values.push(T::try_from_le_slice(min)?);
158
159                let max = max_bytes[i];
160                max_values.push(T::try_from_le_slice(max)?);
161            } else {
162                // need placeholders
163                min_values.push(Default::default());
164                max_values.push(Default::default());
165            }
166        }
167
168        Ok(Self {
169            column_index: ColumnIndex {
170                null_pages,
171                boundary_order,
172                null_counts,
173                repetition_level_histograms,
174                definition_level_histograms,
175                nan_counts,
176            },
177            min_values,
178            max_values,
179        })
180    }
181
182    pub(super) fn try_from_thrift(index: ThriftColumnIndex) -> Result<Self> {
183        Self::try_new(
184            index.null_pages,
185            index.boundary_order,
186            index.null_counts,
187            index.nan_counts,
188            index.repetition_level_histograms,
189            index.definition_level_histograms,
190            index.min_values,
191            index.max_values,
192        )
193    }
194}
195
196impl<T> PrimitiveColumnIndex<T> {
197    /// Returns an array containing the min values for each page.
198    ///
199    /// Values in the returned slice are only valid if [`ColumnIndex::is_null_page()`]
200    /// is `false` for the same index.
201    pub fn min_values(&self) -> &[T] {
202        &self.min_values
203    }
204
205    /// Returns an array containing the max values for each page.
206    ///
207    /// Values in the returned slice are only valid if [`ColumnIndex::is_null_page()`]
208    /// is `false` for the same index.
209    pub fn max_values(&self) -> &[T] {
210        &self.max_values
211    }
212
213    /// Returns an iterator over the min values.
214    ///
215    /// Values may be `None` when [`ColumnIndex::is_null_page()`] is `true`.
216    pub fn min_values_iter(&self) -> impl Iterator<Item = Option<&T>> {
217        self.min_values.iter().enumerate().map(|(i, min)| {
218            if self.is_null_page(i) {
219                None
220            } else {
221                Some(min)
222            }
223        })
224    }
225
226    /// Returns an iterator over the max values.
227    ///
228    /// Values may be `None` when [`ColumnIndex::is_null_page()`] is `true`.
229    pub fn max_values_iter(&self) -> impl Iterator<Item = Option<&T>> {
230        self.max_values.iter().enumerate().map(|(i, min)| {
231            if self.is_null_page(i) {
232                None
233            } else {
234                Some(min)
235            }
236        })
237    }
238
239    /// Returns the min value for the page indexed by `idx`
240    ///
241    /// It is `None` when all values are null
242    #[inline]
243    pub fn min_value(&self, idx: usize) -> Option<&T> {
244        if self.null_pages[idx] {
245            None
246        } else {
247            Some(&self.min_values[idx])
248        }
249    }
250
251    /// Returns the max value for the page indexed by `idx`
252    ///
253    /// It is `None` when all values are null
254    #[inline]
255    pub fn max_value(&self, idx: usize) -> Option<&T> {
256        if self.null_pages[idx] {
257            None
258        } else {
259            Some(&self.max_values[idx])
260        }
261    }
262}
263
264impl<T> Deref for PrimitiveColumnIndex<T> {
265    type Target = ColumnIndex;
266
267    fn deref(&self) -> &Self::Target {
268        &self.column_index
269    }
270}
271
272impl<T: ParquetValueType> WriteThrift for PrimitiveColumnIndex<T> {
273    const ELEMENT_TYPE: ElementType = ElementType::Struct;
274    fn write_thrift<W: std::io::Write>(
275        &self,
276        writer: &mut ThriftCompactOutputProtocol<W>,
277    ) -> Result<()> {
278        self.null_pages.write_thrift_field(writer, 1, 0)?;
279
280        // need to handle min/max manually
281        let len = self.null_pages.len();
282        writer.write_field_begin(FieldType::List, 2, 1)?;
283        writer.write_list_begin(ElementType::Binary, len)?;
284        for i in 0..len {
285            let min = self.min_value(i).map(|m| m.as_bytes()).unwrap_or(&[]);
286            min.write_thrift(writer)?;
287        }
288        writer.write_field_begin(FieldType::List, 3, 2)?;
289        writer.write_list_begin(ElementType::Binary, len)?;
290        for i in 0..len {
291            let max = self.max_value(i).map(|m| m.as_bytes()).unwrap_or(&[]);
292            max.write_thrift(writer)?;
293        }
294        let mut last_field_id = self.boundary_order.write_thrift_field(writer, 4, 3)?;
295        if let Some(null_counts) = &self.null_counts {
296            last_field_id = null_counts.write_thrift_field(writer, 5, last_field_id)?;
297        }
298        if let Some(repetition_level_histograms) = &self.repetition_level_histograms {
299            last_field_id =
300                repetition_level_histograms.write_thrift_field(writer, 6, last_field_id)?;
301        }
302        if let Some(definition_level_histograms) = &self.definition_level_histograms {
303            last_field_id =
304                definition_level_histograms.write_thrift_field(writer, 7, last_field_id)?;
305        }
306        if let Some(nan_counts) = &self.nan_counts {
307            nan_counts.write_thrift_field(writer, 8, last_field_id)?;
308        }
309        writer.write_struct_end()
310    }
311}
312
313/// Column index for byte arrays (fixed length and variable)
314#[derive(Debug, Clone, PartialEq)]
315pub struct ByteArrayColumnIndex {
316    pub(crate) column_index: ColumnIndex,
317    // raw bytes for min and max values
318    pub(crate) min_bytes: Vec<u8>,
319    pub(crate) min_offsets: Vec<usize>,
320    pub(crate) max_bytes: Vec<u8>,
321    pub(crate) max_offsets: Vec<usize>,
322}
323
324impl ByteArrayColumnIndex {
325    #[expect(clippy::too_many_arguments)]
326    pub(crate) fn try_new(
327        null_pages: Vec<bool>,
328        boundary_order: BoundaryOrder,
329        null_counts: Option<Vec<i64>>,
330        nan_counts: Option<Vec<i64>>,
331        repetition_level_histograms: Option<Vec<i64>>,
332        definition_level_histograms: Option<Vec<i64>>,
333        min_values: Vec<&[u8]>,
334        max_values: Vec<&[u8]>,
335    ) -> Result<Self> {
336        let len = null_pages.len();
337
338        if min_values.len() != len || max_values.len() != len {
339            return Err(ParquetError::General(format!(
340                "ColumnIndex min/max length mismatch: expected {len}, got min={} max={}",
341                min_values.len(),
342                max_values.len()
343            )));
344        }
345        if let Some(ref nc) = null_counts
346            && nc.len() != len
347        {
348            return Err(ParquetError::General(format!(
349                "ColumnIndex null_counts length mismatch: expected {len}, got {}",
350                nc.len()
351            )));
352        }
353        if let Some(ref rep) = repetition_level_histograms
354            && len != 0
355            && rep.len() % len != 0
356        {
357            return Err(ParquetError::General(
358                "Invalid repetition_level_histograms length".to_string(),
359            ));
360        }
361        if let Some(ref def) = definition_level_histograms
362            && len != 0
363            && def.len() % len != 0
364        {
365            return Err(ParquetError::General(
366                "Invalid definition_level_histograms length".to_string(),
367            ));
368        }
369
370        let min_len = min_values.iter().map(|&v| v.len()).sum();
371        let max_len = max_values.iter().map(|&v| v.len()).sum();
372        let mut min_bytes = vec![0u8; min_len];
373        let mut max_bytes = vec![0u8; max_len];
374
375        let mut min_offsets = vec![0usize; len + 1];
376        let mut max_offsets = vec![0usize; len + 1];
377
378        let mut min_pos = 0;
379        let mut max_pos = 0;
380
381        for (i, is_null) in null_pages.iter().enumerate().take(len) {
382            if !is_null {
383                let min = min_values[i];
384                let dst = &mut min_bytes[min_pos..min_pos + min.len()];
385                dst.copy_from_slice(min);
386                min_offsets[i] = min_pos;
387                min_pos += min.len();
388
389                let max = max_values[i];
390                let dst = &mut max_bytes[max_pos..max_pos + max.len()];
391                dst.copy_from_slice(max);
392                max_offsets[i] = max_pos;
393                max_pos += max.len();
394            } else {
395                min_offsets[i] = min_pos;
396                max_offsets[i] = max_pos;
397            }
398        }
399
400        min_offsets[len] = min_pos;
401        max_offsets[len] = max_pos;
402
403        Ok(Self {
404            column_index: ColumnIndex {
405                null_pages,
406                boundary_order,
407                null_counts,
408                repetition_level_histograms,
409                definition_level_histograms,
410                nan_counts,
411            },
412            min_bytes,
413            min_offsets,
414            max_bytes,
415            max_offsets,
416        })
417    }
418
419    pub(super) fn try_from_thrift(index: ThriftColumnIndex) -> Result<Self> {
420        Self::try_new(
421            index.null_pages,
422            index.boundary_order,
423            index.null_counts,
424            index.nan_counts,
425            index.repetition_level_histograms,
426            index.definition_level_histograms,
427            index.min_values,
428            index.max_values,
429        )
430    }
431
432    /// Returns the min value for the page indexed by `idx`
433    ///
434    /// It is `None` when all values are null
435    pub fn min_value(&self, idx: usize) -> Option<&[u8]> {
436        if self.null_pages[idx] {
437            None
438        } else {
439            let start = self.min_offsets[idx];
440            let end = self.min_offsets[idx + 1];
441            Some(&self.min_bytes[start..end])
442        }
443    }
444
445    /// Returns the max value for the page indexed by `idx`
446    ///
447    /// It is `None` when all values are null
448    pub fn max_value(&self, idx: usize) -> Option<&[u8]> {
449        if self.null_pages[idx] {
450            None
451        } else {
452            let start = self.max_offsets[idx];
453            let end = self.max_offsets[idx + 1];
454            Some(&self.max_bytes[start..end])
455        }
456    }
457
458    /// Returns an iterator over the min values.
459    ///
460    /// Values may be `None` when [`ColumnIndex::is_null_page()`] is `true`.
461    pub fn min_values_iter(&self) -> impl Iterator<Item = Option<&[u8]>> {
462        (0..self.num_pages() as usize).map(|i| self.min_value(i))
463    }
464
465    /// Returns an iterator over the max values.
466    ///
467    /// Values may be `None` when [`ColumnIndex::is_null_page()`] is `true`.
468    pub fn max_values_iter(&self) -> impl Iterator<Item = Option<&[u8]>> {
469        (0..self.num_pages() as usize).map(|i| self.max_value(i))
470    }
471}
472
473impl Deref for ByteArrayColumnIndex {
474    type Target = ColumnIndex;
475
476    fn deref(&self) -> &Self::Target {
477        &self.column_index
478    }
479}
480
481impl WriteThrift for ByteArrayColumnIndex {
482    const ELEMENT_TYPE: ElementType = ElementType::Struct;
483    fn write_thrift<W: std::io::Write>(
484        &self,
485        writer: &mut ThriftCompactOutputProtocol<W>,
486    ) -> Result<()> {
487        self.null_pages.write_thrift_field(writer, 1, 0)?;
488
489        // need to handle min/max manually
490        let len = self.null_pages.len();
491        writer.write_field_begin(FieldType::List, 2, 1)?;
492        writer.write_list_begin(ElementType::Binary, len)?;
493        for i in 0..len {
494            let min = self.min_value(i).unwrap_or(&[]);
495            min.write_thrift(writer)?;
496        }
497        writer.write_field_begin(FieldType::List, 3, 2)?;
498        writer.write_list_begin(ElementType::Binary, len)?;
499        for i in 0..len {
500            let max = self.max_value(i).unwrap_or(&[]);
501            max.write_thrift(writer)?;
502        }
503        let mut last_field_id = self.boundary_order.write_thrift_field(writer, 4, 3)?;
504        if let Some(null_counts) = &self.null_counts {
505            last_field_id = null_counts.write_thrift_field(writer, 5, last_field_id)?;
506        }
507        if let Some(repetition_level_histograms) = &self.repetition_level_histograms {
508            last_field_id =
509                repetition_level_histograms.write_thrift_field(writer, 6, last_field_id)?;
510        }
511        if let Some(definition_level_histograms) = &self.definition_level_histograms {
512            last_field_id =
513                definition_level_histograms.write_thrift_field(writer, 7, last_field_id)?;
514        }
515        if let Some(nan_counts) = &self.nan_counts {
516            nan_counts.write_thrift_field(writer, 8, last_field_id)?;
517        }
518        writer.write_struct_end()
519    }
520}
521
522// Macro to generate getter functions for ColumnIndexMetaData.
523macro_rules! colidx_enum_func {
524    ($self:ident, $func:ident, $arg:ident) => {{
525        match *$self {
526            Self::BOOLEAN(ref typed) => typed.$func($arg),
527            Self::INT32(ref typed) => typed.$func($arg),
528            Self::INT64(ref typed) => typed.$func($arg),
529            Self::INT96(ref typed) => typed.$func($arg),
530            Self::FLOAT(ref typed) => typed.$func($arg),
531            Self::DOUBLE(ref typed) => typed.$func($arg),
532            Self::BYTE_ARRAY(ref typed) => typed.$func($arg),
533            Self::FIXED_LEN_BYTE_ARRAY(ref typed) => typed.$func($arg),
534        }
535    }};
536    ($self:ident, $func:ident) => {{
537        match *$self {
538            Self::BOOLEAN(ref typed) => typed.$func(),
539            Self::INT32(ref typed) => typed.$func(),
540            Self::INT64(ref typed) => typed.$func(),
541            Self::INT96(ref typed) => typed.$func(),
542            Self::FLOAT(ref typed) => typed.$func(),
543            Self::DOUBLE(ref typed) => typed.$func(),
544            Self::BYTE_ARRAY(ref typed) => typed.$func(),
545            Self::FIXED_LEN_BYTE_ARRAY(ref typed) => typed.$func(),
546        }
547    }};
548}
549
550/// Parsed [`ColumnIndex`] information for a Parquet file.
551///
552/// See [`PageIndex`] for more information.
553///
554/// [`PageIndex`]: crate::file::metadata::PageIndex
555/// [`ColumnIndex`]: https://github.com/apache/parquet-format/blob/master/PageIndex.md
556#[derive(Debug, Clone, PartialEq)]
557#[expect(non_camel_case_types)]
558pub enum ColumnIndexMetaData {
559    /// Boolean type index
560    BOOLEAN(PrimitiveColumnIndex<bool>),
561    /// 32-bit integer type index
562    INT32(PrimitiveColumnIndex<i32>),
563    /// 64-bit integer type index
564    INT64(PrimitiveColumnIndex<i64>),
565    /// 96-bit integer type (timestamp) index
566    INT96(PrimitiveColumnIndex<Int96>),
567    /// 32-bit floating point type index
568    FLOAT(PrimitiveColumnIndex<f32>),
569    /// 64-bit floating point type index
570    DOUBLE(PrimitiveColumnIndex<f64>),
571    /// Byte array type index
572    BYTE_ARRAY(ByteArrayColumnIndex),
573    /// Fixed length byte array type index
574    FIXED_LEN_BYTE_ARRAY(ByteArrayColumnIndex),
575}
576
577impl ColumnIndexMetaData {
578    /// Return min/max elements inside ColumnIndex are ordered or not.
579    pub fn is_sorted(&self) -> bool {
580        // 0:UNORDERED, 1:ASCENDING ,2:DESCENDING,
581        if let Some(order) = self.get_boundary_order() {
582            order != BoundaryOrder::UNORDERED
583        } else {
584            false
585        }
586    }
587
588    /// Get boundary_order of this page index.
589    pub fn get_boundary_order(&self) -> Option<BoundaryOrder> {
590        match self {
591            Self::BOOLEAN(index) => Some(index.boundary_order),
592            Self::INT32(index) => Some(index.boundary_order),
593            Self::INT64(index) => Some(index.boundary_order),
594            Self::INT96(index) => Some(index.boundary_order),
595            Self::FLOAT(index) => Some(index.boundary_order),
596            Self::DOUBLE(index) => Some(index.boundary_order),
597            Self::BYTE_ARRAY(index) => Some(index.boundary_order),
598            Self::FIXED_LEN_BYTE_ARRAY(index) => Some(index.boundary_order),
599        }
600    }
601
602    /// Returns array of null counts, one per page.
603    ///
604    /// Returns `None` if no null counts have been set in the index
605    pub fn null_counts(&self) -> Option<&Vec<i64>> {
606        match self {
607            Self::BOOLEAN(index) => index.null_counts.as_ref(),
608            Self::INT32(index) => index.null_counts.as_ref(),
609            Self::INT64(index) => index.null_counts.as_ref(),
610            Self::INT96(index) => index.null_counts.as_ref(),
611            Self::FLOAT(index) => index.null_counts.as_ref(),
612            Self::DOUBLE(index) => index.null_counts.as_ref(),
613            Self::BYTE_ARRAY(index) => index.null_counts.as_ref(),
614            Self::FIXED_LEN_BYTE_ARRAY(index) => index.null_counts.as_ref(),
615        }
616    }
617
618    /// Returns array of NaN counts, one per page.
619    ///
620    /// Returns `None` if no NaN counts have been set in the index
621    pub fn nan_counts(&self) -> Option<&Vec<i64>> {
622        match self {
623            Self::BOOLEAN(index) => index.nan_counts.as_ref(),
624            Self::INT32(index) => index.nan_counts.as_ref(),
625            Self::INT64(index) => index.nan_counts.as_ref(),
626            Self::INT96(index) => index.nan_counts.as_ref(),
627            Self::FLOAT(index) => index.nan_counts.as_ref(),
628            Self::DOUBLE(index) => index.nan_counts.as_ref(),
629            Self::BYTE_ARRAY(index) => index.nan_counts.as_ref(),
630            Self::FIXED_LEN_BYTE_ARRAY(index) => index.nan_counts.as_ref(),
631        }
632    }
633
634    /// Returns the number of pages
635    pub fn num_pages(&self) -> u64 {
636        colidx_enum_func!(self, num_pages)
637    }
638
639    /// Returns the number of null values in the page indexed by `idx`
640    ///
641    /// Returns `None` if no null counts have been set in the index
642    pub fn null_count(&self, idx: usize) -> Option<i64> {
643        colidx_enum_func!(self, null_count, idx)
644    }
645
646    /// Returns the number of NaN values in the page indexed by `idx`
647    ///
648    /// Returns `None` if no NaN counts have been set in the index
649    pub fn nan_count(&self, idx: usize) -> Option<i64> {
650        colidx_enum_func!(self, nan_count, idx)
651    }
652
653    /// Returns the repetition level histogram for the page indexed by `idx`
654    pub fn repetition_level_histogram(&self, idx: usize) -> Option<&[i64]> {
655        colidx_enum_func!(self, repetition_level_histogram, idx)
656    }
657
658    /// Returns the definition level histogram for the page indexed by `idx`
659    pub fn definition_level_histogram(&self, idx: usize) -> Option<&[i64]> {
660        colidx_enum_func!(self, definition_level_histogram, idx)
661    }
662
663    /// Returns whether the page indexed by `idx` consists of all null values
664    #[inline]
665    pub fn is_null_page(&self, idx: usize) -> bool {
666        colidx_enum_func!(self, is_null_page, idx)
667    }
668}
669
670/// Provides iterators over min and max values of a [`ColumnIndexMetaData`]
671pub trait ColumnIndexIterators {
672    /// Can be one of `bool`, `i32`, `i64`, `Int96`, `f32`, `f64`, [`ByteArray`],
673    /// or [`FixedLenByteArray`]
674    type Item;
675
676    /// Return iterator over the min values for the index
677    fn min_values_iter(colidx: &ColumnIndexMetaData) -> impl Iterator<Item = Option<Self::Item>>;
678
679    /// Return iterator over the max values for the index
680    fn max_values_iter(colidx: &ColumnIndexMetaData) -> impl Iterator<Item = Option<Self::Item>>;
681}
682
683macro_rules! column_index_iters {
684    ($item: ident, $variant: ident, $conv:expr) => {
685        impl ColumnIndexIterators for $item {
686            type Item = $item;
687
688            fn min_values_iter(
689                colidx: &ColumnIndexMetaData,
690            ) -> impl Iterator<Item = Option<Self::Item>> {
691                if let ColumnIndexMetaData::$variant(index) = colidx {
692                    index.min_values_iter().map($conv)
693                } else {
694                    panic!(concat!("Wrong type for ", stringify!($item), " iterator"))
695                }
696            }
697
698            fn max_values_iter(
699                colidx: &ColumnIndexMetaData,
700            ) -> impl Iterator<Item = Option<Self::Item>> {
701                if let ColumnIndexMetaData::$variant(index) = colidx {
702                    index.max_values_iter().map($conv)
703                } else {
704                    panic!(concat!("Wrong type for ", stringify!($item), " iterator"))
705                }
706            }
707        }
708    };
709}
710
711column_index_iters!(bool, BOOLEAN, |v| v.copied());
712column_index_iters!(i32, INT32, |v| v.copied());
713column_index_iters!(i64, INT64, |v| v.copied());
714column_index_iters!(Int96, INT96, |v| v.copied());
715column_index_iters!(f32, FLOAT, |v| v.copied());
716column_index_iters!(f64, DOUBLE, |v| v.copied());
717column_index_iters!(ByteArray, BYTE_ARRAY, |v| v
718    .map(|v| ByteArray::from(v.to_owned())));
719column_index_iters!(FixedLenByteArray, FIXED_LEN_BYTE_ARRAY, |v| v
720    .map(|v| FixedLenByteArray::from(v.to_owned())));
721
722impl WriteThrift for ColumnIndexMetaData {
723    const ELEMENT_TYPE: ElementType = ElementType::Struct;
724
725    fn write_thrift<W: std::io::Write>(
726        &self,
727        writer: &mut ThriftCompactOutputProtocol<W>,
728    ) -> Result<()> {
729        match self {
730            ColumnIndexMetaData::BOOLEAN(index) => index.write_thrift(writer),
731            ColumnIndexMetaData::INT32(index) => index.write_thrift(writer),
732            ColumnIndexMetaData::INT64(index) => index.write_thrift(writer),
733            ColumnIndexMetaData::INT96(index) => index.write_thrift(writer),
734            ColumnIndexMetaData::FLOAT(index) => index.write_thrift(writer),
735            ColumnIndexMetaData::DOUBLE(index) => index.write_thrift(writer),
736            ColumnIndexMetaData::BYTE_ARRAY(index) => index.write_thrift(writer),
737            ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(index) => index.write_thrift(writer),
738        }
739    }
740}
741
742#[cfg(test)]
743mod tests {
744    use super::*;
745
746    #[test]
747    fn test_page_index_min_max_null() {
748        let column_index = PrimitiveColumnIndex {
749            column_index: ColumnIndex {
750                null_pages: vec![false],
751                boundary_order: BoundaryOrder::ASCENDING,
752                null_counts: Some(vec![0]),
753                nan_counts: None,
754                repetition_level_histograms: Some(vec![1, 2]),
755                definition_level_histograms: Some(vec![1, 2, 3]),
756            },
757            min_values: vec![-123],
758            max_values: vec![234],
759        };
760
761        assert_eq!(column_index.min_value(0), Some(&-123));
762        assert_eq!(column_index.max_value(0), Some(&234));
763        assert_eq!(column_index.null_count(0), Some(0));
764        assert_eq!(column_index.repetition_level_histogram(0).unwrap(), &[1, 2]);
765        assert_eq!(
766            column_index.definition_level_histogram(0).unwrap(),
767            &[1, 2, 3]
768        );
769    }
770
771    #[test]
772    fn test_page_index_min_max_null_none() {
773        let column_index: PrimitiveColumnIndex<i32> = PrimitiveColumnIndex::<i32> {
774            column_index: ColumnIndex {
775                null_pages: vec![true],
776                boundary_order: BoundaryOrder::ASCENDING,
777                null_counts: Some(vec![1]),
778                nan_counts: None,
779                repetition_level_histograms: None,
780                definition_level_histograms: Some(vec![1, 0]),
781            },
782            min_values: vec![Default::default()],
783            max_values: vec![Default::default()],
784        };
785
786        assert_eq!(column_index.min_value(0), None);
787        assert_eq!(column_index.max_value(0), None);
788        assert_eq!(column_index.null_count(0), Some(1));
789        assert_eq!(column_index.repetition_level_histogram(0), None);
790        assert_eq!(column_index.definition_level_histogram(0).unwrap(), &[1, 0]);
791    }
792
793    #[test]
794    fn test_invalid_column_index() {
795        let column_index = ThriftColumnIndex {
796            null_pages: vec![true, false],
797            min_values: vec![
798                &[],
799                &[], // this shouldn't be empty as null_pages[1] is false
800            ],
801            max_values: vec![
802                &[],
803                &[], // this shouldn't be empty as null_pages[1] is false
804            ],
805            null_counts: None,
806            nan_counts: None,
807            repetition_level_histograms: None,
808            definition_level_histograms: None,
809            boundary_order: BoundaryOrder::UNORDERED,
810        };
811
812        let err = PrimitiveColumnIndex::<i32>::try_from_thrift(column_index).unwrap_err();
813        assert_eq!(
814            err.to_string(),
815            "Parquet error: error converting value, expected 4 bytes got 0"
816        );
817    }
818
819    #[test]
820    fn test_column_index_rejects_mismatched_min_max_lengths() {
821        // Two pages, but only one min/max entry. The entry itself is valid i32 bytes,
822        // so this specifically checks that lengths must match the number of pages.
823        let column_index = ThriftColumnIndex {
824            null_pages: vec![false, false],
825            min_values: vec![&[1u8, 0, 0, 0]],
826            max_values: vec![&[10u8, 0, 0, 0]],
827            null_counts: None,
828            repetition_level_histograms: None,
829            definition_level_histograms: None,
830            boundary_order: BoundaryOrder::UNORDERED,
831            nan_counts: None,
832        };
833
834        // ColumnIndex arrays must align with the number of pages (null_pages.len()).
835        let err = PrimitiveColumnIndex::<i32>::try_from_thrift(column_index).unwrap_err();
836        // Should fail because min/max lengths don’t match null_pages
837        assert!(err.to_string().contains("length mismatch"));
838    }
839}
parquet/file/page_index/column_index.rs

parquet/file/page_index/
column_index.rs
