parquet/arrow/
bloom_filter.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Check Arrow arrays against Parquet bloom filters
6
7use arrow_array::builder::BooleanBuilder;
8use arrow_array::cast::AsArray;
9use arrow_array::types::{Float32Type, Float64Type, RunEndIndexType};
10use arrow_array::{Array, BooleanArray, RunArray, downcast_run_array};
11use arrow_buffer::{ArrowNativeType, BooleanBuffer};
12use arrow_schema::{DataType, Field};
13
14use crate::arrow::arrow_writer::{
15    FixedLenByteArrayVisitor, PrimitiveVisitor, visit_fixed_len_byte_array, visit_i32, visit_i64,
16};
17use crate::arrow::schema::arrow_to_parquet_type;
18use crate::basic::{ConvertedType, LogicalType, TimeUnit, Type as PhysicalType};
19use crate::bloom_filter::Sbbf;
20use crate::data_type::AsBytes;
21use crate::errors::{ParquetError, Result};
22use crate::schema::types::{ColumnDescriptor, Type};
23
24/// A Parquet bloom filter ([`Sbbf`]) that can be checked with Arrow arrays
25///
26/// [`ArrowWriter`] may store an Arrow value as a different Parquet value, and
27/// the bloom filter holds the stored value, so [`Sbbf::check`] with the Arrow
28/// value can report a value that is present as absent. [`ArrowSbbf::check`]
29/// converts Arrow values in the same way as [`ArrowWriter`] before checking the
30/// filter.
31///
32/// ```
33/// # use std::sync::Arc;
34/// # use bytes::Bytes;
35/// # use arrow_array::{ArrayRef, Int8Array, RecordBatch};
36/// # use parquet::arrow::ArrowWriter;
37/// # use parquet::arrow::bloom_filter::ArrowSbbf;
38/// # use parquet::file::properties::{ReaderProperties, WriterProperties};
39/// # use parquet::file::reader::{FileReader, SerializedFileReader};
40/// # use parquet::file::serialized_reader::ReadOptionsBuilder;
41/// let array: ArrayRef = Arc::new(Int8Array::from(vec![1, 2, 3]));
42/// let batch = RecordBatch::try_from_iter([("c", array)]).unwrap();
43///
44/// let props = WriterProperties::builder()
45///     .set_bloom_filter_enabled(true)
46///     .build();
47/// let mut buffer = Vec::new();
48/// let mut writer = ArrowWriter::try_new(&mut buffer, batch.schema(), Some(props)).unwrap();
49/// writer.write(&batch).unwrap();
50/// writer.close().unwrap();
51///
52/// let options = ReadOptionsBuilder::new()
53///     .with_reader_properties(
54///         ReaderProperties::builder()
55///             .set_read_bloom_filter(true)
56///             .build(),
57///     )
58///     .build();
59/// let reader = SerializedFileReader::new_with_options(Bytes::from(buffer), options).unwrap();
60/// let row_group = reader.get_row_group(0).unwrap();
61/// let sbbf = row_group.get_column_bloom_filter(0).unwrap();
62/// let column = reader.metadata().row_group(0).column(0).column_descr();
63///
64/// // The Int8 values were written as INT32, so checking an i8 misses them
65/// assert!(!sbbf.check(&1_i8));
66///
67/// // Use the column's Arrow type, here Int8, from the file's Arrow schema
68/// let arrow_type = batch.schema().field(0).data_type().clone();
69/// let sbbf = ArrowSbbf::try_new(sbbf, column, arrow_type).unwrap();
70/// let found = sbbf.check(&Int8Array::from(vec![1, 3, 4])).unwrap();
71/// assert_eq!(found, vec![Some(true), Some(true), Some(false)].into());
72/// ```
73///
74/// [`ArrowWriter`]: crate::arrow::ArrowWriter
75#[derive(Debug, Clone)]
76pub struct ArrowSbbf<'a> {
77    sbbf: &'a Sbbf,
78    /// The Arrow type of the arrays to check
79    arrow_type: DataType,
80    physical_type: PhysicalType,
81    /// The length of the column's values if it is a `FIXED_LEN_BYTE_ARRAY`
82    /// column, otherwise 0
83    ///
84    /// Decimals are converted to this length, as other writers can store
85    /// decimals in more bytes than their precision needs.
86    type_length: usize,
87}
88
89impl<'a> ArrowSbbf<'a> {
90    /// Create a new [`ArrowSbbf`] for the bloom filter `sbbf` of `column`
91    ///
92    /// `arrow_type` is the Arrow type of the arrays that will be checked, for
93    /// example the type of the column in the file's Arrow schema.
94    ///
95    /// Returns an error if values of `arrow_type` cannot be stored in `column`,
96    /// if `arrow_type` is a decimal and `column` is not a decimal of the same
97    /// scale, or the other way around, or if `arrow_type` is a time or timestamp
98    /// in a different unit than `column`.
99    pub fn try_new(
100        sbbf: &'a Sbbf,
101        column: &ColumnDescriptor,
102        arrow_type: DataType,
103    ) -> Result<Self> {
104        validate(column.self_type(), &arrow_type)?;
105        Ok(Self {
106            sbbf,
107            arrow_type,
108            physical_type: column.physical_type(),
109            // The schema checks that the length of a FIXED_LEN_BYTE_ARRAY column
110            // is positive, and other columns have no length
111            type_length: column.type_length().try_into().unwrap_or(0),
112        })
113    }
114
115    /// Returns the Arrow type of the arrays this [`ArrowSbbf`] checks
116    pub fn arrow_type(&self) -> &DataType {
117        &self.arrow_type
118    }
119
120    /// Check each value of `array` against the bloom filter
121    ///
122    /// Returns `true` if the value may be present, and `false` if it is
123    /// definitely absent. Nulls in `array` are null in the result.
124    ///
125    /// `array` must have the type returned by [`Self::arrow_type`], or be a
126    /// dictionary or run-end encoded array of values of that type, or a
127    /// [`NullArray`](arrow_array::NullArray).
128    pub fn check(&self, array: &dyn Array) -> Result<BooleanArray> {
129        if let Some(dictionary) = array.as_any_dictionary_opt() {
130            let values = self.check(dictionary.values().as_ref())?;
131            let found = arrow_select::take::take(&values, dictionary.keys(), None)?;
132            return Ok(found.as_boolean().clone());
133        }
134        let value_type = value_type(&self.arrow_type);
135        match array.data_type() {
136            DataType::Null => return Ok(BooleanArray::new_null(array.len())),
137            DataType::RunEndEncoded(_, _) => {
138                return downcast_run_array!(
139                    array => self.check_run_array(array),
140                    _ => unreachable!("RunEndEncoded array is not a RunArray"),
141                );
142            }
143            data_type if data_type != value_type => {
144                return Err(ParquetError::General(format!(
145                    "Cannot check {data_type} values with an ArrowSbbf for {value_type} values"
146                )));
147            }
148            _ => {}
149        }
150
151        // `try_new` checked that ArrowWriter stores values of this type in a
152        // column of this physical type, so the conversions below accept them
153        let sbbf = self.sbbf;
154        let found = match self.physical_type {
155            PhysicalType::INT32 => visit_i32(array, CheckPrimitive { sbbf })?,
156            PhysicalType::INT64 => visit_i64(array, CheckPrimitive { sbbf })?,
157            PhysicalType::FLOAT => {
158                CheckPrimitive { sbbf }.slice(array.as_primitive::<Float32Type>().values())?
159            }
160            PhysicalType::DOUBLE => {
161                CheckPrimitive { sbbf }.slice(array.as_primitive::<Float64Type>().values())?
162            }
163            PhysicalType::BOOLEAN => array
164                .as_boolean()
165                .values()
166                .iter()
167                .map(|v| sbbf.check(&v))
168                .collect(),
169            PhysicalType::BYTE_ARRAY => check_byte_array(sbbf, array),
170            PhysicalType::FIXED_LEN_BYTE_ARRAY => {
171                let visitor = CheckFixedLenByteArray {
172                    sbbf,
173                    len: array.len(),
174                    type_length: self.type_length,
175                };
176                visit_fixed_len_byte_array(array, visitor)?
177            }
178            PhysicalType::INT96 => unreachable!("ArrowWriter does not write INT96 columns"),
179        };
180        Ok(BooleanArray::new(found, array.logical_nulls()))
181    }
182
183    /// Checks the values of a run-end encoded array, and repeats each result for its run
184    fn check_run_array<R: RunEndIndexType>(&self, array: &RunArray<R>) -> Result<BooleanArray> {
185        let found = self.check(array.values_slice().as_ref())?;
186        let mut builder = BooleanBuilder::with_capacity(array.len());
187        let mut run_start = 0;
188        for (i, run_end) in array.run_ends().sliced_values().enumerate() {
189            let run_end = run_end.as_usize();
190            match found.is_valid(i) {
191                true => builder.append_n(run_end - run_start, found.value(i)),
192                false => builder.append_nulls(run_end - run_start),
193            }
194            run_start = run_end;
195        }
196        Ok(builder.finish())
197    }
198}
199
200/// Returns the type of the values of a dictionary or run-end encoded `data_type`,
201/// or `data_type` itself
202fn value_type(data_type: &DataType) -> &DataType {
203    match data_type {
204        DataType::Dictionary(_, values) => value_type(values),
205        DataType::RunEndEncoded(_, values) => value_type(values.data_type()),
206        _ => data_type,
207    }
208}
209
210/// Returns an error unless [`ArrowWriter`] stores values of `arrow_type` in the
211/// same way as values of `column`, so that they can be checked against its
212/// bloom filter
213///
214/// [`ArrowWriter`]: crate::arrow::ArrowWriter
215fn validate(column: &Type, arrow_type: &DataType) -> Result<()> {
216    if !arrow_type.is_nested() {
217        let field = Field::new(column.name(), arrow_type.clone(), true);
218        // Only Date64 depends on `coerce_types`, which is stored as INT32 days
219        // or INT64 milliseconds
220        for coerce_types in [false, true] {
221            let written = arrow_to_parquet_type(&field, coerce_types)?;
222            if is_stored_as(&written, column) {
223                return Ok(());
224            }
225        }
226    }
227    Err(type_mismatch(column, arrow_type))
228}
229
230fn type_mismatch(column: &Type, arrow_type: &DataType) -> ParquetError {
231    ParquetError::General(format!(
232        "Cannot check a bloom filter of type {} with {arrow_type} values",
233        describe(column)
234    ))
235}
236
237/// Returns whether values written to a column of type `written` can be checked
238/// against the bloom filter of a column of type `column`
239///
240/// Besides the physical type, the only type parameters that change the stored
241/// values are the scale of a decimal, the length of a `FIXED_LEN_BYTE_ARRAY`,
242/// and the unit of a time or timestamp, so these must match. Other parameters,
243/// such as a timestamp's timezone, do not change the stored values.
244fn is_stored_as(written: &Type, column: &Type) -> bool {
245    let physical_type = column.get_physical_type();
246    match (decimal_scale(written), decimal_scale(column)) {
247        // Decimals are stored as unscaled integers, which are converted to the
248        // column's physical type and length
249        (Some(written_scale), Some(scale)) => {
250            written_scale == scale && physical_type != PhysicalType::BYTE_ARRAY
251        }
252        (None, None) => {
253            written.get_physical_type() == physical_type
254                && (physical_type != PhysicalType::FIXED_LEN_BYTE_ARRAY
255                    || type_length(written) == type_length(column))
256                // The same values mean different instants in different units.
257                // ArrowWriter writes seconds without a unit, so seconds only
258                // match a column without one.
259                && time_unit(written) == time_unit(column)
260        }
261        _ => false,
262    }
263}
264
265/// Returns the scale of a decimal type, or `None` if `t` is not a decimal
266fn decimal_scale(t: &Type) -> Option<i32> {
267    let info = t.get_basic_info();
268    let decimal = matches!(info.logical_type_ref(), Some(LogicalType::Decimal(_)))
269        || info.converted_type() == ConvertedType::DECIMAL;
270    decimal.then(|| t.get_scale())
271}
272
273/// Returns the unit of a `TIME` or `TIMESTAMP` type, or `None` if `t` has no unit
274fn time_unit(t: &Type) -> Option<TimeUnit> {
275    let info = t.get_basic_info();
276    match info.logical_type_ref() {
277        Some(LogicalType::Time(time) | LogicalType::Timestamp(time)) => Some(time.unit),
278        _ => match info.converted_type() {
279            ConvertedType::TIME_MILLIS | ConvertedType::TIMESTAMP_MILLIS => Some(TimeUnit::MILLIS),
280            ConvertedType::TIME_MICROS | ConvertedType::TIMESTAMP_MICROS => Some(TimeUnit::MICROS),
281            _ => None,
282        },
283    }
284}
285
286fn type_length(t: &Type) -> i32 {
287    match t {
288        Type::PrimitiveType { type_length, .. } => *type_length,
289        Type::GroupType { .. } => -1,
290    }
291}
292
293/// Describes the parts of `column`'s type that determine how its values are stored
294fn describe(column: &Type) -> String {
295    let physical_type = column.get_physical_type();
296    let length = match physical_type {
297        PhysicalType::FIXED_LEN_BYTE_ARRAY => format!("({})", type_length(column)),
298        _ => String::new(),
299    };
300    let decimal = decimal_scale(column)
301        .map(|scale| format!(" DECIMAL({}, {scale})", column.get_precision()))
302        .unwrap_or_default();
303    let unit = time_unit(column)
304        .map(|unit| format!(" ({unit:?})"))
305        .unwrap_or_default();
306    format!("{physical_type}{length}{decimal}{unit}")
307}
308
309/// Checks the `INT32`, `INT64`, `FLOAT` or `DOUBLE` values passed to it against a bloom filter
310struct CheckPrimitive<'a> {
311    sbbf: &'a Sbbf,
312}
313
314impl<T: AsBytes> PrimitiveVisitor<T> for CheckPrimitive<'_> {
315    type Output = BooleanBuffer;
316
317    fn slice(self, values: &[T]) -> Result<BooleanBuffer> {
318        Ok(values.iter().map(|v| self.sbbf.check(v)).collect())
319    }
320
321    fn map<S: Copy>(self, values: &[S], convert: impl Fn(S) -> T) -> Result<BooleanBuffer> {
322        Ok(values
323            .iter()
324            .map(|v| self.sbbf.check(&convert(*v)))
325            .collect())
326    }
327}
328
329/// Checks the `FIXED_LEN_BYTE_ARRAY` values passed to it against a bloom filter
330struct CheckFixedLenByteArray<'a> {
331    sbbf: &'a Sbbf,
332    /// The number of values in the array
333    len: usize,
334    type_length: usize,
335}
336
337impl FixedLenByteArrayVisitor for CheckFixedLenByteArray<'_> {
338    type Output = BooleanBuffer;
339
340    fn decimal_length(&self, _precision: u8, _native_length: usize) -> usize {
341        // Decimals of any precision are converted to the column's length
342        self.type_length
343    }
344
345    fn values(self, length: usize, fill: impl Fn(usize, &mut [u8])) -> Result<BooleanBuffer> {
346        // `ArrowSbbf::try_new` checked that the Arrow type is stored in values
347        // of the column's length
348        debug_assert_eq!(length, self.type_length);
349        let mut value = vec![0; length];
350        Ok((0..self.len)
351            .map(|i| {
352                value.fill(0);
353                fill(i, &mut value);
354                self.sbbf.check(value.as_slice())
355            })
356            .collect())
357    }
358}
359
360/// Checks the values of a `BYTE_ARRAY` column, which are stored as they are
361fn check_byte_array(sbbf: &Sbbf, array: &dyn Array) -> BooleanBuffer {
362    match array.data_type() {
363        DataType::Utf8 => check_bytes(sbbf, array.as_string::<i32>()),
364        DataType::LargeUtf8 => check_bytes(sbbf, array.as_string::<i64>()),
365        DataType::Utf8View => check_bytes(sbbf, array.as_string_view()),
366        DataType::Binary => check_bytes(sbbf, array.as_binary::<i32>()),
367        DataType::LargeBinary => check_bytes(sbbf, array.as_binary::<i64>()),
368        DataType::BinaryView => check_bytes(sbbf, array.as_binary_view()),
369        d => unreachable!("ArrowWriter does not write {d} to BYTE_ARRAY columns"),
370    }
371}
372
373fn check_bytes<'a, T: AsRef<[u8]> + ?Sized + 'a>(
374    sbbf: &Sbbf,
375    values: impl arrow_array::ArrayAccessor<Item = &'a T>,
376) -> BooleanBuffer {
377    (0..values.len())
378        .map(|i| sbbf.check(values.value(i).as_ref()))
379        .collect()
380}
381
382#[cfg(test)]
383mod tests {
384    use super::*;
385    use crate::arrow::ArrowWriter;
386    use crate::file::properties::{ReaderProperties, WriterProperties};
387    use crate::file::reader::{FileReader, SerializedFileReader};
388    use crate::file::serialized_reader::ReadOptionsBuilder;
389    use crate::schema::types::{ColumnDescPtr, ColumnPath};
390    use arrow_array::types::{Int16Type, Int32Type};
391    use arrow_array::*;
392    use arrow_buffer::{IntervalDayTime, i256};
393    use arrow_schema::{IntervalUnit, TimeUnit};
394    use bytes::Bytes;
395    use std::sync::Arc;
396
397    /// Writes `batch` with bloom filters, and returns the filter and descriptor of each column
398    fn write_and_read_bloom_filters(
399        batch: &RecordBatch,
400        coerce_types: bool,
401    ) -> Vec<(Sbbf, ColumnDescPtr)> {
402        let props = WriterProperties::builder()
403            .set_bloom_filter_enabled(true)
404            .set_coerce_types(coerce_types)
405            .build();
406        let mut buffer = Vec::new();
407        let mut writer = ArrowWriter::try_new(&mut buffer, batch.schema(), Some(props)).unwrap();
408        writer.write(batch).unwrap();
409        writer.close().unwrap();
410
411        let options = ReadOptionsBuilder::new()
412            .with_reader_properties(
413                ReaderProperties::builder()
414                    .set_read_bloom_filter(true)
415                    .build(),
416            )
417            .build();
418        let reader = SerializedFileReader::new_with_options(Bytes::from(buffer), options).unwrap();
419        let row_group = reader.get_row_group(0).unwrap();
420        (0..batch.num_columns())
421            .map(|i| {
422                let sbbf = row_group.get_column_bloom_filter(i).unwrap().clone();
423                let column = reader.metadata().row_group(0).column(i).column_descr_ptr();
424                (sbbf, column)
425            })
426            .collect()
427    }
428
429    /// Writes `array` with a bloom filter and returns the filter and column descriptor
430    fn write_and_read_bloom_filter(array: ArrayRef, coerce_types: bool) -> (Sbbf, ColumnDescPtr) {
431        let batch = RecordBatch::try_from_iter([("c", array)]).unwrap();
432        write_and_read_bloom_filters(&batch, coerce_types)
433            .pop()
434            .unwrap()
435    }
436
437    /// A column written by [`ArrowWriter`] with a bloom filter
438    struct Written {
439        sbbf: Sbbf,
440        column: ColumnDescPtr,
441    }
442
443    impl Written {
444        /// Writes `array` with a bloom filter
445        fn new(array: impl Array + 'static) -> Self {
446            Self::with_coerce_types(Arc::new(array), false)
447        }
448
449        fn with_coerce_types(array: ArrayRef, coerce_types: bool) -> Self {
450            let (sbbf, column) = write_and_read_bloom_filter(array, coerce_types);
451            Self { sbbf, column }
452        }
453
454        /// Returns an [`ArrowSbbf`] that checks `arrow_type` values against this column
455        fn arrow_sbbf(&self, arrow_type: DataType) -> ArrowSbbf<'_> {
456            ArrowSbbf::try_new(&self.sbbf, &self.column, arrow_type).unwrap()
457        }
458
459        /// Checks `array` with an [`ArrowSbbf`] for the type of `array`
460        fn check(&self, array: &dyn Array) -> BooleanArray {
461            let arrow_type = array.data_type().clone();
462            self.arrow_sbbf(arrow_type).check(array).unwrap()
463        }
464
465        /// Asserts which values of `array` may be in the column
466        fn assert_found(&self, array: &dyn Array, expected: &[bool]) {
467            assert_eq!(self.check(array), BooleanArray::from(expected.to_vec()));
468        }
469
470        /// Asserts that `arrow_type` values cannot be checked against this column,
471        /// which has the type described by `column_type`
472        fn assert_rejects(&self, arrow_type: DataType, column_type: &str) {
473            let err = ArrowSbbf::try_new(&self.sbbf, &self.column, arrow_type.clone()).unwrap_err();
474            assert_eq!(
475                err.to_string(),
476                format!(
477                    "Parquet error: Cannot check a bloom filter of type {column_type} with {arrow_type} values"
478                )
479            );
480        }
481    }
482
483    /// Writes `written` and checks that every value is found and no value of `absent` is
484    fn assert_round_trip(written: ArrayRef, absent: ArrayRef, coerce_types: bool) {
485        let column = Written::with_coerce_types(written.clone(), coerce_types);
486        column.assert_found(written.as_ref(), &vec![true; written.len()]);
487        column.assert_found(absent.as_ref(), &vec![false; absent.len()]);
488    }
489
490    #[test]
491    fn time_units_must_match() {
492        let written = TimestampMillisecondArray::from(vec![1_000, 2_000]);
493        let column = Written::new(written.clone());
494        column.assert_found(&written, &[true, true]);
495
496        // The same instants in a different unit are different values, so
497        // checking them would report instants in the file as absent
498        for unit in [
499            TimeUnit::Second,
500            TimeUnit::Microsecond,
501            TimeUnit::Nanosecond,
502        ] {
503            column.assert_rejects(DataType::Timestamp(unit, None), "INT64 (MILLIS)");
504        }
505        // Integers have no unit
506        column.assert_rejects(DataType::Int64, "INT64 (MILLIS)");
507
508        let column = Written::new(Time64MicrosecondArray::from(vec![1_000]));
509        column.assert_rejects(DataType::Time64(TimeUnit::Nanosecond), "INT64 (MICROS)");
510
511        // ArrowWriter writes seconds without a unit, so they only match a
512        // column without one
513        let written = TimestampSecondArray::from(vec![1, 2]);
514        let column = Written::new(written.clone());
515        column.assert_found(&written, &[true, true]);
516        column.assert_rejects(DataType::Timestamp(TimeUnit::Millisecond, None), "INT64");
517        let column = Written::new(Time32SecondArray::from(vec![1, 2]));
518        column.assert_rejects(DataType::Time32(TimeUnit::Millisecond), "INT32");
519    }
520
521    #[test]
522    fn narrow_integers() {
523        assert_round_trip(
524            Arc::new(Int8Array::from(vec![i8::MIN, -1, 0, 1, i8::MAX])),
525            Arc::new(Int8Array::from(vec![-2, 2, 100])),
526            false,
527        );
528        assert_round_trip(
529            Arc::new(Int16Array::from(vec![i16::MIN, -1, 0, 1, i16::MAX])),
530            Arc::new(Int16Array::from(vec![-2, 2, 1000])),
531            false,
532        );
533        assert_round_trip(
534            Arc::new(UInt8Array::from(vec![0, 1, 128, u8::MAX])),
535            Arc::new(UInt8Array::from(vec![2, 127, 200])),
536            false,
537        );
538        assert_round_trip(
539            Arc::new(UInt16Array::from(vec![0, 1, 32768, u16::MAX])),
540            Arc::new(UInt16Array::from(vec![2, 32767, 60000])),
541            false,
542        );
543    }
544
545    #[test]
546    fn unsigned_integers_above_signed_max() {
547        assert_round_trip(
548            Arc::new(UInt32Array::from(vec![0, i32::MAX as u32 + 1, u32::MAX])),
549            Arc::new(UInt32Array::from(vec![1, u32::MAX - 1])),
550            false,
551        );
552        assert_round_trip(
553            Arc::new(UInt64Array::from(vec![0, i64::MAX as u64 + 1, u64::MAX])),
554            Arc::new(UInt64Array::from(vec![1, u64::MAX - 1])),
555            false,
556        );
557    }
558
559    #[test]
560    fn date64() {
561        let day = 86_400_000;
562        let written: ArrayRef = Arc::new(Date64Array::from(vec![0, day, 365 * day]));
563        let absent: ArrayRef = Arc::new(Date64Array::from(vec![2 * day, 366 * day]));
564        // Stored as INT64 milliseconds
565        assert_round_trip(written.clone(), absent.clone(), false);
566        // Stored as INT32 days
567        assert_round_trip(written, absent, true);
568    }
569
570    #[test]
571    fn date64_stored_as_days() {
572        // Milliseconds are divided by a day and rounded towards zero
573        let written: ArrayRef = Arc::new(Date64Array::from(vec![5 * 86_400_000, -1]));
574        let column = Written::with_coerce_types(written.clone(), true);
575        assert!(column.sbbf.check(&5_i32));
576        assert!(column.sbbf.check(&0_i32));
577        column.assert_found(written.as_ref(), &[true, true]);
578    }
579
580    #[test]
581    fn decimals() {
582        // Stored as INT32, INT64 and FIXED_LEN_BYTE_ARRAY depending on precision
583        for precision in [5, 15, 30] {
584            assert_round_trip(
585                Arc::new(decimal128(vec![-12345, 0, 12345], precision, 2)),
586                Arc::new(decimal128(vec![-1, 1, 54321], precision, 2)),
587                false,
588            );
589        }
590    }
591
592    fn decimal128(values: Vec<i128>, precision: u8, scale: i8) -> Decimal128Array {
593        Decimal128Array::from(values)
594            .with_precision_and_scale(precision, scale)
595            .unwrap()
596    }
597
598    #[test]
599    fn decimal_precision_differs_from_column() {
600        // Precision 30 is stored as a 13 byte FIXED_LEN_BYTE_ARRAY
601        let column = Written::new(decimal128(vec![12345, -12345], 30, 2));
602        assert_eq!(column.column.type_length(), 13);
603
604        // The values are converted to the column's width, whatever the array's precision
605        for precision in [20, 30, 38] {
606            let values = decimal128(vec![12345, -12345, 54321], precision, 2);
607            column.assert_found(&values, &[true, true, false]);
608        }
609        let decimal256 = Decimal256Array::from(vec![i256::from(12345), i256::from(-12345)])
610            .with_precision_and_scale(76, 2)
611            .unwrap();
612        column.assert_found(&decimal256, &[true, true]);
613
614        // Narrower decimal types are sign-extended to 13 bytes
615        let decimal32 = Decimal32Array::from(vec![12345, -12345, 54321])
616            .with_precision_and_scale(9, 2)
617            .unwrap();
618        column.assert_found(&decimal32, &[true, true, false]);
619        let decimal64 = Decimal64Array::from(vec![12345, -12345, 54321])
620            .with_precision_and_scale(18, 2)
621            .unwrap();
622        column.assert_found(&decimal64, &[true, true, false]);
623
624        // Values that need more than 13 bytes are truncated to their low 13
625        // bytes, like ArrowWriter does. These are the bytes of 12345 and -12345
626        let wide = 1_i128 << 104;
627        let values = decimal128(vec![12345 + wide, -12345 - wide], 38, 2);
628        column.assert_found(&values, &[true, true]);
629    }
630
631    #[test]
632    fn decimal_column_longer_than_i256() {
633        // ArrowWriter stores decimals in at most 32 bytes, but another writer can
634        // use a longer FIXED_LEN_BYTE_ARRAY, to which values are sign-extended
635        let column = Type::primitive_type_builder("c", PhysicalType::FIXED_LEN_BYTE_ARRAY)
636            .with_length(33)
637            .with_logical_type(Some(LogicalType::decimal(2, 10)))
638            .with_precision(10)
639            .with_scale(2)
640            .build()
641            .unwrap();
642        let column = ColumnDescriptor::new(Arc::new(column), 1, 0, ColumnPath::from("c"));
643        let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap();
644        for (sign, value) in [(0, 12345), (0xFF, -12345)] {
645            let mut bytes = vec![sign];
646            bytes.extend_from_slice(&i256::from(value).to_be_bytes());
647            sbbf.insert(bytes.as_slice());
648        }
649        let column = Written {
650            sbbf,
651            column: Arc::new(column),
652        };
653        column.assert_found(
654            &decimal128(vec![12345, -12345, 54321], 10, 2),
655            &[true, true, false],
656        );
657    }
658
659    #[test]
660    fn narrower_decimal_type_than_column() {
661        // Precision 12 is stored as INT64
662        let written = Decimal64Array::from(vec![12345, -12345])
663            .with_precision_and_scale(12, 2)
664            .unwrap();
665        let column = Written::new(written);
666        assert_eq!(column.column.physical_type(), PhysicalType::INT64);
667
668        let decimal32 = Decimal32Array::from(vec![12345, -12345, 54321])
669            .with_precision_and_scale(9, 2)
670            .unwrap();
671        column.assert_found(&decimal32, &[true, true, false]);
672    }
673
674    #[test]
675    fn decimal_too_large_for_precision() {
676        // Arrow does not validate precision, so these values need more bytes than
677        // the column has. They are truncated in the same way as ArrowWriter
678        // truncates them, for INT32, INT64 and FIXED_LEN_BYTE_ARRAY columns.
679        for precision in [5, 15, 20] {
680            let written = decimal128(vec![i128::MAX / 3, -(i128::MAX / 3)], precision, 2);
681            Written::new(written.clone()).assert_found(&written, &[true, true]);
682        }
683    }
684
685    #[test]
686    fn decimal_scale_differs_from_column() {
687        // Stored as INT32, INT64 and FIXED_LEN_BYTE_ARRAY depending on precision
688        for (precision, column_type) in [
689            (5, "INT32 DECIMAL(5, 2)"),
690            (15, "INT64 DECIMAL(15, 2)"),
691            (30, "FIXED_LEN_BYTE_ARRAY(13) DECIMAL(30, 2)"),
692        ] {
693            let written = decimal128(vec![12345], precision, 2);
694            let column = Written::new(written.clone());
695            column.assert_found(&written, &[true]);
696
697            // 123.45 at scale 3 is 123450, so checking 12345 would check 12.345
698            column.assert_rejects(DataType::Decimal128(precision, 3), column_type);
699        }
700
701        // Nor can decimal values be checked against a column that is not a decimal
702        let column = Written::new(Int32Array::from(vec![12345]));
703        column.assert_rejects(DataType::Decimal128(5, 0), "INT32");
704    }
705
706    #[test]
707    fn decimal_column_requires_decimal_values() {
708        // Stored as INT32, INT64 and FIXED_LEN_BYTE_ARRAY depending on precision
709        for (precision, column_type) in [
710            (5, "INT32 DECIMAL(5, 2)"),
711            (15, "INT64 DECIMAL(15, 2)"),
712            (30, "FIXED_LEN_BYTE_ARRAY(13) DECIMAL(30, 2)"),
713        ] {
714            let column = Written::new(decimal128(vec![500], precision, 2));
715            // 500 at scale 2 is 5.00, so checking the integer 500 or 5 would be ambiguous
716            column.assert_rejects(DataType::Int32, column_type);
717        }
718
719        let column = Written::new(decimal128(vec![500], 30, 2));
720        column.assert_rejects(
721            DataType::FixedSizeBinary(13),
722            "FIXED_LEN_BYTE_ARRAY(13) DECIMAL(30, 2)",
723        );
724    }
725
726    #[test]
727    fn fixed_size_binary_length_differs_from_column() {
728        let written = FixedSizeBinaryArray::try_from_iter([b"abcd", b"efgh"].into_iter()).unwrap();
729        let column = Written::new(written.clone());
730        column.assert_found(&written, &[true, true]);
731        column.assert_rejects(DataType::FixedSizeBinary(3), "FIXED_LEN_BYTE_ARRAY(4)");
732    }
733
734    #[test]
735    fn float16() {
736        let f16 = half::f16::from_f32;
737        assert_round_trip(
738            Arc::new(Float16Array::from(vec![f16(-1.5), f16(0.0), f16(2.25)])),
739            Arc::new(Float16Array::from(vec![f16(1.5), f16(3.0)])),
740            false,
741        );
742    }
743
744    #[test]
745    fn intervals() {
746        assert_round_trip(
747            Arc::new(IntervalYearMonthArray::from(vec![-1, 0, 13])),
748            Arc::new(IntervalYearMonthArray::from(vec![1, 12])),
749            false,
750        );
751        let interval = IntervalDayTime::new;
752        assert_round_trip(
753            Arc::new(IntervalDayTimeArray::from(vec![
754                interval(1, 2),
755                interval(-3, 4),
756            ])),
757            Arc::new(IntervalDayTimeArray::from(vec![
758                interval(2, 1),
759                interval(0, 0),
760            ])),
761            false,
762        );
763    }
764
765    #[test]
766    fn strings() {
767        assert_round_trip(
768            Arc::new(StringArray::from(vec!["a", "bc", ""])),
769            Arc::new(StringArray::from(vec!["b", "abc"])),
770            false,
771        );
772    }
773
774    #[test]
775    fn dictionary_and_nulls() {
776        let column = Written::new(Int8Array::from(vec![1, 2, 3]));
777        let sbbf = column.arrow_sbbf(DataType::Int8);
778
779        let found = sbbf
780            .check(&Int8Array::from(vec![Some(1), None, Some(4)]))
781            .unwrap();
782        assert_eq!(
783            found,
784            BooleanArray::from(vec![Some(true), None, Some(false)])
785        );
786
787        let dictionary = DictionaryArray::<Int32Type>::new(
788            Int32Array::from(vec![Some(1), None, Some(0), Some(1)]),
789            Arc::new(Int8Array::from(vec![4, 3])),
790        );
791        let found = sbbf.check(&dictionary).unwrap();
792        assert_eq!(
793            found,
794            BooleanArray::from(vec![Some(true), None, Some(false), Some(true)])
795        );
796
797        let found = sbbf.check(&NullArray::new(2)).unwrap();
798        assert_eq!(found, BooleanArray::new_null(2));
799    }
800
801    #[test]
802    fn null_array() {
803        let columns: [ArrayRef; 2] = [
804            Arc::new(Float64Array::from(vec![1.5])),
805            Arc::new(StringArray::from(vec!["a"])),
806        ];
807        for written in columns {
808            let arrow_type = written.data_type().clone();
809            let column = Written::with_coerce_types(written, false);
810            let found = column
811                .arrow_sbbf(arrow_type)
812                .check(&NullArray::new(2))
813                .unwrap();
814            assert_eq!(found, BooleanArray::from(vec![None, None]));
815        }
816    }
817
818    #[test]
819    fn run_end_encoded() {
820        let run_ends = Int32Array::from(vec![2, 3, 5, 6]);
821        let values = Int16Array::from(vec![1, -2, 7, 3]);
822        let written = RunArray::<Int32Type>::try_new(&run_ends, &values).unwrap();
823        let column = Written::new(written);
824        let sbbf = column.arrow_sbbf(DataType::Int16);
825
826        let values = Int16Array::from(vec![Some(-2), Some(4), None, Some(1)]);
827        let array =
828            RunArray::<Int16Type>::try_new(&Int16Array::from(vec![1, 3, 4, 6]), &values).unwrap();
829        let found = sbbf.check(&array).unwrap();
830        assert_eq!(
831            found,
832            BooleanArray::from(vec![
833                Some(true),
834                Some(false),
835                Some(false),
836                None,
837                Some(true),
838                Some(true)
839            ])
840        );
841
842        let found = sbbf.check(&array.slice(2, 3)).unwrap();
843        assert_eq!(
844            found,
845            BooleanArray::from(vec![Some(false), None, Some(true)])
846        );
847    }
848
849    #[test]
850    fn unsupported_type() {
851        let column = Written::new(Int8Array::from(vec![1, 2, 3]));
852        column.assert_rejects(DataType::Utf8, "INT32");
853        column.assert_rejects(DataType::new_list(DataType::Int8, true), "INT32");
854
855        let column =
856            Written::new(FixedSizeBinaryArray::try_from_iter([b"abcd"].into_iter()).unwrap());
857        column.assert_rejects(DataType::Int32, "FIXED_LEN_BYTE_ARRAY(4)");
858        column.assert_rejects(
859            DataType::Interval(IntervalUnit::MonthDayNano),
860            "FIXED_LEN_BYTE_ARRAY(4)",
861        );
862    }
863
864    #[test]
865    fn array_type_differs_from_arrow_type() {
866        let column = Written::new(Int8Array::from(vec![1, 2, 3]));
867        let err = column
868            .arrow_sbbf(DataType::Int8)
869            .check(&Int16Array::from(vec![1]))
870            .unwrap_err();
871        assert_eq!(
872            err.to_string(),
873            "Parquet error: Cannot check Int16 values with an ArrowSbbf for Int8 values"
874        );
875
876        // A dictionary or run-end encoded type is checked by its values
877        let dictionary = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Int8));
878        let found = column
879            .arrow_sbbf(dictionary)
880            .check(&Int8Array::from(vec![1, 4]))
881            .unwrap();
882        assert_eq!(found, BooleanArray::from(vec![true, false]));
883    }
884
885    /// Returns `values` with a null inserted at index 2
886    fn with_null<T>(values: [T; 4]) -> Vec<Option<T>> {
887        let mut values: Vec<_> = values.into_iter().map(Some).collect();
888        values.insert(2, None);
889        values
890    }
891
892    /// Returns a column of each Arrow type that [`ArrowWriter`] can write to a
893    /// primitive column, with edge values and a null in the third row
894    fn writer_type_columns() -> Vec<(&'static str, ArrayRef)> {
895        let day = 86_400_000;
896        let f16 = half::f16::from_f32;
897        let fsb = |values: Vec<Option<&[u8; 4]>>| -> ArrayRef {
898            Arc::new(
899                FixedSizeBinaryArray::try_from_sparse_iter_with_size(values.into_iter(), 4)
900                    .unwrap(),
901            )
902        };
903        // The last value is too large for the precision, so the writer truncates
904        // it. No value is 0, which a wrong truncation could produce.
905        let decimal32 = |precision| -> ArrayRef {
906            let values = with_null([-12345, 1, 12345, i32::MAX / 3]);
907            Arc::new(
908                Decimal32Array::from(values)
909                    .with_precision_and_scale(precision, 2)
910                    .unwrap(),
911            )
912        };
913        let decimal64 = |precision| -> ArrayRef {
914            let values = with_null([-12345, 1, 12345, i64::MAX / 3]);
915            Arc::new(
916                Decimal64Array::from(values)
917                    .with_precision_and_scale(precision, 2)
918                    .unwrap(),
919            )
920        };
921        let decimal128 = |precision| -> ArrayRef {
922            let values = with_null([-12345, 1, 12345, i128::MAX / 3]);
923            Arc::new(
924                Decimal128Array::from(values)
925                    .with_precision_and_scale(precision, 2)
926                    .unwrap(),
927            )
928        };
929        let decimal256 = |precision| -> ArrayRef {
930            let large = i256::MAX.wrapping_div(i256::from(3));
931            let values = with_null([i256::from(-12345), i256::from(1), i256::from(12345), large]);
932            Arc::new(
933                Decimal256Array::from(values)
934                    .with_precision_and_scale(precision, 2)
935                    .unwrap(),
936            )
937        };
938        let strings = with_null(["", "a", "longer than twelve bytes", "é"]);
939        let binaries: Vec<_> = strings.iter().map(|s| s.map(str::as_bytes)).collect();
940        let interval = IntervalDayTime::new;
941        let day_times = with_null([
942            interval(1, 2),
943            interval(-3, 4),
944            interval(0, 0),
945            interval(i32::MAX, i32::MIN),
946        ]);
947        let int16s = Int16Array::from(vec![
948            Some(9),
949            Some(-7),
950            None,
951            Some(3),
952            Some(i16::MIN),
953            Some(5),
954        ]);
955        // The dictionaries have no repeated values, so the last rows have
956        // values that are not in the first rows
957        let keys = Int32Array::from(with_null([0, 1, 2, 3]));
958        let ree_int8 = RunArray::<Int32Type>::try_new(
959            &Int32Array::from(vec![2, 3, 5]),
960            &Int8Array::from(vec![Some(-3), None, Some(i8::MAX)]),
961        );
962        let ree_utf8 = RunArray::<Int16Type>::try_new(
963            &Int16Array::from(vec![1, 3, 5]),
964            &StringArray::from(vec![Some("a"), None, Some("b")]),
965        );
966
967        let time32s = with_null([0, 1, 3600, 86399]);
968        let time64s = with_null([0, 1, 3600, 86399]);
969        let i64s = with_null([-1, 0, 1, i64::MAX]);
970        let float16s = with_null([f16(-1.5), f16(0.0), f16(-0.0), half::f16::NAN]);
971        // Stored as days, rounded towards zero, with coerce_types
972        let date64s = with_null([-1, 0, 5 * day + 1, -5 * day - 1]);
973        let timestamp_ms = TimestampMillisecondArray::from(i64s.clone()).with_timezone("+01:00");
974
975        vec![
976            (
977                "int8",
978                Arc::new(Int8Array::from(with_null([i8::MIN, -1, 1, i8::MAX]))),
979            ),
980            (
981                "int16",
982                Arc::new(Int16Array::from(with_null([i16::MIN, -1, 1, i16::MAX]))),
983            ),
984            (
985                "int32",
986                Arc::new(Int32Array::from(with_null([i32::MIN, -1, 1, i32::MAX]))),
987            ),
988            (
989                "int64",
990                Arc::new(Int64Array::from(with_null([i64::MIN, -1, 1, i64::MAX]))),
991            ),
992            (
993                "uint8",
994                Arc::new(UInt8Array::from(with_null([0, 1, 128, u8::MAX]))),
995            ),
996            (
997                "uint16",
998                Arc::new(UInt16Array::from(with_null([0, 1, 32768, u16::MAX]))),
999            ),
1000            (
1001                "uint32",
1002                Arc::new(UInt32Array::from(with_null([0, 1, 1 << 31, u32::MAX]))),
1003            ),
1004            (
1005                "uint64",
1006                Arc::new(UInt64Array::from(with_null([0, 1, 1 << 63, u64::MAX]))),
1007            ),
1008            ("float16", Arc::new(Float16Array::from(float16s))),
1009            (
1010                "float32",
1011                Arc::new(Float32Array::from(with_null([-1.5, 0.0, -0.0, f32::NAN]))),
1012            ),
1013            (
1014                "float64",
1015                Arc::new(Float64Array::from(with_null([
1016                    -1.5,
1017                    0.0,
1018                    f64::INFINITY,
1019                    f64::NAN,
1020                ]))),
1021            ),
1022            (
1023                "bool",
1024                Arc::new(BooleanArray::from(with_null([true, false, true, false]))),
1025            ),
1026            (
1027                "date32",
1028                Arc::new(Date32Array::from(with_null([-1, 0, 1, 20000]))),
1029            ),
1030            ("date64", Arc::new(Date64Array::from(date64s))),
1031            (
1032                "time32_s",
1033                Arc::new(Time32SecondArray::from(time32s.clone())),
1034            ),
1035            ("time32_ms", Arc::new(Time32MillisecondArray::from(time32s))),
1036            (
1037                "time64_us",
1038                Arc::new(Time64MicrosecondArray::from(time64s.clone())),
1039            ),
1040            ("time64_ns", Arc::new(Time64NanosecondArray::from(time64s))),
1041            (
1042                "timestamp_s",
1043                Arc::new(TimestampSecondArray::from(i64s.clone())),
1044            ),
1045            ("timestamp_ms", Arc::new(timestamp_ms)),
1046            (
1047                "timestamp_us",
1048                Arc::new(TimestampMicrosecondArray::from(i64s.clone())),
1049            ),
1050            (
1051                "timestamp_ns",
1052                Arc::new(TimestampNanosecondArray::from(i64s.clone())),
1053            ),
1054            (
1055                "duration_s",
1056                Arc::new(DurationSecondArray::from(i64s.clone())),
1057            ),
1058            (
1059                "duration_ms",
1060                Arc::new(DurationMillisecondArray::from(i64s.clone())),
1061            ),
1062            (
1063                "duration_us",
1064                Arc::new(DurationMicrosecondArray::from(i64s.clone())),
1065            ),
1066            ("duration_ns", Arc::new(DurationNanosecondArray::from(i64s))),
1067            // Stored as INT32, INT64 or FIXED_LEN_BYTE_ARRAY depending on precision
1068            ("decimal32_5", decimal32(5)),
1069            ("decimal32_9", decimal32(9)),
1070            ("decimal64_5", decimal64(5)),
1071            ("decimal64_18", decimal64(18)),
1072            ("decimal128_5", decimal128(5)),
1073            ("decimal128_15", decimal128(15)),
1074            ("decimal128_20", decimal128(20)),
1075            ("decimal128_38", decimal128(38)),
1076            ("decimal256_5", decimal256(5)),
1077            ("decimal256_15", decimal256(15)),
1078            ("decimal256_30", decimal256(30)),
1079            ("decimal256_76", decimal256(76)),
1080            (
1081                "interval_year_month",
1082                Arc::new(IntervalYearMonthArray::from(with_null([
1083                    -1,
1084                    0,
1085                    13,
1086                    i32::MAX,
1087                ]))),
1088            ),
1089            (
1090                "interval_day_time",
1091                Arc::new(IntervalDayTimeArray::from(day_times)),
1092            ),
1093            (
1094                "fixed_size_binary",
1095                fsb(with_null([
1096                    b"abcd",
1097                    b"\0\0\0\0",
1098                    b"\xff\xfe\xfd\xfc",
1099                    b"abce",
1100                ])),
1101            ),
1102            ("utf8", Arc::new(StringArray::from(strings.clone()))),
1103            (
1104                "large_utf8",
1105                Arc::new(LargeStringArray::from(strings.clone())),
1106            ),
1107            ("utf8_view", Arc::new(StringViewArray::from(strings))),
1108            ("binary", Arc::new(BinaryArray::from(binaries.clone()))),
1109            (
1110                "large_binary",
1111                Arc::new(LargeBinaryArray::from(binaries.clone())),
1112            ),
1113            ("binary_view", Arc::new(BinaryViewArray::from(binaries))),
1114            ("sliced_int16", Arc::new(int16s.slice(1, 5))),
1115            // Dictionary and run-end encoded arrays are written as their values
1116            (
1117                "dictionary_int16",
1118                Arc::new(DictionaryArray::new(
1119                    keys.clone(),
1120                    Arc::new(Int16Array::from(vec![-1, i16::MAX, 7, i16::MIN])),
1121                )),
1122            ),
1123            (
1124                "dictionary_decimal",
1125                Arc::new(DictionaryArray::new(
1126                    keys.clone(),
1127                    Arc::new(
1128                        Decimal128Array::from(vec![-12345, 1, 12345, i128::MAX / 3])
1129                            .with_precision_and_scale(30, 2)
1130                            .unwrap(),
1131                    ),
1132                )),
1133            ),
1134            (
1135                "dictionary_fixed_size_binary",
1136                Arc::new(DictionaryArray::new(
1137                    keys.clone(),
1138                    fsb(vec![
1139                        Some(b"abcd"),
1140                        Some(b"wxyz"),
1141                        Some(b"efgh"),
1142                        Some(b"ijkl"),
1143                    ]),
1144                )),
1145            ),
1146            (
1147                "dictionary_utf8",
1148                Arc::new(DictionaryArray::new(
1149                    keys,
1150                    Arc::new(StringArray::from(vec![
1151                        "a",
1152                        "longer than twelve bytes",
1153                        "b",
1154                        "c",
1155                    ])),
1156                )),
1157            ),
1158            ("ree_int8", Arc::new(ree_int8.unwrap())),
1159            ("ree_utf8", Arc::new(ree_utf8.unwrap())),
1160        ]
1161    }
1162
1163    /// Writes a column of each Arrow type that [`ArrowWriter`] can write to a
1164    /// primitive column, and checks that [`ArrowSbbf`] finds every written value
1165    /// and does not find values that were not written
1166    #[test]
1167    fn every_writer_type() {
1168        let columns = writer_type_columns();
1169        let batch = RecordBatch::try_from_iter(columns.clone()).unwrap();
1170        // The last two rows of each column have values that are not in the
1171        // first three rows, except for booleans, which only have two values
1172        let (head, tail) = (batch.slice(0, 3), batch.slice(3, 2));
1173        for coerce_types in [false, true] {
1174            let all = write_and_read_bloom_filters(&batch, coerce_types);
1175            let first_rows = write_and_read_bloom_filters(&head, coerce_types);
1176            for (i, (name, _)) in columns.iter().enumerate() {
1177                let context = format!("{name} with coerce_types {coerce_types}");
1178                let check = |(sbbf, column): &(Sbbf, ColumnDescPtr), array: &ArrayRef| {
1179                    ArrowSbbf::try_new(sbbf, column, array.data_type().clone())
1180                        .and_then(|sbbf| sbbf.check(array.as_ref()))
1181                        .unwrap_or_else(|e| panic!("{context}: {e}"))
1182                };
1183                // Every value is found, and nulls are null
1184                let array = batch.column(i);
1185                let found = check(&all[i], array);
1186                let nulls = array.logical_nulls();
1187                let valid = |i| nulls.as_ref().is_none_or(|nulls| nulls.is_valid(i));
1188                let expected: BooleanArray =
1189                    (0..array.len()).map(|i| valid(i).then_some(true)).collect();
1190                assert_eq!(found, expected, "{context}");
1191
1192                // Values that were not written are not found
1193                if *name != "bool" {
1194                    let found = check(&first_rows[i], tail.column(i));
1195                    assert_eq!(found, BooleanArray::from(vec![false, false]), "{context}");
1196                }
1197            }
1198        }
1199    }
1200}
parquet/arrow/bloom_filter.rs

parquet/arrow/
bloom_filter.rs
