Skip to main content

arrow_select/
nullif.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Implements the `nullif` function for Arrow arrays.
6
7use arrow_array::{Array, ArrayRef, BooleanArray, UInt64Array, make_array};
8use arrow_buffer::buffer::bitwise_bin_op_helper;
9use arrow_buffer::{BooleanBuffer, NullBuffer, bitwise_unary_op_helper};
10use arrow_schema::{ArrowError, DataType};
11
12/// Returns a new array with the same values and the validity bit to false where
13/// the corresponding element of `right` is true.
14///
15/// This can be used to implement SQL `NULLIF`
16///
17/// # Example
18/// ```
19/// # use arrow_array::{Int32Array, BooleanArray};
20/// # use arrow_array::cast::AsArray;
21/// # use arrow_array::types::Int32Type;
22/// # use arrow_select::nullif::nullif;
23/// // input is [null, 8, 1, 9]
24/// let a = Int32Array::from(vec![None, Some(8), Some(1), Some(9)]);
25/// // use nullif to set index 1 to null
26/// let bool_array = BooleanArray::from(vec![Some(false), Some(true), Some(false), None]);
27/// let nulled = nullif(&a, &bool_array).unwrap();
28/// // The resulting array is [null, null, 1, 9]
29/// assert_eq!(nulled.as_primitive(), &Int32Array::from(vec![None, None, Some(1), Some(9)]));
30/// ```
31pub fn nullif(left: &dyn Array, right: &BooleanArray) -> Result<ArrayRef, ArrowError> {
32    if left.len() != right.len() {
33        return Err(ArrowError::ComputeError(
34            "Cannot perform comparison operation on arrays of different length".to_string(),
35        ));
36    }
37    let len = left.len();
38
39    if len == 0 || left.data_type() == &DataType::Null {
40        return Ok(make_array(left.to_data()));
41    }
42
43    // Compute right_values & right_bitmap. True bits are positions that should
44    // become null; nulls in `right` do not introduce nulls.
45    let should_null = match right.nulls() {
46        Some(nulls) => right.values() & nulls.inner(),
47        None => right.values().clone(),
48    };
49
50    // REE & Union have cases where they technically shouldn't be allowed to hold
51    // nulls based on nullability of their children
52    match left.data_type() {
53        DataType::RunEndEncoded(_, values) => {
54            if !values.is_nullable() && should_null.has_true() {
55                return Err(ArrowError::ComputeError(
56                    "Cannot introduce nulls into a RunEndEncoded array with a non-nullable values field".into(),
57                ));
58            }
59            return nullif_take(left, &should_null);
60        }
61        DataType::Union(_, _) => return nullif_take(left, &should_null),
62        _ => {}
63    }
64
65    let left_data = left.to_data();
66
67    // left=0 (null)   right=null       output bitmap=null
68    // left=0          right=1          output bitmap=null
69    // left=1 (set)    right=null       output bitmap=set   (passthrough)
70    // left=1          right=1 & comp=true    output bitmap=null
71    // left=1          right=1 & comp=false   output bitmap=set
72    //
73    // Thus: result = left null bitmap & (!right_values | !right_bitmap)
74    //              OR left null bitmap & !(right_values & right_bitmap)
75
76    // Compute left null bitmap & !should_null
77
78    let (combined, null_count) = match left_data.nulls() {
79        Some(left) => {
80            let mut valid_count = 0;
81            let b = bitwise_bin_op_helper(
82                left.buffer(),
83                left.offset(),
84                should_null.inner(),
85                should_null.offset(),
86                len,
87                |l, r| {
88                    let t = l & !r;
89                    valid_count += t.count_ones() as usize;
90                    t
91                },
92            );
93            (b, len - valid_count)
94        }
95        None => {
96            let mut null_count = 0;
97            let buffer =
98                bitwise_unary_op_helper(should_null.inner(), should_null.offset(), len, |b| {
99                    let t = !b;
100                    null_count += t.count_zeros() as usize;
101                    t
102                });
103            (buffer, null_count)
104        }
105    };
106
107    let combined = BooleanBuffer::new(combined, 0, len);
108    // Safety:
109    // Counted nulls whilst computing
110    let nulls = unsafe { NullBuffer::new_unchecked(combined, null_count) };
111    let data = left_data.into_builder().nulls(Some(nulls));
112
113    // SAFETY:
114    // Only altered null mask
115    Ok(make_array(unsafe { data.build_unchecked() }))
116}
117
118/// Applies `nullif` to arrays that represent logical nulls in their children.
119fn nullif_take(left: &dyn Array, should_null: &BooleanBuffer) -> Result<ArrayRef, ArrowError> {
120    let indices = UInt64Array::from_iter((0..left.len()).map(|index| {
121        if should_null.value(index) {
122            None
123        } else {
124            Some(index as u64)
125        }
126    }));
127    crate::take::take(left, &indices, None)
128}
129
130#[cfg(test)]
131mod tests {
132    use super::*;
133    use arrow_array::builder::{BooleanBuilder, Int32Builder, StructBuilder};
134    use arrow_array::cast::AsArray;
135    use arrow_array::types::Int16Type;
136    use arrow_array::types::Int32Type;
137    use arrow_array::{
138        Int16Array, Int32Array, NullArray, RunArray, StringArray, StructArray, UnionArray,
139    };
140    use arrow_buffer::{RunEndBuffer, ScalarBuffer};
141    use arrow_data::ArrayData;
142    use arrow_schema::{Field, Fields, UnionFields, UnionMode};
143    use rand::prelude::StdRng;
144    use rand::{RngExt, SeedableRng};
145    use std::sync::Arc;
146
147    #[test]
148    fn test_nullif_int_array() {
149        let a = Int32Array::from(vec![Some(15), None, Some(8), Some(1), Some(9)]);
150        let comp = BooleanArray::from(vec![Some(false), None, Some(true), Some(false), None]);
151        let res = nullif(&a, &comp).unwrap();
152
153        let expected = Int32Array::from(vec![
154            Some(15),
155            None,
156            None, // comp true, slot 2 turned into null
157            Some(1),
158            // Even though comp array / right is null, should still pass through original value
159            // comp true, slot 2 turned into null
160            Some(9),
161        ]);
162
163        let res = res.as_primitive::<Int32Type>();
164        assert_eq!(&expected, res);
165    }
166
167    #[test]
168    fn test_nullif_null_array() {
169        assert_eq!(
170            nullif(&NullArray::new(0), &BooleanArray::new_null(0))
171                .unwrap()
172                .as_ref(),
173            &NullArray::new(0)
174        );
175
176        assert_eq!(
177            nullif(
178                &NullArray::new(3),
179                &BooleanArray::from(vec![Some(false), Some(true), None]),
180            )
181            .unwrap()
182            .as_ref(),
183            &NullArray::new(3)
184        );
185    }
186
187    #[test]
188    fn test_nullif_run_end_encoded() {
189        let values = Int32Array::from(vec![10, 20]);
190        let ree = RunArray::<Int16Type>::try_new(&Int16Array::from(vec![1, 2]), &values).unwrap();
191        let mask = BooleanArray::from(vec![Some(false), Some(true)]);
192
193        let result = nullif(&ree, &mask).unwrap();
194        let result = result.as_run::<Int16Type>();
195        assert_eq!(result.logical_null_count(), 1);
196        assert_eq!(
197            result
198                .values()
199                .as_primitive::<Int32Type>()
200                .iter()
201                .collect::<Vec<_>>(),
202            vec![Some(10), None]
203        );
204    }
205
206    #[test]
207    fn test_nullif_run_end_encoded_mask_nulls() {
208        let values = Int32Array::from(vec![10, 20, 30]);
209        let ree =
210            RunArray::<Int16Type>::try_new(&Int16Array::from(vec![1, 2, 3]), &values).unwrap();
211        let mask = BooleanArray::from(vec![Some(true), Some(false), None, Some(true)]);
212        let mask = mask.slice(1, 3); // Some(false), None, Some(true)
213
214        let result = nullif(&ree, &mask).unwrap();
215        let result = result.as_run::<Int16Type>();
216        assert_eq!(result.logical_null_count(), 1);
217        assert_eq!(
218            result
219                .values()
220                .as_primitive::<Int32Type>()
221                .iter()
222                .collect::<Vec<_>>(),
223            vec![Some(10), Some(20), None]
224        );
225    }
226
227    #[test]
228    fn test_nullif_run_end_encoded_non_nullable_values() {
229        let ree = unsafe {
230            RunArray::<Int16Type>::new_unchecked(
231                DataType::RunEndEncoded(
232                    Arc::new(Field::new("run_ends", DataType::Int16, false)),
233                    Arc::new(Field::new("values", DataType::Int32, false)),
234                ),
235                RunEndBuffer::new(vec![1_i16, 2].into(), 0, 2),
236                Arc::new(Int32Array::from(vec![10, 20])),
237            )
238        };
239        let mask = BooleanArray::from(vec![Some(false), Some(true)]);
240
241        let error = nullif(&ree, &mask).unwrap_err();
242        assert_eq!(
243            error.to_string(),
244            "Compute error: Cannot introduce nulls into a RunEndEncoded array with a non-nullable values field"
245        );
246    }
247
248    #[test]
249    fn test_nullif_union() {
250        let fields =
251            UnionFields::try_new(vec![0], vec![Field::new("i", DataType::Int32, true)]).unwrap();
252        let union = UnionArray::try_new(
253            fields,
254            ScalarBuffer::from(vec![0_i8, 0]),
255            Some(ScalarBuffer::from(vec![0_i32, 1])),
256            vec![Arc::new(Int32Array::from(vec![10, 20]))],
257        )
258        .unwrap();
259        let mask = BooleanArray::from(vec![Some(false), Some(true)]);
260
261        let result = nullif(&union, &mask).unwrap();
262        let result = result.as_union();
263        assert_eq!(result.logical_null_count(), 1);
264        assert_eq!(result.type_ids(), &ScalarBuffer::from(vec![0_i8, 0]));
265        assert_eq!(
266            result
267                .child(0)
268                .as_primitive::<Int32Type>()
269                .iter()
270                .collect::<Vec<_>>(),
271            vec![Some(10), None]
272        );
273        assert!(matches!(
274            result.data_type(),
275            DataType::Union(_, UnionMode::Dense)
276        ));
277    }
278
279    #[test]
280    fn test_nullif_union_without_nullable_child() {
281        let fields =
282            UnionFields::try_new(vec![0], vec![Field::new("i", DataType::Int32, false)]).unwrap();
283        let union = UnionArray::try_new(
284            fields,
285            ScalarBuffer::from(vec![0_i8, 0]),
286            None,
287            vec![Arc::new(Int32Array::from(vec![10, 20]))],
288        )
289        .unwrap();
290        let mask = BooleanArray::from(vec![Some(false), Some(true)]);
291
292        let error = nullif(&union, &mask).unwrap_err();
293        assert_eq!(
294            error.to_string(),
295            "Compute error: Cannot take null indices from a union with no field that can represent nulls"
296        );
297    }
298
299    #[test]
300    fn test_nullif_int_array_offset() {
301        let a = Int32Array::from(vec![None, Some(15), Some(8), Some(1), Some(9)]);
302        let a = a.slice(1, 3); // Some(15), Some(8), Some(1)
303        let comp = BooleanArray::from(vec![
304            Some(false),
305            Some(false),
306            Some(false),
307            None,
308            Some(true),
309            Some(false),
310            None,
311        ]);
312        let comp = comp.slice(2, 3); // Some(false), None, Some(true)
313        let res = nullif(&a, &comp).unwrap();
314
315        let expected = Int32Array::from(vec![
316            Some(15), // False => keep it
317            Some(8),  // None => keep it
318            None,     // true => None
319        ]);
320        let res = res.as_primitive::<Int32Type>();
321        assert_eq!(&expected, res)
322    }
323
324    #[test]
325    fn test_nullif_string() {
326        let s = StringArray::from_iter([
327            Some("hello"),
328            None,
329            Some("world"),
330            Some("a"),
331            Some("b"),
332            None,
333            None,
334        ]);
335        let select = BooleanArray::from_iter([
336            Some(true),
337            Some(true),
338            Some(false),
339            Some(true),
340            Some(false),
341            Some(false),
342            None,
343        ]);
344
345        let a = nullif(&s, &select).unwrap();
346        let r: Vec<_> = a.as_string::<i32>().iter().collect();
347        assert_eq!(
348            r,
349            vec![None, None, Some("world"), None, Some("b"), None, None]
350        );
351
352        let s = s.slice(2, 3);
353        let select = select.slice(1, 3);
354        let a = nullif(&s, &select).unwrap();
355        let r: Vec<_> = a.as_string::<i32>().iter().collect();
356        assert_eq!(r, vec![None, Some("a"), None]);
357    }
358
359    #[test]
360    fn test_nullif_int_large_left_offset() {
361        let a = Int32Array::from(vec![
362            Some(-1), // 0
363            Some(-1),
364            Some(-1),
365            Some(-1),
366            Some(-1),
367            Some(-1),
368            Some(-1),
369            Some(-1),
370            Some(-1), // 8
371            Some(-1),
372            Some(-1),
373            Some(-1),
374            Some(-1),
375            Some(-1),
376            Some(-1),
377            Some(-1),
378            None,     // 16
379            Some(15), // 17
380            Some(8),
381            Some(1),
382            Some(9),
383        ]);
384        let a = a.slice(17, 3); // Some(15), Some(8), Some(1)
385
386        let comp = BooleanArray::from(vec![
387            Some(false),
388            Some(false),
389            Some(false),
390            None,
391            Some(true),
392            Some(false),
393            None,
394        ]);
395        let comp = comp.slice(2, 3); // Some(false), None, Some(true)
396        let res = nullif(&a, &comp).unwrap();
397        let res = res.as_primitive::<Int32Type>();
398
399        let expected = Int32Array::from(vec![
400            Some(15), // False => keep it
401            Some(8),  // None => keep it
402            None,     // true => None
403        ]);
404        assert_eq!(&expected, res)
405    }
406
407    #[test]
408    fn test_nullif_int_large_right_offset() {
409        let a = Int32Array::from(vec![
410            None,     // 0
411            Some(15), // 1
412            Some(8),
413            Some(1),
414            Some(9),
415        ]);
416        let a = a.slice(1, 3); // Some(15), Some(8), Some(1)
417
418        let comp = BooleanArray::from(vec![
419            Some(false), // 0
420            Some(false),
421            Some(false),
422            Some(false),
423            Some(false),
424            Some(false),
425            Some(false),
426            Some(false),
427            Some(false), // 8
428            Some(false),
429            Some(false),
430            Some(false),
431            Some(false),
432            Some(false),
433            Some(false),
434            Some(false),
435            Some(false), // 16
436            Some(false), // 17
437            Some(false), // 18
438            None,
439            Some(true),
440            Some(false),
441            None,
442        ]);
443        let comp = comp.slice(18, 3); // Some(false), None, Some(true)
444        let res = nullif(&a, &comp).unwrap();
445        let res = res.as_primitive::<Int32Type>();
446
447        let expected = Int32Array::from(vec![
448            Some(15), // False => keep it
449            Some(8),  // None => keep it
450            None,     // true => None
451        ]);
452        assert_eq!(&expected, res)
453    }
454
455    #[test]
456    fn test_nullif_boolean_offset() {
457        let a = BooleanArray::from(vec![
458            None,       // 0
459            Some(true), // 1
460            Some(false),
461            Some(true),
462            Some(true),
463        ]);
464        let a = a.slice(1, 3); // Some(true), Some(false), Some(true)
465
466        let comp = BooleanArray::from(vec![
467            Some(false), // 0
468            Some(false), // 1
469            Some(false), // 2
470            None,
471            Some(true),
472            Some(false),
473            None,
474        ]);
475        let comp = comp.slice(2, 3); // Some(false), None, Some(true)
476        let res = nullif(&a, &comp).unwrap();
477        let res = res.as_boolean();
478
479        let expected = BooleanArray::from(vec![
480            Some(true),  // False => keep it
481            Some(false), // None => keep it
482            None,        // true => None
483        ]);
484        assert_eq!(&expected, res)
485    }
486
487    struct Foo {
488        a: Option<i32>,
489        b: Option<bool>,
490        /// Whether the entry should be valid.
491        is_valid: bool,
492    }
493
494    impl Foo {
495        fn new_valid(a: i32, b: bool) -> Foo {
496            Self {
497                a: Some(a),
498                b: Some(b),
499                is_valid: true,
500            }
501        }
502
503        fn new_null() -> Foo {
504            Self {
505                a: None,
506                b: None,
507                is_valid: false,
508            }
509        }
510    }
511
512    /// Struct Array equality is a bit weird -- we need to have the *child values*
513    /// correct even if the enclosing struct indicates it is null. But we
514    /// also need the top level is_valid bits to be correct.
515    fn create_foo_struct(values: Vec<Foo>) -> StructArray {
516        let mut struct_array = StructBuilder::new(
517            Fields::from(vec![
518                Field::new("a", DataType::Int32, true),
519                Field::new("b", DataType::Boolean, true),
520            ]),
521            vec![
522                Box::new(Int32Builder::with_capacity(values.len())),
523                Box::new(BooleanBuilder::with_capacity(values.len())),
524            ],
525        );
526
527        for value in values {
528            struct_array
529                .field_builder::<Int32Builder>(0)
530                .unwrap()
531                .append_option(value.a);
532            struct_array
533                .field_builder::<BooleanBuilder>(1)
534                .unwrap()
535                .append_option(value.b);
536            struct_array.append(value.is_valid);
537        }
538
539        struct_array.finish()
540    }
541
542    #[test]
543    fn test_nullif_struct_slices() {
544        let struct_array = create_foo_struct(vec![
545            Foo::new_valid(7, true),
546            Foo::new_valid(15, false),
547            Foo::new_valid(8, true),
548            Foo::new_valid(12, false),
549            Foo::new_null(),
550            Foo::new_null(),
551            Foo::new_valid(42, true),
552        ]);
553
554        // Some({a: 15, b: false}), Some({a: 8, b: true}), Some({a: 12, b: false}),
555        // None, None
556        let struct_array = struct_array.slice(1, 5);
557        let comp = BooleanArray::from(vec![
558            Some(false), // 0
559            Some(false), // 1
560            Some(false), // 2
561            None,
562            Some(true),
563            Some(false),
564            None,
565        ]);
566        let comp = comp.slice(2, 5); // Some(false), None, Some(true), Some(false), None
567        let res = nullif(&struct_array, &comp).unwrap();
568        let res = res.as_struct();
569
570        let expected = create_foo_struct(vec![
571            // Some(false) -> keep
572            Foo::new_valid(15, false),
573            // None -> keep
574            Foo::new_valid(8, true),
575            // Some(true) -> null out. But child values are still there.
576            Foo {
577                a: Some(12),
578                b: Some(false),
579                is_valid: false,
580            },
581            // Some(false) -> keep, but was null
582            Foo::new_null(),
583            // None -> keep, but was null
584            Foo::new_null(),
585        ]);
586
587        assert_eq!(&expected, res);
588    }
589
590    #[test]
591    fn test_nullif_no_nulls() {
592        let a = Int32Array::from(vec![Some(15), Some(7), Some(8), Some(1), Some(9)]);
593        let comp = BooleanArray::from(vec![Some(false), None, Some(true), Some(false), None]);
594        let res = nullif(&a, &comp).unwrap();
595        let res = res.as_primitive::<Int32Type>();
596
597        let expected = Int32Array::from(vec![Some(15), Some(7), None, Some(1), Some(9)]);
598        assert_eq!(res, &expected);
599    }
600
601    #[test]
602    fn nullif_empty() {
603        let a = Int32Array::from(ArrayData::new_empty(&DataType::Int32));
604        let mask = BooleanArray::from(ArrayData::new_empty(&DataType::Boolean));
605        let res = nullif(&a, &mask).unwrap();
606        assert_eq!(res.as_ref(), &a);
607    }
608
609    fn test_nullif(values: &Int32Array, filter: &BooleanArray) {
610        let expected: Int32Array = values
611            .iter()
612            .zip(filter.iter())
613            .map(|(a, b)| match b {
614                Some(true) => None,
615                Some(false) | None => a,
616            })
617            .collect();
618
619        let r = nullif(values, filter).unwrap();
620        let r_data = r.to_data();
621        r_data.validate().unwrap();
622
623        assert_eq!(
624            r.as_ref(),
625            &expected,
626            "expected nulls: {:#?}\n\n\
627        result nulls:   {:#?}\n\n\\
628        expected values: {:#?}\n\n\
629        result values:   {:#?}",
630            expected.nulls(),
631            r.nulls(),
632            expected.values(),
633            r.as_primitive::<Int32Type>().values()
634        );
635        validate_nulls(expected.nulls());
636        validate_nulls(r.nulls());
637    }
638
639    /// Ensures that the null count matches the actual number of nulls.
640    fn validate_nulls(nulls: Option<&NullBuffer>) {
641        let Some(nulls) = nulls else {
642            return;
643        };
644        let mut actual_null_count = 0;
645        for i in 0..nulls.len() {
646            if nulls.is_null(i) {
647                actual_null_count += 1;
648            }
649        }
650        assert_eq!(actual_null_count, nulls.null_count());
651    }
652
653    #[test]
654    #[cfg_attr(miri, ignore)] // Takes too long
655    fn nullif_fuzz() {
656        let mut rng = StdRng::seed_from_u64(7337);
657
658        let arrays = [
659            Int32Array::from(vec![0; 1024]), // no nulls
660            (0..1024) // 50% nulls
661                .map(|_| rng.random_bool(0.5).then_some(1))
662                .collect(),
663        ];
664
665        for a in arrays {
666            let a_slices = [
667                (0, 128),
668                (0, 129),
669                (64, 64),
670                (0, 64),
671                (32, 32),
672                (0, 0),
673                (32, 0),
674                (5, 800),
675                (33, 53),
676                (77, 101),
677            ];
678            for (a_offset, a_length) in a_slices {
679                let a = a.slice(a_offset, a_length);
680
681                for i in 1..65 {
682                    let b_start_offset = rng.random_range(0..i);
683                    let b_end_offset = rng.random_range(0..i);
684
685                    // b with 50% nulls
686                    let b: BooleanArray = (0..a_length + b_start_offset + b_end_offset)
687                        .map(|_| rng.random_bool(0.5).then(|| rng.random_bool(0.5)))
688                        .collect();
689                    let b_sliced = b.slice(b_start_offset, a_length);
690                    test_nullif(&a, &b_sliced);
691
692                    // b with no nulls (and no null buffer)
693                    let b = remove_null_buffer(&b);
694                    let b_sliced = b.slice(b_start_offset, a_length);
695                    test_nullif(&a, &b_sliced);
696
697                    // b with no nulls (but with a null buffer)
698                    let b = remove_null_values(&b);
699                    let b_sliced = b.slice(b_start_offset, a_length);
700                    test_nullif(&a, &b_sliced);
701                }
702            }
703        }
704    }
705
706    /// Returns a new BooleanArray with no null buffer
707    fn remove_null_buffer(array: &BooleanArray) -> BooleanArray {
708        BooleanArray::new(array.values().clone(), None)
709    }
710
711    /// Returns a new BooleanArray with a null buffer where all values are valid
712    fn remove_null_values(array: &BooleanArray) -> BooleanArray {
713        let len = array.len();
714        let new_nulls = NullBuffer::from_iter(std::iter::repeat_n(true, len));
715        BooleanArray::new(array.values().clone(), Some(new_nulls))
716    }
717}