1use arrow_array::{Array, ArrayRef, BooleanArray, UInt64Array, make_array};
8use arrow_buffer::buffer::bitwise_bin_op_helper;
9use arrow_buffer::{BooleanBuffer, NullBuffer, bitwise_unary_op_helper};
10use arrow_schema::{ArrowError, DataType};
11
12pub fn nullif(left: &dyn Array, right: &BooleanArray) -> Result<ArrayRef, ArrowError> {
32 if left.len() != right.len() {
33 return Err(ArrowError::ComputeError(
34 "Cannot perform comparison operation on arrays of different length".to_string(),
35 ));
36 }
37 let len = left.len();
38
39 if len == 0 || left.data_type() == &DataType::Null {
40 return Ok(make_array(left.to_data()));
41 }
42
43 let should_null = match right.nulls() {
46 Some(nulls) => right.values() & nulls.inner(),
47 None => right.values().clone(),
48 };
49
50 match left.data_type() {
53 DataType::RunEndEncoded(_, values) => {
54 if !values.is_nullable() && should_null.has_true() {
55 return Err(ArrowError::ComputeError(
56 "Cannot introduce nulls into a RunEndEncoded array with a non-nullable values field".into(),
57 ));
58 }
59 return nullif_take(left, &should_null);
60 }
61 DataType::Union(_, _) => return nullif_take(left, &should_null),
62 _ => {}
63 }
64
65 let left_data = left.to_data();
66
67 let (combined, null_count) = match left_data.nulls() {
79 Some(left) => {
80 let mut valid_count = 0;
81 let b = bitwise_bin_op_helper(
82 left.buffer(),
83 left.offset(),
84 should_null.inner(),
85 should_null.offset(),
86 len,
87 |l, r| {
88 let t = l & !r;
89 valid_count += t.count_ones() as usize;
90 t
91 },
92 );
93 (b, len - valid_count)
94 }
95 None => {
96 let mut null_count = 0;
97 let buffer =
98 bitwise_unary_op_helper(should_null.inner(), should_null.offset(), len, |b| {
99 let t = !b;
100 null_count += t.count_zeros() as usize;
101 t
102 });
103 (buffer, null_count)
104 }
105 };
106
107 let combined = BooleanBuffer::new(combined, 0, len);
108 let nulls = unsafe { NullBuffer::new_unchecked(combined, null_count) };
111 let data = left_data.into_builder().nulls(Some(nulls));
112
113 Ok(make_array(unsafe { data.build_unchecked() }))
116}
117
118fn nullif_take(left: &dyn Array, should_null: &BooleanBuffer) -> Result<ArrayRef, ArrowError> {
120 let indices = UInt64Array::from_iter((0..left.len()).map(|index| {
121 if should_null.value(index) {
122 None
123 } else {
124 Some(index as u64)
125 }
126 }));
127 crate::take::take(left, &indices, None)
128}
129
130#[cfg(test)]
131mod tests {
132 use super::*;
133 use arrow_array::builder::{BooleanBuilder, Int32Builder, StructBuilder};
134 use arrow_array::cast::AsArray;
135 use arrow_array::types::Int16Type;
136 use arrow_array::types::Int32Type;
137 use arrow_array::{
138 Int16Array, Int32Array, NullArray, RunArray, StringArray, StructArray, UnionArray,
139 };
140 use arrow_buffer::{RunEndBuffer, ScalarBuffer};
141 use arrow_data::ArrayData;
142 use arrow_schema::{Field, Fields, UnionFields, UnionMode};
143 use rand::prelude::StdRng;
144 use rand::{RngExt, SeedableRng};
145 use std::sync::Arc;
146
147 #[test]
148 fn test_nullif_int_array() {
149 let a = Int32Array::from(vec![Some(15), None, Some(8), Some(1), Some(9)]);
150 let comp = BooleanArray::from(vec![Some(false), None, Some(true), Some(false), None]);
151 let res = nullif(&a, &comp).unwrap();
152
153 let expected = Int32Array::from(vec![
154 Some(15),
155 None,
156 None, Some(1),
158 Some(9),
161 ]);
162
163 let res = res.as_primitive::<Int32Type>();
164 assert_eq!(&expected, res);
165 }
166
167 #[test]
168 fn test_nullif_null_array() {
169 assert_eq!(
170 nullif(&NullArray::new(0), &BooleanArray::new_null(0))
171 .unwrap()
172 .as_ref(),
173 &NullArray::new(0)
174 );
175
176 assert_eq!(
177 nullif(
178 &NullArray::new(3),
179 &BooleanArray::from(vec![Some(false), Some(true), None]),
180 )
181 .unwrap()
182 .as_ref(),
183 &NullArray::new(3)
184 );
185 }
186
187 #[test]
188 fn test_nullif_run_end_encoded() {
189 let values = Int32Array::from(vec![10, 20]);
190 let ree = RunArray::<Int16Type>::try_new(&Int16Array::from(vec![1, 2]), &values).unwrap();
191 let mask = BooleanArray::from(vec![Some(false), Some(true)]);
192
193 let result = nullif(&ree, &mask).unwrap();
194 let result = result.as_run::<Int16Type>();
195 assert_eq!(result.logical_null_count(), 1);
196 assert_eq!(
197 result
198 .values()
199 .as_primitive::<Int32Type>()
200 .iter()
201 .collect::<Vec<_>>(),
202 vec![Some(10), None]
203 );
204 }
205
206 #[test]
207 fn test_nullif_run_end_encoded_mask_nulls() {
208 let values = Int32Array::from(vec![10, 20, 30]);
209 let ree =
210 RunArray::<Int16Type>::try_new(&Int16Array::from(vec![1, 2, 3]), &values).unwrap();
211 let mask = BooleanArray::from(vec![Some(true), Some(false), None, Some(true)]);
212 let mask = mask.slice(1, 3); let result = nullif(&ree, &mask).unwrap();
215 let result = result.as_run::<Int16Type>();
216 assert_eq!(result.logical_null_count(), 1);
217 assert_eq!(
218 result
219 .values()
220 .as_primitive::<Int32Type>()
221 .iter()
222 .collect::<Vec<_>>(),
223 vec![Some(10), Some(20), None]
224 );
225 }
226
227 #[test]
228 fn test_nullif_run_end_encoded_non_nullable_values() {
229 let ree = unsafe {
230 RunArray::<Int16Type>::new_unchecked(
231 DataType::RunEndEncoded(
232 Arc::new(Field::new("run_ends", DataType::Int16, false)),
233 Arc::new(Field::new("values", DataType::Int32, false)),
234 ),
235 RunEndBuffer::new(vec![1_i16, 2].into(), 0, 2),
236 Arc::new(Int32Array::from(vec![10, 20])),
237 )
238 };
239 let mask = BooleanArray::from(vec![Some(false), Some(true)]);
240
241 let error = nullif(&ree, &mask).unwrap_err();
242 assert_eq!(
243 error.to_string(),
244 "Compute error: Cannot introduce nulls into a RunEndEncoded array with a non-nullable values field"
245 );
246 }
247
248 #[test]
249 fn test_nullif_union() {
250 let fields =
251 UnionFields::try_new(vec![0], vec![Field::new("i", DataType::Int32, true)]).unwrap();
252 let union = UnionArray::try_new(
253 fields,
254 ScalarBuffer::from(vec![0_i8, 0]),
255 Some(ScalarBuffer::from(vec![0_i32, 1])),
256 vec![Arc::new(Int32Array::from(vec![10, 20]))],
257 )
258 .unwrap();
259 let mask = BooleanArray::from(vec![Some(false), Some(true)]);
260
261 let result = nullif(&union, &mask).unwrap();
262 let result = result.as_union();
263 assert_eq!(result.logical_null_count(), 1);
264 assert_eq!(result.type_ids(), &ScalarBuffer::from(vec![0_i8, 0]));
265 assert_eq!(
266 result
267 .child(0)
268 .as_primitive::<Int32Type>()
269 .iter()
270 .collect::<Vec<_>>(),
271 vec![Some(10), None]
272 );
273 assert!(matches!(
274 result.data_type(),
275 DataType::Union(_, UnionMode::Dense)
276 ));
277 }
278
279 #[test]
280 fn test_nullif_union_without_nullable_child() {
281 let fields =
282 UnionFields::try_new(vec![0], vec![Field::new("i", DataType::Int32, false)]).unwrap();
283 let union = UnionArray::try_new(
284 fields,
285 ScalarBuffer::from(vec![0_i8, 0]),
286 None,
287 vec![Arc::new(Int32Array::from(vec![10, 20]))],
288 )
289 .unwrap();
290 let mask = BooleanArray::from(vec![Some(false), Some(true)]);
291
292 let error = nullif(&union, &mask).unwrap_err();
293 assert_eq!(
294 error.to_string(),
295 "Compute error: Cannot take null indices from a union with no field that can represent nulls"
296 );
297 }
298
299 #[test]
300 fn test_nullif_int_array_offset() {
301 let a = Int32Array::from(vec![None, Some(15), Some(8), Some(1), Some(9)]);
302 let a = a.slice(1, 3); let comp = BooleanArray::from(vec![
304 Some(false),
305 Some(false),
306 Some(false),
307 None,
308 Some(true),
309 Some(false),
310 None,
311 ]);
312 let comp = comp.slice(2, 3); let res = nullif(&a, &comp).unwrap();
314
315 let expected = Int32Array::from(vec![
316 Some(15), Some(8), None, ]);
320 let res = res.as_primitive::<Int32Type>();
321 assert_eq!(&expected, res)
322 }
323
324 #[test]
325 fn test_nullif_string() {
326 let s = StringArray::from_iter([
327 Some("hello"),
328 None,
329 Some("world"),
330 Some("a"),
331 Some("b"),
332 None,
333 None,
334 ]);
335 let select = BooleanArray::from_iter([
336 Some(true),
337 Some(true),
338 Some(false),
339 Some(true),
340 Some(false),
341 Some(false),
342 None,
343 ]);
344
345 let a = nullif(&s, &select).unwrap();
346 let r: Vec<_> = a.as_string::<i32>().iter().collect();
347 assert_eq!(
348 r,
349 vec![None, None, Some("world"), None, Some("b"), None, None]
350 );
351
352 let s = s.slice(2, 3);
353 let select = select.slice(1, 3);
354 let a = nullif(&s, &select).unwrap();
355 let r: Vec<_> = a.as_string::<i32>().iter().collect();
356 assert_eq!(r, vec![None, Some("a"), None]);
357 }
358
359 #[test]
360 fn test_nullif_int_large_left_offset() {
361 let a = Int32Array::from(vec![
362 Some(-1), Some(-1),
364 Some(-1),
365 Some(-1),
366 Some(-1),
367 Some(-1),
368 Some(-1),
369 Some(-1),
370 Some(-1), Some(-1),
372 Some(-1),
373 Some(-1),
374 Some(-1),
375 Some(-1),
376 Some(-1),
377 Some(-1),
378 None, Some(15), Some(8),
381 Some(1),
382 Some(9),
383 ]);
384 let a = a.slice(17, 3); let comp = BooleanArray::from(vec![
387 Some(false),
388 Some(false),
389 Some(false),
390 None,
391 Some(true),
392 Some(false),
393 None,
394 ]);
395 let comp = comp.slice(2, 3); let res = nullif(&a, &comp).unwrap();
397 let res = res.as_primitive::<Int32Type>();
398
399 let expected = Int32Array::from(vec![
400 Some(15), Some(8), None, ]);
404 assert_eq!(&expected, res)
405 }
406
407 #[test]
408 fn test_nullif_int_large_right_offset() {
409 let a = Int32Array::from(vec![
410 None, Some(15), Some(8),
413 Some(1),
414 Some(9),
415 ]);
416 let a = a.slice(1, 3); let comp = BooleanArray::from(vec![
419 Some(false), Some(false),
421 Some(false),
422 Some(false),
423 Some(false),
424 Some(false),
425 Some(false),
426 Some(false),
427 Some(false), Some(false),
429 Some(false),
430 Some(false),
431 Some(false),
432 Some(false),
433 Some(false),
434 Some(false),
435 Some(false), Some(false), Some(false), None,
439 Some(true),
440 Some(false),
441 None,
442 ]);
443 let comp = comp.slice(18, 3); let res = nullif(&a, &comp).unwrap();
445 let res = res.as_primitive::<Int32Type>();
446
447 let expected = Int32Array::from(vec![
448 Some(15), Some(8), None, ]);
452 assert_eq!(&expected, res)
453 }
454
455 #[test]
456 fn test_nullif_boolean_offset() {
457 let a = BooleanArray::from(vec![
458 None, Some(true), Some(false),
461 Some(true),
462 Some(true),
463 ]);
464 let a = a.slice(1, 3); let comp = BooleanArray::from(vec![
467 Some(false), Some(false), Some(false), None,
471 Some(true),
472 Some(false),
473 None,
474 ]);
475 let comp = comp.slice(2, 3); let res = nullif(&a, &comp).unwrap();
477 let res = res.as_boolean();
478
479 let expected = BooleanArray::from(vec![
480 Some(true), Some(false), None, ]);
484 assert_eq!(&expected, res)
485 }
486
487 struct Foo {
488 a: Option<i32>,
489 b: Option<bool>,
490 is_valid: bool,
492 }
493
494 impl Foo {
495 fn new_valid(a: i32, b: bool) -> Foo {
496 Self {
497 a: Some(a),
498 b: Some(b),
499 is_valid: true,
500 }
501 }
502
503 fn new_null() -> Foo {
504 Self {
505 a: None,
506 b: None,
507 is_valid: false,
508 }
509 }
510 }
511
512 fn create_foo_struct(values: Vec<Foo>) -> StructArray {
516 let mut struct_array = StructBuilder::new(
517 Fields::from(vec![
518 Field::new("a", DataType::Int32, true),
519 Field::new("b", DataType::Boolean, true),
520 ]),
521 vec![
522 Box::new(Int32Builder::with_capacity(values.len())),
523 Box::new(BooleanBuilder::with_capacity(values.len())),
524 ],
525 );
526
527 for value in values {
528 struct_array
529 .field_builder::<Int32Builder>(0)
530 .unwrap()
531 .append_option(value.a);
532 struct_array
533 .field_builder::<BooleanBuilder>(1)
534 .unwrap()
535 .append_option(value.b);
536 struct_array.append(value.is_valid);
537 }
538
539 struct_array.finish()
540 }
541
542 #[test]
543 fn test_nullif_struct_slices() {
544 let struct_array = create_foo_struct(vec![
545 Foo::new_valid(7, true),
546 Foo::new_valid(15, false),
547 Foo::new_valid(8, true),
548 Foo::new_valid(12, false),
549 Foo::new_null(),
550 Foo::new_null(),
551 Foo::new_valid(42, true),
552 ]);
553
554 let struct_array = struct_array.slice(1, 5);
557 let comp = BooleanArray::from(vec![
558 Some(false), Some(false), Some(false), None,
562 Some(true),
563 Some(false),
564 None,
565 ]);
566 let comp = comp.slice(2, 5); let res = nullif(&struct_array, &comp).unwrap();
568 let res = res.as_struct();
569
570 let expected = create_foo_struct(vec![
571 Foo::new_valid(15, false),
573 Foo::new_valid(8, true),
575 Foo {
577 a: Some(12),
578 b: Some(false),
579 is_valid: false,
580 },
581 Foo::new_null(),
583 Foo::new_null(),
585 ]);
586
587 assert_eq!(&expected, res);
588 }
589
590 #[test]
591 fn test_nullif_no_nulls() {
592 let a = Int32Array::from(vec![Some(15), Some(7), Some(8), Some(1), Some(9)]);
593 let comp = BooleanArray::from(vec![Some(false), None, Some(true), Some(false), None]);
594 let res = nullif(&a, &comp).unwrap();
595 let res = res.as_primitive::<Int32Type>();
596
597 let expected = Int32Array::from(vec![Some(15), Some(7), None, Some(1), Some(9)]);
598 assert_eq!(res, &expected);
599 }
600
601 #[test]
602 fn nullif_empty() {
603 let a = Int32Array::from(ArrayData::new_empty(&DataType::Int32));
604 let mask = BooleanArray::from(ArrayData::new_empty(&DataType::Boolean));
605 let res = nullif(&a, &mask).unwrap();
606 assert_eq!(res.as_ref(), &a);
607 }
608
609 fn test_nullif(values: &Int32Array, filter: &BooleanArray) {
610 let expected: Int32Array = values
611 .iter()
612 .zip(filter.iter())
613 .map(|(a, b)| match b {
614 Some(true) => None,
615 Some(false) | None => a,
616 })
617 .collect();
618
619 let r = nullif(values, filter).unwrap();
620 let r_data = r.to_data();
621 r_data.validate().unwrap();
622
623 assert_eq!(
624 r.as_ref(),
625 &expected,
626 "expected nulls: {:#?}\n\n\
627 result nulls: {:#?}\n\n\\
628 expected values: {:#?}\n\n\
629 result values: {:#?}",
630 expected.nulls(),
631 r.nulls(),
632 expected.values(),
633 r.as_primitive::<Int32Type>().values()
634 );
635 validate_nulls(expected.nulls());
636 validate_nulls(r.nulls());
637 }
638
639 fn validate_nulls(nulls: Option<&NullBuffer>) {
641 let Some(nulls) = nulls else {
642 return;
643 };
644 let mut actual_null_count = 0;
645 for i in 0..nulls.len() {
646 if nulls.is_null(i) {
647 actual_null_count += 1;
648 }
649 }
650 assert_eq!(actual_null_count, nulls.null_count());
651 }
652
653 #[test]
654 #[cfg_attr(miri, ignore)] fn nullif_fuzz() {
656 let mut rng = StdRng::seed_from_u64(7337);
657
658 let arrays = [
659 Int32Array::from(vec![0; 1024]), (0..1024) .map(|_| rng.random_bool(0.5).then_some(1))
662 .collect(),
663 ];
664
665 for a in arrays {
666 let a_slices = [
667 (0, 128),
668 (0, 129),
669 (64, 64),
670 (0, 64),
671 (32, 32),
672 (0, 0),
673 (32, 0),
674 (5, 800),
675 (33, 53),
676 (77, 101),
677 ];
678 for (a_offset, a_length) in a_slices {
679 let a = a.slice(a_offset, a_length);
680
681 for i in 1..65 {
682 let b_start_offset = rng.random_range(0..i);
683 let b_end_offset = rng.random_range(0..i);
684
685 let b: BooleanArray = (0..a_length + b_start_offset + b_end_offset)
687 .map(|_| rng.random_bool(0.5).then(|| rng.random_bool(0.5)))
688 .collect();
689 let b_sliced = b.slice(b_start_offset, a_length);
690 test_nullif(&a, &b_sliced);
691
692 let b = remove_null_buffer(&b);
694 let b_sliced = b.slice(b_start_offset, a_length);
695 test_nullif(&a, &b_sliced);
696
697 let b = remove_null_values(&b);
699 let b_sliced = b.slice(b_start_offset, a_length);
700 test_nullif(&a, &b_sliced);
701 }
702 }
703 }
704 }
705
706 fn remove_null_buffer(array: &BooleanArray) -> BooleanArray {
708 BooleanArray::new(array.values().clone(), None)
709 }
710
711 fn remove_null_values(array: &BooleanArray) -> BooleanArray {
713 let len = array.len();
714 let new_nulls = NullBuffer::from_iter(std::iter::repeat_n(true, len));
715 BooleanArray::new(array.values().clone(), Some(new_nulls))
716 }
717}