Data manipulation¶
Recipes related to compare, filtering or transforming data.
Compare Vectors for Field Equality¶
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.compare.TypeEqualsVisitor;
import org.apache.arrow.memory.RootAllocator;
try(
BufferAllocator allocator = new RootAllocator();
IntVector right = new IntVector("int", allocator);
) {
right.allocateNew(3);
right.set(0, 10);
right.set(1, 20);
right.set(2, 30);
right.setValueCount(3);
IntVector left1 = new IntVector("int", allocator);
IntVector left2 = new IntVector("int2", allocator);
TypeEqualsVisitor visitor = new TypeEqualsVisitor(right);
System.out.println(visitor.equals(left1));
System.out.println(visitor.equals(left2));
}
true
false
Compare Vectors Equality¶
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.compare.VectorEqualsVisitor;
try(
BufferAllocator allocator = new RootAllocator();
IntVector vector1 = new IntVector("vector1", allocator);
IntVector vector2 = new IntVector("vector1", allocator);
IntVector vector3 = new IntVector("vector1", allocator)
) {
vector1.allocateNew(1);
vector1.set(0, 10);
vector1.setValueCount(1);
vector2.allocateNew(1);
vector2.set(0, 10);
vector2.setValueCount(1);
vector3.allocateNew(1);
vector3.set(0, 20);
vector3.setValueCount(1);
VectorEqualsVisitor visitor = new VectorEqualsVisitor();
System.out.println(visitor.vectorEquals(vector1, vector2));
System.out.println(visitor.vectorEquals(vector1, vector3));
}
true
false
Compare Values on the Array¶
Comparing two values at the given indices in the vectors:
import org.apache.arrow.algorithm.sort.DefaultVectorComparators;
import org.apache.arrow.algorithm.sort.VectorValueComparator;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.VarCharVector;
import org.apache.arrow.memory.RootAllocator;
try(
BufferAllocator allocator = new RootAllocator();
VarCharVector vec = new VarCharVector("valueindexcomparator", allocator);
) {
vec.allocateNew(3);
vec.setValueCount(3);
vec.set(0, "ba".getBytes());
vec.set(1, "abc".getBytes());
vec.set(2, "aa".getBytes());
VectorValueComparator<VarCharVector> valueComparator = DefaultVectorComparators.createDefaultComparator(vec);
valueComparator.attachVector(vec);
System.out.println(valueComparator.compare(0, 1) > 0);
System.out.println(valueComparator.compare(1, 2) < 0);
}
true
false
Consider that if we need our own comparator we could extend VectorValueComparator and override compareNotNull method as needed
Search Values on the Array¶
Linear Search - O(n)¶
Algorithm: org.apache.arrow.algorithm.search.VectorSearcher#linearSearch - O(n)
import org.apache.arrow.algorithm.search.VectorSearcher;
import org.apache.arrow.algorithm.sort.DefaultVectorComparators;
import org.apache.arrow.algorithm.sort.VectorValueComparator;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.memory.RootAllocator;
try(
BufferAllocator allocator = new RootAllocator();
IntVector linearSearchVector = new IntVector("linearSearchVector", allocator);
) {
linearSearchVector.allocateNew(10);
linearSearchVector.setValueCount(10);
for (int i = 0; i < 10; i++) {
linearSearchVector.set(i, i);
}
VectorValueComparator<IntVector> comparatorInt = DefaultVectorComparators.createDefaultComparator(linearSearchVector);
int result = VectorSearcher.linearSearch(linearSearchVector, comparatorInt, linearSearchVector, 3);
System.out.println(result);
}
3
Binary Search - O(log(n))¶
Algorithm: org.apache.arrow.algorithm.search.VectorSearcher#binarySearch - O(log(n))
import org.apache.arrow.algorithm.search.VectorSearcher;
import org.apache.arrow.algorithm.sort.DefaultVectorComparators;
import org.apache.arrow.algorithm.sort.VectorValueComparator;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.memory.RootAllocator;
try(
BufferAllocator allocator = new RootAllocator();
IntVector binarySearchVector = new IntVector("", allocator);
) {
binarySearchVector.allocateNew(10);
binarySearchVector.setValueCount(10);
for (int i = 0; i < 10; i++) {
binarySearchVector.set(i, i);
}
VectorValueComparator<IntVector> comparatorInt = DefaultVectorComparators.createDefaultComparator(binarySearchVector);
int result = VectorSearcher.binarySearch(binarySearchVector, comparatorInt, binarySearchVector, 3);
System.out.println(result);
}
3
Sort Values on the Array¶
In-place Sorter - O(nlog(n))¶
Sorting by manipulating the original vector. Algorithm: org.apache.arrow.algorithm.sort.FixedWidthInPlaceVectorSorter - O(nlog(n))
import org.apache.arrow.algorithm.sort.DefaultVectorComparators;
import org.apache.arrow.algorithm.sort.FixedWidthInPlaceVectorSorter;
import org.apache.arrow.algorithm.sort.VectorValueComparator;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.memory.RootAllocator;
try(
BufferAllocator allocator = new RootAllocator();
IntVector intVectorNotSorted = new IntVector("intvectornotsorted", allocator);
) {
intVectorNotSorted.allocateNew(3);
intVectorNotSorted.setValueCount(3);
intVectorNotSorted.set(0, 10);
intVectorNotSorted.set(1, 8);
intVectorNotSorted.setNull(2);
FixedWidthInPlaceVectorSorter<IntVector> sorter = new FixedWidthInPlaceVectorSorter<IntVector>();
VectorValueComparator<IntVector> comparator = DefaultVectorComparators.createDefaultComparator(intVectorNotSorted);
sorter.sortInPlace(intVectorNotSorted, comparator);
System.out.println(intVectorNotSorted);
}
[null, 8, 10]
Out-place Sorter - O(nlog(n))¶
Sorting by copies vector elements to a new vector in sorted order - O(nlog(n)) Algorithm: : org.apache.arrow.algorithm.sort.FixedWidthInPlaceVectorSorter. FixedWidthOutOfPlaceVectorSorter & VariableWidthOutOfPlaceVectorSor
import org.apache.arrow.algorithm.sort.DefaultVectorComparators;
import org.apache.arrow.algorithm.sort.FixedWidthOutOfPlaceVectorSorter;
import org.apache.arrow.algorithm.sort.OutOfPlaceVectorSorter;
import org.apache.arrow.algorithm.sort.VectorValueComparator;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.memory.RootAllocator;
try(
BufferAllocator allocator = new RootAllocator();
IntVector intVectorNotSorted = new IntVector("intvectornotsorted", allocator);
IntVector intVectorSorted = (IntVector) intVectorNotSorted.getField()
.getFieldType().createNewSingleVector("new-out-of-place-sorter",
allocator, null);
) {
intVectorNotSorted.allocateNew(3);
intVectorNotSorted.setValueCount(3);
intVectorNotSorted.set(0, 10);
intVectorNotSorted.set(1, 8);
intVectorNotSorted.setNull(2);
OutOfPlaceVectorSorter<IntVector> sorterOutOfPlaceSorter = new FixedWidthOutOfPlaceVectorSorter<>();
VectorValueComparator<IntVector> comparatorOutOfPlaceSorter = DefaultVectorComparators.createDefaultComparator(intVectorNotSorted);
intVectorSorted.allocateNew(intVectorNotSorted.getValueCount());
intVectorSorted.setValueCount(intVectorNotSorted.getValueCount());
sorterOutOfPlaceSorter.sortOutOfPlace(intVectorNotSorted, intVectorSorted, comparatorOutOfPlaceSorter);
System.out.println(intVectorSorted);
}
[null, 8, 10]