Struct DictionaryArray
pub struct DictionaryArray<K>where
K: ArrowDictionaryKeyType,{
data_type: DataType,
keys: PrimitiveArray<K>,
values: Arc<dyn Array>,
is_ordered: bool,
}
Expand description
An array of dictionary encoded values
This is mostly used to represent strings or a limited set of primitive types as integers, for example when doing NLP analysis or representing chromosomes by name.
DictionaryArray
are represented using a keys
array and a
values
array, which may be different lengths. The keys
array
stores indexes in the values
array which holds
the corresponding logical value, as shown here:
┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
┌─────────────────┐ ┌─────────┐ │ ┌─────────────────┐
│ │ A │ │ 0 │ │ A │ values[keys[0]]
├─────────────────┤ ├─────────┤ │ ├─────────────────┤
│ │ D │ │ 2 │ │ B │ values[keys[1]]
├─────────────────┤ ├─────────┤ │ ├─────────────────┤
│ │ B │ │ 2 │ │ B │ values[keys[2]]
└─────────────────┘ ├─────────┤ │ ├─────────────────┤
│ │ 1 │ │ D │ values[keys[3]]
├─────────┤ │ ├─────────────────┤
│ │ 1 │ │ D │ values[keys[4]]
├─────────┤ │ ├─────────────────┤
│ │ 0 │ │ A │ values[keys[5]]
└─────────┘ │ └─────────────────┘
│ values keys
─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘
Logical array
Contents
DictionaryArray
length = 6
§Example: From Nullable Data
let test = vec!["a", "a", "b", "c"];
let array : DictionaryArray<Int8Type> = test.iter().map(|&x| if x == "b" {None} else {Some(x)}).collect();
assert_eq!(array.keys(), &Int8Array::from(vec![Some(0), Some(0), None, Some(1)]));
§Example: From Non-Nullable Data
let test = vec!["a", "a", "b", "c"];
let array : DictionaryArray<Int8Type> = test.into_iter().collect();
assert_eq!(array.keys(), &Int8Array::from(vec![0, 0, 1, 2]));
§Example: From Existing Arrays
// You can form your own DictionaryArray by providing the
// values (dictionary) and keys (indexes into the dictionary):
let values = StringArray::from_iter_values(["a", "b", "c"]);
let keys = Int8Array::from_iter_values([0, 0, 1, 2]);
let array = DictionaryArray::<Int8Type>::try_new(keys, Arc::new(values)).unwrap();
let expected: DictionaryArray::<Int8Type> = vec!["a", "a", "b", "c"].into_iter().collect();
assert_eq!(&array, &expected);
§Example: Using Builder
let mut builder = StringDictionaryBuilder::<Int32Type>::new();
builder.append_value("a");
builder.append_null();
builder.append_value("a");
builder.append_value("b");
let array = builder.finish();
let values: Vec<_> = array.downcast_dict::<StringArray>().unwrap().into_iter().collect();
assert_eq!(&values, &[Some("a"), None, Some("a"), Some("b")]);
Fields§
§data_type: DataType
§keys: PrimitiveArray<K>
§values: Arc<dyn Array>
§is_ordered: bool
Implementations§
§impl<K> DictionaryArray<K>where
K: ArrowDictionaryKeyType,
impl<K> DictionaryArray<K>where
K: ArrowDictionaryKeyType,
pub fn new(
keys: PrimitiveArray<K>,
values: Arc<dyn Array>,
) -> DictionaryArray<K>
pub fn new( keys: PrimitiveArray<K>, values: Arc<dyn Array>, ) -> DictionaryArray<K>
Attempt to create a new DictionaryArray with a specified keys (indexes into the dictionary) and values (dictionary) array.
§Panics
Panics if Self::try_new
returns an error
pub fn try_new(
keys: PrimitiveArray<K>,
values: Arc<dyn Array>,
) -> Result<DictionaryArray<K>, ArrowError>
pub fn try_new( keys: PrimitiveArray<K>, values: Arc<dyn Array>, ) -> Result<DictionaryArray<K>, ArrowError>
Attempt to create a new DictionaryArray with a specified keys (indexes into the dictionary) and values (dictionary) array.
§Errors
Returns an error if any keys[i] >= values.len() || keys[i] < 0
pub fn new_scalar<T>(value: Scalar<T>) -> Scalar<DictionaryArray<K>>where
T: Array + 'static,
pub fn new_scalar<T>(value: Scalar<T>) -> Scalar<DictionaryArray<K>>where
T: Array + 'static,
Create a new Scalar
from value
pub unsafe fn new_unchecked(
keys: PrimitiveArray<K>,
values: Arc<dyn Array>,
) -> DictionaryArray<K>
pub unsafe fn new_unchecked( keys: PrimitiveArray<K>, values: Arc<dyn Array>, ) -> DictionaryArray<K>
Create a new DictionaryArray
without performing validation
§Safety
Safe provided Self::try_new
would not return an error
pub fn into_parts(self) -> (PrimitiveArray<K>, Arc<dyn Array>)
pub fn into_parts(self) -> (PrimitiveArray<K>, Arc<dyn Array>)
Deconstruct this array into its constituent parts
pub fn keys(&self) -> &PrimitiveArray<K>
pub fn keys(&self) -> &PrimitiveArray<K>
Return an array view of the keys of this dictionary as a PrimitiveArray.
pub fn lookup_key(
&self,
value: &str,
) -> Option<<K as ArrowPrimitiveType>::Native>
pub fn lookup_key( &self, value: &str, ) -> Option<<K as ArrowPrimitiveType>::Native>
If value
is present in values
(aka the dictionary),
returns the corresponding key (index into the values
array). Otherwise returns None
.
Panics if values
is not a StringArray
.
pub fn value_type(&self) -> DataType
pub fn value_type(&self) -> DataType
Returns a clone of the value type of this list.
pub fn is_ordered(&self) -> bool
pub fn is_ordered(&self) -> bool
Currently exists for compatibility purposes with Arrow IPC.
pub fn keys_iter(&self) -> impl Iterator<Item = Option<usize>>
pub fn keys_iter(&self) -> impl Iterator<Item = Option<usize>>
Return an iterator over the keys (indexes into the dictionary)
pub fn key(&self, i: usize) -> Option<usize>
pub fn key(&self, i: usize) -> Option<usize>
Return the value of keys
(the dictionary key) at index i
,
cast to usize
, None
if the value at i
is NULL
.
pub fn slice(&self, offset: usize, length: usize) -> DictionaryArray<K>
pub fn slice(&self, offset: usize, length: usize) -> DictionaryArray<K>
Returns a zero-copy slice of this array with the indicated offset and length.
pub fn downcast_dict<V>(&self) -> Option<TypedDictionaryArray<'_, K, V>>where
V: 'static,
pub fn downcast_dict<V>(&self) -> Option<TypedDictionaryArray<'_, K, V>>where
V: 'static,
Downcast this dictionary to a TypedDictionaryArray
use arrow_array::{Array, ArrayAccessor, DictionaryArray, StringArray, types::Int32Type};
let orig = [Some("a"), Some("b"), None];
let dictionary = DictionaryArray::<Int32Type>::from_iter(orig);
let typed = dictionary.downcast_dict::<StringArray>().unwrap();
assert_eq!(typed.value(0), "a");
assert_eq!(typed.value(1), "b");
assert!(typed.is_null(2));
pub fn with_values(&self, values: Arc<dyn Array>) -> DictionaryArray<K>
pub fn with_values(&self, values: Arc<dyn Array>) -> DictionaryArray<K>
Returns a new dictionary with the same keys as the current instance but with a different set of dictionary values
This can be used to perform an operation on the values of a dictionary
§Panics
Panics if values
has a length less than the current values
// Construct a Dict(Int32, Int8)
let mut builder = PrimitiveDictionaryBuilder::<Int32Type, Int8Type>::with_capacity(2, 200);
for i in 0..100 {
builder.append(i % 2).unwrap();
}
let dictionary = builder.finish();
// Perform a widening cast of dictionary values
let typed_dictionary = dictionary.downcast_dict::<Int8Array>().unwrap();
let values: Int64Array = typed_dictionary.values().unary(|x| x as i64);
// Create a Dict(Int32,
let new = dictionary.with_values(Arc::new(values));
// Verify values are as expected
let new_typed = new.downcast_dict::<Int64Array>().unwrap();
for i in 0..100 {
assert_eq!(new_typed.value(i), (i % 2) as i64)
}
pub fn into_primitive_dict_builder<V>(
self,
) -> Result<PrimitiveDictionaryBuilder<K, V>, DictionaryArray<K>>where
V: ArrowPrimitiveType,
pub fn into_primitive_dict_builder<V>(
self,
) -> Result<PrimitiveDictionaryBuilder<K, V>, DictionaryArray<K>>where
V: ArrowPrimitiveType,
Returns PrimitiveDictionaryBuilder
of this dictionary array for mutating
its keys and values if the underlying data buffer is not shared by others.
pub fn unary_mut<F, V>(
self,
op: F,
) -> Result<DictionaryArray<K>, DictionaryArray<K>>where
V: ArrowPrimitiveType,
F: Fn(<V as ArrowPrimitiveType>::Native) -> <V as ArrowPrimitiveType>::Native,
pub fn unary_mut<F, V>(
self,
op: F,
) -> Result<DictionaryArray<K>, DictionaryArray<K>>where
V: ArrowPrimitiveType,
F: Fn(<V as ArrowPrimitiveType>::Native) -> <V as ArrowPrimitiveType>::Native,
Applies an unary and infallible function to a mutable dictionary array. Mutable dictionary array means that the buffers are not shared with other arrays. As a result, this mutates the buffers directly without allocating new buffers.
§Implementation
This will apply the function for all dictionary values, including those on null slots. This implies that the operation must be infallible for any value of the corresponding type or this function may panic.
§Example
let values = Int32Array::from(vec![Some(10), Some(20), None]);
let keys = Int8Array::from_iter_values([0, 0, 1, 2]);
let dictionary = DictionaryArray::<Int8Type>::try_new(keys, Arc::new(values)).unwrap();
let c = dictionary.unary_mut::<_, Int32Type>(|x| x + 1).unwrap();
let typed = c.downcast_dict::<Int32Array>().unwrap();
assert_eq!(typed.value(0), 11);
assert_eq!(typed.value(1), 11);
assert_eq!(typed.value(2), 21);
pub fn occupancy(&self) -> BooleanBuffer
pub fn occupancy(&self) -> BooleanBuffer
Computes an occupancy mask for this dictionary’s values
For each value in Self::values
the corresponding bit will be set in the
returned mask if it is referenced by a key in this DictionaryArray
Trait Implementations§
§impl<K> AnyDictionaryArray for DictionaryArray<K>where
K: ArrowDictionaryKeyType,
impl<K> AnyDictionaryArray for DictionaryArray<K>where
K: ArrowDictionaryKeyType,
§impl<T> Array for DictionaryArray<T>where
T: ArrowDictionaryKeyType,
impl<T> Array for DictionaryArray<T>where
T: ArrowDictionaryKeyType,
§fn slice(&self, offset: usize, length: usize) -> Arc<dyn Array>
fn slice(&self, offset: usize, length: usize) -> Arc<dyn Array>
§fn shrink_to_fit(&mut self)
fn shrink_to_fit(&mut self)
§fn offset(&self) -> usize
fn offset(&self) -> usize
0
. Read more§fn nulls(&self) -> Option<&NullBuffer>
fn nulls(&self) -> Option<&NullBuffer>
§fn logical_nulls(&self) -> Option<NullBuffer>
fn logical_nulls(&self) -> Option<NullBuffer>
NullBuffer
that represents the logical
null values of this array, if any. Read more§fn logical_null_count(&self) -> usize
fn logical_null_count(&self) -> usize
§fn is_nullable(&self) -> bool
fn is_nullable(&self) -> bool
false
if the array is guaranteed to not contain any logical nulls Read more§fn get_buffer_memory_size(&self) -> usize
fn get_buffer_memory_size(&self) -> usize
§fn get_array_memory_size(&self) -> usize
fn get_array_memory_size(&self) -> usize
get_buffer_memory_size()
and
includes the overhead of the data structures that contain the pointers to the various buffers.§fn null_count(&self) -> usize
fn null_count(&self) -> usize
§impl<K> Clone for DictionaryArray<K>where
K: ArrowDictionaryKeyType,
impl<K> Clone for DictionaryArray<K>where
K: ArrowDictionaryKeyType,
§fn clone(&self) -> DictionaryArray<K>
fn clone(&self) -> DictionaryArray<K>
1.0.0 · Source§fn clone_from(&mut self, source: &Self)
fn clone_from(&mut self, source: &Self)
source
. Read more§impl<T> Debug for DictionaryArray<T>where
T: ArrowDictionaryKeyType,
impl<T> Debug for DictionaryArray<T>where
T: ArrowDictionaryKeyType,
§impl<T> From<ArrayData> for DictionaryArray<T>where
T: ArrowDictionaryKeyType,
impl<T> From<ArrayData> for DictionaryArray<T>where
T: ArrowDictionaryKeyType,
Constructs a DictionaryArray
from an array data reference.
§fn from(data: ArrayData) -> DictionaryArray<T>
fn from(data: ArrayData) -> DictionaryArray<T>
§impl<T> From<DictionaryArray<T>> for ArrayDatawhere
T: ArrowDictionaryKeyType,
impl<T> From<DictionaryArray<T>> for ArrayDatawhere
T: ArrowDictionaryKeyType,
§fn from(array: DictionaryArray<T>) -> ArrayData
fn from(array: DictionaryArray<T>) -> ArrayData
§impl<'a, T> FromIterator<&'a str> for DictionaryArray<T>where
T: ArrowDictionaryKeyType,
impl<'a, T> FromIterator<&'a str> for DictionaryArray<T>where
T: ArrowDictionaryKeyType,
Constructs a DictionaryArray
from an iterator of strings.
§Example:
use arrow_array::{DictionaryArray, PrimitiveArray, StringArray, types::Int8Type};
let test = vec!["a", "a", "b", "c"];
let array: DictionaryArray<Int8Type> = test.into_iter().collect();
assert_eq!(
"DictionaryArray {keys: PrimitiveArray<Int8>\n[\n 0,\n 0,\n 1,\n 2,\n] values: StringArray\n[\n \"a\",\n \"b\",\n \"c\",\n]}\n",
format!("{:?}", array)
);
§fn from_iter<I>(iter: I) -> DictionaryArray<T>where
I: IntoIterator<Item = &'a str>,
fn from_iter<I>(iter: I) -> DictionaryArray<T>where
I: IntoIterator<Item = &'a str>,
§impl<'a, T> FromIterator<Option<&'a str>> for DictionaryArray<T>where
T: ArrowDictionaryKeyType,
impl<'a, T> FromIterator<Option<&'a str>> for DictionaryArray<T>where
T: ArrowDictionaryKeyType,
Constructs a DictionaryArray
from an iterator of optional strings.
§Example:
use arrow_array::{DictionaryArray, PrimitiveArray, StringArray, types::Int8Type};
let test = vec!["a", "a", "b", "c"];
let array: DictionaryArray<Int8Type> = test
.iter()
.map(|&x| if x == "b" { None } else { Some(x) })
.collect();
assert_eq!(
"DictionaryArray {keys: PrimitiveArray<Int8>\n[\n 0,\n 0,\n null,\n 1,\n] values: StringArray\n[\n \"a\",\n \"c\",\n]}\n",
format!("{:?}", array)
);