Struct ParquetObjectReader

pub struct ParquetObjectReader {
    store: Arc<dyn ObjectStore>,
    meta: ObjectMeta,
    metadata_size_hint: Option<usize>,
    preload_column_index: bool,
    preload_offset_index: bool,
    runtime: Option<Handle>,
}

Reads Parquet files in object storage using [ObjectStore].

// Populate configuration from environment
let storage_container = Arc::new(MicrosoftAzureBuilder::from_env().build().unwrap());
let location = Path::from("path/to/blob.parquet");
let meta = storage_container.head(&location).await.unwrap();
println!("Found Blob with {}B at {}", meta.size, meta.location);

// Show Parquet metadata
let reader = ParquetObjectReader::new(storage_container, meta);
let builder = ParquetRecordBatchStreamBuilder::new(reader).await.unwrap();
print_parquet_metadata(&mut stdout(), builder.metadata());

Fields

store: Arc<dyn ObjectStore>

meta: ObjectMeta

metadata_size_hint: Option<usize>

preload_column_index: bool

preload_offset_index: bool

runtime: Option<Handle>

Implementations

impl ParquetObjectReader

pub fn new(store: Arc<dyn ObjectStore>, meta: ObjectMeta) -> Self

Creates a new ParquetObjectReader for the provided [ObjectStore] and [ObjectMeta]

[ObjectMeta] can be obtained using [ObjectStore::list] or [ObjectStore::head]

pub fn with_footer_size_hint(self, hint: usize) -> Self

Provide a hint as to the size of the parquet file’s footer, see fetch_parquet_metadata

pub fn with_preload_column_index(self, preload_column_index: bool) -> Self

Load the Column Index as part of Self::get_metadata

pub fn with_preload_offset_index(self, preload_offset_index: bool) -> Self

Load the Offset Index as part of Self::get_metadata

pub fn with_runtime(self, handle: Handle) -> Self

Perform IO on the provided tokio runtime

Tokio is a cooperative scheduler, and relies on tasks yielding in a timely manner to service IO. Therefore, running IO and CPU-bound tasks, such as parquet decoding, on the same tokio runtime can lead to degraded throughput, dropped connections and other issues. For more information see here.

fn spawn<F, O, E>(&self, f: F) -> BoxFuture<'_, Result<O>>

where F: for<'a> FnOnce(&'a Arc<dyn ObjectStore>, &'a Path) -> BoxFuture<'a, Result<O, E>> + Send + 'static, O: Send + 'static, E: Into<ParquetError> + Send + 'static,

Trait Implementations

impl AsyncFileReader for ParquetObjectReader

fn get_bytes(&mut self, range: Range<usize>) -> BoxFuture<'_, Result<Bytes>>

Retrieve the bytes in range

fn get_byte_ranges( &mut self, ranges: Vec<Range<usize>>, ) -> BoxFuture<'_, Result<Vec<Bytes>>>

where Self: Send,

Retrieve multiple byte ranges. The default implementation will call get_bytes sequentially

fn get_metadata(&mut self) -> BoxFuture<'_, Result<Arc<ParquetMetaData>>>

Provides asynchronous access to the ParquetMetaData of a parquet file, allowing fine-grained control over how metadata is sourced, in particular allowing for caching, pre-fetching, catalog metadata, etc…

impl Clone for ParquetObjectReader

fn clone(&self) -> ParquetObjectReader

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for ParquetObjectReader

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.