Struct ParquetObjectReader

pub struct ParquetObjectReader {
    store: Arc<dyn ObjectStore>,
    path: Path,
    file_size: Option<usize>,
    metadata_size_hint: Option<usize>,
    preload_column_index: bool,
    preload_offset_index: bool,
    runtime: Option<Handle>,
}

Reads Parquet files in object storage using [ObjectStore].

// Populate configuration from environment
let storage_container = Arc::new(MicrosoftAzureBuilder::from_env().build().unwrap());
let location = Path::from("path/to/blob.parquet");
let meta = storage_container.head(&location).await.unwrap();
println!("Found Blob with {}B at {}", meta.size, meta.location);

// Show Parquet metadata
let reader = ParquetObjectReader::new(storage_container, meta.location).with_file_size(meta.size.try_into().unwrap());
let builder = ParquetRecordBatchStreamBuilder::new(reader).await.unwrap();
print_parquet_metadata(&mut stdout(), builder.metadata());

Fields

store: Arc<dyn ObjectStore>

path: Path

file_size: Option<usize>

metadata_size_hint: Option<usize>

preload_column_index: bool

preload_offset_index: bool

runtime: Option<Handle>

Implementations

impl ParquetObjectReader

pub fn new(store: Arc<dyn ObjectStore>, path: Path) -> Self

Creates a new ParquetObjectReader for the provided [ObjectStore] and [Path].

pub fn with_footer_size_hint(self, hint: usize) -> Self

Provide a hint as to the size of the parquet file’s footer, see fetch_parquet_metadata

pub fn with_file_size(self, file_size: usize) -> Self

Provide the byte size of this file.

If provided, the file size will ensure that only bounded range requests are used. If file size is not provided, the reader will use suffix range requests to fetch the metadata.

Providing this size up front is an important optimization to avoid extra calls when the underlying store does not support suffix range requests.

The file size can be obtained using [ObjectStore::list] or [ObjectStore::head].

pub fn with_preload_column_index(self, preload_column_index: bool) -> Self

Load the Column Index as part of Self::get_metadata

pub fn with_preload_offset_index(self, preload_offset_index: bool) -> Self

Load the Offset Index as part of Self::get_metadata

pub fn with_runtime(self, handle: Handle) -> Self

Perform IO on the provided tokio runtime

Tokio is a cooperative scheduler, and relies on tasks yielding in a timely manner to service IO. Therefore, running IO and CPU-bound tasks, such as parquet decoding, on the same tokio runtime can lead to degraded throughput, dropped connections and other issues. For more information see here.

fn spawn<F, O, E>(&self, f: F) -> BoxFuture<'_, Result<O>>

where F: for<'a> FnOnce(&'a Arc<dyn ObjectStore>, &'a Path) -> BoxFuture<'a, Result<O, E>> + Send + 'static, O: Send + 'static, E: Into<ParquetError> + Send + 'static,

Trait Implementations

impl AsyncFileReader for ParquetObjectReader

fn get_bytes(&mut self, range: Range<usize>) -> BoxFuture<'_, Result<Bytes>>

Retrieve the bytes in range

fn get_byte_ranges( &mut self, ranges: Vec<Range<usize>>, ) -> BoxFuture<'_, Result<Vec<Bytes>>>

where Self: Send,

Retrieve multiple byte ranges. The default implementation will call get_bytes sequentially

fn get_metadata<'a>( &'a mut self, options: Option<&'a ArrowReaderOptions>, ) -> BoxFuture<'a, Result<Arc<ParquetMetaData>>>

Provides asynchronous access to the ParquetMetaData of a parquet file, allowing fine-grained control over how metadata is sourced, in particular allowing for caching, pre-fetching, catalog metadata, etc… ArrowReaderOptions may be provided to supply decryption parameters

impl Clone for ParquetObjectReader

fn clone(&self) -> ParquetObjectReader

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for ParquetObjectReader

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl MetadataSuffixFetch for &mut ParquetObjectReader

fn fetch_suffix(&mut self, suffix: usize) -> BoxFuture<'_, Result<Bytes>>

Return a future that fetches the last n bytes asynchronously