parquet/arrow/async_reader/
metadata.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use crate::arrow::async_reader::AsyncFileReader;
6use crate::errors::Result;
7use bytes::Bytes;
8use futures::future::BoxFuture;
9use std::ops::Range;
10
11/// A data source that can be used with [`ParquetMetaDataReader`] to load [`ParquetMetaData`]
12///
13/// Note that implementation is provided for [`AsyncFileReader`].
14///
15/// # Example `MetadataFetch` for a custom async data source
16///
17/// ```rust
18/// # use parquet::errors::Result;
19/// # use parquet::arrow::async_reader::MetadataFetch;
20/// # use bytes::Bytes;
21/// # use std::ops::Range;
22/// # use std::io::SeekFrom;
23/// # use futures::future::BoxFuture;
24/// # use futures::FutureExt;
25/// # use tokio::io::{AsyncReadExt, AsyncSeekExt};
26/// // Adapter that implements the API for reading bytes from an async source (in
27/// // this case a tokio::fs::File)
28/// struct TokioFileMetadata {
29///     file: tokio::fs::File,
30/// }
31/// impl MetadataFetch for TokioFileMetadata {
32///     fn fetch(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes>> {
33///         // return a future that fetches data in range
34///         async move {
35///             let len = (range.end - range.start).try_into().unwrap();
36///             let mut buf = vec![0; len]; // target buffer
37///             // seek to the start of the range and read the data
38///             self.file.seek(SeekFrom::Start(range.start)).await?;
39///             self.file.read_exact(&mut buf).await?;
40///             Ok(Bytes::from(buf)) // convert to Bytes
41///         }
42///             .boxed() // turn into BoxedFuture, using FutureExt::boxed
43///     }
44/// }
45///```
46///
47/// [`ParquetMetaDataReader`]: crate::file::metadata::reader::ParquetMetaDataReader
48/// [`ParquetMetaData`]: crate::file::metadata::ParquetMetaData
49pub trait MetadataFetch {
50    /// Return a future that fetches the specified range of bytes asynchronously
51    ///
52    /// Note the returned type is a boxed future, often created by
53    /// [`FutureExt::boxed`]. See the trait documentation for an example
54    ///
55    /// [`FutureExt::boxed`]: futures::FutureExt::boxed
56    fn fetch(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes>>;
57}
58
59impl<T: AsyncFileReader> MetadataFetch for &mut T {
60    fn fetch(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes>> {
61        self.get_bytes(range)
62    }
63}
64
65/// A data source that can be used with [`ParquetMetaDataReader`] to load [`ParquetMetaData`] via suffix
66/// requests, without knowing the file size
67///
68/// [`ParquetMetaDataReader`]: crate::file::metadata::reader::ParquetMetaDataReader
69/// [`ParquetMetaData`]: crate::file::metadata::ParquetMetaData
70pub trait MetadataSuffixFetch: MetadataFetch {
71    /// Return a future that fetches the last `n` bytes asynchronously
72    ///
73    /// Note the returned type is a boxed future, often created by
74    /// [`FutureExt::boxed`]. See the trait documentation for an example
75    ///
76    /// [`FutureExt::boxed`]: futures::FutureExt::boxed
77    fn fetch_suffix(&mut self, suffix: usize) -> BoxFuture<'_, Result<Bytes>>;
78}
parquet/arrow/async_reader/metadata.rs

parquet/arrow/async_reader/
metadata.rs
