Skip to main content

arrow_avro/reader/async_reader/
async_file_reader.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use crate::errors::AvroError;
6use bytes::Bytes;
7use futures::FutureExt;
8use futures::future::BoxFuture;
9use std::io::SeekFrom;
10use std::ops::Range;
11use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt};
12
13/// The asynchronous interface used by [`super::AsyncAvroFileReader`] to read avro files
14///
15/// Notes:
16///
17/// 1. There is a default implementation for types that implement [`AsyncRead`]
18///    and [`AsyncSeek`], for example [`tokio::fs::File`].
19///
20/// 2. Implementations for remote storage, such as the [`object_store`] crate,
21///    can implement this interface directly, typically by pairing a store
22///    handle with an object path and delegating [`Self::get_bytes`] and
23///    [`Self::get_byte_ranges`] to ranged reads. [`super::SpawnedReader`] can
24///    wrap such a reader to perform its I/O on a dedicated tokio runtime.
25///
26/// [`object_store`]: https://crates.io/crates/object_store
27///
28/// # Example: implementing `AsyncFileReader` for the `object_store` crate
29///
30/// ```no_run
31/// # use std::ops::Range;
32/// # use std::sync::Arc;
33/// use arrow_avro::errors::AvroError;
34/// use arrow_avro::reader::AsyncFileReader;
35/// use bytes::Bytes;
36/// use futures::FutureExt;
37/// use futures::future::BoxFuture;
38/// use object_store::path::Path;
39/// use object_store::{ObjectStore, ObjectStoreExt};
40///
41/// #[derive(Clone, Debug)]
42/// struct ObjectStoreReader {
43///     store: Arc<dyn ObjectStore>,
44///     path: Path,
45/// }
46///
47/// impl AsyncFileReader for ObjectStoreReader {
48///     fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes, AvroError>> {
49///         async move {
50///             self.store
51///                 .get_range(&self.path, range)
52///                 .await
53///                 .map_err(|e| AvroError::General(e.to_string()))
54///         }
55///         .boxed()
56///     }
57///
58///     fn get_byte_ranges(
59///         &mut self,
60///         ranges: Vec<Range<u64>>,
61///     ) -> BoxFuture<'_, Result<Vec<Bytes>, AvroError>> {
62///         async move {
63///             self.store
64///                 .get_ranges(&self.path, &ranges)
65///                 .await
66///                 .map_err(|e| AvroError::General(e.to_string()))
67///         }
68///         .boxed()
69///     }
70/// }
71/// ```
72///
73/// [`tokio::fs::File`]: https://docs.rs/tokio/latest/tokio/fs/struct.File.html
74pub trait AsyncFileReader: Send {
75    /// Retrieve the bytes in `range`
76    fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes, AvroError>>;
77
78    /// Retrieve multiple byte ranges. The default implementation will call `get_bytes` sequentially
79    fn get_byte_ranges(
80        &mut self,
81        ranges: Vec<Range<u64>>,
82    ) -> BoxFuture<'_, Result<Vec<Bytes>, AvroError>> {
83        async move {
84            let mut result = Vec::with_capacity(ranges.len());
85
86            for range in ranges {
87                let data = self.get_bytes(range).await?;
88                result.push(data);
89            }
90
91            Ok(result)
92        }
93        .boxed()
94    }
95}
96
97/// This allows Box<dyn AsyncFileReader + '_> to be used as an AsyncFileReader,
98impl AsyncFileReader for Box<dyn AsyncFileReader + '_> {
99    fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes, AvroError>> {
100        self.as_mut().get_bytes(range)
101    }
102
103    fn get_byte_ranges(
104        &mut self,
105        ranges: Vec<Range<u64>>,
106    ) -> BoxFuture<'_, Result<Vec<Bytes>, AvroError>> {
107        self.as_mut().get_byte_ranges(ranges)
108    }
109}
110
111impl<T: AsyncRead + AsyncSeek + Unpin + Send> AsyncFileReader for T {
112    fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes, AvroError>> {
113        async move {
114            self.seek(SeekFrom::Start(range.start)).await?;
115
116            let to_read = range.end - range.start;
117            let mut buffer = Vec::with_capacity(to_read as usize);
118            let read = self.take(to_read).read_to_end(&mut buffer).await?;
119            if read as u64 != to_read {
120                return Err(AvroError::EOF(format!(
121                    "expected to read {to_read} bytes, got {read}"
122                )));
123            }
124
125            Ok(buffer.into())
126        }
127        .boxed()
128    }
129}