arrow_avro/reader/async_reader/async_file_reader.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use crate::errors::AvroError;
6use bytes::Bytes;
7use futures::FutureExt;
8use futures::future::BoxFuture;
9use std::io::SeekFrom;
10use std::ops::Range;
11use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt};
12
13/// The asynchronous interface used by [`super::AsyncAvroFileReader`] to read avro files
14///
15/// Notes:
16///
17/// 1. There is a default implementation for types that implement [`AsyncRead`]
18/// and [`AsyncSeek`], for example [`tokio::fs::File`].
19///
20/// 2. Implementations for remote storage, such as the [`object_store`] crate,
21/// can implement this interface directly, typically by pairing a store
22/// handle with an object path and delegating [`Self::get_bytes`] and
23/// [`Self::get_byte_ranges`] to ranged reads. [`super::SpawnedReader`] can
24/// wrap such a reader to perform its I/O on a dedicated tokio runtime.
25///
26/// [`object_store`]: https://crates.io/crates/object_store
27///
28/// # Example: implementing `AsyncFileReader` for the `object_store` crate
29///
30/// ```no_run
31/// # use std::ops::Range;
32/// # use std::sync::Arc;
33/// use arrow_avro::errors::AvroError;
34/// use arrow_avro::reader::AsyncFileReader;
35/// use bytes::Bytes;
36/// use futures::FutureExt;
37/// use futures::future::BoxFuture;
38/// use object_store::path::Path;
39/// use object_store::{ObjectStore, ObjectStoreExt};
40///
41/// #[derive(Clone, Debug)]
42/// struct ObjectStoreReader {
43/// store: Arc<dyn ObjectStore>,
44/// path: Path,
45/// }
46///
47/// impl AsyncFileReader for ObjectStoreReader {
48/// fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes, AvroError>> {
49/// async move {
50/// self.store
51/// .get_range(&self.path, range)
52/// .await
53/// .map_err(|e| AvroError::General(e.to_string()))
54/// }
55/// .boxed()
56/// }
57///
58/// fn get_byte_ranges(
59/// &mut self,
60/// ranges: Vec<Range<u64>>,
61/// ) -> BoxFuture<'_, Result<Vec<Bytes>, AvroError>> {
62/// async move {
63/// self.store
64/// .get_ranges(&self.path, &ranges)
65/// .await
66/// .map_err(|e| AvroError::General(e.to_string()))
67/// }
68/// .boxed()
69/// }
70/// }
71/// ```
72///
73/// [`tokio::fs::File`]: https://docs.rs/tokio/latest/tokio/fs/struct.File.html
74pub trait AsyncFileReader: Send {
75 /// Retrieve the bytes in `range`
76 fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes, AvroError>>;
77
78 /// Retrieve multiple byte ranges. The default implementation will call `get_bytes` sequentially
79 fn get_byte_ranges(
80 &mut self,
81 ranges: Vec<Range<u64>>,
82 ) -> BoxFuture<'_, Result<Vec<Bytes>, AvroError>> {
83 async move {
84 let mut result = Vec::with_capacity(ranges.len());
85
86 for range in ranges {
87 let data = self.get_bytes(range).await?;
88 result.push(data);
89 }
90
91 Ok(result)
92 }
93 .boxed()
94 }
95}
96
97/// This allows Box<dyn AsyncFileReader + '_> to be used as an AsyncFileReader,
98impl AsyncFileReader for Box<dyn AsyncFileReader + '_> {
99 fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes, AvroError>> {
100 self.as_mut().get_bytes(range)
101 }
102
103 fn get_byte_ranges(
104 &mut self,
105 ranges: Vec<Range<u64>>,
106 ) -> BoxFuture<'_, Result<Vec<Bytes>, AvroError>> {
107 self.as_mut().get_byte_ranges(ranges)
108 }
109}
110
111impl<T: AsyncRead + AsyncSeek + Unpin + Send> AsyncFileReader for T {
112 fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes, AvroError>> {
113 async move {
114 self.seek(SeekFrom::Start(range.start)).await?;
115
116 let to_read = range.end - range.start;
117 let mut buffer = Vec::with_capacity(to_read as usize);
118 let read = self.take(to_read).read_to_end(&mut buffer).await?;
119 if read as u64 != to_read {
120 return Err(AvroError::EOF(format!(
121 "expected to read {to_read} bytes, got {read}"
122 )));
123 }
124
125 Ok(buffer.into())
126 }
127 .boxed()
128 }
129}