Skip to main content

arrow_avro/reader/async_reader/
store.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use crate::errors::AvroError;
6use crate::reader::async_reader::AsyncFileReader;
7use bytes::Bytes;
8use futures::future::BoxFuture;
9use futures::{FutureExt, TryFutureExt};
10use object_store::ObjectStore;
11use object_store::ObjectStoreExt;
12use object_store::path::Path;
13use std::ops::Range;
14use std::sync::Arc;
15use tokio::runtime::Handle;
16
17/// An implementation of an AsyncFileReader using the [`ObjectStore`] API.
18#[deprecated(
19    since = "59.2.0",
20    note = "Implement `AsyncFileReader` directly instead; see the example on the `AsyncFileReader` trait documentation and `arrow-avro/examples/object_store.rs`. Use `SpawnedReader` to perform I/O on a dedicated runtime. See also https://github.com/apache/arrow-rs/issues/10308"
21)]
22#[derive(Clone, Debug)]
23pub struct AvroObjectReader {
24    store: Arc<dyn ObjectStore>,
25    path: Path,
26    runtime: Option<Handle>,
27}
28
29#[expect(deprecated)]
30impl AvroObjectReader {
31    /// Creates a new [`Self`] from a store implementation and file location.
32    pub fn new(store: Arc<dyn ObjectStore>, path: Path) -> Self {
33        Self {
34            store,
35            path,
36            runtime: None,
37        }
38    }
39
40    /// Perform IO on the provided tokio runtime
41    ///
42    /// Tokio is a cooperative scheduler, and relies on tasks yielding in a timely manner
43    /// to service IO. Therefore, running IO and CPU-bound tasks, such as avro decoding,
44    /// on the same tokio runtime can lead to degraded throughput, dropped connections and
45    /// other issues. For more information see [here].
46    ///
47    /// [here]: https://www.influxdata.com/blog/using-rustlangs-async-tokio-runtime-for-cpu-bound-tasks/
48    #[deprecated(
49        since = "59.2.0",
50        note = "Wrap the reader in a `SpawnedReader` instead, e.g. `SpawnedReader::new(reader, handle)`. See also https://github.com/apache/arrow-rs/issues/10308"
51    )]
52    pub fn with_runtime(self, handle: Handle) -> Self {
53        Self {
54            runtime: Some(handle),
55            ..self
56        }
57    }
58
59    fn spawn<F, O, E>(&self, f: F) -> BoxFuture<'_, Result<O, AvroError>>
60    where
61        F: for<'a> FnOnce(&'a Arc<dyn ObjectStore>, &'a Path) -> BoxFuture<'a, Result<O, E>>
62            + Send
63            + 'static,
64        O: Send + 'static,
65        E: Into<AvroError> + Send + 'static,
66    {
67        match &self.runtime {
68            Some(handle) => {
69                let path = self.path.clone();
70                let store = Arc::clone(&self.store);
71                handle
72                    .spawn(async move { f(&store, &path).await })
73                    .map_ok_or_else(
74                        |e| match e.try_into_panic() {
75                            Err(e) => Err(AvroError::External(Box::new(e))),
76                            Ok(p) => std::panic::resume_unwind(p),
77                        },
78                        |res| res.map_err(Into::into),
79                    )
80                    .boxed()
81            }
82            None => f(&self.store, &self.path).map_err(Into::into).boxed(),
83        }
84    }
85}
86
87#[expect(deprecated)]
88impl AsyncFileReader for AvroObjectReader {
89    fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes, AvroError>> {
90        self.spawn(|store, path| async move { store.get_range(path, range).await }.boxed())
91    }
92
93    fn get_byte_ranges(
94        &mut self,
95        ranges: Vec<Range<u64>>,
96    ) -> BoxFuture<'_, Result<Vec<Bytes>, AvroError>>
97    where
98        Self: Send,
99    {
100        self.spawn(|store, path| async move { store.get_ranges(path, &ranges).await }.boxed())
101    }
102}
103
104#[cfg(test)]
105mod tests {
106    use super::*;
107    use object_store::memory::InMemory;
108
109    fn find_object_store_error(err: &AvroError) -> Option<&object_store::Error> {
110        let mut source: Option<&(dyn std::error::Error + 'static)> = Some(err);
111        while let Some(e) = source {
112            if let Some(os) = e.downcast_ref::<object_store::Error>() {
113                return Some(os);
114            }
115            source = e.source();
116        }
117        None
118    }
119
120    #[tokio::test]
121    async fn test_get_bytes_preserves_object_store_error_source() {
122        let store = Arc::new(InMemory::new());
123        #[expect(deprecated)]
124        let mut reader = AvroObjectReader::new(store, Path::from("missing.avro"));
125        let err = reader.get_bytes(0..10).await.unwrap_err();
126        assert!(
127            matches!(
128                find_object_store_error(&err),
129                Some(object_store::Error::NotFound { .. })
130            ),
131            "expected NotFound in source chain, got: {err}"
132        );
133    }
134
135    #[tokio::test]
136    async fn test_get_bytes_on_runtime_preserves_object_store_error_source() {
137        let store = Arc::new(InMemory::new());
138        #[expect(deprecated)]
139        let mut reader = AvroObjectReader::new(store, Path::from("missing.avro"))
140            .with_runtime(Handle::current());
141        let err = reader.get_bytes(0..10).await.unwrap_err();
142        assert!(
143            matches!(
144                find_object_store_error(&err),
145                Some(object_store::Error::NotFound { .. })
146            ),
147            "expected NotFound in source chain, got: {err}"
148        );
149    }
150}