Skip to main content

parquet/arrow/arrow_reader/
metrics.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! [ArrowReaderMetrics] for collecting metrics about the Arrow reader
6
7use std::sync::Arc;
8use std::sync::atomic::AtomicUsize;
9
10/// This enum represents the state of Arrow reader metrics collection.
11///
12/// The inner metrics are stored in an `Arc<ArrowReaderMetricsInner>`
13/// so cloning the `ArrowReaderMetrics` enum will not clone the inner metrics.
14///
15/// To access metrics, create an `ArrowReaderMetrics` via [`ArrowReaderMetrics::enabled()`]
16/// and configure the `ArrowReaderBuilder` with a clone.
17#[derive(Debug, Clone)]
18pub enum ArrowReaderMetrics {
19    /// Metrics are not collected (default)
20    Disabled,
21    /// Metrics are collected and stored in an `Arc`.
22    ///
23    /// Create this via [`ArrowReaderMetrics::enabled()`].
24    Enabled(Arc<ArrowReaderMetricsInner>),
25}
26
27impl ArrowReaderMetrics {
28    /// Creates a new instance of [`ArrowReaderMetrics::Disabled`]
29    pub fn disabled() -> Self {
30        Self::Disabled
31    }
32
33    /// Creates a new instance of [`ArrowReaderMetrics::Enabled`]
34    pub fn enabled() -> Self {
35        Self::Enabled(Arc::new(ArrowReaderMetricsInner::new()))
36    }
37
38    /// Predicate Cache: number of records read directly from the inner reader
39    ///
40    /// This is the total number of records read from the inner reader (that is
41    /// actually decoding). It measures the amount of work that could not be
42    /// avoided with caching.
43    ///
44    /// It returns the number of records read across all columns, so if you read
45    /// 2 columns each with 100 records, this will return 200.
46    ///
47    ///
48    /// Returns None if metrics are disabled.
49    pub fn records_read_from_inner(&self) -> Option<usize> {
50        match self {
51            Self::Disabled => None,
52            Self::Enabled(inner) => Some(
53                inner
54                    .records_read_from_inner
55                    .load(std::sync::atomic::Ordering::Relaxed),
56            ),
57        }
58    }
59
60    /// Predicate Cache: number of records read from the cache
61    ///
62    /// This is the total number of records read from the cache actually
63    /// decoding). It measures the amount of work that was avoided with caching.
64    ///
65    /// It returns the number of records read across all columns, so if you read
66    /// 2 columns each with 100 records from the cache, this will return 200.
67    ///
68    /// Only records served from the shared (global) predicate cache are
69    /// counted. A reader's temporary local buffer of batches it has already
70    /// decoded is left out of this metric.
71    ///
72    /// Returns None if metrics are disabled.
73    pub fn records_read_from_cache(&self) -> Option<usize> {
74        match self {
75            Self::Disabled => None,
76            Self::Enabled(inner) => Some(
77                inner
78                    .records_read_from_cache
79                    .load(std::sync::atomic::Ordering::Relaxed),
80            ),
81        }
82    }
83
84    /// Increments the count of records read from the inner reader
85    pub(crate) fn increment_inner_reads(&self, count: usize) {
86        let Self::Enabled(inner) = self else {
87            return;
88        };
89        inner
90            .records_read_from_inner
91            .fetch_add(count, std::sync::atomic::Ordering::Relaxed);
92    }
93
94    /// Increments the count of records read from the cache
95    pub(crate) fn increment_cache_reads(&self, count: usize) {
96        let Self::Enabled(inner) = self else {
97            return;
98        };
99
100        inner
101            .records_read_from_cache
102            .fetch_add(count, std::sync::atomic::Ordering::Relaxed);
103    }
104}
105
106/// Holds the actual metrics for the Arrow reader.
107///
108/// Please see [`ArrowReaderMetrics`] for the public interface.
109#[derive(Debug)]
110pub struct ArrowReaderMetricsInner {
111    // Metrics for Predicate Cache
112    /// Total number of records read from the inner reader (uncached)
113    records_read_from_inner: AtomicUsize,
114    /// Total number of records read from previously cached pages
115    records_read_from_cache: AtomicUsize,
116}
117
118impl ArrowReaderMetricsInner {
119    /// Creates a new instance of `ArrowReaderMetricsInner`
120    pub(crate) fn new() -> Self {
121        Self {
122            records_read_from_inner: AtomicUsize::new(0),
123            records_read_from_cache: AtomicUsize::new(0),
124        }
125    }
126}