parquet/arrow/arrow_reader/metrics.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! [ArrowReaderMetrics] for collecting metrics about the Arrow reader
6
7use std::sync::Arc;
8use std::sync::atomic::AtomicUsize;
9
10/// This enum represents the state of Arrow reader metrics collection.
11///
12/// The inner metrics are stored in an `Arc<ArrowReaderMetricsInner>`
13/// so cloning the `ArrowReaderMetrics` enum will not clone the inner metrics.
14///
15/// To access metrics, create an `ArrowReaderMetrics` via [`ArrowReaderMetrics::enabled()`]
16/// and configure the `ArrowReaderBuilder` with a clone.
17#[derive(Debug, Clone)]
18pub enum ArrowReaderMetrics {
19 /// Metrics are not collected (default)
20 Disabled,
21 /// Metrics are collected and stored in an `Arc`.
22 ///
23 /// Create this via [`ArrowReaderMetrics::enabled()`].
24 Enabled(Arc<ArrowReaderMetricsInner>),
25}
26
27impl ArrowReaderMetrics {
28 /// Creates a new instance of [`ArrowReaderMetrics::Disabled`]
29 pub fn disabled() -> Self {
30 Self::Disabled
31 }
32
33 /// Creates a new instance of [`ArrowReaderMetrics::Enabled`]
34 pub fn enabled() -> Self {
35 Self::Enabled(Arc::new(ArrowReaderMetricsInner::new()))
36 }
37
38 /// Predicate Cache: number of records read directly from the inner reader
39 ///
40 /// This is the total number of records read from the inner reader (that is
41 /// actually decoding). It measures the amount of work that could not be
42 /// avoided with caching.
43 ///
44 /// It returns the number of records read across all columns, so if you read
45 /// 2 columns each with 100 records, this will return 200.
46 ///
47 ///
48 /// Returns None if metrics are disabled.
49 pub fn records_read_from_inner(&self) -> Option<usize> {
50 match self {
51 Self::Disabled => None,
52 Self::Enabled(inner) => Some(
53 inner
54 .records_read_from_inner
55 .load(std::sync::atomic::Ordering::Relaxed),
56 ),
57 }
58 }
59
60 /// Predicate Cache: number of records read from the cache
61 ///
62 /// This is the total number of records read from the cache actually
63 /// decoding). It measures the amount of work that was avoided with caching.
64 ///
65 /// It returns the number of records read across all columns, so if you read
66 /// 2 columns each with 100 records from the cache, this will return 200.
67 ///
68 /// Only records served from the shared (global) predicate cache are
69 /// counted. A reader's temporary local buffer of batches it has already
70 /// decoded is left out of this metric.
71 ///
72 /// Returns None if metrics are disabled.
73 pub fn records_read_from_cache(&self) -> Option<usize> {
74 match self {
75 Self::Disabled => None,
76 Self::Enabled(inner) => Some(
77 inner
78 .records_read_from_cache
79 .load(std::sync::atomic::Ordering::Relaxed),
80 ),
81 }
82 }
83
84 /// Increments the count of records read from the inner reader
85 pub(crate) fn increment_inner_reads(&self, count: usize) {
86 let Self::Enabled(inner) = self else {
87 return;
88 };
89 inner
90 .records_read_from_inner
91 .fetch_add(count, std::sync::atomic::Ordering::Relaxed);
92 }
93
94 /// Increments the count of records read from the cache
95 pub(crate) fn increment_cache_reads(&self, count: usize) {
96 let Self::Enabled(inner) = self else {
97 return;
98 };
99
100 inner
101 .records_read_from_cache
102 .fetch_add(count, std::sync::atomic::Ordering::Relaxed);
103 }
104}
105
106/// Holds the actual metrics for the Arrow reader.
107///
108/// Please see [`ArrowReaderMetrics`] for the public interface.
109#[derive(Debug)]
110pub struct ArrowReaderMetricsInner {
111 // Metrics for Predicate Cache
112 /// Total number of records read from the inner reader (uncached)
113 records_read_from_inner: AtomicUsize,
114 /// Total number of records read from previously cached pages
115 records_read_from_cache: AtomicUsize,
116}
117
118impl ArrowReaderMetricsInner {
119 /// Creates a new instance of `ArrowReaderMetricsInner`
120 pub(crate) fn new() -> Self {
121 Self {
122 records_read_from_inner: AtomicUsize::new(0),
123 records_read_from_cache: AtomicUsize::new(0),
124 }
125 }
126}