hotshot_query_service/
metrics.rs

1#![allow(dead_code)]
2
3// Copyright (c) 2022 Espresso Systems (espressosys.com)
4// This file is part of the HotShot Query Service library.
5//
6// This program is free software: you can redistribute it and/or modify it under the terms of the GNU
7// General Public License as published by the Free Software Foundation, either version 3 of the
8// License, or (at your option) any later version.
9// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
10// even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11// General Public License for more details.
12// You should have received a copy of the GNU General Public License along with this program. If not,
13// see <https://www.gnu.org/licenses/>.
14
15use std::{
16    collections::HashMap,
17    sync::{Arc, RwLock},
18};
19
20use hotshot_types::traits::metrics;
21use itertools::Itertools;
22use prometheus::{
23    Encoder, HistogramVec, Opts, Registry, TextEncoder,
24    core::{AtomicU64, GenericCounter, GenericCounterVec, GenericGauge, GenericGaugeVec},
25};
26use snafu::Snafu;
27use tracing::warn;
28
29#[derive(Debug, Snafu)]
30pub enum MetricsError {
31    NoSuchSubgroup {
32        path: Vec<String>,
33    },
34    NoSuchMetric {
35        namespace: Vec<String>,
36        name: String,
37    },
38    Prometheus {
39        source: prometheus::Error,
40    },
41}
42
43impl From<prometheus::Error> for MetricsError {
44    fn from(source: prometheus::Error) -> Self {
45        Self::Prometheus { source }
46    }
47}
48
49/// A Prometheus-based implementation of a [Metrics](metrics::Metrics) registry.
50///
51/// [PrometheusMetrics] provides a collection of metrics including [Counter], [Gauge], and
52/// [Histogram]. These metrics can be created and associated with a [PrometheusMetrics] collection
53/// and then used as handles for updating and populating. The [PrometheusMetrics] registry can then
54/// be used to collect all of the associated metrics and export them in the Prometheus text format.
55///
56/// This implementation provides a few features beyond the basic [prometheus] features. It supports
57/// hierarchical namespaces; any [PrometheusMetrics] can be used to derive a subgroup with a certain
58/// name. The subgroup is then related to the parent, and any [PrometheusMetrics] in the tree of
59/// related groups can be used to collect _all_ registered metrics. The namespacing will be
60/// reflected in the fully qualified name of each metric in the Prometheus output. The subgroup
61/// relationship is pure and deterministic -- calling
62/// [get_subgroup](PrometheusMetrics::get_subgroup) with the same subgroup name will always return a
63/// handle to the same underlying [PrometheusMetrics] object.
64///
65/// [PrometheusMetrics] also supports querying for individual metrics by name, unlike
66/// [prometheus::Registry]. This provides a programming interface for inspecting the values of
67/// specific metrics at run-time, if that is preferable to exporting all metrics wholesale.
68#[derive(Clone, Debug, Default)]
69pub struct PrometheusMetrics {
70    metrics: Registry,
71    namespace: Vec<String>,
72    children: Arc<RwLock<HashMap<String, PrometheusMetrics>>>,
73    counters: Arc<RwLock<HashMap<String, Counter>>>,
74    gauges: Arc<RwLock<HashMap<String, Gauge>>>,
75    histograms: Arc<RwLock<HashMap<String, Histogram>>>,
76    counter_families: Arc<RwLock<HashMap<String, CounterFamily>>>,
77    gauge_families: Arc<RwLock<HashMap<String, GaugeFamily>>>,
78    histogram_families: Arc<RwLock<HashMap<String, HistogramFamily>>>,
79}
80
81impl PrometheusMetrics {
82    /// Get a counter in this sub-group by name.
83    pub fn get_counter(&self, name: &str) -> Result<Counter, MetricsError> {
84        self.get_metric(&self.counters, name)
85    }
86
87    /// Get a gauge in this sub-group by name.
88    pub fn get_gauge(&self, name: &str) -> Result<Gauge, MetricsError> {
89        self.get_metric(&self.gauges, name)
90    }
91
92    /// Get a histogram in this sub-group by name.
93    pub fn get_histogram(&self, name: &str) -> Result<Histogram, MetricsError> {
94        self.get_metric(&self.histograms, name)
95    }
96
97    /// Get a counter family in this sub-group by name.
98    pub fn get_counter_family(&self, name: &str) -> Result<CounterFamily, MetricsError> {
99        self.get_metric(&self.counter_families, name)
100    }
101
102    /// Get a gauge family in this sub-group by name.
103    pub fn gauge_family(&self, name: &str) -> Result<GaugeFamily, MetricsError> {
104        self.get_metric(&self.gauge_families, name)
105    }
106
107    /// Get a histogram family in this sub-group by name.
108    pub fn get_histogram_family(&self, name: &str) -> Result<HistogramFamily, MetricsError> {
109        self.get_metric(&self.histogram_families, name)
110    }
111
112    /// Borrow the underlying [`prometheus::Registry`].
113    ///
114    /// Useful for callers that need to scrape the full set of metric families
115    /// (e.g. a remote-write push task) instead of going through the text-format
116    /// [`Metrics::export`] path. The registry is `Clone` (it shares state via
117    /// `Arc` internally), so callers can take an owned copy if needed.
118    pub fn registry(&self) -> &Registry {
119        &self.metrics
120    }
121
122    /// Get a (possibly nested) subgroup of this group by its path.
123    pub fn get_subgroup<I>(&self, path: I) -> Result<PrometheusMetrics, MetricsError>
124    where
125        I: IntoIterator,
126        I::Item: AsRef<str>,
127    {
128        let mut curr = self.clone();
129        for seg in path.into_iter() {
130            let next = curr
131                .children
132                .read()
133                .unwrap()
134                .get(seg.as_ref())
135                .ok_or_else(|| MetricsError::NoSuchSubgroup {
136                    path: {
137                        let mut path = curr.namespace.clone();
138                        path.push(seg.as_ref().to_string());
139                        path
140                    },
141                })?
142                .clone();
143            curr = next;
144        }
145        Ok(curr)
146    }
147
148    fn get_metric<M: Clone>(
149        &self,
150        metrics: &Arc<RwLock<HashMap<String, M>>>,
151        name: &str,
152    ) -> Result<M, MetricsError> {
153        metrics
154            .read()
155            .unwrap()
156            .get(name)
157            .cloned()
158            .ok_or_else(|| MetricsError::NoSuchMetric {
159                namespace: self.namespace.clone(),
160                name: name.to_string(),
161            })
162    }
163
164    fn metric_opts(&self, name: String, unit_label: Option<String>) -> Opts {
165        let help = unit_label.unwrap_or_else(|| name.clone());
166        let mut opts = Opts::new(name, help);
167        let mut group_names = self.namespace.iter();
168        if let Some(namespace) = group_names.next() {
169            opts = opts
170                .namespace(namespace.clone())
171                .subsystem(group_names.join("_"));
172        }
173        opts
174    }
175}
176
177impl tide_disco::metrics::Metrics for PrometheusMetrics {
178    type Error = MetricsError;
179
180    fn export(&self) -> Result<String, Self::Error> {
181        let encoder = TextEncoder::new();
182        let metric_families = self.metrics.gather();
183        let mut buffer = vec![];
184        encoder.encode(&metric_families, &mut buffer)?;
185        String::from_utf8(buffer).map_err(|err| MetricsError::Prometheus {
186            source: prometheus::Error::Msg(format!(
187                "could not convert Prometheus output to UTF-8: {err}"
188            )),
189        })
190    }
191}
192
193impl metrics::Metrics for PrometheusMetrics {
194    fn create_counter(
195        &self,
196        name: String,
197        unit_label: Option<String>,
198    ) -> Box<dyn metrics::Counter> {
199        let counter = Counter::new(&self.metrics, self.metric_opts(name.clone(), unit_label));
200        self.counters.write().unwrap().insert(name, counter.clone());
201        Box::new(counter)
202    }
203
204    fn create_gauge(&self, name: String, unit_label: Option<String>) -> Box<dyn metrics::Gauge> {
205        let gauge = Gauge::new(&self.metrics, self.metric_opts(name.clone(), unit_label));
206        self.gauges.write().unwrap().insert(name, gauge.clone());
207        Box::new(gauge)
208    }
209
210    fn create_histogram(
211        &self,
212        name: String,
213        unit_label: Option<String>,
214    ) -> Box<dyn metrics::Histogram> {
215        let histogram = Histogram::new(&self.metrics, self.metric_opts(name.clone(), unit_label));
216        self.histograms
217            .write()
218            .unwrap()
219            .insert(name, histogram.clone());
220        Box::new(histogram)
221    }
222
223    fn create_text(&self, name: String) {
224        self.create_gauge(name, None).set(1);
225    }
226
227    fn counter_family(&self, name: String, labels: Vec<String>) -> Box<dyn metrics::CounterFamily> {
228        let family =
229            CounterFamily::new(&self.metrics, self.metric_opts(name.clone(), None), &labels);
230        self.counter_families
231            .write()
232            .unwrap()
233            .insert(name, family.clone());
234        Box::new(family)
235    }
236
237    fn gauge_family(&self, name: String, labels: Vec<String>) -> Box<dyn metrics::GaugeFamily> {
238        let family = GaugeFamily::new(&self.metrics, self.metric_opts(name.clone(), None), &labels);
239        self.gauge_families
240            .write()
241            .unwrap()
242            .insert(name, family.clone());
243        Box::new(family)
244    }
245
246    fn histogram_family(
247        &self,
248        name: String,
249        labels: Vec<String>,
250    ) -> Box<dyn metrics::HistogramFamily> {
251        let family =
252            HistogramFamily::new(&self.metrics, self.metric_opts(name.clone(), None), &labels);
253        self.histogram_families
254            .write()
255            .unwrap()
256            .insert(name, family.clone());
257        Box::new(family)
258    }
259
260    fn text_family(&self, name: String, labels: Vec<String>) -> Box<dyn metrics::TextFamily> {
261        Box::new(TextFamily::new(
262            &self.metrics,
263            self.metric_opts(name.clone(), None),
264            &labels,
265        ))
266    }
267
268    fn subgroup(&self, subgroup_name: String) -> Box<dyn metrics::Metrics> {
269        Box::new(
270            self.children
271                .write()
272                .unwrap()
273                .entry(subgroup_name.clone())
274                .or_insert_with(|| Self {
275                    metrics: self.metrics.clone(),
276                    namespace: {
277                        let mut namespace = self.namespace.clone();
278                        namespace.push(subgroup_name);
279                        namespace
280                    },
281                    ..Default::default()
282                })
283                .clone(),
284        )
285    }
286}
287
288/// A [Counter](metrics::Counter) metric.
289#[derive(Clone, Debug)]
290pub struct Counter(GenericCounter<AtomicU64>);
291
292impl Counter {
293    fn new(registry: &Registry, opts: Opts) -> Self {
294        let counter = GenericCounter::with_opts(opts).unwrap();
295        registry.register(Box::new(counter.clone())).unwrap();
296        Self(counter)
297    }
298
299    pub fn get(&self) -> usize {
300        self.0.get() as usize
301    }
302}
303
304impl metrics::Counter for Counter {
305    fn add(&self, amount: usize) {
306        self.0.inc_by(amount as u64);
307    }
308}
309
310/// A [Gauge](metrics::Gauge) metric.
311#[derive(Clone, Debug)]
312pub struct Gauge(GenericGauge<AtomicU64>);
313
314impl Gauge {
315    fn new(registry: &Registry, opts: Opts) -> Self {
316        let gauge = GenericGauge::with_opts(opts).unwrap();
317        registry.register(Box::new(gauge.clone())).unwrap();
318        Self(gauge)
319    }
320
321    pub fn get(&self) -> usize {
322        self.0.get() as usize
323    }
324}
325
326impl metrics::Gauge for Gauge {
327    fn set(&self, amount: usize) {
328        self.0.set(amount as u64);
329    }
330
331    fn update(&self, delta: i64) {
332        if delta >= 0 {
333            self.0.add(delta as u64);
334        } else {
335            self.0.sub(-delta as u64);
336        }
337    }
338}
339
340/// A [Histogram](metrics::Histogram) metric.
341#[derive(Clone, Debug)]
342pub struct Histogram(prometheus::Histogram);
343
344impl Histogram {
345    fn new(registry: &Registry, opts: Opts) -> Self {
346        let histogram = prometheus::Histogram::with_opts(opts.into()).unwrap();
347        registry.register(Box::new(histogram.clone())).unwrap();
348        Self(histogram)
349    }
350
351    pub fn sample_count(&self) -> usize {
352        self.0.get_sample_count() as usize
353    }
354
355    pub fn sum(&self) -> f64 {
356        self.0.get_sample_sum()
357    }
358
359    pub fn mean(&self) -> f64 {
360        self.sum() / (self.sample_count() as f64)
361    }
362}
363
364impl metrics::Histogram for Histogram {
365    fn add_point(&self, point: f64) {
366        self.0.observe(point);
367    }
368}
369
370/// A [CounterFamily](metrics::CounterFamily) metric.
371#[derive(Clone, Debug)]
372pub struct CounterFamily(GenericCounterVec<AtomicU64>);
373
374impl CounterFamily {
375    fn new(registry: &Registry, opts: Opts, labels: &[String]) -> Self {
376        let labels = labels.iter().map(String::as_str).collect::<Vec<_>>();
377        let family = GenericCounterVec::new(opts, &labels).unwrap();
378        registry.register(Box::new(family.clone())).unwrap();
379        Self(family)
380    }
381
382    pub fn get(&self, label_values: &[impl AsRef<str>]) -> Counter {
383        let labels = label_values.iter().map(AsRef::as_ref).collect::<Vec<_>>();
384        Counter(self.0.get_metric_with_label_values(&labels).unwrap())
385    }
386}
387
388impl metrics::MetricsFamily<Box<dyn metrics::Counter>> for CounterFamily {
389    fn create(&self, labels: Vec<String>) -> Box<dyn metrics::Counter> {
390        Box::new(self.get(&labels))
391    }
392
393    fn destroy(&self, labels: &[&str]) {
394        if let Err(err) = self.0.remove_label_values(labels) {
395            warn!(%err, "failed to remove prometheus counter")
396        }
397    }
398}
399
400/// A [GaugeFamily](metrics::GaugeFamily) metric.
401#[derive(Clone, Debug)]
402pub struct GaugeFamily(GenericGaugeVec<AtomicU64>);
403
404impl GaugeFamily {
405    fn new(registry: &Registry, opts: Opts, labels: &[String]) -> Self {
406        let labels = labels.iter().map(String::as_str).collect::<Vec<_>>();
407        let family = GenericGaugeVec::new(opts, &labels).unwrap();
408        registry.register(Box::new(family.clone())).unwrap();
409        Self(family)
410    }
411
412    pub fn get(&self, label_values: &[impl AsRef<str>]) -> Gauge {
413        let labels = label_values.iter().map(AsRef::as_ref).collect::<Vec<_>>();
414        Gauge(self.0.get_metric_with_label_values(&labels).unwrap())
415    }
416}
417
418impl metrics::MetricsFamily<Box<dyn metrics::Gauge>> for GaugeFamily {
419    fn create(&self, labels: Vec<String>) -> Box<dyn metrics::Gauge> {
420        Box::new(self.get(&labels))
421    }
422
423    fn destroy(&self, labels: &[&str]) {
424        if let Err(err) = self.0.remove_label_values(labels) {
425            warn!(%err, "failed to remove prometheus gauge")
426        }
427    }
428}
429
430/// A [HistogramFamily](metrics::HistogramFamily) metric.
431#[derive(Clone, Debug)]
432pub struct HistogramFamily(HistogramVec);
433
434impl HistogramFamily {
435    fn new(registry: &Registry, opts: Opts, labels: &[String]) -> Self {
436        let labels = labels.iter().map(String::as_str).collect::<Vec<_>>();
437        let family = HistogramVec::new(opts.into(), &labels).unwrap();
438        registry.register(Box::new(family.clone())).unwrap();
439        Self(family)
440    }
441
442    pub fn get(&self, label_values: &[impl AsRef<str>]) -> Histogram {
443        let labels = label_values.iter().map(AsRef::as_ref).collect::<Vec<_>>();
444        Histogram(self.0.get_metric_with_label_values(&labels).unwrap())
445    }
446}
447
448impl metrics::MetricsFamily<Box<dyn metrics::Histogram>> for HistogramFamily {
449    fn create(&self, labels: Vec<String>) -> Box<dyn metrics::Histogram> {
450        Box::new(self.get(&labels))
451    }
452
453    fn destroy(&self, labels: &[&str]) {
454        if let Err(err) = self.0.remove_label_values(labels) {
455            warn!(%err, "failed to remove prometheus histogram")
456        }
457    }
458}
459
460/// A [TextFamily](metrics::TextFamily) metric.
461#[derive(Clone, Debug)]
462pub struct TextFamily(GaugeFamily);
463
464impl TextFamily {
465    fn new(registry: &Registry, opts: Opts, labels: &[String]) -> Self {
466        Self(GaugeFamily::new(registry, opts, labels))
467    }
468}
469
470impl metrics::MetricsFamily<()> for TextFamily {
471    fn create(&self, labels: Vec<String>) {
472        self.0.create(labels).set(1);
473    }
474
475    fn destroy(&self, labels: &[&str]) {
476        self.0.destroy(labels)
477    }
478}
479
480#[cfg(test)]
481mod test {
482    use metrics::Metrics;
483    use tide_disco::metrics::Metrics as _;
484
485    use super::*;
486
487    #[test_log::test]
488    fn test_prometheus_metrics() {
489        let metrics = PrometheusMetrics::default();
490
491        // Register one metric of each type.
492        let counter = metrics.create_counter("counter".into(), None);
493        let gauge = metrics.create_gauge("gauge".into(), None);
494        let histogram = metrics.create_histogram("histogram".into(), None);
495        metrics.create_text("text".into());
496
497        // Set the metric values.
498        counter.add(20);
499        gauge.set(42);
500        histogram.add_point(20f64);
501
502        // Check the values.
503        assert_eq!(metrics.get_counter("counter").unwrap().get(), 20);
504        assert_eq!(metrics.get_gauge("gauge").unwrap().get(), 42);
505        assert_eq!(
506            metrics.get_histogram("histogram").unwrap().sample_count(),
507            1
508        );
509        assert_eq!(metrics.get_histogram("histogram").unwrap().sum(), 20f64);
510        assert_eq!(metrics.get_histogram("histogram").unwrap().mean(), 20f64);
511
512        // Set the metric values again, to be sure they update properly.
513        counter.add(22);
514        gauge.set(100);
515        histogram.add_point(22f64);
516
517        // Check the updated values.
518        assert_eq!(metrics.get_counter("counter").unwrap().get(), 42);
519        assert_eq!(metrics.get_gauge("gauge").unwrap().get(), 100);
520        assert_eq!(
521            metrics.get_histogram("histogram").unwrap().sample_count(),
522            2
523        );
524        assert_eq!(metrics.get_histogram("histogram").unwrap().sum(), 42f64);
525        assert_eq!(metrics.get_histogram("histogram").unwrap().mean(), 21f64);
526
527        // Export to a Prometheus string.
528        let string = metrics.export().unwrap();
529        // Make sure the output makes sense.
530        let lines = string.lines().collect::<Vec<_>>();
531        assert!(lines.contains(&"counter 42"));
532        assert!(lines.contains(&"gauge 100"));
533        assert!(lines.contains(&"histogram_sum 42"));
534        assert!(lines.contains(&"histogram_count 2"));
535        assert!(lines.contains(&"text 1"));
536    }
537
538    #[test_log::test]
539    fn test_namespace() {
540        let metrics = PrometheusMetrics::default();
541        let subgroup1 = metrics.subgroup("subgroup1".into());
542        let subgroup2 = subgroup1.subgroup("subgroup2".into());
543        let counter = subgroup2.create_counter("counter".into(), None);
544        subgroup2.create_text("text".into());
545        counter.add(42);
546
547        // Check namespacing.
548        assert_eq!(
549            metrics.get_subgroup(["subgroup1"]).unwrap().namespace,
550            ["subgroup1"]
551        );
552        assert_eq!(
553            metrics
554                .get_subgroup(["subgroup1", "subgroup2"])
555                .unwrap()
556                .namespace,
557            ["subgroup1", "subgroup2"]
558        );
559        assert_eq!(
560            metrics
561                .get_subgroup(["subgroup1"])
562                .unwrap()
563                .get_subgroup(["subgroup2"])
564                .unwrap()
565                .namespace,
566            ["subgroup1", "subgroup2"]
567        );
568
569        // Check different ways of accessing the counter.
570        assert_eq!(
571            metrics
572                .get_subgroup(["subgroup1", "subgroup2"])
573                .unwrap()
574                .get_counter("counter")
575                .unwrap()
576                .get(),
577            42
578        );
579        assert_eq!(
580            metrics
581                .get_subgroup(["subgroup1"])
582                .unwrap()
583                .get_subgroup(["subgroup2"])
584                .unwrap()
585                .get_counter("counter")
586                .unwrap()
587                .get(),
588            42
589        );
590
591        // Check fully-qualified counter name in export.
592        assert!(
593            metrics
594                .export()
595                .unwrap()
596                .lines()
597                .contains(&"subgroup1_subgroup2_counter 42")
598        );
599
600        // Check fully-qualified text name in export.
601        assert!(
602            metrics
603                .export()
604                .unwrap()
605                .lines()
606                .contains(&"subgroup1_subgroup2_text 1")
607        );
608    }
609
610    #[test_log::test]
611    fn test_labels() {
612        let metrics = PrometheusMetrics::default();
613
614        let http_count = metrics.counter_family("http".into(), vec!["method".into()]);
615        let get_count = http_count.create(vec!["GET".into()]);
616        let post_count = http_count.create(vec!["POST".into()]);
617        get_count.add(1);
618        post_count.add(2);
619
620        metrics
621            .text_family("version".into(), vec!["semver".into(), "rev".into()])
622            .create(vec!["0.1.0".into(), "d1b650a7".into()]);
623
624        assert_eq!(
625            metrics
626                .get_counter_family("http")
627                .unwrap()
628                .get(&["GET"])
629                .get(),
630            1
631        );
632        assert_eq!(
633            metrics
634                .get_counter_family("http")
635                .unwrap()
636                .get(&["POST"])
637                .get(),
638            2
639        );
640
641        // Export to a Prometheus string.
642        let string = metrics.export().unwrap();
643        // Make sure the output makes sense.
644        let lines = string.lines().collect::<Vec<_>>();
645        assert!(lines.contains(&"http{method=\"GET\"} 1"), "{lines:?}");
646        assert!(lines.contains(&"http{method=\"POST\"} 2"), "{lines:?}");
647        assert!(
648            lines.contains(&"version{rev=\"d1b650a7\",semver=\"0.1.0\"} 1"),
649            "{lines:?}"
650        );
651    }
652
653    #[test_log::test]
654    fn test_destroy() {
655        let metrics = PrometheusMetrics::default();
656
657        let counters = metrics.counter_family("requests".into(), vec!["peer".into()]);
658        counters.create(vec!["alice".into()]).add(1);
659        counters.create(vec!["bob".into()]).add(2);
660
661        let gauges = Metrics::gauge_family(&metrics, "queue".into(), vec!["peer".into()]);
662        gauges.create(vec!["alice".into()]).set(7);
663        gauges.create(vec!["bob".into()]).set(9);
664
665        let histograms = metrics.histogram_family("latency".into(), vec!["peer".into()]);
666        histograms.create(vec!["alice".into()]).add_point(1.0);
667        histograms.create(vec!["bob".into()]).add_point(2.0);
668
669        let texts = metrics.text_family("version".into(), vec!["peer".into()]);
670        texts.create(vec!["alice".into()]);
671        texts.create(vec!["bob".into()]);
672
673        // Before destroy: both peers are present in the export.
674        let before = metrics.export().unwrap();
675        assert!(before.contains("requests{peer=\"alice\"} 1"), "{before}");
676        assert!(before.contains("requests{peer=\"bob\"} 2"), "{before}");
677        assert!(before.contains("queue{peer=\"alice\"} 7"), "{before}");
678        assert!(before.contains("queue{peer=\"bob\"} 9"), "{before}");
679        assert!(before.contains("latency_count{peer=\"alice\"}"), "{before}");
680        assert!(before.contains("latency_count{peer=\"bob\"}"), "{before}");
681        assert!(before.contains("version{peer=\"alice\"} 1"), "{before}");
682        assert!(before.contains("version{peer=\"bob\"} 1"), "{before}");
683
684        // Destroy alice from every family.
685        counters.destroy(&["alice"]);
686        gauges.destroy(&["alice"]);
687        histograms.destroy(&["alice"]);
688        texts.destroy(&["alice"]);
689
690        // After destroy: alice is gone, bob is untouched.
691        let after = metrics.export().unwrap();
692        assert!(!after.contains("peer=\"alice\""), "{after}");
693        assert!(after.contains("requests{peer=\"bob\"} 2"), "{after}");
694        assert!(after.contains("queue{peer=\"bob\"} 9"), "{after}");
695        assert!(after.contains("latency_count{peer=\"bob\"}"), "{after}");
696        assert!(after.contains("version{peer=\"bob\"} 1"), "{after}");
697
698        // Destroying a non-existent variant is a no-op (just logs a warning).
699        counters.destroy(&["nobody"]);
700        gauges.destroy(&["nobody"]);
701        histograms.destroy(&["nobody"]);
702        texts.destroy(&["nobody"]);
703    }
704}
hotshot_query_service/metrics.rs

hotshot_query_service/
metrics.rs
