Add exemplar support

.github/workflows/ci.yml

@@ -41,3 +41,4 @@ jobs:
       - run: cargo build --package example-axum
       - run: cargo build --package example-custom-metrics
       - run: cargo build --package example-full-api
+      - run: cargo build --package example-tracing

CHANGELOG.md

@@ -14,6 +14,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `ResultLabels` derive macro allows to specify on an enum whether variants should
   always be "ok", or "error" for the success rate metrics of functions using them. (#61)
 - Support the official `prometheus-client` crate for producing metrics
+- Exemplar support when using the feature flags `exemplars-tracing` and `prometheus-client`.
+  Autometrics now provides a `tracing_subscriber::Layer` that makes the `trace_id` available
+  to the library, and it will automatically be added to the counter and histogram metrics
+  as an exemplar
 
 ### Changed
 

autometrics/Cargo.toml

@@ -24,6 +24,7 @@ prometheus-exporter = [
   "dep:prometheus",
   "thiserror"
 ]
+exemplars-tracing = ["tracing", "tracing-subscriber"]
 custom-objective-percentile = []
 custom-objective-latency = []
 
@@ -48,11 +49,18 @@ thiserror = { version = "1", optional = true }
 # Used for prometheus-client feature
 prometheus-client = { version = "0.21.1", optional = true }
 
+# Used for exemplars-tracing feature
+tracing = { version = "0.1", optional = true }
+tracing-subscriber = { version = "0.3", default-features = false, optional = true }
+
 [dev-dependencies]
 axum = { version = "0.6", features = ["tokio"] }
 http = "0.2"
 tokio = { version = "1", features = ["full"] }
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", default-features = false }
 trybuild = "1.0"
+uuid = { version = "1", features = ["v4"] }
 vergen = { version = "8.1", features = ["git", "gitcl"] }
 
 [package.metadata.docs.rs]

autometrics/README.md

@@ -110,6 +110,7 @@ fn main() {
 ### Feature flags
 
 - `prometheus-exporter` - exports a Prometheus metrics collector and exporter (compatible with any of the Metrics Libraries)
+- `exemplars-tracing` - extract the `trace_id` field from [`tracing::Span`](https://docs.rs/tracing/latest/tracing/struct.Span.html)s and attach it as an [exemplar](https://grafana.com/docs/grafana/latest/fundamentals/exemplars/) for the metrics produced by Autometrics. This is currently only supported with the `prometheus-client` feature, because only that crate has support for exemplars. Note that Prometheus must be specifically [configured](https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage) to use the experimental exemplars feature.
 - `custom-objective-latency` - by default, Autometrics only supports a fixed set of latency thresholds for objectives. Enable this to use custom latency thresholds. Note, however, that the custom latency **must** match one of the buckets configured for your histogram or the alerts will not work. This is not currently compatible with the `prometheus` or `prometheus-exporter` feature.
 - `custom-objective-percentile` by default, Autometrics only supports a fixed set of objective percentiles. Enable this to use a custom percentile. Note, however, that using custom percentiles requires generating a different recording and alerting rules file using the CLI + Sloth (see [here](https://github.com/autometrics-dev/autometrics-rs/tree/main/autometrics-cli)).
 

autometrics/src/integrations/mod.rs

@@ -0,0 +1,7 @@
+#[cfg(feature = "prometheus-client")]
+pub mod prometheus_client {
+    pub use crate::tracker::prometheus_client::REGISTRY;
+}
+
+#[cfg(feature = "exemplars-tracing")]
+pub mod tracing;

autometrics/src/integrations/tracing.rs

@@ -0,0 +1,95 @@
+//! Tracing integration for autometrics
+//!
+//! This module enables autometrics to use the `trace_id` field from the current span as an exemplar.
+//! Exemplars are a newer Prometheus / OpenMetrics / OpenTelemetry feature that allows you to associate
+//! specific traces with a given metric. This enables you to dig into the specifics that produced
+//! a certain metric by looking at a detailed example.
+//!
+//! # Example
+//! ```rust
+//! use autometrics::{autometrics, integrations::tracing::AutometricsLayer};
+//! use tracing::{instrument, trace};
+//! use tracing_subscriber::prelude::*;
+//! use uuid::Uuid;
+//!
+//! #[autometrics]
+//! #[instrument(fields(trace_id = %Uuid::new_v4())]
+//! fn my_function() {
+//!     trace!("Hello world!");
+//! }
+//!
+//! fn main() {
+//!     tracing_subscriber::fmt::fmt()
+//!         .finish()
+//!         .with(AutometricsLayer::default())
+//!         .init();
+//! }
+//! ```
+
+use tracing::field::{Field, Visit};
+use tracing::{span::Attributes, Id, Subscriber};
+use tracing_subscriber::layer::{Context, Layer};
+use tracing_subscriber::registry::{LookupSpan, Registry};
+
+#[derive(Clone, Hash, PartialEq, Eq, Debug, Default)]
+#[cfg_attr(
+    feature = "prometheus-client",
+    derive(prometheus_client::encoding::EncodeLabelSet)
+)]
+pub(crate) struct TraceLabel {
+    trace_id: String,
+}
+
+impl Visit for TraceLabel {
+    fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
+        if field.name() == "trace_id" {
+            self.trace_id = format!("{:?}", value);
+        }
+    }
+}
+
+/// Get the exemplar from the current tracing span
+pub(crate) fn get_exemplar() -> Option<TraceLabel> {
+    let span = tracing::span::Span::current();
+
+    span.with_subscriber(|(id, sub)| {
+        sub.downcast_ref::<Registry>()
+            .and_then(|reg| reg.span(id))
+            .and_then(|span| {
+                span.scope()
+                    .find_map(|span| span.extensions().get::<TraceLabel>().cloned())
+            })
+    })
+    .flatten()
+}
+
+/// A tracing [`Layer`] that enables autometrics to use the `trace_id` field from the current span
+/// as an exemplar for Prometheus metrics.
+///
+/// # Example
+/// ```rust
+/// use autometrics::integrations::tracing::AutometricsLayer;
+///
+/// fn main() {
+///     tracing_subscriber::fmt::fmt()
+///         .finish()
+///         .with(AutometricsLayer::default())
+///         .init();
+/// }
+/// ```
+#[derive(Clone, Default)]
+pub struct AutometricsLayer();
+
+impl<S: Subscriber + for<'lookup> LookupSpan<'lookup>> Layer<S> for AutometricsLayer {
+    fn on_new_span(&self, attrs: &Attributes<'_>, id: &Id, ctx: Context<'_, S>) {
+        let mut trace_label = TraceLabel::default();
+        attrs.values().record(&mut trace_label);
+
+        if !trace_label.trace_id.is_empty() {
+            if let Some(span) = ctx.span(id) {
+                let mut ext = span.extensions_mut();
+                ext.insert(trace_label);
+            }
+        }
+    }
+}

autometrics/src/lib.rs

@@ -6,6 +6,8 @@
 #![doc = include_str!("../README.md")]
 
 mod constants;
+/// Functionality specific to the libraries used to collect metrics
+pub mod integrations;
 mod labels;
 pub mod objectives;
 #[cfg(feature = "prometheus-exporter")]
@@ -186,14 +188,6 @@ pub use autometrics_macros::ResultLabels;
 #[cfg(feature = "prometheus-exporter")]
 pub use self::prometheus_exporter::*;
 
-/// Functionality specific to the libraries used to collect metrics
-pub mod integrations {
-    #[cfg(feature = "prometheus-client")]
-    pub mod prometheus_client {
-        pub use crate::tracker::prometheus_client::REGISTRY;
-    }
-}
-
 /// We use the histogram buckets recommended by the OpenTelemetry specification
 /// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/sdk.md#explicit-bucket-histogram-aggregation
 #[cfg(any(

autometrics/src/tracker/mod.rs

@@ -35,6 +35,9 @@ pub use self::prometheus_client::PrometheusClientTracker;
 ))]
 compile_error!("Only one of the metrics, opentelemetry, prometheus, or prometheus-client features can be enabled at a time");
 
+#[cfg(all(feature = "exemplars-tracing", not(feature = "prometheus-client")))]
+compile_error!("The exemplars-tracing feature can only be used with the `prometheus-client` metrics library because that is the only one that currently supports exemplars");
+
 pub trait TrackMetrics {
     fn set_build_info(build_info_labels: &BuildInfoLabels);
     fn start(gauge_labels: Option<&GaugeLabels>) -> Self;

autometrics/src/tracker/prometheus_client.rs

@@ -1,25 +1,39 @@
 use super::TrackMetrics;
+#[cfg(feature = "exemplars-tracing")]
+use crate::integrations::tracing::{get_exemplar, TraceLabel};
 use crate::labels::{BuildInfoLabels, CounterLabels, GaugeLabels, HistogramLabels};
 use crate::{constants::*, HISTOGRAM_BUCKETS};
 use once_cell::sync::Lazy;
-use prometheus_client::metrics::{
-    counter::Counter, family::Family, gauge::Gauge, histogram::Histogram,
-};
+#[cfg(feature = "exemplars-tracing")]
+use prometheus_client::metrics::exemplar::{CounterWithExemplar, HistogramWithExemplars};
+#[cfg(not(feature = "exemplars-tracing"))]
+use prometheus_client::metrics::{counter::Counter, histogram::Histogram};
+use prometheus_client::metrics::{family::Family, gauge::Gauge};
 use prometheus_client::registry::Registry;
 use std::time::Instant;
 
+#[cfg(feature = "exemplars-tracing")]
+type CounterType = CounterWithExemplar<TraceLabel>;
+#[cfg(not(feature = "exemplars-tracing"))]
+type CounterType = Counter;
+
+#[cfg(feature = "exemplars-tracing")]
+type HistogramType = HistogramWithExemplars<TraceLabel>;
+#[cfg(not(feature = "exemplars-tracing"))]
+type HistogramType = Histogram;
+
 static REGISTRY_AND_METRICS: Lazy<(Registry, Metrics)> = Lazy::new(|| {
     let mut registry = <Registry>::default();
 
-    let counter = Family::<CounterLabels, Counter>::default();
+    let counter = Family::<CounterLabels, CounterType>::default();
     registry.register(
         COUNTER_NAME_PROMETHEUS,
         COUNTER_DESCRIPTION,
         counter.clone(),
     );
 
-    let histogram = Family::<HistogramLabels, Histogram>::new_with_constructor(|| {
-        Histogram::new(HISTOGRAM_BUCKETS.into_iter())
+    let histogram = Family::<HistogramLabels, HistogramType>::new_with_constructor(|| {
+        HistogramType::new(HISTOGRAM_BUCKETS.into_iter())
     });
     registry.register(
         HISTOGRAM_NAME_PROMETHEUS,
@@ -45,10 +59,11 @@ static REGISTRY_AND_METRICS: Lazy<(Registry, Metrics)> = Lazy::new(|| {
 });
 /// The [`Registry`] used to collect metrics when the `prometheus-client` feature is enabled
 pub static REGISTRY: Lazy<&Registry> = Lazy::new(|| &REGISTRY_AND_METRICS.0);
+static METRICS: Lazy<&Metrics> = Lazy::new(|| &REGISTRY_AND_METRICS.1);
 
 struct Metrics {
-    counter: Family<CounterLabels, Counter>,
-    histogram: Family<HistogramLabels, Histogram>,
+    counter: Family<CounterLabels, CounterType>,
+    histogram: Family<HistogramLabels, HistogramType>,
     gauge: Family<GaugeLabels, Gauge>,
     build_info: Family<BuildInfoLabels, Gauge>,
 }
@@ -60,20 +75,12 @@ pub struct PrometheusClientTracker {
 
 impl TrackMetrics for PrometheusClientTracker {
     fn set_build_info(build_info_labels: &BuildInfoLabels) {
-        REGISTRY_AND_METRICS
-            .1
-            .build_info
-            .get_or_create(&build_info_labels)
-            .set(1);
+        METRICS.build_info.get_or_create(&build_info_labels).set(1);
     }
 
     fn start(gauge_labels: Option<&GaugeLabels>) -> Self {
         if let Some(gauge_labels) = gauge_labels {
-            REGISTRY_AND_METRICS
-                .1
-                .gauge
-                .get_or_create(&gauge_labels)
-                .inc();
+            METRICS.gauge.get_or_create(&gauge_labels).inc();
         }
         Self {
             gauge_labels: gauge_labels.cloned(),
@@ -82,22 +89,24 @@ impl TrackMetrics for PrometheusClientTracker {
     }
 
     fn finish(self, counter_labels: &CounterLabels, histogram_labels: &HistogramLabels) {
-        REGISTRY_AND_METRICS
-            .1
-            .counter
-            .get_or_create(&counter_labels)
-            .inc();
-        REGISTRY_AND_METRICS
-            .1
-            .histogram
-            .get_or_create(&histogram_labels)
-            .observe(self.start_time.elapsed().as_secs_f64());
+        #[cfg(feature = "exemplars-tracing")]
+        let exemplar = get_exemplar();
+
+        let counter = METRICS.counter.get_or_create(&counter_labels);
+        #[cfg(feature = "exemplars-tracing")]
+        counter.inc_by(1, exemplar.clone());
+        #[cfg(not(feature = "exemplars-tracing"))]
+        counter.inc();
+
+        let histogram = METRICS.histogram.get_or_create(&histogram_labels);
+        let duration = self.start_time.elapsed().as_secs_f64();
+        #[cfg(feature = "exemplars-tracing")]
+        histogram.observe(duration, exemplar);
+        #[cfg(not(feature = "exemplars-tracing"))]
+        histogram.observe(duration);
+
         if let Some(gauge_labels) = self.gauge_labels {
-            REGISTRY_AND_METRICS
-                .1
-                .gauge
-                .get_or_create(&gauge_labels)
-                .dec();
+            METRICS.gauge.get_or_create(&gauge_labels).dec();
         }
     }
 }

examples/README.md

@@ -11,6 +11,7 @@ cargo run --package example-{name of example}
 
 - [axum](./axum) - Use autometrics to instrument HTTP handlers
 - [custom-metrics](./custom-metrics/) - Define your own custom metrics alongside the ones generated by autometrics (using any of the metrics collection crates)
+- [tracing](./tracing/) - Use the `trace_id` from `tracing::Span`s as Prometheus exemplars
 
 ## Full Example
 

examples/tracing/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "example-tracing"
+version = "0.0.0"
+publish = false
+edition = "2021"
+
+[dependencies]
+autometrics = { path = "../../autometrics", features = ["prometheus-client", "prometheus-exporter", "exemplars-tracing"] }
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+uuid = { version = "1.3", features = ["v4"] }

examples/tracing/README.md

@@ -0,0 +1,7 @@
+# Tracing Exemplars Example
+
+This example demonstrates how Autometrics can pick up the `trace_id` from [`tracing::Span`](https://docs.rs/tracing/latest/tracing/struct.Span.html)s and attach them to the metrics as [exemplars](https://grafana.com/docs/grafana/latest/fundamentals/exemplars/).
+
+> **Note**
+>
+> Prometheus must be [specifically configured](https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage) to enable the experimental exemplars feature.

examples/tracing/src/main.rs

@@ -0,0 +1,33 @@
+use autometrics::{autometrics, encode_global_metrics, integrations::tracing::AutometricsLayer};
+use tracing::{instrument, trace};
+use tracing_subscriber::{prelude::*, EnvFilter};
+use uuid::Uuid;
+
+// Autometrics looks for a field called `trace_id` and attaches
+// that as an exemplar for the metrics it generates.
+#[autometrics]
+#[instrument(fields(trace_id = %Uuid::new_v4(), foo = "bar"))]
+fn outer_function() {
+    trace!("Outer function called");
+    inner_function("hello")
+}
+
+#[autometrics]
+#[instrument]
+fn inner_function(param: &str) {
+    trace!("Inner function called");
+}
+
+fn main() {
+    tracing_subscriber::fmt::fmt()
+        .finish()
+        .with(EnvFilter::from_default_env())
+        .with(AutometricsLayer::default())
+        .init();
+
+    for _i in 0..10 {
+        outer_function();
+    }
+
+    println!("{}", encode_global_metrics().unwrap());
+}