Add exemplar support

.github/workflows/ci.yml

@@ -26,8 +26,8 @@ jobs:
 
       # Run the tests with each of the different metrics libraries
       - run: cargo test --features=prometheus-exporter,metrics
-      - run: cargo test --features=prometheus-exporter,prometheus --tests
-      - run: cargo test --features=prometheus-exporter,prometheus-client --tests
+      - run: cargo test --features=prometheus-exporter,prometheus
+      - run: cargo test --features=prometheus-exporter,prometheus-client,exemplars-tracing
       # The tests using opentelemetry are currently failing because of this issue:
       # https://github.com/open-telemetry/opentelemetry-rust/issues/1070
       # We should remove the continue-on-error once that is fixed
@@ -41,3 +41,4 @@ jobs:
       - run: cargo build --package example-axum
       - run: cargo build --package example-custom-metrics
       - run: cargo build --package example-full-api
+      - run: cargo build --package example-exemplars-tracing

CHANGELOG.md

@@ -14,6 +14,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `ResultLabels` derive macro allows to specify on an enum whether variants should
   always be "ok", or "error" for the success rate metrics of functions using them. (#61)
 - Support the official `prometheus-client` crate for producing metrics
+- Exemplar support when using the feature flags `exemplars-tracing` and `prometheus-client`.
+  Autometrics now provides a `tracing_subscriber::Layer` that makes the specific `Span` fields
+  available to the library, and autometrics will automatically attach those fields as exemplars
+  on the counter and histogram metrics
 
 ### Changed
 

README.md

@@ -55,6 +55,7 @@ https://github.com/autometrics-dev/autometrics-rs/assets/3262610/966ed140-1d6c-4
 - [🚨 Define alerts](https://docs.rs/autometrics/latest/autometrics/objectives/index.html) using SLO best practices directly in your source code
 - [📊 Grafana dashboards](https://github.com/autometrics-dev#5-configuring-prometheus) work out of the box to visualize the performance of instrumented functions & SLOs
 - [⚙️ Configurable](https://docs.rs/autometrics/latest/autometrics/#metrics-libraries) metric collection library ([`opentelemetry`](https://crates.io/crates/opentelemetry), [`prometheus`](https://crates.io/crates/prometheus), [`prometheus-client`](https://crates.io/crates/prometheus-client) or [`metrics`](https://crates.io/crates/metrics))
+- [📍 Attach exemplars](https://docs.rs/autometrics/latest/autometrics/exemplars/index.html) to connect metrics with distributed traces
 - ⚡ Minimal runtime overhead
 
 See [Why Autometrics?](https://github.com/autometrics-dev#4-why-autometrics) for more details on the ideas behind autometrics.

autometrics/Cargo.toml

@@ -24,6 +24,7 @@ prometheus-exporter = [
   "dep:prometheus",
   "thiserror"
 ]
+exemplars-tracing = ["tracing", "tracing-subscriber"]
 custom-objective-percentile = []
 custom-objective-latency = []
 
@@ -48,11 +49,18 @@ thiserror = { version = "1", optional = true }
 # Used for prometheus-client feature
 prometheus-client = { version = "0.21.1", optional = true }
 
+# Used for exemplars-tracing feature
+tracing = { version = "0.1", optional = true }
+tracing-subscriber = { version = "0.3", default-features = false, features = ["registry"], optional = true }
+
 [dev-dependencies]
 axum = { version = "0.6", features = ["tokio"] }
 http = "0.2"
 tokio = { version = "1", features = ["full"] }
+tracing = "0.1"
+tracing-subscriber = "0.3"
 trybuild = "1.0"
+uuid = { version = "1", features = ["v4"] }
 vergen = { version = "8.1", features = ["git", "gitcl"] }
 
 [package.metadata.docs.rs]

autometrics/README.md

@@ -54,6 +54,7 @@ pub async fn main() {
 - [🚨 Define alerts](https://docs.rs/autometrics/latest/autometrics/objectives/index.html) using SLO best practices directly in your source code
 - [📊 Grafana dashboards](https://github.com/autometrics-dev#5-configuring-prometheus) work out of the box to visualize the performance of instrumented functions & SLOs
 - [⚙️ Configurable](#metrics-libraries) metric collection library ([`opentelemetry`](https://crates.io/crates/opentelemetry), [`prometheus`](https://crates.io/crates/prometheus), [`prometheus-client`](https://crates.io/crates/prometheus-client) or [`metrics`](https://crates.io/crates/metrics))
+- [📍 Attach exemplars](https://docs.rs/autometrics/latest/autometrics/exemplars/index.html) to connect metrics with distributed traces
 - ⚡ Minimal runtime overhead
 
 See [Why Autometrics?](https://github.com/autometrics-dev#4-why-autometrics) for more details on the ideas behind autometrics.
@@ -110,6 +111,7 @@ fn main() {
 ### Feature flags
 
 - `prometheus-exporter` - exports a Prometheus metrics collector and exporter (compatible with any of the Metrics Libraries)
+- `exemplars-tracing` - extract fields from [`tracing::Span`](https://docs.rs/tracing/latest/tracing/struct.Span.html)s and attach them as [exemplars](https://grafana.com/docs/grafana/latest/fundamentals/exemplars/) for the metrics produced by Autometrics. This is currently only supported with the `prometheus-client` feature due to lack of support in the other metrics libraries. Note that Prometheus must be specifically [configured](https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage) to enable the exemplars feature.
 - `custom-objective-latency` - by default, Autometrics only supports a fixed set of latency thresholds for objectives. Enable this to use custom latency thresholds. Note, however, that the custom latency **must** match one of the buckets configured for your histogram or the alerts will not work. This is not currently compatible with the `prometheus` or `prometheus-exporter` feature.
 - `custom-objective-percentile` by default, Autometrics only supports a fixed set of objective percentiles. Enable this to use a custom percentile. Note, however, that using custom percentiles requires generating a different recording and alerting rules file using the CLI + Sloth (see [here](https://github.com/autometrics-dev/autometrics-rs/tree/main/autometrics-cli)).
 

autometrics/src/exemplars/mod.rs

@@ -0,0 +1,49 @@
+//! Connect metrics to traces using exemplars
+//!
+//! Exemplars are a newer Prometheus / OpenMetrics / OpenTelemetry feature that allows you to associate
+//! specific traces or samples with a given metric. This enables you to investigate what caused metrics
+//! to change by looking at individual examples that contributed to the metrics.
+//!
+//! Autometrics integrates with tracing libraries to extract details from the
+//! current span context and attach them as exemplars to the metrics it generates.
+//!
+//! See each of the submodules for details on how to integrate with each tracing library.
+//!
+//! # Supported Metrics Libraries
+//!
+//! Exemplars are currently only supported with the `prometheus-client` metrics library,
+//! because that is the only one that currently supports producing metrics with exemplars.
+//!
+//! # Exposing Metrics to Prometheus with Exemplars
+//!
+//! To enable Prometheus to scrape metrics with exemplars you must:
+//! 1. Run Prometheus with the [`--enable-feature=exemplar-storage`](https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage) flag
+//! 2. Set the `Content-Type` header on the response from the Prometheus metrics endpoint to indicate
+//!    it is using the OpenMetrics exposition format instead of the default Prometheus format.
+//!   ```http
+//!   Content-Type: application/openmetrics-text; version=1.0.0; charset=utf-8
+//!   ```
+//!
+//!  ```rust
+//!   use http::{header::CONTENT_TYPE, Response};
+//!
+//!   /// Expose the metrics to Prometheus in the OpenMetrics format
+//!   async fn get_metrics() -> Response<String> {
+//!       match autometrics::encode_global_metrics() {
+//!           Ok(metrics) => Response::builder()
+//!               .header(
+//!                   CONTENT_TYPE,
+//!                   "application/openmetrics-text; version=1.0.0; charset=utf-8",
+//!               )
+//!               .body(metrics)
+//!               .unwrap(),
+//!           Err(err) => Response::builder()
+//!               .status(500)
+//!               .body(err.to_string())
+//!               .unwrap(),
+//!       }
+//!   }
+//!   ```
+
+#[cfg(feature = "exemplars-tracing")]
+pub mod tracing;

autometrics/src/exemplars/tracing.rs

@@ -0,0 +1,123 @@
+//! Extract fields from [`tracing::Span`]s as exemplars.
+//!
+//! This module enables autometrics to use fields from the current [`Span`] as exemplar labels.
+//!
+//! # Example
+//!
+//! ```rust
+//! use autometrics::autometrics;
+//! use autometrics::exemplars::tracing::AutometricsExemplarExtractor;
+//! use tracing::{instrument, trace};
+//! use tracing_subscriber::prelude::*;
+//! use uuid::Uuid;
+//!
+//! #[autometrics]
+//! #[instrument(fields(trace_id = %Uuid::new_v4()))]
+//! fn my_function() {
+//!     trace!("Hello world!");
+//! }
+//!
+//! fn main() {
+//!     tracing_subscriber::fmt::fmt()
+//!         .finish()
+//!         .with(AutometricsExemplarExtractor::from_fields(&["trace_id"]))
+//!         .init();
+//! }
+//! ```
+//!
+//! [`Span`]: tracing::Span
+
+use std::collections::HashMap;
+use tracing::field::{Field, Visit};
+use tracing::{span::Attributes, Id, Subscriber};
+use tracing_subscriber::layer::{Context, Layer};
+use tracing_subscriber::registry::{LookupSpan, Registry};
+
+pub(crate) type TraceLabels = HashMap<&'static str, String>;
+
+/// Get the exemplar from the current tracing span
+pub(crate) fn get_exemplar() -> Option<TraceLabels> {
+    let span = tracing::span::Span::current();
+
+    span.with_subscriber(|(id, sub)| {
+        sub.downcast_ref::<Registry>()
+            .and_then(|reg| reg.span(id))
+            .and_then(|span| {
+                span.scope()
+                    .find_map(|span| span.extensions().get::<TraceLabels>().cloned())
+            })
+    })
+    .flatten()
+}
+
+/// A [`tracing_subscriber::Layer`] that enables autometrics to use fields from the current span as exemplars for
+/// the metrics it produces.
+///
+/// # Example
+/// ```rust
+/// use autometrics::exemplars::tracing::AutometricsExemplarExtractor;
+/// use tracing_subscriber::prelude::*;
+///
+/// fn main() {
+///     tracing_subscriber::fmt::fmt()
+///         .finish()
+///         .with(AutometricsExemplarExtractor::from_fields(&["trace_id"]))
+///         .init();
+/// }
+/// ```
+#[derive(Clone)]
+pub struct AutometricsExemplarExtractor {
+    fields: &'static [&'static str],
+}
+
+impl AutometricsExemplarExtractor {
+    /// Create a new [`AutometricsExemplarExtractor`] that will extract the given fields from the current [`Span`] scope
+    /// to use as the labels for the exemplars.
+    ///
+    /// [`Span`]: tracing::Span
+    pub fn from_fields(fields: &'static [&'static str]) -> Self {
+        Self { fields }
+    }
+}
+
+impl<S: Subscriber + for<'lookup> LookupSpan<'lookup>> Layer<S> for AutometricsExemplarExtractor {
+    fn on_new_span(&self, attrs: &Attributes<'_>, id: &Id, ctx: Context<'_, S>) {
+        let mut visitor = TraceLabelVisitor::new(self.fields);
+        attrs.values().record(&mut visitor);
+
+        if !visitor.labels.is_empty() {
+            if let Some(span) = ctx.span(id) {
+                let mut ext = span.extensions_mut();
+                ext.insert(visitor.labels);
+            }
+        }
+    }
+}
+
+struct TraceLabelVisitor {
+    fields: &'static [&'static str],
+    labels: TraceLabels,
+}
+
+impl TraceLabelVisitor {
+    fn new(fields: &'static [&'static str]) -> Self {
+        Self {
+            fields,
+            labels: HashMap::with_capacity(fields.len()),
+        }
+    }
+}
+
+impl Visit for TraceLabelVisitor {
+    fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
+        if self.fields.contains(&field.name()) && !self.labels.contains_key(field.name()) {
+            self.labels.insert(field.name(), format!("{:?}", value));
+        }
+    }
+
+    fn record_str(&mut self, field: &Field, value: &str) {
+        if self.fields.contains(&field.name()) && !self.labels.contains_key(field.name()) {
+            self.labels.insert(field.name(), value.to_string());
+        }
+    }
+}

autometrics/src/integrations/mod.rs

@@ -0,0 +1,9 @@
+//! Functionality that is specific to an optional dependency
+
+/// Access the [`Registry`] used to collect metrics when the `prometheus-client` feature is enabled
+///
+/// [`Registry`]: ::prometheus_client::registry::Registry
+#[cfg(feature = "prometheus-client")]
+pub mod prometheus_client {
+    pub use crate::tracker::prometheus_client::REGISTRY;
+}

autometrics/src/lib.rs

@@ -6,6 +6,9 @@
 #![doc = include_str!("../README.md")]
 
 mod constants;
+#[cfg(feature = "exemplars-tracing")]
+pub mod exemplars;
+pub mod integrations;
 mod labels;
 pub mod objectives;
 #[cfg(feature = "prometheus-exporter")]
@@ -186,14 +189,6 @@ pub use autometrics_macros::ResultLabels;
 #[cfg(feature = "prometheus-exporter")]
 pub use self::prometheus_exporter::*;
 
-/// Functionality specific to the libraries used to collect metrics
-pub mod integrations {
-    #[cfg(feature = "prometheus-client")]
-    pub mod prometheus_client {
-        pub use crate::tracker::prometheus_client::REGISTRY;
-    }
-}
-
 /// We use the histogram buckets recommended by the OpenTelemetry specification
 /// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/sdk.md#explicit-bucket-histogram-aggregation
 #[cfg(any(

autometrics/src/tracker/mod.rs

@@ -18,23 +18,29 @@ pub use self::prometheus::PrometheusTracker;
 #[cfg(feature = "prometheus-client")]
 pub use self::prometheus_client::PrometheusClientTracker;
 
-#[cfg(any(
-    all(
-        feature = "metrics",
-        any(
+#[cfg(all(
+    not(doc),
+    any(
+        all(
+            feature = "metrics",
+            any(
+                feature = "opentelemetry",
+                feature = "prometheus",
+                feature = "prometheus-client"
+            )
+        ),
+        all(
             feature = "opentelemetry",
-            feature = "prometheus",
-            feature = "prometheus-client"
-        )
-    ),
-    all(
-        feature = "opentelemetry",
-        any(feature = "prometheus", feature = "prometheus-client")
-    ),
-    all(feature = "prometheus", feature = "prometheus-client")
+            any(feature = "prometheus", feature = "prometheus-client")
+        ),
+        all(feature = "prometheus", feature = "prometheus-client")
+    )
 ))]
 compile_error!("Only one of the metrics, opentelemetry, prometheus, or prometheus-client features can be enabled at a time");
 
+#[cfg(all(feature = "exemplars-tracing", not(feature = "prometheus-client")))]
+compile_error!("The exemplars-tracing feature can only be used with the `prometheus-client` metrics library because that is the only one that currently supports exemplars");
+
 pub trait TrackMetrics {
     fn set_build_info(build_info_labels: &BuildInfoLabels);
     fn start(gauge_labels: Option<&GaugeLabels>) -> Self;