Support OpenTelemetry span context for exemplars

.github/workflows/ci.yml

@@ -28,6 +28,7 @@ jobs:
       - run: cargo test --features=prometheus-exporter,metrics
       - run: cargo test --features=prometheus-exporter,prometheus
       - run: cargo test --features=prometheus-exporter,prometheus-client,exemplars-tracing
+      - run: cargo test --features=prometheus-exporter,prometheus-client,exemplars-tracing-opentelemetry
       # The tests using opentelemetry are currently failing because of this issue:
       # https://github.com/open-telemetry/opentelemetry-rust/issues/1070
       # We should remove the continue-on-error once that is fixed

autometrics/Cargo.toml

@@ -14,7 +14,7 @@ readme = "README.md"
 
 [features]
 metrics = ["dep:metrics"]
-opentelemetry = ["opentelemetry_api"]
+opentelemetry = ["opentelemetry_api/metrics"]
 prometheus = ["dep:prometheus"]
 prometheus-client = ["dep:prometheus-client"]
 prometheus-exporter = [
@@ -25,17 +25,24 @@ prometheus-exporter = [
   "dep:prometheus",
   "thiserror"
 ]
-exemplars-tracing = ["tracing", "tracing-subscriber"]
+exemplars-tracing = ["tracing", "tracing-subscriber", "_exemplars"]
+exemplars-tracing-opentelemetry = [
+  "opentelemetry_api/trace",
+  "tracing",
+  "tracing-opentelemetry",
+  "_exemplars"
+]
 custom-objective-percentile = []
 custom-objective-latency = []
+_exemplars = []
 
 [dependencies]
 autometrics-macros = { workspace = true }
 spez = "0.1.2"
 once_cell = "1.17"
 
 # Used for opentelemetry feature
-opentelemetry_api = { version = "0.19.0", default-features = false, features = ["metrics"], optional = true }
+opentelemetry_api = { version = "0.19.0", default-features = false, optional = true }
 
 # Use for metrics feature
 metrics = { version = "0.21", default-features = false, optional = true }
@@ -55,11 +62,16 @@ prometheus-client = { version = "0.21.1", optional = true }
 tracing = { version = "0.1", optional = true }
 tracing-subscriber = { version = "0.3", default-features = false, features = ["registry"], optional = true }
 
+# Used for exemplars-tracing-opentelemetry feature
+tracing-opentelemetry = { version = "0.19", default-features = false, optional = true }
+
 [dev-dependencies]
 axum = { version = "0.6", features = ["tokio"] }
 http = "0.2"
+opentelemetry_sdk = "0.19"
 tokio = { version = "1", features = ["full"] }
 tracing = "0.1"
+tracing-opentelemetry = "0.19"
 tracing-subscriber = "0.3"
 trybuild = "1.0"
 uuid = { version = "1", features = ["v4"] }

autometrics/src/exemplars/mod.rs

@@ -18,32 +18,44 @@
 //!
 //! To enable Prometheus to scrape metrics with exemplars you must:
 //! 1. Run Prometheus with the [`--enable-feature=exemplar-storage`](https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage) flag
-//! 2. Set the `Content-Type` header on the response from the Prometheus metrics endpoint to indicate
-//!    it is using the OpenMetrics exposition format instead of the default Prometheus format.
+//! 2. Export the metrics to Prometheus using [`prometheus_exporter::encode_http_response`] or
+//!   make sure to manually set the `Content-Type` header to indicate it is using the the OpenMetrics format:
 //!   ```http
 //!   Content-Type: application/openmetrics-text; version=1.0.0; charset=utf-8
 //!   ```
-//!
-//!  ```rust
-//!   use http::{header::CONTENT_TYPE, Response};
-//!
-//!   /// Expose the metrics to Prometheus in the OpenMetrics format
-//!   async fn get_metrics() -> Response<String> {
-//!       match autometrics::encode_global_metrics() {
-//!           Ok(metrics) => Response::builder()
-//!               .header(
-//!                   CONTENT_TYPE,
-//!                   "application/openmetrics-text; version=1.0.0; charset=utf-8",
-//!               )
-//!               .body(metrics)
-//!               .unwrap(),
-//!           Err(err) => Response::builder()
-//!               .status(500)
-//!               .body(err.to_string())
-//!               .unwrap(),
-//!       }
-//!   }
-//!   ```
+
+use std::collections::HashMap;
 
 #[cfg(feature = "exemplars-tracing")]
 pub mod tracing;
+#[cfg(feature = "exemplars-tracing-opentelemetry")]
+mod tracing_opentelemetry;
+
+#[cfg(all(
+    feature = "exemplars-tracing-opentelemetry",
+    feature = "exemplars-tracing"
+))]
+compile_error!("Only one of the exemplars-tracing and exemplars-tracing-opentelemetry features can be enabled at a time");
+
+#[cfg(not(any(
+    feature = "exemplars-tracing-opentelemetry",
+    feature = "exemplars-tracing"
+)))]
+compile_error!(
+    "One of the exemplars-tracing-opentelemetry or exemplars-tracing features must be enabled"
+);
+
+#[cfg(not(feature = "prometheus-client"))]
+compile_error!("Exemplars can only be used with the `prometheus-client` metrics library because that is the only one that currently supports producing metrics with exemplars");
+
+pub(crate) type TraceLabels = HashMap<&'static str, String>;
+pub(crate) fn get_exemplar() -> Option<TraceLabels> {
+    #[cfg(feature = "exemplars-tracing-opentelemetry")]
+    {
+        tracing_opentelemetry::get_exemplar()
+    }
+    #[cfg(feature = "exemplars-tracing")]
+    {
+        tracing::get_exemplar()
+    }
+}

autometrics/src/exemplars/tracing.rs

@@ -27,14 +27,12 @@
 //!
 //! [`Span`]: tracing::Span
 
-use std::collections::HashMap;
+use super::TraceLabels;
 use tracing::field::{Field, Visit};
 use tracing::{span::Attributes, Id, Subscriber};
 use tracing_subscriber::layer::{Context, Layer};
 use tracing_subscriber::registry::{LookupSpan, Registry};
 
-pub(crate) type TraceLabels = HashMap<&'static str, String>;
-
 /// Get the exemplar from the current tracing span
 pub(crate) fn get_exemplar() -> Option<TraceLabels> {
     let span = tracing::span::Span::current();
@@ -103,7 +101,7 @@ impl TraceLabelVisitor {
     fn new(fields: &'static [&'static str]) -> Self {
         Self {
             fields,
-            labels: HashMap::with_capacity(fields.len()),
+            labels: TraceLabels::with_capacity(fields.len()),
         }
     }
 }

autometrics/src/exemplars/tracing_opentelemetry.rs

@@ -0,0 +1,25 @@
+use super::TraceLabels;
+use opentelemetry_api::trace::TraceContextExt;
+use std::iter::FromIterator;
+use tracing::Span;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
+
+pub fn get_exemplar() -> Option<TraceLabels> {
+    // Get the OpenTelemetry Context from the tracing span
+    let context = Span::current().context();
+
+    // Now get the OpenTelemetry "span" from the Context
+    // (it's confusing because the word "span" is used by both tracing and OpenTelemetry
+    // to mean slightly different things)
+    let span = context.span();
+    let span_context = span.span_context();
+
+    if span_context.is_valid() {
+        Some(TraceLabels::from_iter([
+            ("trace_id", span_context.trace_id().to_string()),
+            ("span_id", span_context.span_id().to_string()),
+        ]))
+    } else {
+        None
+    }
+}

autometrics/src/lib.rs

@@ -6,7 +6,7 @@
 #![doc = include_str!("../README.md")]
 
 mod constants;
-#[cfg(feature = "exemplars-tracing")]
+#[cfg(feature = "_exemplars")]
 pub mod exemplars;
 pub mod integrations;
 mod labels;

autometrics/src/prometheus_exporter.rs

@@ -31,10 +31,10 @@ use opentelemetry_sdk::metrics::{controllers, processors, selectors};
 use prometheus::TextEncoder;
 use thiserror::Error;
 
-#[cfg(not(feature = "exemplars-tracing"))]
+#[cfg(not(feature = "_exemplars"))]
 /// Prometheus text format content type
 const RESPONSE_CONTENT_TYPE: &str = "text/plain; version=0.0.4";
-#[cfg(feature = "exemplars-tracing")]
+#[cfg(feature = "_exemplars")]
 /// OpenMetrics content type
 const RESPONSE_CONTENT_TYPE: &str = "application/openmetrics-text; version=1.0.0; charset=utf-8";
 

autometrics/src/tracker/mod.rs

@@ -38,9 +38,6 @@ pub use self::prometheus_client::PrometheusClientTracker;
 ))]
 compile_error!("Only one of the metrics, opentelemetry, prometheus, or prometheus-client features can be enabled at a time");
 
-#[cfg(all(feature = "exemplars-tracing", not(feature = "prometheus-client")))]
-compile_error!("The exemplars-tracing feature can only be used with the `prometheus-client` metrics library because that is the only one that currently supports exemplars");
-
 pub trait TrackMetrics {
     fn set_build_info(build_info_labels: &BuildInfoLabels);
     fn start(gauge_labels: Option<&GaugeLabels>) -> Self;

autometrics/src/tracker/prometheus_client.rs

@@ -1,26 +1,24 @@
 use super::TrackMetrics;
-#[cfg(feature = "exemplars-tracing")]
-use crate::exemplars::tracing::get_exemplar;
+#[cfg(feature = "_exemplars")]
+use crate::exemplars::get_exemplar;
 use crate::labels::{BuildInfoLabels, CounterLabels, GaugeLabels, HistogramLabels};
 use crate::{constants::*, HISTOGRAM_BUCKETS};
 use once_cell::sync::Lazy;
-#[cfg(feature = "exemplars-tracing")]
-use prometheus_client::metrics::exemplar::{CounterWithExemplar, HistogramWithExemplars};
-#[cfg(not(feature = "exemplars-tracing"))]
-use prometheus_client::metrics::{counter::Counter, histogram::Histogram};
 use prometheus_client::metrics::{family::Family, gauge::Gauge};
 use prometheus_client::registry::Registry;
 use std::time::Instant;
 
-#[cfg(feature = "exemplars-tracing")]
-type CounterType = CounterWithExemplar<Vec<(&'static str, String)>>;
-#[cfg(not(feature = "exemplars-tracing"))]
-type CounterType = Counter;
+#[cfg(feature = "_exemplars")]
+type CounterType =
+    prometheus_client::metrics::exemplar::CounterWithExemplar<Vec<(&'static str, String)>>;
+#[cfg(not(feature = "_exemplars"))]
+type CounterType = prometheus_client::metrics::counter::Counter;
 
-#[cfg(feature = "exemplars-tracing")]
-type HistogramType = HistogramWithExemplars<Vec<(&'static str, String)>>;
-#[cfg(not(feature = "exemplars-tracing"))]
-type HistogramType = Histogram;
+#[cfg(feature = "_exemplars")]
+type HistogramType =
+    prometheus_client::metrics::exemplar::HistogramWithExemplars<Vec<(&'static str, String)>>;
+#[cfg(not(feature = "_exemplars"))]
+type HistogramType = prometheus_client::metrics::histogram::Histogram;
 
 static REGISTRY_AND_METRICS: Lazy<(Registry, Metrics)> = Lazy::new(|| {
     let mut registry = <Registry>::default();
@@ -89,18 +87,18 @@ impl TrackMetrics for PrometheusClientTracker {
     }
 
     fn finish(self, counter_labels: &CounterLabels, histogram_labels: &HistogramLabels) {
-        #[cfg(feature = "exemplars-tracing")]
+        #[cfg(feature = "_exemplars")]
         let exemplar = get_exemplar().map(|exemplar| exemplar.into_iter().collect::<Vec<_>>());
 
         METRICS.counter.get_or_create(&counter_labels).inc_by(
             1,
-            #[cfg(feature = "exemplars-tracing")]
+            #[cfg(feature = "_exemplars")]
             exemplar.clone(),
         );
 
         METRICS.histogram.get_or_create(&histogram_labels).observe(
             self.start_time.elapsed().as_secs_f64(),
-            #[cfg(feature = "exemplars-tracing")]
+            #[cfg(feature = "_exemplars")]
             exemplar,
         );
 

autometrics/tests/exemplars_test.rs

@@ -1,21 +1,21 @@
-#![cfg(all(feature = "exemplars-tracing", feature = "prometheus-exporter"))]
+#![cfg(feature = "prometheus-exporter")]
 
-use autometrics::exemplars::tracing::AutometricsExemplarExtractor;
 use autometrics::{autometrics, prometheus_exporter};
-use tracing::instrument;
-use tracing_subscriber::prelude::*;
+use tracing_opentelemetry::OpenTelemetryLayer;
+use tracing_subscriber::{prelude::*, Registry};
 
+#[cfg(feature = "exemplars-tracing")]
 #[test]
 fn single_field() {
     prometheus_exporter::init();
 
     #[autometrics]
-    #[instrument(fields(trace_id = "test_trace_id"))]
+    #[tracing::instrument(fields(trace_id = "test_trace_id"))]
     fn single_field_fn() {}
 
-    let subscriber = tracing_subscriber::fmt::fmt()
-        .finish()
-        .with(AutometricsExemplarExtractor::from_fields(&["trace_id"]));
+    let subscriber = tracing_subscriber::fmt::fmt().finish().with(
+        autometrics::exemplars::tracing::AutometricsExemplarExtractor::from_fields(&["trace_id"]),
+    );
     tracing::subscriber::with_default(subscriber, || single_field_fn());
 
     let metrics = prometheus_exporter::encode_to_string().unwrap();
@@ -26,20 +26,20 @@ fn single_field() {
     }))
 }
 
+#[cfg(feature = "exemplars-tracing")]
 #[test]
 fn multiple_fields() {
     prometheus_exporter::init();
 
     #[autometrics]
-    #[instrument(fields(trace_id = "test_trace_id", foo = 99))]
+    #[tracing::instrument(fields(trace_id = "test_trace_id", foo = 99))]
     fn multiple_fields_fn() {}
 
-    let subscriber =
-        tracing_subscriber::fmt::fmt()
-            .finish()
-            .with(AutometricsExemplarExtractor::from_fields(&[
-                "trace_id", "foo",
-            ]));
+    let subscriber = tracing_subscriber::fmt::fmt().finish().with(
+        autometrics::exemplars::tracing::AutometricsExemplarExtractor::from_fields(&[
+            "trace_id", "foo",
+        ]),
+    );
     tracing::subscriber::with_default(subscriber, || multiple_fields_fn());
 
     let metrics = prometheus_exporter::encode_to_string().unwrap();
@@ -51,3 +51,31 @@ fn multiple_fields() {
                 || line.ends_with(r#"} 1 # {foo="99",trace_id="test_trace_id"} 1.0"#))
     }))
 }
+
+#[cfg(feature = "exemplars-tracing-opentelemetry")]
+#[test]
+fn tracing_opentelemetry_context() {
+    prometheus_exporter::init();
+
+    let tracer = opentelemetry_sdk::export::trace::stdout::new_pipeline()
+        .with_writer(std::io::sink())
+        .install_simple();
+    // This adds the OpenTelemetry Context to every tracing Span
+    let otel_layer = OpenTelemetryLayer::new(tracer);
+    let subscriber = Registry::default().with(otel_layer);
+
+    #[autometrics]
+    fn opentelemetry_context_fn() {}
+
+    tracing::subscriber::with_default(subscriber, || {
+        // Create a new span and execute the function inside it
+        tracing::info_span!("my_span").in_scope(opentelemetry_context_fn);
+    });
+
+    let metrics = prometheus_exporter::encode_to_string().unwrap();
+    assert!(metrics.lines().any(|line| {
+        line.starts_with("function_calls_count_total{")
+            && line.contains(r#"function="opentelemetry_context_fn""#)
+            && (line.contains(r#"trace_id=""#) || line.contains(r#"span_id=""#))
+    }))
+}

examples/exemplars-tracing-opentelemetry/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "example-exemplars-tracing-opentelemetry"
+version = "0.0.0"
+publish = false
+edition = "2021"
+
+[dependencies]
+autometrics = { path = "../../autometrics", features = [
+  "prometheus-client",
+  "prometheus-exporter",
+  "exemplars-tracing-opentelemetry"
+  ] }
+autometrics-example-util = { path = "../util" }
+axum = { version = "0.6", features = ["json"] }
+axum-tracing-opentelemetry = "0.10"
+opentelemetry = "0.19"
+reqwest = { version = "0.11", features = ["json"] }
+tokio = { version = "1", features = ["full"] }
+tracing = "0.1"
+tracing-opentelemetry = "0.19"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }

examples/exemplars-tracing-opentelemetry/README.md

@@ -0,0 +1,7 @@
+# OpenTelemetry + Tracing Exemplars Example
+
+This example demonstrates how Autometrics can pick up the `trace_id` and `span_id` from the [`opentelemetry::Context`](https://docs.rs/opentelemetry/latest/opentelemetry/struct.Context.html) and attach them to the metrics as [exemplars](https://grafana.com/docs/grafana/latest/fundamentals/exemplars/).
+
+> **Note**
+>
+> Prometheus must be [specifically configured](https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage) to enable the experimental exemplars feature.