Support OpenTelemetry span context for exemplars

.github/workflows/ci.yml

@@ -28,6 +28,7 @@ jobs:
       - run: cargo test --features=prometheus-exporter,metrics
       - run: cargo test --features=prometheus-exporter,prometheus
       - run: cargo test --features=prometheus-exporter,prometheus-client,exemplars-tracing
+      - run: cargo test --features=prometheus-exporter,prometheus-client,exemplars-tracing-opentelemetry
       # The tests using opentelemetry are currently failing because of this issue:
       # https://github.com/open-telemetry/opentelemetry-rust/issues/1070
       # We should remove the continue-on-error once that is fixed

CHANGELOG.md

@@ -11,13 +11,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Support the official `prometheus-client` crate for producing metrics
+- Exemplar support when using the feature flags `exemplars-tracing` or `exemplars-tracing-opentelemetry`.
+  Autometrics can now extract fields from the current span and attach them as exemplars on the
+  counter and histogram metrics
 - `ResultLabels` derive macro allows to specify on an enum whether variants should
   always be "ok", or "error" for the success rate metrics of functions using them. (#61)
-- Support the official `prometheus-client` crate for producing metrics
-- Exemplar support when using the feature flags `exemplars-tracing` and `prometheus-client`.
-  Autometrics now provides a `tracing_subscriber::Layer` that makes the specific `Span` fields
-  available to the library, and autometrics will automatically attach those fields as exemplars
-  on the counter and histogram metrics
 - The `prometheus_exporter` module contains all functions related to the `prometheus-exporter` feature
 - `prometheus_exporter::encode_http_response` function returns an `http::Response` with the metrics.
   This is especially recommended when using exemplars, because it automatically uses the OpenMetrics

README.md

@@ -59,7 +59,7 @@ https://github.com/autometrics-dev/autometrics-rs/assets/3262610/966ed140-1d6c-4
 - [🚨 Define alerts](https://docs.rs/autometrics/latest/autometrics/objectives/index.html) using SLO best practices directly in your source code
 - [📊 Grafana dashboards](https://github.com/autometrics-dev#5-configuring-prometheus) work out of the box to visualize the performance of instrumented functions & SLOs
 - [⚙️ Configurable](https://docs.rs/autometrics/latest/autometrics/#metrics-libraries) metric collection library ([`opentelemetry`](https://crates.io/crates/opentelemetry), [`prometheus`](https://crates.io/crates/prometheus), [`prometheus-client`](https://crates.io/crates/prometheus-client) or [`metrics`](https://crates.io/crates/metrics))
-- [📍 Attach exemplars](https://docs.rs/autometrics/latest/autometrics/exemplars/index.html) to connect metrics with distributed traces
+- [📍 Attach exemplars](https://docs.rs/autometrics/latest/autometrics/exemplars/index.html) to connect metrics with traces
 - ⚡ Minimal runtime overhead
 
 See [Why Autometrics?](https://github.com/autometrics-dev#4-why-autometrics) for more details on the ideas behind autometrics.

autometrics/Cargo.toml

@@ -14,7 +14,7 @@ readme = "README.md"
 
 [features]
 metrics = ["dep:metrics"]
-opentelemetry = ["opentelemetry_api"]
+opentelemetry = ["opentelemetry_api/metrics"]
 prometheus = ["dep:prometheus"]
 prometheus-client = ["dep:prometheus-client"]
 prometheus-exporter = [
@@ -25,17 +25,24 @@ prometheus-exporter = [
   "dep:prometheus",
   "thiserror"
 ]
-exemplars-tracing = ["tracing", "tracing-subscriber"]
+exemplars-tracing = ["tracing", "tracing-subscriber", "_exemplars"]
+exemplars-tracing-opentelemetry = [
+  "opentelemetry_api/trace",
+  "tracing",
+  "tracing-opentelemetry",
+  "_exemplars"
+]
 custom-objective-percentile = []
 custom-objective-latency = []
+_exemplars = []
 
 [dependencies]
 autometrics-macros = { workspace = true }
 spez = "0.1.2"
 once_cell = "1.17"
 
 # Used for opentelemetry feature
-opentelemetry_api = { version = "0.19.0", default-features = false, features = ["metrics"], optional = true }
+opentelemetry_api = { version = "0.19.0", default-features = false, optional = true }
 
 # Use for metrics feature
 metrics = { version = "0.21", default-features = false, optional = true }
@@ -55,11 +62,16 @@ prometheus-client = { version = "0.21.1", optional = true }
 tracing = { version = "0.1", optional = true }
 tracing-subscriber = { version = "0.3", default-features = false, features = ["registry"], optional = true }
 
+# Used for exemplars-tracing-opentelemetry feature
+tracing-opentelemetry = { version = "0.19", default-features = false, optional = true }
+
 [dev-dependencies]
 axum = { version = "0.6", features = ["tokio"] }
 http = "0.2"
+opentelemetry_sdk = "0.19"
 tokio = { version = "1", features = ["full"] }
 tracing = "0.1"
+tracing-opentelemetry = "0.19"
 tracing-subscriber = "0.3"
 trybuild = "1.0"
 uuid = { version = "1", features = ["v4"] }

autometrics/README.md

@@ -49,7 +49,7 @@ pub async fn main() {
 - [🚨 Define alerts](https://docs.rs/autometrics/latest/autometrics/objectives/index.html) using SLO best practices directly in your source code
 - [📊 Grafana dashboards](https://github.com/autometrics-dev#5-configuring-prometheus) work out of the box to visualize the performance of instrumented functions & SLOs
 - [⚙️ Configurable](#metrics-libraries) metric collection library ([`opentelemetry`](https://crates.io/crates/opentelemetry), [`prometheus`](https://crates.io/crates/prometheus), [`prometheus-client`](https://crates.io/crates/prometheus-client) or [`metrics`](https://crates.io/crates/metrics))
-- [📍 Attach exemplars](https://docs.rs/autometrics/latest/autometrics/exemplars/index.html) to connect metrics with distributed traces
+- [📍 Attach exemplars](https://docs.rs/autometrics/latest/autometrics/exemplars/index.html) to connect metrics with traces
 - ⚡ Minimal runtime overhead
 
 See [Why Autometrics?](https://github.com/autometrics-dev#4-why-autometrics) for more details on the ideas behind autometrics.
@@ -106,11 +106,10 @@ fn main() {
 ### Feature flags
 
 - `prometheus-exporter` - exports a Prometheus metrics collector and exporter (compatible with any of the Metrics Libraries)
-- `exemplars-tracing` - extract fields from [`tracing::Span`](https://docs.rs/tracing/latest/tracing/struct.Span.html)s and attach them as [exemplars](https://grafana.com/docs/grafana/latest/fundamentals/exemplars/) for the metrics produced by Autometrics. This is currently only supported with the `prometheus-client` feature due to lack of support in the other metrics libraries. Note that Prometheus must be specifically [configured](https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage) to enable the exemplars feature.
 - `custom-objective-latency` - by default, Autometrics only supports a fixed set of latency thresholds for objectives. Enable this to use custom latency thresholds. Note, however, that the custom latency **must** match one of the buckets configured for your histogram or the alerts will not work. This is not currently compatible with the `prometheus` or `prometheus-exporter` feature.
 - `custom-objective-percentile` by default, Autometrics only supports a fixed set of objective percentiles. Enable this to use a custom percentile. Note, however, that using custom percentiles requires generating a different recording and alerting rules file using the CLI + Sloth (see [here](https://github.com/autometrics-dev/autometrics-rs/tree/main/autometrics-cli)).
 
-#### Metrics Libraries
+#### Metrics libraries
 
 **Required:** Configure the crate that autometrics will use to produce metrics by using one of the following feature flags:
 
@@ -122,3 +121,10 @@ fn main() {
 - `metrics` - use the [metrics](https://crates.io/crates/metrics) crate for producing metrics
 - `prometheus` - use the [prometheus](https://crates.io/crates/prometheus) crate for producing metrics
 - `prometheus-client` - use the official [prometheus-client](https://crates.io/crates/prometheus-client) crate for producing metrics
+
+#### Exemplars (for integrating metrics with traces)
+
+See the crate docs for [exemplars](https://docs.rs/autometrics/latest/autometrics/exemplars/index.html) for details about these features.
+
+- `exemplars-tracing` - extract arbitrary fields from `tracing::Span`s
+- `exemplars-tracing-opentelemetry` - extract the `trace_id` and `span_id` from the `opentelemetry::Context`, which is attached to `tracing::Span`s by the `tracing-opentelemetry` crate

autometrics/src/exemplars/mod.rs

@@ -5,45 +5,104 @@
 //! to change by looking at individual examples that contributed to the metrics.
 //!
 //! Autometrics integrates with tracing libraries to extract details from the
-//! current span context and attach them as exemplars to the metrics it generates.
+//! current span context and automatically attach them as exemplars to the generated metrics.
 //!
-//! See each of the submodules for details on how to integrate with each tracing library.
-//!
-//! # Supported Metrics Libraries
+//! # Supported metrics libraries
 //!
 //! Exemplars are currently only supported with the `prometheus-client` metrics library,
 //! because that is the only one that currently supports producing metrics with exemplars.
 //!
-//! # Exposing Metrics to Prometheus with Exemplars
+//! # Exposing metrics to Prometheus with exemplars
 //!
 //! To enable Prometheus to scrape metrics with exemplars you must:
 //! 1. Run Prometheus with the [`--enable-feature=exemplar-storage`](https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage) flag
-//! 2. Set the `Content-Type` header on the response from the Prometheus metrics endpoint to indicate
-//!    it is using the OpenMetrics exposition format instead of the default Prometheus format.
+//! 2. Export the metrics to Prometheus using the provided [`prometheus_exporter::encode_http_response`] or
+//!   make sure to manually set the `Content-Type` header to indicate it is using the the OpenMetrics format,
+//!   rather than the default Prometheus format:
 //!   ```http
 //!   Content-Type: application/openmetrics-text; version=1.0.0; charset=utf-8
 //!   ```
 //!
-//!  ```rust
-//!   use http::{header::CONTENT_TYPE, Response};
-//!
-//!   /// Expose the metrics to Prometheus in the OpenMetrics format
-//!   async fn get_metrics() -> Response<String> {
-//!       match autometrics::encode_global_metrics() {
-//!           Ok(metrics) => Response::builder()
-//!               .header(
-//!                   CONTENT_TYPE,
-//!                   "application/openmetrics-text; version=1.0.0; charset=utf-8",
-//!               )
-//!               .body(metrics)
-//!               .unwrap(),
-//!           Err(err) => Response::builder()
-//!               .status(500)
-//!               .body(err.to_string())
-//!               .unwrap(),
-//!       }
-//!   }
-//!   ```
+//! [`prometheus_exporter::encode_http_response`]: crate::prometheus_exporter::encode_http_response
+//!
+//! # Tracing libraries
+//!
+//! ## [`tracing`](https://crates.io/crates/tracing)
+//!
+//! See the [`tracing` submodule docs](tracing).
+//!
+//! ## [`tracing-opentelemetry`](https://crates.io/crates/tracing-opentelemetry)
+//!
+//! Extract exemplars from the OpenTelemetry Context attached to the current tracing Span.
+//!
+//! This works in the following way:
+//! 1. Add the [`tracing_opentelemetry::OpenTelemetryLayer`] to your tracing subscriber
+//! 2. That layer ensures that there is an [`opentelemetry::Context`] attached to every [`tracing::Span`]
+//! 3. Spans can be manually created or created for every function using the [`tracing::instrument`] macro
+//! 4. Autometrics extracts the `trace_id` and `span_id` from the `Context` and attaches them as exemplars to the generated metrics
+//!
+//! ### Example
+//!
+//! ```rust
+//! # use autometrics::autometrics;
+//! use tracing_subscriber::prelude::*;
+//! use tracing_opentelemetry::OpenTelemetryLayer;
+//!
+//! let tracer = opentelemetry_sdk::export::trace::stdout::new_pipeline().install_simple();
+//! let otel_layer = OpenTelemetryLayer::new(tracer);
+//!
+//! // Create a tracing subscriber with the OpenTelemetry layer
+//! tracing_subscriber::fmt()
+//!   .finish()
+//!   .with(otel_layer)
+//!   .init();
+//!
+//! #[autometrics]
+//! #[tracing::instrument]
+//! async fn my_function() {
+//!   // This function produces metrics with exemplars
+//!   // that contain a trace_id and span_id
+//! }
+//! ```
+//!
+//! [`tracing_opentelemetry::OpenTelemetryLayer`]: https://docs.rs/tracing-opentelemetry/latest/tracing_opentelemetry/struct.OpenTelemetryLayer.html
+//! [`opentelemetry::Context`]: https://docs.rs/opentelemetry/latest/opentelemetry/struct.Context.html
+//! [`tracing::Span`]: https://docs.rs/tracing/latest/tracing/struct.Span.html
+//! [`tracing::instrument`]: https://docs.rs/tracing/latest/tracing/attr.instrument.html
+
+use std::collections::HashMap;
 
 #[cfg(feature = "exemplars-tracing")]
 pub mod tracing;
+#[cfg(feature = "exemplars-tracing-opentelemetry")]
+mod tracing_opentelemetry;
+
+#[cfg(all(
+    not(doc),
+    feature = "exemplars-tracing-opentelemetry",
+    feature = "exemplars-tracing"
+))]
+compile_error!("Only one of the exemplars-tracing and exemplars-tracing-opentelemetry features can be enabled at a time");
+
+#[cfg(not(any(
+    feature = "exemplars-tracing-opentelemetry",
+    feature = "exemplars-tracing"
+)))]
+compile_error!(
+    "One of the exemplars-tracing-opentelemetry or exemplars-tracing features must be enabled"
+);
+
+#[cfg(not(feature = "prometheus-client"))]
+compile_error!("Exemplars can only be used with the `prometheus-client` metrics library because that is the only one that currently supports producing metrics with exemplars");
+
+pub(crate) type TraceLabels = HashMap<&'static str, String>;
+pub(crate) fn get_exemplar() -> Option<TraceLabels> {
+    #[cfg(feature = "exemplars-tracing-opentelemetry")]
+    {
+        tracing_opentelemetry::get_exemplar()
+    }
+    #[cfg(feature = "exemplars-tracing")]
+    {
+        tracing::get_exemplar()
+    }
+}

autometrics/src/exemplars/tracing.rs

@@ -27,14 +27,12 @@
 //!
 //! [`Span`]: tracing::Span
 
-use std::collections::HashMap;
+use super::TraceLabels;
 use tracing::field::{Field, Visit};
 use tracing::{span::Attributes, Id, Subscriber};
 use tracing_subscriber::layer::{Context, Layer};
 use tracing_subscriber::registry::{LookupSpan, Registry};
 
-pub(crate) type TraceLabels = HashMap<&'static str, String>;
-
 /// Get the exemplar from the current tracing span
 pub(crate) fn get_exemplar() -> Option<TraceLabels> {
     let span = tracing::span::Span::current();
@@ -103,7 +101,7 @@ impl TraceLabelVisitor {
     fn new(fields: &'static [&'static str]) -> Self {
         Self {
             fields,
-            labels: HashMap::with_capacity(fields.len()),
+            labels: TraceLabels::with_capacity(fields.len()),
         }
     }
 }

autometrics/src/exemplars/tracing_opentelemetry.rs

@@ -0,0 +1,25 @@
+use super::TraceLabels;
+use opentelemetry_api::trace::TraceContextExt;
+use std::iter::FromIterator;
+use tracing::Span;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
+
+pub fn get_exemplar() -> Option<TraceLabels> {
+    // Get the OpenTelemetry Context from the tracing span
+    let context = Span::current().context();
+
+    // Now get the OpenTelemetry "span" from the Context
+    // (it's confusing because the word "span" is used by both tracing and OpenTelemetry
+    // to mean slightly different things)
+    let span = context.span();
+    let span_context = span.span_context();
+
+    if span_context.is_valid() {
+        Some(TraceLabels::from_iter([
+            ("trace_id", span_context.trace_id().to_string()),
+            ("span_id", span_context.span_id().to_string()),
+        ]))
+    } else {
+        None
+    }
+}

autometrics/src/lib.rs

@@ -6,7 +6,7 @@
 #![doc = include_str!("../README.md")]
 
 mod constants;
-#[cfg(feature = "exemplars-tracing")]
+#[cfg(feature = "_exemplars")]
 pub mod exemplars;
 pub mod integrations;
 mod labels;
@@ -191,6 +191,7 @@ pub use autometrics_macros::ResultLabels;
     since = "0.5.0",
     note = "Use autometrics::prometheus_exporter::encode_to_string instead. This will be removed in v0.6"
 )]
+/// Replaced by [`prometheus_exporter::encode_to_string`]
 pub fn encode_global_metrics() -> Result<String, prometheus_exporter::EncodingError> {
     prometheus_exporter::encode_to_string()
 }
@@ -199,6 +200,7 @@ pub fn encode_global_metrics() -> Result<String, prometheus_exporter::EncodingEr
     since = "0.5.0",
     note = "Use autometrics::prometheus_exporter::init instead. This will be removed in v0.6"
 )]
+/// Replaced by [`prometheus_exporter::init`]
 pub fn global_metrics_exporter() -> prometheus_exporter::GlobalPrometheus {
     prometheus_exporter::init();
     prometheus_exporter::GLOBAL_EXPORTER.clone()

autometrics/src/prometheus_exporter.rs

@@ -1,4 +1,4 @@
-//! Helper functions for easily collecting and exporting metrics to Prometheus.
+//! Helper functions for easily collecting and exporting metrics to Prometheus
 //!
 //! You do not need this module if you are already collecting custom metrics and exporting them to Prometheus.
 //!
@@ -31,10 +31,10 @@ use opentelemetry_sdk::metrics::{controllers, processors, selectors};
 use prometheus::TextEncoder;
 use thiserror::Error;
 
-#[cfg(not(feature = "exemplars-tracing"))]
+#[cfg(not(feature = "_exemplars"))]
 /// Prometheus text format content type
 const RESPONSE_CONTENT_TYPE: &str = "text/plain; version=0.0.4";
-#[cfg(feature = "exemplars-tracing")]
+#[cfg(feature = "_exemplars")]
 /// OpenMetrics content type
 const RESPONSE_CONTENT_TYPE: &str = "application/openmetrics-text; version=1.0.0; charset=utf-8";
 

autometrics/src/tracker/mod.rs

@@ -38,9 +38,6 @@ pub use self::prometheus_client::PrometheusClientTracker;
 ))]
 compile_error!("Only one of the metrics, opentelemetry, prometheus, or prometheus-client features can be enabled at a time");
 
-#[cfg(all(feature = "exemplars-tracing", not(feature = "prometheus-client")))]
-compile_error!("The exemplars-tracing feature can only be used with the `prometheus-client` metrics library because that is the only one that currently supports exemplars");
-
 pub trait TrackMetrics {
     fn set_build_info(build_info_labels: &BuildInfoLabels);
     fn start(gauge_labels: Option<&GaugeLabels>) -> Self;

autometrics/src/tracker/prometheus.rs

@@ -25,9 +25,7 @@ static COUNTER: Lazy<IntCounterVec> = Lazy::new(|| {
             OBJECTIVE_PERCENTILE_PROMETHEUS,
         ]
     )
-    .expect(&format!(
-        "Failed to register {COUNTER_NAME_PROMETHEUS} counter"
-    ))
+    .expect("Failed to register function_calls_count_total counter")
 });
 static HISTOGRAM: Lazy<HistogramVec> = Lazy::new(|| {
     let opts = histogram_opts!(