Scrape metrics with Prometheus in exemplars example

autometrics/src/exemplars/mod.rs

@@ -1,12 +1,28 @@
-//! Attach exemplars to the generated metrics (used for connecting metrics to traces)
+//! Connect metrics to traces using exemplars
+//!
+//! Exemplars are a newer Prometheus / OpenMetrics / OpenTelemetry feature that allows you to associate
+//! specific traces or samples with a given metric. This enables you to investigate what caused metrics
+//! to change by looking at individual examples that contributed to the metrics.
 //!
 //! Autometrics integrates with tracing libraries to extract details from the
 //! current span context and attach them as exemplars to the metrics it generates.
 //!
 //! See each of the submodules for details on how to integrate with each tracing library.
 //!
-//! **Note:** This is currently only supported with the `prometheus-client` metrics library,
+//! # Supported Metrics Libraries
+//!
+//! Exemplars are currently only supported with the `prometheus-client` metrics library,
 //! because that is the only one that currently supports producing metrics with exemplars.
+//!
+//! # Exposing Metrics to Prometheus with Exemplars
+//!
+//! To enable Prometheus to scrape metrics with exemplars you must:
+//! 1. Run Prometheus with the [`--enable-feature=exemplar-storage`](https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage) flag
+//! 2. Set the `Content-Type` header on the response from the Prometheus metrics endpoint to indicate
+//!    it is using the OpenMetrics exposition format instead of the default Prometheus format.
+//!   ```http
+//!   Content-Type: application/openmetrics-text; version=1.0.0; charset=utf-8
+//!   ```
 
 #[cfg(feature = "exemplars-tracing")]
 pub mod tracing;

autometrics/src/exemplars/tracing.rs

@@ -2,10 +2,6 @@
 //!
 //! This module enables autometrics to use fields such as `trace_id` from the current [`Span`] as exemplars.
 //!
-//! Exemplars are a newer Prometheus / OpenMetrics / OpenTelemetry feature that allows you to associate
-//! specific traces or samples with a given metric. This enables you to investigate what caused metrics
-//! to change by looking at individual examples that contributed to the metrics.
-//!
 //! # Example
 //!
 //! ```rust

examples/axum/src/main.rs

@@ -54,7 +54,7 @@ pub async fn get_metrics() -> (StatusCode, String) {
 pub async fn main() {
     // Run Prometheus and generate random traffic for the app
     // (You would not actually do this in production, but it makes it easier to see the example in action)
-    let _prometheus = run_prometheus();
+    let _prometheus = run_prometheus(false);
     tokio::spawn(generate_random_traffic());
 
     let app = Router::new()

examples/exemplars-tracing/Cargo.toml

@@ -5,7 +5,15 @@ publish = false
 edition = "2021"
 
 [dependencies]
-autometrics = { path = "../../autometrics", features = ["prometheus-client", "prometheus-exporter", "exemplars-tracing"] }
+autometrics = { path = "../../autometrics", features = [
+  "prometheus-client",
+  "prometheus-exporter",
+  "exemplars-tracing"
+  ] }
+autometrics-example-util = { path = "../util" }
+axum = { version = "0.6", features = ["json"] }
+reqwest = { version = "0.11", features = ["json"] }
+tokio = { version = "1", features = ["full"] }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 uuid = { version = "1.3", features = ["v4"] }

examples/exemplars-tracing/src/main.rs

@@ -1,6 +1,10 @@
 use autometrics::{
     autometrics, encode_global_metrics, exemplars::tracing::AutometricsExemplarExtractor,
+    global_metrics_exporter,
 };
+use autometrics_example_util::run_prometheus;
+use axum::{http::header::CONTENT_TYPE, response::Response, routing::get, Router};
+use std::{net::SocketAddr, time::Duration};
 use tracing::{instrument, trace};
 use tracing_subscriber::{prelude::*, EnvFilter};
 use uuid::Uuid;
@@ -9,9 +13,10 @@ use uuid::Uuid;
 // that as an exemplar for the metrics it generates.
 #[autometrics]
 #[instrument(fields(trace_id = %Uuid::new_v4()))]
-fn outer_function() {
+async fn outer_function() -> String {
     trace!("Outer function called");
-    inner_function("hello")
+    inner_function("hello");
+    "Hello world!".to_string()
 }
 
 // This function will also have the `trace_id` attached as an exemplar
@@ -22,14 +27,60 @@ fn inner_function(param: &str) {
     trace!("Inner function called");
 }
 
-fn main() {
+/// Expose the metrics to Prometheus in the OpenMetrics format
+///
+/// Note that setting the content type is necessary in order for
+/// Prometheus to scrape the metrics with the exemplars.
+async fn get_metrics() -> Response<String> {
+    match encode_global_metrics() {
+        Ok(metrics) => Response::builder()
+            .header(
+                CONTENT_TYPE,
+                "application/openmetrics-text; version=1.0.0; charset=utf-8",
+            )
+            .body(metrics)
+            .unwrap(),
+        Err(err) => Response::builder()
+            .status(500)
+            .body(err.to_string())
+            .unwrap(),
+    }
+}
+
+#[tokio::main]
+async fn main() {
+    // Run Prometheus with flag --enable-feature=exemplars-storage
+    let _prometheus = run_prometheus(true);
+    tokio::spawn(generate_random_traffic());
+
+    let _ = global_metrics_exporter();
     tracing_subscriber::fmt::fmt()
         .finish()
         .with(EnvFilter::from_default_env())
+        // Set up the tracing layer to expose the `trace_id` field to Autometrics.
         .with(AutometricsExemplarExtractor::from_fields(&["trace_id"]))
         .init();
 
-    outer_function();
+    let app = Router::new()
+        .route("/", get(outer_function))
+        .route("/metrics", get(get_metrics));
+
+    let addr = SocketAddr::from(([127, 0, 0, 1], 3000));
+    let server = axum::Server::bind(&addr);
+
+    println!("\nVisit the following URL to see one of the charts along with the exemplars:");
+    println!("http://localhost:9090/graph?g0.expr=%23%20Rate%20of%20calls%20to%20the%20%60outer_function%60%20function%20per%20second%2C%20averaged%20over%205%20minute%20windows%0A%0Asum%20by%20(function%2C%20module%2C%20commit%2C%20version)%20(rate(function_calls_count_total%7Bfunction%3D%22outer_function%22%7D%5B5m%5D)%20*%20on%20(instance%2C%20job)%20group_left(version%2C%20commit)%20last_over_time(build_info%5B1s%5D))&g0.tab=0&g0.stacked=0&g0.show_exemplars=1&g0.range_input=1h");
+
+    server
+        .serve(app.into_make_service())
+        .await
+        .expect("Error starting example API server");
+}
 
-    println!("{}", encode_global_metrics().unwrap());
+pub async fn generate_random_traffic() {
+    let client = reqwest::Client::new();
+    loop {
+        client.get("http://localhost:3000").send().await.ok();
+        tokio::time::sleep(Duration::from_millis(100)).await
+    }
 }

examples/util/src/lib.rs

@@ -16,9 +16,14 @@ impl Drop for ChildGuard {
     }
 }
 
-pub fn run_prometheus() -> ChildGuard {
+pub fn run_prometheus(enable_exemplars: bool) -> ChildGuard {
+    let mut args = vec!["--config.file", PROMETHEUS_CONFIG_PATH];
+    if enable_exemplars {
+        args.push("--enable-feature=exemplar-storage");
+    }
+
     match Command::new("prometheus")
-        .args(["--config.file", PROMETHEUS_CONFIG_PATH])
+        .args(&args)
         .stdout(Stdio::null())
         .stderr(Stdio::null())
         .spawn()
@@ -31,9 +36,13 @@ pub fn run_prometheus() -> ChildGuard {
         }
         Ok(child) => {
             eprintln!(
-                "Running Prometheus on port 9090 (using config file: {})\n",
-                PROMETHEUS_CONFIG_PATH
+                "Running Prometheus on port 9090 (using config file: {PROMETHEUS_CONFIG_PATH})",
             );
+            if enable_exemplars {
+                eprintln!(
+                    "Exemplars are enabled (using the flag: --enable-feature=exemplar-storage)"
+                );
+            }
             ChildGuard(child)
         }
     }