[sdk-metrics] Add experimental envvar for setting ExemplarFilter for histograms

docs/metrics/customizing-the-sdk/README.md

@@ -372,6 +372,9 @@ OpenTelemetry SDK comes with the following `ExemplarFilters` (defined on
 The `SetExemplarFilter` extension method on `MeterProviderBuilder` can be used
 to set the desired `ExemplarFilterType` and enable `Exemplar` collection:
 
+> [!NOTE]
+> The `SetExemplarFilter` API was added in the `1.9.0` release.
+
 ```csharp
 using OpenTelemetry;
 using OpenTelemetry.Metrics;
@@ -382,6 +385,24 @@ using var meterProvider = Sdk.CreateMeterProviderBuilder()
     .Build();
 ```
 
+It is also possible to configure the `ExemplarFilter` by using following
+environmental variables:
+
+> [!NOTE]
+> Programmatically calling `SetExemplarFilter` will override any defaults set
+  using environment variables or configuration.
+
+| Environment variable       | Description                                        | Notes |
+| -------------------------- | -------------------------------------------------- |-------|
+| `OTEL_METRICS_EXEMPLAR_FILTER` | Sets the default `ExemplarFilter` to use for all metrics. | Added in `1.9.0` |
+| `OTEL_DOTNET_EXPERIMENTAL_METRICS_EXEMPLAR_FILTER_HISTOGRAMS`        | Sets the default `ExemplarFilter` to use for histogram metrics. If not set than `OTEL_METRICS_EXEMPLAR_FILTER` also applies to histograms. | Experimental key (may be removed or changed in the future). Added in `1.9.0` |
+
+Allowed values:
+
+* `always_off`: Eqivalent to `ExemplarFilterType.AlwaysOff`
+* `always_on`: Eqivalent to `ExemplarFilterType.AlwaysOn`
+* `trace_based`: Eqivalent to `ExemplarFilterType.TraceBased`
+
 #### ExemplarReservoir
 
 `ExemplarReservoir` receives the measurements sampled by the `ExemplarFilter`
@@ -398,7 +419,8 @@ metrics except Histograms with buckets. It has a fixed reservoir pool, and
 implements the equivalent of [naive
 reservoir](https://en.wikipedia.org/wiki/Reservoir_sampling). The reservoir pool
 size (currently defaulting to 1) determines the maximum number of exemplars
-stored.
+stored. Exponential histograms use a `SimpleFixedSizeExemplarReservoir` with a
+pool size equal to the number of buckets up to a max of `20`.
 
 > [!NOTE]
 > Currently there is no ability to change or configure `ExemplarReservoir`.

examples/AspNetCore/Program.cs

@@ -84,9 +84,7 @@
         // Ensure the MeterProvider subscribes to any custom Meters.
         builder
             .AddMeter(Instrumentation.MeterName)
-#if EXPOSE_EXPERIMENTAL_FEATURES
             .SetExemplarFilter(ExemplarFilterType.TraceBased)
-#endif
             .AddRuntimeInstrumentation()
             .AddHttpClientInstrumentation()
             .AddAspNetCoreInstrumentation();

src/OpenTelemetry/CHANGELOG.md

@@ -25,6 +25,13 @@
   supported in stable builds.
   ([#5607](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5607))
 
+* The `ExemplarFilter` used by SDK `MeterProvider`s for histogram metrics can
+  now be controlled via the experimental
+  `OTEL_DOTNET_EXPERIMENTAL_METRICS_EXEMPLAR_FILTER_HISTOGRAMS` environment
+  variable. The supported values are: `always_off`, `always_on`, and
+  `trace_based`.
+  ([#5611](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5611))
+
 ## 1.8.1
 
 Released 2024-Apr-17

src/OpenTelemetry/Metrics/MeterProviderSdk.cs

@@ -16,6 +16,7 @@ internal sealed class MeterProviderSdk : MeterProvider
     internal const string EmitOverFlowAttributeConfigKey = "OTEL_DOTNET_EXPERIMENTAL_METRICS_EMIT_OVERFLOW_ATTRIBUTE";
     internal const string ReclaimUnusedMetricPointsConfigKey = "OTEL_DOTNET_EXPERIMENTAL_METRICS_RECLAIM_UNUSED_METRIC_POINTS";
     internal const string ExemplarFilterConfigKey = "OTEL_METRICS_EXEMPLAR_FILTER";
+    internal const string ExemplarFilterHistogramsConfigKey = "OTEL_DOTNET_EXPERIMENTAL_METRICS_EXEMPLAR_FILTER_HISTOGRAMS";
 
     internal readonly IServiceProvider ServiceProvider;
     internal readonly IDisposable? OwnedServiceProvider;
@@ -24,6 +25,7 @@ internal sealed class MeterProviderSdk : MeterProvider
     internal bool EmitOverflowAttribute;
     internal bool ReclaimUnusedMetricPoints;
     internal ExemplarFilterType? ExemplarFilter;
+    internal ExemplarFilterType? ExemplarFilterForHistograms;
     internal Action? OnCollectObservableInstruments;
 
     private readonly List<object> instrumentations = new();
@@ -83,7 +85,8 @@ internal MeterProviderSdk(
                 state.CardinalityLimit,
                 this.EmitOverflowAttribute,
                 this.ReclaimUnusedMetricPoints,
-                this.ExemplarFilter);
+                this.ExemplarFilter,
+                this.ExemplarFilterForHistograms);
 
             if (this.reader == null)
             {
@@ -490,37 +493,70 @@ private void ApplySpecificationConfigurationKeys(IConfiguration configuration)
             OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent("Reclaim unused metric point feature enabled via configuration.");
         }
 
+        var hasProgrammaticExemplarFilterValue = this.ExemplarFilter.HasValue;
+
         if (configuration.TryGetStringValue(ExemplarFilterConfigKey, out var configValue))
         {
-            if (this.ExemplarFilter.HasValue)
+            if (hasProgrammaticExemplarFilterValue)
             {
                 OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent(
                     $"Exemplar filter configuration value '{configValue}' has been ignored because a value '{this.ExemplarFilter}' was set programmatically.");
                 return;
             }
 
-            ExemplarFilterType? exemplarFilter;
+            if (!TryParseExemplarFilterFromConfigurationValue(configValue, out var exemplarFilter))
+            {
+                OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent($"Exemplar filter configuration was found but the value '{configValue}' is invalid and will be ignored.");
+                return;
+            }
+
+            this.ExemplarFilter = exemplarFilter;
+
+            OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent($"Exemplar filter set to '{exemplarFilter}' from configuration.");
+        }
+
+        if (configuration.TryGetStringValue(ExemplarFilterHistogramsConfigKey, out configValue))
+        {
+            if (hasProgrammaticExemplarFilterValue)
+            {
+                OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent(
+                    $"Exemplar filter histogram configuration value '{configValue}' has been ignored because a value '{this.ExemplarFilter}' was set programmatically.");
+                return;
+            }
+
+            if (!TryParseExemplarFilterFromConfigurationValue(configValue, out var exemplarFilter))
+            {
+                OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent($"Exemplar filter histogram configuration was found but the value '{configValue}' is invalid and will be ignored.");
+                return;
+            }
+
+            this.ExemplarFilterForHistograms = exemplarFilter;
+
+            OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent($"Exemplar filter for histograms set to '{exemplarFilter}' from configuration.");
+        }
+
+        static bool TryParseExemplarFilterFromConfigurationValue(string? configValue, out ExemplarFilterType? exemplarFilter)
+        {
             if (string.Equals("always_off", configValue, StringComparison.OrdinalIgnoreCase))
             {
                 exemplarFilter = ExemplarFilterType.AlwaysOff;
+                return true;
             }
-            else if (string.Equals("always_on", configValue, StringComparison.OrdinalIgnoreCase))
+
+            if (string.Equals("always_on", configValue, StringComparison.OrdinalIgnoreCase))
             {
                 exemplarFilter = ExemplarFilterType.AlwaysOn;
+                return true;
             }
-            else if (string.Equals("trace_based", configValue, StringComparison.OrdinalIgnoreCase))
+
+            if (string.Equals("trace_based", configValue, StringComparison.OrdinalIgnoreCase))
             {
                 exemplarFilter = ExemplarFilterType.TraceBased;
+                return true;
             }
-            else
-            {
-                OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent($"Exemplar filter configuration was found but the value '{configValue}' is invalid and will be ignored.");
-                return;
-            }
-
-            this.ExemplarFilter = exemplarFilter;
 
-            OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent($"Exemplar filter set to '{exemplarFilter}' from configuration.");
+            exemplarFilter = null;
+            return false;
         }
     }
 }

src/OpenTelemetry/Metrics/Metric.cs

@@ -125,12 +125,7 @@ internal Metric(
             aggType = AggregationType.LongGauge;
             this.MetricType = MetricType.LongGauge;
         }
-        else if (instrumentIdentity.InstrumentType == typeof(Histogram<long>)
-            || instrumentIdentity.InstrumentType == typeof(Histogram<int>)
-            || instrumentIdentity.InstrumentType == typeof(Histogram<short>)
-            || instrumentIdentity.InstrumentType == typeof(Histogram<byte>)
-            || instrumentIdentity.InstrumentType == typeof(Histogram<float>)
-            || instrumentIdentity.InstrumentType == typeof(Histogram<double>))
+        else if (instrumentIdentity.IsHistogram)
         {
             var explicitBucketBounds = instrumentIdentity.HistogramBucketBounds;
             var exponentialMaxSize = instrumentIdentity.ExponentialHistogramMaxSize;

src/OpenTelemetry/Metrics/MetricReaderExt.cs

@@ -25,6 +25,7 @@ public abstract partial class MetricReader
     private bool emitOverflowAttribute;
     private bool reclaimUnusedMetricPoints;
     private ExemplarFilterType? exemplarFilter;
+    private ExemplarFilterType? exemplarFilterForHistograms;
 
     internal static void DeactivateMetric(Metric metric)
     {
@@ -54,6 +55,11 @@ internal virtual List<Metric> AddMetricWithNoViews(Instrument instrument)
         Debug.Assert(this.metrics != null, "this.metrics was null");
 
         var metricStreamIdentity = new MetricStreamIdentity(instrument!, metricStreamConfiguration: null);
+
+        var exemplarFilter = metricStreamIdentity.IsHistogram
+            ? this.exemplarFilterForHistograms ?? this.exemplarFilter
+            : this.exemplarFilter;
+
         lock (this.instrumentCreationLock)
         {
             if (this.TryGetExistingMetric(in metricStreamIdentity, out var existingMetric))
@@ -72,7 +78,13 @@ internal virtual List<Metric> AddMetricWithNoViews(Instrument instrument)
                 Metric? metric = null;
                 try
                 {
-                    metric = new Metric(metricStreamIdentity, this.GetAggregationTemporality(metricStreamIdentity.InstrumentType), this.cardinalityLimit, this.emitOverflowAttribute, this.reclaimUnusedMetricPoints, this.exemplarFilter);
+                    metric = new Metric(
+                        metricStreamIdentity,
+                        this.GetAggregationTemporality(metricStreamIdentity.InstrumentType),
+                        this.cardinalityLimit,
+                        this.emitOverflowAttribute,
+                        this.reclaimUnusedMetricPoints,
+                        exemplarFilter);
                 }
                 catch (NotSupportedException nse)
                 {
@@ -114,6 +126,10 @@ internal virtual List<Metric> AddMetricWithViews(Instrument instrument, List<Met
                 var metricStreamConfig = metricStreamConfigs[i];
                 var metricStreamIdentity = new MetricStreamIdentity(instrument!, metricStreamConfig);
 
+                var exemplarFilter = metricStreamIdentity.IsHistogram
+                    ? this.exemplarFilterForHistograms ?? this.exemplarFilter
+                    : this.exemplarFilter;
+
                 if (!MeterProviderBuilderSdk.IsValidInstrumentName(metricStreamIdentity.InstrumentName))
                 {
                     OpenTelemetrySdkEventSource.Log.MetricInstrumentIgnored(
@@ -150,7 +166,7 @@ internal virtual List<Metric> AddMetricWithViews(Instrument instrument, List<Met
                         metricStreamConfig?.CardinalityLimit ?? this.cardinalityLimit,
                         this.emitOverflowAttribute,
                         this.reclaimUnusedMetricPoints,
-                        this.exemplarFilter,
+                        exemplarFilter,
                         metricStreamConfig?.ExemplarReservoirFactory);
 
                     this.instrumentIdentityToMetric[metricStreamIdentity] = metric;
@@ -170,7 +186,8 @@ internal void ApplyParentProviderSettings(
         int cardinalityLimit,
         bool emitOverflowAttribute,
         bool reclaimUnusedMetricPoints,
-        ExemplarFilterType? exemplarFilter)
+        ExemplarFilterType? exemplarFilter,
+        ExemplarFilterType? exemplarFilterForHistograms)
     {
         this.metricLimit = metricLimit;
         this.metrics = new Metric[metricLimit];
@@ -179,6 +196,7 @@ internal void ApplyParentProviderSettings(
         this.emitOverflowAttribute = emitOverflowAttribute;
         this.reclaimUnusedMetricPoints = reclaimUnusedMetricPoints;
         this.exemplarFilter = exemplarFilter;
+        this.exemplarFilterForHistograms = exemplarFilterForHistograms;
     }
 
     private bool TryGetExistingMetric(in MetricStreamIdentity metricStreamIdentity, [NotNullWhen(true)] out Metric? existingMetric)

src/OpenTelemetry/Metrics/MetricStreamIdentity.cs

@@ -115,6 +115,14 @@ public MetricStreamIdentity(Instrument instrument, MetricStreamConfiguration? me
 
     public bool HistogramRecordMinMax { get; }
 
+    public bool IsHistogram =>
+        this.InstrumentType == typeof(Histogram<long>)
+        || this.InstrumentType == typeof(Histogram<int>)
+        || this.InstrumentType == typeof(Histogram<short>)
+        || this.InstrumentType == typeof(Histogram<byte>)
+        || this.InstrumentType == typeof(Histogram<float>)
+        || this.InstrumentType == typeof(Histogram<double>);
+
     public static bool operator ==(MetricStreamIdentity metricIdentity1, MetricStreamIdentity metricIdentity2) => metricIdentity1.Equals(metricIdentity2);
 
     public static bool operator !=(MetricStreamIdentity metricIdentity1, MetricStreamIdentity metricIdentity2) => !metricIdentity1.Equals(metricIdentity2);

test/OpenTelemetry.Tests/Metrics/MetricExemplarTests.cs

@@ -34,6 +34,7 @@ public void TestExemplarFilterSetFromConfiguration(
             configBuilder.AddInMemoryCollection(new Dictionary<string, string?>
             {
                 [MeterProviderSdk.ExemplarFilterConfigKey] = configValue,
+                [MeterProviderSdk.ExemplarFilterHistogramsConfigKey] = configValue,
             });
         }
 
@@ -52,6 +53,14 @@ public void TestExemplarFilterSetFromConfiguration(
 
         Assert.NotNull(meterProviderSdk);
         Assert.Equal((ExemplarFilterType?)expectedValue, meterProviderSdk.ExemplarFilter);
+        if (programmaticValue.HasValue)
+        {
+            Assert.False(meterProviderSdk.ExemplarFilterForHistograms.HasValue);
+        }
+        else
+        {
+            Assert.Equal((ExemplarFilterType?)expectedValue, meterProviderSdk.ExemplarFilterForHistograms);
+        }
     }
 
     [Theory]
@@ -260,9 +269,10 @@ static void ValidateSecondPhase(
     }
 
     [Theory]
-    [InlineData(MetricReaderTemporalityPreference.Cumulative)]
-    [InlineData(MetricReaderTemporalityPreference.Delta)]
-    public void TestExemplarsHistogramWithBuckets(MetricReaderTemporalityPreference temporality)
+    [InlineData(MetricReaderTemporalityPreference.Cumulative, null)]
+    [InlineData(MetricReaderTemporalityPreference.Delta, null)]
+    [InlineData(MetricReaderTemporalityPreference.Delta, "always_on")]
+    public void TestExemplarsHistogramWithBuckets(MetricReaderTemporalityPreference temporality, string? configValue)
     {
         DateTime testStartTime = DateTime.UtcNow;
         var exportedItems = new List<Metric>();
@@ -275,31 +285,48 @@ public void TestExemplarsHistogramWithBuckets(MetricReaderTemporalityPreference
 
         var buckets = new double[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
 
-        using var container = this.BuildMeterProvider(out var meterProvider, builder => builder
-            .AddMeter(meter.Name)
-            .SetExemplarFilter(ExemplarFilterType.AlwaysOn)
-            .AddView(i =>
+        var configBuilder = new ConfigurationBuilder();
+        if (!string.IsNullOrEmpty(configValue))
+        {
+            configBuilder.AddInMemoryCollection(new Dictionary<string, string?>
+            {
+                [MeterProviderSdk.ExemplarFilterHistogramsConfigKey] = configValue,
+            });
+        }
+
+        using var container = this.BuildMeterProvider(out var meterProvider, builder =>
+        {
+            if (string.IsNullOrEmpty(configValue))
             {
-                if (i.Name.StartsWith("histogramWithBucketsAndMinMax"))
+                builder.SetExemplarFilter(ExemplarFilterType.AlwaysOn);
+            }
+
+            builder
+                .ConfigureServices(s => s.AddSingleton<IConfiguration>(configBuilder.Build()))
+                .AddMeter(meter.Name)
+                .AddView(i =>
                 {
-                    return new ExplicitBucketHistogramConfiguration
+                    if (i.Name.StartsWith("histogramWithBucketsAndMinMax"))
                     {
-                        Boundaries = buckets,
-                    };
-                }
-                else
-                {
-                    return new ExplicitBucketHistogramConfiguration
+                        return new ExplicitBucketHistogramConfiguration
+                        {
+                            Boundaries = buckets,
+                        };
+                    }
+                    else
                     {
-                        Boundaries = buckets,
-                        RecordMinMax = false,
-                    };
-                }
-            })
-            .AddInMemoryExporter(exportedItems, metricReaderOptions =>
-            {
-                metricReaderOptions.TemporalityPreference = temporality;
-            }));
+                        return new ExplicitBucketHistogramConfiguration
+                        {
+                            Boundaries = buckets,
+                            RecordMinMax = false,
+                        };
+                    }
+                })
+                .AddInMemoryExporter(exportedItems, metricReaderOptions =>
+                {
+                    metricReaderOptions.TemporalityPreference = temporality;
+                });
+        });
 
         var measurementValues = buckets
             /* 2000 is here to test overflow measurement */