diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/Diagnostics/AzureMonitorExporterEventSource.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/Diagnostics/AzureMonitorExporterEventSource.cs index 8affcf29db13..e2d1bb9eb3e8 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/Diagnostics/AzureMonitorExporterEventSource.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/Diagnostics/AzureMonitorExporterEventSource.cs @@ -491,5 +491,50 @@ public void CustomerSdkStatsInitializationFailed(Exception ex) [Event(50, Message = "Invalid sampler argument '{1}' for sampler '{0}'. Ignoring.", Level = EventLevel.Warning)] public void InvalidSamplerArgument(string samplerType, string samplerArg) => WriteEvent(50, samplerType, samplerArg); + + [Event(51, Message = "Failure to calculate CPU Counter. Unexpected negative timespan: PreviousCollectedTime: {0}. RecentCollectedTime: {1}. Not user actionable.", Level = EventLevel.Error)] + public void ProcessCountersUnexpectedNegativeTimeSpan(long previousCollectedTime, long recentCollectedTime) => WriteEvent(51, previousCollectedTime, recentCollectedTime); + + [Event(52, Message = "Failure to calculate CPU Counter. Unexpected negative value: PreviousCollectedValue: {0}. RecentCollectedValue: {1}. Not user actionable.", Level = EventLevel.Error)] + public void ProcessCountersUnexpectedNegativeValue(long previousCollectedValue, long recentCollectedValue) => WriteEvent(52, previousCollectedValue, recentCollectedValue); + + [Event(53, Message = "Calculated Cpu Counter: Period: {0}. DiffValue: {1}. CalculatedValue: {2}. ProcessorCount: {3}. NormalizedValue: {4}", Level = EventLevel.Verbose)] + public void ProcessCountersCpuCounter(long period, long diffValue, double calculatedValue, int processorCount, double normalizedValue) => WriteEvent(53, period, diffValue, calculatedValue, processorCount, normalizedValue); + + [NonEvent] + public void FailedToCollectProcessPrivateBytes(System.Exception ex) + { + if (IsEnabled(EventLevel.Error)) + { + FailedToCollectProcessPrivateBytes(ex.FlattenException().ToInvariantString()); + } + } + + [Event(54, Message = "Failed to collect Process Private Bytes due to an exception. {0}", Level = EventLevel.Warning)] + public void FailedToCollectProcessPrivateBytes(string exceptionMessage) => WriteEvent(54, exceptionMessage); + + [NonEvent] + public void FailedToCalculateRequestRate(Exception ex) + { + if (IsEnabled(EventLevel.Warning)) + { + FailedToCalculateRequestRate(ex.FlattenException().ToInvariantString()); + } + } + + [Event(55, Message = "Failed to calculate request rate due to an exception. {0}", Level = EventLevel.Warning)] + public void FailedToCalculateRequestRate(string exceptionMessage) => WriteEvent(55, exceptionMessage); + + [NonEvent] + public void FailedToCalculateExceptionRate(Exception ex) + { + if (IsEnabled(EventLevel.Warning)) + { + FailedToCalculateExceptionRate(ex.FlattenException().ToInvariantString()); + } + } + + [Event(56, Message = "Failed to calculate exception rate due to an exception. {0}", Level = EventLevel.Warning)] + public void FailedToCalculateExceptionRate(string exceptionMessage) => WriteEvent(56, exceptionMessage); } } diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/PerfCounterConstants.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/PerfCounterConstants.cs new file mode 100644 index 000000000000..9aeea059fa6c --- /dev/null +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/PerfCounterConstants.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Monitor.OpenTelemetry.Exporter.Internals +{ + internal static class PerfCounterConstants + { + internal const string PerfCounterMeterName = "PerfCounterMeter"; + internal const string RequestRateInstrumentationName = "RequestCounterRate"; + internal const string ProcessCpuInstrumentationName = "ProcessCpu"; + internal const string ProcessCpuNormalizedInstrumentationName = "ProcessCpuNormalized"; + internal const string ProcessPrivateBytesInstrumentationName = "ProcessPrivateBytes"; + internal const string ExceptionRateName = "AzureMonitorExceptionRate"; + + // Breeze perf counter names + internal const string ExceptionRateMetricIdValue = "\\.NET CLR Exceptions(??APP_CLR_PROC??)\\# of Exceps Thrown / sec"; + internal const string RequestRateMetricIdValue = "\\ASP.NET Applications(??APP_W3SVC_PROC??)\\Requests/Sec"; + internal const string ProcessCpuMetricIdValue = "\\Process(??APP_WIN32_PROC??)\\% Processor Time"; + internal const string ProcessCpuNormalizedMetricIdValue = "\\Process(??APP_WIN32_PROC??)\\% Processor Time Normalized"; + internal const string ProcessPrivateBytesMetricIdValue = "\\Process(??APP_WIN32_PROC??)\\Private Bytes"; + } +} diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/StandardMetricsExtractionProcessor.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/StandardMetricsExtractionProcessor.cs index 8866e1202695..0b5266dca88d 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/StandardMetricsExtractionProcessor.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/StandardMetricsExtractionProcessor.cs @@ -1,10 +1,13 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using System; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.Metrics; +using System.Diagnostics.Tracing; +using System.Threading; +using Azure.Monitor.OpenTelemetry.Exporter.Internals.Diagnostics; using Azure.Monitor.OpenTelemetry.Exporter.Models; using OpenTelemetry; using OpenTelemetry.Metrics; @@ -13,17 +16,53 @@ namespace Azure.Monitor.OpenTelemetry.Exporter.Internals { internal sealed class StandardMetricsExtractionProcessor : BaseProcessor { + [ThreadStatic] private static bool t_handlingFirstChanceException; private bool _disposed; private AzureMonitorResource? _resource; + internal readonly MeterProvider? _meterProvider; - private readonly Meter _meter; + private readonly Meter _standardMetricMeter; + private readonly Meter _perfCounterMeter; + private readonly Histogram _requestDuration; private readonly Histogram _dependencyDuration; + private readonly ObservableGauge _processPrivateBytesGauge; + private readonly ObservableGauge _processCpuGauge; + private readonly ObservableGauge _processCpuNormalizedGauge; + private readonly ObservableGauge _requestRateGauge; + private readonly ObservableGauge _exceptionRateGauge; + + private readonly Process _process = Process.GetCurrentProcess(); + private readonly int _processorCount = Environment.ProcessorCount; + private DateTimeOffset _cachedCollectedTime = DateTimeOffset.MinValue; + private long _cachedCollectedValue = 0; + + private DateTimeOffset _lastCpuCalculationTime = DateTimeOffset.MinValue; + private double _cachedRawCpuValue = 0; + private double _cachedNormalizedCpuValue = 0; + private bool _cpuCalculationValid = false; + + // Request rate tracking + private long _requestCount = 0; + private DateTimeOffset _lastRequestRateCalculationTime = DateTimeOffset.UtcNow; + private long _lastRequestCount = 0; + + // Exception rate tracking + private long _exceptionCount = 0; + private DateTimeOffset _lastExceptionRateCalculationTime = DateTimeOffset.UtcNow; + private long _lastExceptionCount = 0; + internal static readonly IReadOnlyDictionary s_standardMetricNameMapping = new Dictionary() { [StandardMetricConstants.RequestDurationInstrumentName] = StandardMetricConstants.RequestDurationMetricIdValue, [StandardMetricConstants.DependencyDurationInstrumentName] = StandardMetricConstants.DependencyDurationMetricIdValue, + + [PerfCounterConstants.RequestRateInstrumentationName] = PerfCounterConstants.RequestRateMetricIdValue, + [PerfCounterConstants.ProcessPrivateBytesInstrumentationName] = PerfCounterConstants.ProcessPrivateBytesMetricIdValue, + [PerfCounterConstants.ExceptionRateName] = PerfCounterConstants.ExceptionRateMetricIdValue, + [PerfCounterConstants.ProcessCpuInstrumentationName] = PerfCounterConstants.ProcessCpuMetricIdValue, + [PerfCounterConstants.ProcessCpuNormalizedInstrumentationName] = PerfCounterConstants.ProcessCpuNormalizedMetricIdValue, }; internal AzureMonitorResource? StandardMetricResource => _resource ??= ParentProvider?.GetResource().CreateAzureMonitorResource(); @@ -32,12 +71,38 @@ internal StandardMetricsExtractionProcessor(AzureMonitorMetricExporter metricExp { _meterProvider = Sdk.CreateMeterProviderBuilder() .AddMeter(StandardMetricConstants.StandardMetricMeterName) + .AddMeter(PerfCounterConstants.PerfCounterMeterName) .AddReader(new PeriodicExportingMetricReader(metricExporter) { TemporalityPreference = MetricReaderTemporalityPreference.Delta }) .Build(); - _meter = new Meter(StandardMetricConstants.StandardMetricMeterName); - _requestDuration = _meter.CreateHistogram(StandardMetricConstants.RequestDurationInstrumentName); - _dependencyDuration = _meter.CreateHistogram(StandardMetricConstants.DependencyDurationInstrumentName); + + _standardMetricMeter = new Meter(StandardMetricConstants.StandardMetricMeterName); + _requestDuration = _standardMetricMeter.CreateHistogram(StandardMetricConstants.RequestDurationInstrumentName); + _dependencyDuration = _standardMetricMeter.CreateHistogram(StandardMetricConstants.DependencyDurationInstrumentName); + + _perfCounterMeter = new Meter(PerfCounterConstants.PerfCounterMeterName); + _requestRateGauge = _perfCounterMeter.CreateObservableGauge(PerfCounterConstants.RequestRateInstrumentationName, () => GetRequestRate()); + _processPrivateBytesGauge = _perfCounterMeter.CreateObservableGauge(PerfCounterConstants.ProcessPrivateBytesInstrumentationName, () => GetProcessPrivateBytes()); + _processCpuGauge = _perfCounterMeter.CreateObservableGauge(PerfCounterConstants.ProcessCpuInstrumentationName, () => GetProcessCPU()); + _processCpuNormalizedGauge = _perfCounterMeter.CreateObservableGauge(PerfCounterConstants.ProcessCpuNormalizedInstrumentationName, () => GetProcessCPUNormalized()); + _exceptionRateGauge = _perfCounterMeter.CreateObservableGauge(PerfCounterConstants.ExceptionRateName, () => GetExceptionRate()); + + AppDomain.CurrentDomain.FirstChanceException += (source, e) => + { + // Avoid recursion if the listener itself throws an exception while recording the measurement + // in its `OnMeasurementRecorded` callback. + if (t_handlingFirstChanceException) + return; + + t_handlingFirstChanceException = true; + + // Increment exception count for rate calculation + Interlocked.Increment(ref _exceptionCount); + + t_handlingFirstChanceException = false; + }; + + InitializeCpuBaseline(); } public override void OnEnd(Activity activity) @@ -49,6 +114,9 @@ public override void OnEnd(Activity activity) activity.SetTag("_MS.ProcessedByMetricExtractors", "(Name: X,Ver:'1.1')"); ReportRequestDurationMetric(activity); } + + // Increment request count for rate calculation + Interlocked.Increment(ref _requestCount); } if (activity.Kind == ActivityKind.Client || activity.Kind == ActivityKind.Internal || activity.Kind == ActivityKind.Producer) { @@ -133,6 +201,202 @@ private void ReportDependencyDurationMetric(Activity activity) activityTagsProcessor.Return(); } + private long GetProcessPrivateBytes() + { + try + { + _process.Refresh(); + return _process.PrivateMemorySize64; + } + catch (Exception ex) + { + // Log to event source. + AzureMonitorExporterEventSource.Log.FailedToCollectProcessPrivateBytes(ex); + return 0; + } + } + + private double GetProcessCPU() + { + EnsureCpuCalculation(); + return _cachedRawCpuValue; + } + + private double GetProcessCPUNormalized() + { + EnsureCpuCalculation(); + return _cachedNormalizedCpuValue; + } + + private double GetRequestRate() + { + try + { + var now = DateTimeOffset.UtcNow; + var currentRequestCount = Interlocked.Read(ref _requestCount); + + // Calculate rate for the duration since last calculation + var timeDifferenceSeconds = (now - _lastRequestRateCalculationTime).TotalSeconds; + var requestDifference = currentRequestCount - _lastRequestCount; + + double currentRate = 0; + if (timeDifferenceSeconds > 0) + { + currentRate = requestDifference / timeDifferenceSeconds; + } + + // Update for next calculation + _lastRequestRateCalculationTime = now; + _lastRequestCount = currentRequestCount; + + return Math.Max(0, currentRate); + } + catch (Exception ex) + { + AzureMonitorExporterEventSource.Log.FailedToCalculateRequestRate(ex); + return 0; + } + } + + private double GetExceptionRate() + { + try + { + var now = DateTimeOffset.UtcNow; + var currentExceptionCount = Interlocked.Read(ref _exceptionCount); + + // Calculate rate for the duration since last calculation + var timeDifferenceSeconds = (now - _lastExceptionRateCalculationTime).TotalSeconds; + var exceptionDifference = currentExceptionCount - _lastExceptionCount; + + double currentRate = 0; + if (timeDifferenceSeconds > 0) + { + currentRate = exceptionDifference / timeDifferenceSeconds; + } + + // Update for next calculation + _lastExceptionRateCalculationTime = now; + _lastExceptionCount = currentExceptionCount; + + return Math.Max(0, currentRate); + } + catch (Exception ex) + { + AzureMonitorExporterEventSource.Log.FailedToCalculateExceptionRate(ex); + return 0; + } + } + + private void EnsureCpuCalculation() + { + var now = DateTimeOffset.UtcNow; + + // Check if we need to recalculate (59-second cache to align with ~60-second collection intervals) + if (!_cpuCalculationValid || (now - _lastCpuCalculationTime).TotalSeconds > 59) + { + try + { + if (TryCalculateCPUCounter(out double rawValue, out double normalizedValue)) + { + _cachedRawCpuValue = rawValue; + _cachedNormalizedCpuValue = normalizedValue; + _cpuCalculationValid = true; + } + else + { + _cachedRawCpuValue = 0; + _cachedNormalizedCpuValue = 0; + _cpuCalculationValid = false; + } + } + catch + { + _cachedRawCpuValue = 0; + _cachedNormalizedCpuValue = 0; + _cpuCalculationValid = false; + } + + _lastCpuCalculationTime = now; + } + } + + private bool TryCalculateCPUCounter(out double rawValue, out double normalizedValue) + { + var previousCollectedValue = _cachedCollectedValue; + var previousCollectedTime = _cachedCollectedTime; + + // Refresh process data to get current CPU time + _process.Refresh(); + + var recentCollectedValue = _cachedCollectedValue = _process.TotalProcessorTime.Ticks; + var recentCollectedTime = _cachedCollectedTime = DateTimeOffset.UtcNow; + + if (previousCollectedTime == DateTimeOffset.MinValue) + { + rawValue = default; + normalizedValue = default; + return false; + } + + var period = recentCollectedTime.Ticks - previousCollectedTime.Ticks; + if (period <= 0) + { + AzureMonitorExporterEventSource.Log.ProcessCountersUnexpectedNegativeTimeSpan( + previousCollectedTime: previousCollectedTime.Ticks, + recentCollectedTime: recentCollectedTime.Ticks); + + rawValue = default; + normalizedValue = default; + return false; + } + + var diff = recentCollectedValue - previousCollectedValue; + if (diff < 0) + { + AzureMonitorExporterEventSource.Log.ProcessCountersUnexpectedNegativeValue( + previousCollectedValue: previousCollectedValue, + recentCollectedValue: recentCollectedValue); + + rawValue = default; + normalizedValue = default; + return false; + } + + // Calculate raw CPU percentage (can exceed 100% on multi-core systems) + rawValue = (double)diff * 100.0 / period; + + // Calculate normalized CPU percentage (0-100%) + normalizedValue = rawValue / _processorCount; + + // Clamp to reasonable bounds + rawValue = Math.Max(0, Math.Min(100 * _processorCount, rawValue)); + normalizedValue = Math.Max(0, Math.Min(100, normalizedValue)); + + AzureMonitorExporterEventSource.Log.ProcessCountersCpuCounter( + period: period, + diffValue: diff, + calculatedValue: rawValue, + processorCount: _processorCount, + normalizedValue: normalizedValue); + + return true; + } + + private void InitializeCpuBaseline() + { + try + { + _process.Refresh(); + _cachedCollectedValue = _process.TotalProcessorTime.Ticks; + _cachedCollectedTime = DateTimeOffset.UtcNow; + } + catch + { + // If initialization fails, keep defaults and let first measurement be zero + } + } + protected override void Dispose(bool disposing) { if (!_disposed) @@ -142,7 +406,9 @@ protected override void Dispose(bool disposing) try { _meterProvider?.Dispose(); - _meter?.Dispose(); + _standardMetricMeter?.Dispose(); + _perfCounterMeter?.Dispose(); + _process?.Dispose(); } catch (Exception) { diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/StandardMetricTests.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/StandardMetricTests.cs index 8bbb12c9d5cf..3afefcc2c989 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/StandardMetricTests.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/StandardMetricTests.cs @@ -50,10 +50,10 @@ public void ValidateRequestDurationMetric() standardMetricCustomProcessor._meterProvider?.ForceFlush(); - Assert.Single(metricTelemetryItems); - - var metricTelemetry = metricTelemetryItems.Last()!; - Assert.Equal("MetricData", metricTelemetry.Data.BaseType); + // Find the specific Request Duration metric among possibly many perf counter metrics. + var metricTelemetry = GetMetricTelemetry(metricTelemetryItems, StandardMetricConstants.RequestDurationMetricIdValue); + Assert.NotNull(metricTelemetry); + Assert.Equal("MetricData", metricTelemetry!.Data.BaseType); var metricData = (MetricsData)metricTelemetry.Data.BaseData; Assert.True(metricData.Properties.TryGetValue(StandardMetricConstants.RequestSuccessKey, out var isSuccess)); Assert.Equal("True", isSuccess); @@ -99,10 +99,9 @@ public void ValidateRequestDurationMetricNew() standardMetricCustomProcessor._meterProvider?.ForceFlush(); - Assert.Single(metricTelemetryItems); - - var metricTelemetry = metricTelemetryItems.Last()!; - Assert.Equal("MetricData", metricTelemetry.Data.BaseType); + var metricTelemetry = GetMetricTelemetry(metricTelemetryItems, StandardMetricConstants.RequestDurationMetricIdValue); + Assert.NotNull(metricTelemetry); + Assert.Equal("MetricData", metricTelemetry!.Data.BaseType); var metricData = (MetricsData)metricTelemetry.Data.BaseData; Assert.True(metricData.Properties.TryGetValue(StandardMetricConstants.RequestSuccessKey, out var isSuccess)); Assert.Equal("True", isSuccess); @@ -151,10 +150,9 @@ public void ValidateRequestDurationMetricConsumerKind() standardMetricCustomProcessor._meterProvider?.ForceFlush(); - Assert.Single(metricTelemetryItems); - - var metricTelemetry = metricTelemetryItems.Last()!; - Assert.Equal("MetricData", metricTelemetry.Data.BaseType); + var metricTelemetry = GetMetricTelemetry(metricTelemetryItems, StandardMetricConstants.RequestDurationMetricIdValue); + Assert.NotNull(metricTelemetry); + Assert.Equal("MetricData", metricTelemetry!.Data.BaseType); var metricData = (MetricsData)metricTelemetry.Data.BaseData; Assert.True(metricData.Properties.TryGetValue(StandardMetricConstants.RequestSuccessKey, out var isSuccess)); Assert.Equal("True", isSuccess); @@ -208,10 +206,9 @@ public void ValidateDependencyDurationMetric(bool isAzureSDK) standardMetricCustomProcessor._meterProvider?.ForceFlush(); - Assert.Single(metricTelemetryItems); - - var metricTelemetry = metricTelemetryItems.Last()!; - Assert.Equal("MetricData", metricTelemetry.Data.BaseType); + var metricTelemetry = GetMetricTelemetry(metricTelemetryItems, StandardMetricConstants.DependencyDurationMetricIdValue); + Assert.NotNull(metricTelemetry); + Assert.Equal("MetricData", metricTelemetry!.Data.BaseType); var metricData = (MetricsData)metricTelemetry.Data.BaseData; Assert.True(metricData.Properties.TryGetValue(StandardMetricConstants.DependencySuccessKey, out var isSuccess)); Assert.Equal("True", isSuccess); @@ -277,10 +274,9 @@ public void ValidateDependencyDurationMetricForProducerKind(bool isAzureSDKSpan) standardMetricCustomProcessor._meterProvider?.ForceFlush(); - Assert.Single(metricTelemetryItems); - - var metricTelemetry = metricTelemetryItems.Last()!; - Assert.Equal("MetricData", metricTelemetry.Data.BaseType); + var metricTelemetry = GetMetricTelemetry(metricTelemetryItems, StandardMetricConstants.DependencyDurationMetricIdValue); + Assert.NotNull(metricTelemetry); + Assert.Equal("MetricData", metricTelemetry!.Data.BaseType); var metricData = (MetricsData)metricTelemetry.Data.BaseData; Assert.True(metricData.Properties.TryGetValue(StandardMetricConstants.DependencySuccessKey, out var isSuccess)); Assert.Equal("True", isSuccess); @@ -345,10 +341,9 @@ public void ValidateDependencyDurationMetricNew(bool isAzureSDK) standardMetricCustomProcessor._meterProvider?.ForceFlush(); - Assert.Single(metricTelemetryItems); - - var metricTelemetry = metricTelemetryItems.Last()!; - Assert.Equal("MetricData", metricTelemetry.Data.BaseType); + var metricTelemetry = GetMetricTelemetry(metricTelemetryItems, StandardMetricConstants.DependencyDurationMetricIdValue); + Assert.NotNull(metricTelemetry); + Assert.Equal("MetricData", metricTelemetry!.Data.BaseType); var metricData = (MetricsData)metricTelemetry.Data.BaseData; Assert.True(metricData.Properties.TryGetValue(StandardMetricConstants.DependencySuccessKey, out var isSuccess)); Assert.Equal("True", isSuccess); @@ -405,10 +400,10 @@ public void ValidateNullStatusCode(ActivityKind kind) standardMetricCustomProcessor._meterProvider?.ForceFlush(); - // Standard Metrics + Resource Metrics. - Assert.Single(metricTelemetryItems); - var metricTelemetry = metricTelemetryItems.Last()!; - Assert.Equal("MetricData", metricTelemetry.Data.BaseType); + var metricIdToFind = kind == ActivityKind.Client ? StandardMetricConstants.DependencyDurationMetricIdValue : StandardMetricConstants.RequestDurationMetricIdValue; + var metricTelemetry = GetMetricTelemetry(metricTelemetryItems, metricIdToFind); + Assert.NotNull(metricTelemetry); + Assert.Equal("MetricData", metricTelemetry!.Data.BaseType); var metricData = (MetricsData)metricTelemetry.Data.BaseData; if (kind == ActivityKind.Client) @@ -453,10 +448,10 @@ public void ValidateNullStatusCodeNew(ActivityKind kind) standardMetricCustomProcessor._meterProvider?.ForceFlush(); - // Standard Metrics + Resource Metrics. - Assert.Single(metricTelemetryItems); - var metricTelemetry = metricTelemetryItems.Last()!; - Assert.Equal("MetricData", metricTelemetry.Data.BaseType); + var metricIdToFind = kind == ActivityKind.Client ? StandardMetricConstants.DependencyDurationMetricIdValue : StandardMetricConstants.RequestDurationMetricIdValue; + var metricTelemetry = GetMetricTelemetry(metricTelemetryItems, metricIdToFind); + Assert.NotNull(metricTelemetry); + Assert.Equal("MetricData", metricTelemetry!.Data.BaseType); var metricData = (MetricsData)metricTelemetry.Data.BaseData; if (kind == ActivityKind.Client) @@ -471,6 +466,81 @@ public void ValidateNullStatusCodeNew(ActivityKind kind) } } + [Fact] + public void ValidatePerfCounterMetrics() + { + // This test validates the presence of perf counter based metrics emitted via _perfCounterMeter + // in StandardMetricsExtractionProcessor: Request Rate, Process Private Bytes, CPU, Normalized CPU, Exception Rate. + var activitySource = new ActivitySource(nameof(StandardMetricTests.ValidatePerfCounterMetrics)); + var traceTelemetryItems = new List(); + var metricTelemetryItems = new List(); + + var standardMetricCustomProcessor = new StandardMetricsExtractionProcessor(new AzureMonitorMetricExporter(new MockTransmitter(metricTelemetryItems))); + + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .SetSampler(new AlwaysOnSampler()) + .AddSource(nameof(StandardMetricTests.ValidatePerfCounterMetrics)) + .AddProcessor(standardMetricCustomProcessor) + .AddProcessor(new BatchActivityExportProcessor(new AzureMonitorTraceExporter(new AzureMonitorExporterOptions(), new MockTransmitter(traceTelemetryItems)))) + .Build(); + + // Generate multiple request activities to increment the RequestRate counter. + for (int i = 0; i < 5; i++) + { + using var a = activitySource.StartActivity("Req", ActivityKind.Server); + a?.SetTag(SemanticConventions.AttributeHttpStatusCode, 200); + } + + // Generate a few thrown (and caught) exceptions to trigger System.Runtime "dotnet.exceptions" counter where available. + for (int i = 0; i < 3; i++) + { + try + { + throw new InvalidOperationException("Test exception for exception rate metric"); + } + catch + { + // Swallow - we just need the throw to increment the counter. + } + } + + tracerProvider?.ForceFlush(); + WaitForActivityExport(traceTelemetryItems); + + standardMetricCustomProcessor._meterProvider?.ForceFlush(); + + // We expect multiple metric telemetry items now (at least one per perf counter plus request duration histogram). + Assert.True(metricTelemetryItems.Count >= 2, "Expected multiple metric telemetry items including perf counters."); + + // Helper local function to find metric by mapped name. + MetricsData? FindMetric(string expectedName) => metricTelemetryItems + .Select(ti => (MetricsData)ti.Data.BaseData) + .FirstOrDefault(md => md.Metrics.Count > 0 && md.Metrics[0].Name == expectedName); + + var requestRate = FindMetric(PerfCounterConstants.RequestRateMetricIdValue); + Assert.NotNull(requestRate); + Assert.True(requestRate!.Metrics[0].Value >= 1, "Request rate should be >= 1"); + + var privateBytes = FindMetric(PerfCounterConstants.ProcessPrivateBytesMetricIdValue); + Assert.NotNull(privateBytes); + Assert.True(privateBytes!.Metrics[0].Value >= 0); + + var cpu = FindMetric(PerfCounterConstants.ProcessCpuMetricIdValue); + Assert.NotNull(cpu); + Assert.True(cpu!.Metrics[0].Value >= 0); + + var cpuNormalized = FindMetric(PerfCounterConstants.ProcessCpuNormalizedMetricIdValue); + Assert.NotNull(cpuNormalized); + Assert.True(cpuNormalized!.Metrics[0].Value >= 0); + + // Exception rate metric may not be available on all target frameworks/runtimes; assert only if present. + var exceptionRate = FindMetric(PerfCounterConstants.ExceptionRateMetricIdValue); + if (exceptionRate != null) + { + Assert.True(exceptionRate.Metrics[0].Value >= 0); + } + } + private void WaitForActivityExport(List traceTelemetryItems) { var result = SpinWait.SpinUntil( @@ -483,5 +553,21 @@ private void WaitForActivityExport(List traceTelemetryItems) Assert.True(result, $"{nameof(WaitForActivityExport)} failed."); } + + private TelemetryItem? GetMetricTelemetry(List metricTelemetryItems, string metricName) + { + foreach (var item in metricTelemetryItems) + { + if (item.Data.BaseType == "MetricData") + { + var data = (MetricsData)item.Data.BaseData; + if (data.Metrics.Count > 0 && data.Metrics[0].Name == metricName) + { + return item; + } + } + } + return null; + } } }