Add support for System.Diagnostics.Metrics in dotnet-monitor's collec…

…tor (#3587)

* [Dotnet Monitor] Ignore - Adding System.Diagnostics.Metrics Support (#3529)

* Got basic counter rate end-to-end working - currently in a broken state as I investigate other types of metrics

* Leftovers from previous commit

* Gauges working for systems diagnostics metrics

* Added in histogram, adding in options for maxHistograms and maxTimeSeries

* Added in error payloads for logging purposes

* Temporarily changed visibility for testing - this may be reverted later

* Now handling multiple sessions

* Added filtering for counters, instead of allowing all counters for a provider to go through.

* Handle observable... errors

* Some cleanup, added error event check

* Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20221129.1 (#3528)

[main] Update dependencies from dotnet/source-build-reference-packages

Co-authored-by: dotnet-maestro[bot] <42748379+dotnet-maestro[bot]@users.noreply.github.com>

* [Dotnet Monitor] Revert To Logging a single payload (not a list) (#3538)

* [Feature branch changes, no review] Fixup counter apis (#3559)

* Fix ICountersLogger contract

* MetadataUpdates

* Fixup api protection levels

* Fixup tests and add CounterEnded payload

* Fixup metadata parsing

* [Dotnet-Monitor] [Feature Branch] Switched quantile to Percentile (#3567)

* Minor branch cleanup

* PR for feature branch

* PR feedback

* Pr feedback feedback

* Convert Histogram to single payload

* Tweaks to account for new All flag

* Fixed build/test failures from Wiktor's changes

* Fix issue with empty quantiles

* Initial PR feedback (#3620)

* Fixes outdated naming in test

---------

Co-authored-by: kkeirstead <[email protected]>
Co-authored-by: dotnet-maestro[bot] <42748379+dotnet-maestro[bot]@users.noreply.github.com>
Co-authored-by: kkeirstead <[email protected]>

src/Microsoft.Diagnostics.Monitoring.EventPipe/Configuration/MetricSourceConfiguration.cs

@@ -13,39 +13,83 @@
 
 namespace Microsoft.Diagnostics.Monitoring.EventPipe
 {
+    [Flags]
+    public enum MetricType
+    {
+        EventCounter = 0x1,
+        Meter = 0x2,
+        All = 0xFF
+    }
+
+    public sealed class MetricEventPipeProvider
+    {
+        public string Provider { get; set; }
+
+        public float? IntervalSeconds { get; set; }
+
+        public MetricType Type { get; set; } = MetricType.All;
+    }
+
     public sealed class MetricSourceConfiguration : MonitoringSourceConfiguration
     {
         private readonly IList<EventPipeProvider> _eventPipeProviders;
+        public string SessionId { get; private set; }
 
-        public MetricSourceConfiguration(float metricIntervalSeconds, IEnumerable<string> customProviderNames)
+        public MetricSourceConfiguration(float metricIntervalSeconds, IEnumerable<string> eventCounterProviderNames)
+            : this(metricIntervalSeconds, CreateProviders(eventCounterProviderNames?.Any() == true ? eventCounterProviderNames : DefaultMetricProviders))
         {
-            RequestRundown = false;
-            if (customProviderNames == null)
-            {
-                throw new ArgumentNullException(nameof(customProviderNames));
-            }
-            MetricIntervalSeconds = metricIntervalSeconds.ToString(CultureInfo.InvariantCulture);
+        }
 
-            IEnumerable<string> providers = null;
-            if (customProviderNames.Any())
+        public MetricSourceConfiguration(float metricIntervalSeconds, IEnumerable<MetricEventPipeProvider> providers, int maxHistograms = 20, int maxTimeSeries = 1000)
+        {
+            if (providers == null)
             {
-                providers = customProviderNames;
+                throw new ArgumentNullException(nameof(providers));
             }
-            else
+
+            RequestRundown = false;
+
+            _eventPipeProviders = providers.Where(provider => provider.Type.HasFlag(MetricType.EventCounter))
+                .Select((MetricEventPipeProvider provider) => new EventPipeProvider(provider.Provider,
+                    EventLevel.Informational,
+                    (long)ClrTraceEventParser.Keywords.None,
+                    new Dictionary<string, string>()
+                    {
+                        {
+                            "EventCounterIntervalSec", (provider.IntervalSeconds ?? metricIntervalSeconds).ToString(CultureInfo.InvariantCulture)
+                        }
+                    })).ToList();
+
+            IEnumerable<MetricEventPipeProvider> meterProviders = providers.Where(provider => provider.Type.HasFlag(MetricType.Meter));
+
+            if (meterProviders.Any())
             {
-                providers = new[] { SystemRuntimeEventSourceName, MicrosoftAspNetCoreHostingEventSourceName, GrpcAspNetCoreServer };
-            }
+                const long TimeSeriesValuesEventKeyword = 0x2;
+                string metrics = string.Join(',', meterProviders.Select(p => p.Provider));
+
+                SessionId = Guid.NewGuid().ToString();
 
-            _eventPipeProviders = providers.Select((string provider) => new EventPipeProvider(provider,
-               EventLevel.Informational,
-               (long)ClrTraceEventParser.Keywords.None,
-               new Dictionary<string, string>()
-               {
-                    { "EventCounterIntervalSec", MetricIntervalSeconds }
-               })).ToList();
+                EventPipeProvider metricsEventSourceProvider =
+                    new EventPipeProvider(MonitoringSourceConfiguration.SystemDiagnosticsMetricsProviderName, EventLevel.Informational, TimeSeriesValuesEventKeyword,
+                        new Dictionary<string, string>()
+                        {
+                            { "SessionId", SessionId },
+                            { "Metrics", metrics },
+                            { "RefreshInterval", metricIntervalSeconds.ToString(CultureInfo.InvariantCulture) },
+                            { "MaxTimeSeries", maxTimeSeries.ToString(CultureInfo.InvariantCulture) },
+                            { "MaxHistograms", maxHistograms.ToString(CultureInfo.InvariantCulture) }
+                        }
+                    );
+
+                _eventPipeProviders = _eventPipeProviders.Append(metricsEventSourceProvider).ToArray();
+            }
         }
 
-        private string MetricIntervalSeconds { get; }
+        private static IEnumerable<MetricEventPipeProvider> CreateProviders(IEnumerable<string> providers) =>
+            providers.Select(provider => new MetricEventPipeProvider {
+                Provider = provider,
+                Type = MetricType.EventCounter
+            });
 
         public override IList<EventPipeProvider> GetProviders() => _eventPipeProviders;
     }

src/Microsoft.Diagnostics.Monitoring.EventPipe/Configuration/MonitoringSourceConfiguration.cs

@@ -27,6 +27,7 @@ public abstract class MonitoringSourceConfiguration
         public const string TplEventSource = "System.Threading.Tasks.TplEventSource";
         public const string SampleProfilerProviderName = "Microsoft-DotNETCore-SampleProfiler";
         public const string EventPipeProviderName = "Microsoft-DotNETCore-EventPipe";
+        public const string SystemDiagnosticsMetricsProviderName = "System.Diagnostics.Metrics";
 
         public static IEnumerable<string> DefaultMetricProviders => new[] { SystemRuntimeEventSourceName, MicrosoftAspNetCoreHostingEventSourceName, GrpcAspNetCoreServer };
 

src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/CounterFilter.cs

@@ -38,12 +38,20 @@ public void AddFilter(string providerName, string[] counters)
 
         public IEnumerable<string> GetProviders() => _enabledCounters.Keys;
 
+        public int IntervalSeconds => _intervalMilliseconds / 1000;
+
         public bool IsIncluded(string providerName, string counterName, int intervalMilliseconds)
         {
             if (_intervalMilliseconds != intervalMilliseconds)
             {
                 return false;
             }
+
+            return IsIncluded(providerName, counterName);
+        }
+
+        public bool IsIncluded(string providerName, string counterName)
+        {
             if (_enabledCounters.Count == 0)
             {
                 return true;

src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/CounterPayload.cs

@@ -5,17 +5,15 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
+using System.Linq;
 
 namespace Microsoft.Diagnostics.Monitoring.EventPipe
 {
+    /// <summary>
+    /// TODO This is currently a duplication of the src\Tools\dotnet-counters\CounterPayload.cs stack. The two will be unified in a separate change.
+    /// </summary>
     internal class CounterPayload : ICounterPayload
     {
-#if NETSTANDARD
-        private static readonly IReadOnlyDictionary<string, string> Empty = new ReadOnlyDictionary<string, string>(new Dictionary<string, string>(0));
-#else
-        private static readonly IReadOnlyDictionary<string, string> Empty = System.Collections.Immutable.ImmutableDictionary<string, string>.Empty;
-#endif
-
         public CounterPayload(DateTime timestamp,
             string provider,
             string name,
@@ -24,7 +22,7 @@ public CounterPayload(DateTime timestamp,
             double value,
             CounterType counterType,
             float interval,
-            Dictionary<string, string> metadata)
+            string metadata)
         {
             Timestamp = timestamp;
             Name = name;
@@ -34,14 +32,27 @@ public CounterPayload(DateTime timestamp,
             CounterType = counterType;
             Provider = provider;
             Interval = interval;
-            Metadata = metadata ?? Empty;
+            Metadata = metadata;
+            EventType = EventType.Gauge;
+        }
+
+        // Copied from dotnet-counters
+        public CounterPayload(string providerName, string name, string displayName, string displayUnits, string metadata, double value, DateTime timestamp, string type, EventType eventType)
+        {
+            Provider = providerName;
+            Name = name;
+            Metadata = metadata;
+            Value = value;
+            Timestamp = timestamp;
+            CounterType = (CounterType)Enum.Parse(typeof(CounterType), type);
+            EventType = eventType;
         }
 
         public string Namespace { get; }
 
         public string Name { get; }
 
-        public string DisplayName { get; }
+        public string DisplayName { get; protected set; }
 
         public string Unit { get; }
 
@@ -55,6 +66,81 @@ public CounterPayload(DateTime timestamp,
 
         public string Provider { get; }
 
-        public IReadOnlyDictionary<string, string> Metadata { get; }
+        public string Metadata { get; }
+
+        public EventType EventType { get; set; }
+    }
+
+    internal class GaugePayload : CounterPayload
+    {
+        public GaugePayload(string providerName, string name, string displayName, string displayUnits, string metadata, double value, DateTime timestamp) :
+            base(providerName, name, displayName, displayUnits, metadata, value, timestamp, "Metric", EventType.Gauge)
+        {
+            // In case these properties are not provided, set them to appropriate values.
+            string counterName = string.IsNullOrEmpty(displayName) ? name : displayName;
+            DisplayName = !string.IsNullOrEmpty(displayUnits) ? $"{counterName} ({displayUnits})" : counterName;
+        }
+    }
+
+    internal class CounterEndedPayload : CounterPayload
+    {
+        public CounterEndedPayload(string providerName, string name, string displayName, DateTime timestamp)
+            : base(providerName, name, displayName, string.Empty, null, 0.0, timestamp, "Metric", EventType.CounterEnded)
+        {
+
+        }
+    }
+
+    internal class RatePayload : CounterPayload
+    {
+        public RatePayload(string providerName, string name, string displayName, string displayUnits, string metadata, double value, double intervalSecs, DateTime timestamp) :
+            base(providerName, name, displayName, displayUnits, metadata, value, timestamp, "Rate", EventType.Rate)
+        {
+            // In case these properties are not provided, set them to appropriate values.
+            string counterName = string.IsNullOrEmpty(displayName) ? name : displayName;
+            string unitsName = string.IsNullOrEmpty(displayUnits) ? "Count" : displayUnits;
+            string intervalName = intervalSecs.ToString() + " sec";
+            DisplayName = $"{counterName} ({unitsName} / {intervalName})";
+        }
+    }
+
+    internal class PercentilePayload : CounterPayload
+    {
+        public PercentilePayload(string providerName, string name, string displayName, string displayUnits, string metadata, IEnumerable<Quantile> quantiles, DateTime timestamp) :
+            base(providerName, name, displayName, displayUnits, metadata, 0.0, timestamp, "Metric", EventType.Histogram)
+        {
+            // In case these properties are not provided, set them to appropriate values.
+            string counterName = string.IsNullOrEmpty(displayName) ? name : displayName;
+            DisplayName = !string.IsNullOrEmpty(displayUnits) ? $"{counterName} ({displayUnits})" : counterName;
+            Quantiles = quantiles.ToArray();
+        }
+
+        public Quantile[] Quantiles { get; }
+    }
+
+    internal record struct Quantile(double Percentage, double Value);
+
+    internal class ErrorPayload : CounterPayload
+    {
+        public ErrorPayload(string errorMessage) : this(errorMessage, DateTime.UtcNow) 
+        {
+        }
+
+        public ErrorPayload(string errorMessage, DateTime timestamp) :
+            base(string.Empty, string.Empty, string.Empty, string.Empty, null, 0.0, timestamp, "Metric", EventType.Error)
+        {
+            ErrorMessage = errorMessage;
+        }
+
+        public string ErrorMessage { get; private set; }
+    }
+
+    internal enum EventType : int
+    {
+        Rate,
+        Gauge,
+        Histogram,
+        Error,
+        CounterEnded
     }
 }

src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/CounterUtilities.cs

@@ -0,0 +1,55 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using System;
+
+namespace Microsoft.Diagnostics.Monitoring.EventPipe
+{
+    internal static class CounterUtilities
+    {
+        //The metadata payload is formatted as a string of comma separated key:value pairs.
+        //This limitation means that metadata values cannot include commas; otherwise, the
+        //metadata will be parsed incorrectly. If a value contains a comma, then all metadata
+        //is treated as invalid and excluded from the payload.
+        public static IDictionary<string, string> GetMetadata(string metadataPayload, char kvSeparator = ':')
+        {
+            var metadataDict = new Dictionary<string, string>();
+
+            ReadOnlySpan<char> metadata = metadataPayload;
+
+            while (!metadata.IsEmpty) {
+                int commaIndex = metadata.IndexOf(',');
+
+                ReadOnlySpan<char> kvPair;
+
+                if (commaIndex < 0) {
+                    kvPair = metadata;
+                    metadata = default;
+                }
+                else {
+                    kvPair = metadata[..commaIndex];
+                    metadata = metadata.Slice(commaIndex + 1);
+                }
+
+                int colonIndex = kvPair.IndexOf(kvSeparator);
+                if (colonIndex < 0) {
+                    metadataDict.Clear();
+                    break;
+                }
+
+                string metadataKey = kvPair[..colonIndex].ToString();
+                string metadataValue = kvPair.Slice(colonIndex + 1).ToString();
+                metadataDict[metadataKey] = metadataValue;
+            }
+
+            return metadataDict;
+        }
+
+        public static string AppendPercentile(string tags, double quantile) => AppendPercentile(tags, FormattableString.Invariant($"Percentile={(int)(100 * quantile)}"));
+
+        private static string AppendPercentile(string tags, string percentile) => string.IsNullOrEmpty(tags) ? percentile : string.Concat(tags, ",", percentile);
+    }
+}
+

src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/ICounterPayload.cs

@@ -33,8 +33,17 @@ internal interface ICounterPayload
 
         DateTime Timestamp { get; }
 
+        /// <summary>
+        /// The interval between counters. Note this is the actual measure of time elapsed, not the requested interval.
+        /// </summary>
         float Interval { get; }
 
-        IReadOnlyDictionary<string, string> Metadata { get; }
+        /// <summary>
+        /// Optional metadata for counters. Note that normal counters use ':' as a separator character, while System.Diagnostics.Metrics use ';'.
+        /// We do not immediately convert string to Dictionary, since dotnet-counters does not need this conversion.
+        /// </summary>
+        string Metadata { get; }
+
+        EventType EventType { get; set; }
     }
 }

src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/ICountersLogger.cs

@@ -2,20 +2,16 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
-using System.Collections.Generic;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 
 namespace Microsoft.Diagnostics.Monitoring.EventPipe
 {
     internal interface ICountersLogger
     {
-        //TODO Consider making these async.
-
         void Log(ICounterPayload counter);
-        void PipelineStarted();
-        void PipelineStopped();
+
+        Task PipelineStarted(CancellationToken token);
+        Task PipelineStopped(CancellationToken token);
     }
 }