Add Parameter Filter (dotnet#2314)

Can filter benchmarks to only run cases with matching parameter values

Author: Tohron

.gitignore

@@ -55,6 +55,7 @@ bld/
 
 # Visual Studio 2017 auto generated files
 Generated\ Files/
+**/*Properties/launchSettings.json
 
 # scenario intermediate directories
 src/scenarios/*/pub

src/benchmarks/micro/Program.cs

@@ -21,6 +21,7 @@ static int Main(string[] args)
             int? partitionIndex;
             List<string> exclusionFilterValue;
             List<string> categoryExclusionFilterValue;
+            Dictionary<string, string> parameterFilterValue;
             bool getDiffableDisasm;
 
             // Parse and remove any additional parameters that we need that aren't part of BDN
@@ -29,6 +30,7 @@ static int Main(string[] args)
                 argsList = CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-index", out partitionIndex);
                 argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--exclusion-filter", out exclusionFilterValue);
                 argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--category-exclusion-filter", out categoryExclusionFilterValue);
+                argsList = CommandLineOptions.ParseAndRemovePairsParameter(argsList, "--parameter-filter", out parameterFilterValue);
                 CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--disasm-diff", out getDiffableDisasm);
 
                 CommandLineOptions.ValidatePartitionParameters(partitionCount, partitionIndex);
@@ -49,6 +51,7 @@ static int Main(string[] args)
                         partitionIndex: partitionIndex,
                         exclusionFilterValue: exclusionFilterValue,
                         categoryExclusionFilterValue: categoryExclusionFilterValue,
+                        parameterFilterValue: parameterFilterValue,
                         getDiffableDisasm: getDiffableDisasm)
                     .AddValidator(new NoWasmValidator(Categories.NoWASM)))
                 .ToExitCode();

src/harness/BenchmarkDotNet.Extensions/CommandLineOptions.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Numerics;
 
 namespace BenchmarkDotNet.Extensions
 {
@@ -53,6 +54,29 @@ public static List<string> ParseAndRemoveStringsParameter(List<string> argsList,
             return argsList;
         }
 
+        public static List<string> ParseAndRemovePairsParameter(List<string> argsList, string parameter, out Dictionary<string, string> pairValues)
+        {
+            int parameterIndex = argsList.IndexOf(parameter);
+            pairValues = new Dictionary<string, string>();
+
+            if (parameterIndex + 1 < argsList.Count)
+            {
+                while (parameterIndex + 1 < argsList.Count && !argsList[parameterIndex + 1].StartsWith("-"))
+                {
+                    string[] pair = argsList[parameterIndex + 1].Split(":".ToCharArray(), 2);
+                    pairValues.Add(pair[0], pair[1]);
+                    argsList.RemoveAt(parameterIndex + 1);
+                }
+            }
+            //We only want to remove the --exclusion-filter if it exists
+            if (parameterIndex != -1)
+            {
+                argsList.RemoveAt(parameterIndex);
+            }
+
+            return argsList;
+        }
+
         public static void ParseAndRemoveBooleanParameter(List<string> argsList, string parameter, out bool parameterValue)
         {
             int parameterIndex = argsList.IndexOf(parameter);

src/harness/BenchmarkDotNet.Extensions/ParameterFilter.cs

@@ -0,0 +1,33 @@
+using BenchmarkDotNet.Filters;
+using BenchmarkDotNet.Parameters;
+using BenchmarkDotNet.Running;
+using System.Collections.Generic;
+using System.Linq;
+
+public class ParameterFilter : IFilter
+{
+    private readonly Dictionary<string, string> _parameterValues;
+
+    public ParameterFilter(Dictionary<string, string> parameterValues)
+    {
+        _parameterValues = parameterValues;
+    }
+
+    public bool Predicate(BenchmarkCase benchmarkCase)
+    {
+        Dictionary<string, object> items = benchmarkCase.Parameters.Items.ToDictionary<ParameterInstance, string, object>(instance => instance.Name, instance => instance.Value);
+
+        if (_parameterValues == null)
+            return true;
+
+        foreach (string key in _parameterValues.Keys)
+        {
+            if (!items.Keys.Contains(key))
+                return false;
+            if (!items[key].ToString().Equals(_parameterValues[key]))
+                return false;
+        }
+        
+        return true;
+    }
+}

src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs

@@ -25,6 +25,7 @@ public static IConfig Create(
             int? partitionIndex = null,
             List<string> exclusionFilterValue = null,
             List<string> categoryExclusionFilterValue = null,
+            Dictionary<string, string> parameterFilterValue = null,
             Job job = null,
             bool getDiffableDisasm = false)
         {
@@ -54,6 +55,7 @@ public static IConfig Create(
                 .AddFilter(new PartitionFilter(partitionCount, partitionIndex))
                 .AddFilter(new ExclusionFilter(exclusionFilterValue))
                 .AddFilter(new CategoryExclusionFilter(categoryExclusionFilterValue))
+                .AddFilter(new ParameterFilter(parameterFilterValue))
                 .AddExporter(JsonExporter.Full) // make sure we export to Json
                 .AddColumn(StatisticColumn.Median, StatisticColumn.Min, StatisticColumn.Max)
                 .AddValidator(TooManyTestCasesValidator.FailOnError)