Add integration tests and fix some bugs

src/BenchmarkDotNet/Configs/ConfigExtensions.cs

@@ -7,6 +7,7 @@
 using BenchmarkDotNet.Analysers;
 using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.EventHandlers;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Filters;
 using BenchmarkDotNet.Jobs;
@@ -110,6 +111,7 @@ public static class ConfigExtensions
         [Obsolete("This method will soon be removed, please start using .AddLogicalGroupRules() instead.")]
         [EditorBrowsable(EditorBrowsableState.Never)] public static IConfig With(this IConfig config, params BenchmarkLogicalGroupRule[] rules) => config.AddLogicalGroupRules(rules);
         [PublicAPI] public static ManualConfig AddLogicalGroupRules(this IConfig config, params BenchmarkLogicalGroupRule[] rules) => config.With(c => c.AddLogicalGroupRules(rules));
+        [PublicAPI] public static ManualConfig AddEventHandler(this IConfig config, params IEventHandler[] eventHandlers) => config.With(c => AddEventHandler(c, eventHandlers));
 
         [PublicAPI] public static ManualConfig HideColumns(this IConfig config, params string[] columnNames) => config.With(c => c.HideColumns(columnNames));
         [PublicAPI] public static ManualConfig HideColumns(this IConfig config, params IColumn[] columns) => config.With(c => c.HideColumns(columns));

src/BenchmarkDotNet/Configs/DebugConfig.cs

@@ -4,6 +4,7 @@
 using BenchmarkDotNet.Analysers;
 using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.EventHandlers;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Filters;
 using BenchmarkDotNet.Jobs;
@@ -61,7 +62,7 @@ public abstract class DebugConfig : IConfig
         public IEnumerable<IDiagnoser> GetDiagnosers() => Array.Empty<IDiagnoser>();
         public IEnumerable<IAnalyser> GetAnalysers() => Array.Empty<IAnalyser>();
         public IEnumerable<HardwareCounter> GetHardwareCounters() => Array.Empty<HardwareCounter>();
-        public IEnumerable<IBenchmarkEventHandler> GetEventHandlers() => Array.Empty<IBenchmarkEventHandler>();
+        public IEnumerable<IEventHandler> GetEventHandlers() => Array.Empty<IEventHandler>();
         public IEnumerable<IFilter> GetFilters() => Array.Empty<IFilter>();
         public IEnumerable<IColumnHidingRule> GetColumnHidingRules() => Array.Empty<IColumnHidingRule>();
 

src/BenchmarkDotNet/Configs/DefaultConfig.cs

@@ -5,6 +5,7 @@
 using BenchmarkDotNet.Analysers;
 using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.EventHandlers;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Exporters.Csv;
 using BenchmarkDotNet.Filters;
@@ -108,7 +109,7 @@ public string ArtifactsPath
 
         public IEnumerable<IFilter> GetFilters() => Array.Empty<IFilter>();
 
-        public IEnumerable<IBenchmarkEventHandler> GetEventHandlers() => Array.Empty<IBenchmarkEventHandler>();
+        public IEnumerable<IEventHandler> GetEventHandlers() => Array.Empty<IEventHandler>();
 
         public IEnumerable<IColumnHidingRule> GetColumnHidingRules() => Array.Empty<IColumnHidingRule>();
     }

src/BenchmarkDotNet/Configs/IConfig.cs

@@ -4,6 +4,7 @@
 using BenchmarkDotNet.Analysers;
 using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.EventHandlers;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Filters;
 using BenchmarkDotNet.Jobs;
@@ -27,7 +28,7 @@ public interface IConfig
         IEnumerable<HardwareCounter> GetHardwareCounters();
         IEnumerable<IFilter> GetFilters();
         IEnumerable<BenchmarkLogicalGroupRule> GetLogicalGroupRules();
-        IEnumerable<IBenchmarkEventHandler> GetEventHandlers();
+        IEnumerable<IEventHandler> GetEventHandlers();
         IEnumerable<IColumnHidingRule> GetColumnHidingRules();
 
         IOrderer? Orderer { get; }

src/BenchmarkDotNet/Configs/ImmutableConfig.cs

@@ -6,6 +6,7 @@
 using BenchmarkDotNet.Analysers;
 using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.EventHandlers;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Filters;
 using BenchmarkDotNet.Jobs;
@@ -31,7 +32,7 @@ public sealed class ImmutableConfig : IConfig
         private readonly ImmutableHashSet<HardwareCounter> hardwareCounters;
         private readonly ImmutableHashSet<IFilter> filters;
         private readonly ImmutableArray<BenchmarkLogicalGroupRule> rules;
-        private readonly ImmutableHashSet<IBenchmarkEventHandler> eventHandlers;
+        private readonly ImmutableHashSet<IEventHandler> eventHandlers;
         private readonly ImmutableArray<IColumnHidingRule> columnHidingRules;
 
         internal ImmutableConfig(
@@ -46,7 +47,7 @@ internal ImmutableConfig(
             ImmutableArray<BenchmarkLogicalGroupRule> uniqueRules,
             ImmutableArray<IColumnHidingRule> uniqueColumnHidingRules,
             ImmutableHashSet<Job> uniqueRunnableJobs,
-            ImmutableHashSet<IBenchmarkEventHandler> uniqueEventHandlers,
+            ImmutableHashSet<IEventHandler> uniqueEventHandlers,
             ConfigUnionRule unionRule,
             string artifactsPath,
             CultureInfo cultureInfo,
@@ -99,15 +100,15 @@ internal ImmutableConfig(
         public IEnumerable<HardwareCounter> GetHardwareCounters() => hardwareCounters;
         public IEnumerable<IFilter> GetFilters() => filters;
         public IEnumerable<BenchmarkLogicalGroupRule> GetLogicalGroupRules() => rules;
-        public IEnumerable<IBenchmarkEventHandler> GetEventHandlers() => eventHandlers;
+        public IEnumerable<IEventHandler> GetEventHandlers() => eventHandlers;
         public IEnumerable<IColumnHidingRule> GetColumnHidingRules() => columnHidingRules;
 
         public ILogger GetCompositeLogger() => new CompositeLogger(loggers);
         public IExporter GetCompositeExporter() => new CompositeExporter(exporters);
         public IValidator GetCompositeValidator() => new CompositeValidator(validators);
         public IAnalyser GetCompositeAnalyser() => new CompositeAnalyser(analysers);
         public IDiagnoser GetCompositeDiagnoser() => new CompositeDiagnoser(diagnosers);
-        public IBenchmarkEventHandler GetCompositeEventHandler() => new CompositeBenchmarkEventHandler(eventHandlers);
+        public IEventHandler GetCompositeEventHandler() => new CompositeEventHandler(eventHandlers);
 
         public bool HasMemoryDiagnoser() => diagnosers.OfType<MemoryDiagnoser>().Any();
 

src/BenchmarkDotNet/Configs/ManualConfig.cs

@@ -6,6 +6,7 @@
 using BenchmarkDotNet.Analysers;
 using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.EventHandlers;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Extensions;
 using BenchmarkDotNet.Filters;
@@ -33,7 +34,7 @@ public class ManualConfig : IConfig
         private readonly HashSet<HardwareCounter> hardwareCounters = new HashSet<HardwareCounter>();
         private readonly List<IFilter> filters = new List<IFilter>();
         private readonly List<BenchmarkLogicalGroupRule> logicalGroupRules = new List<BenchmarkLogicalGroupRule>();
-        private readonly List<IBenchmarkEventHandler> eventHandlers = new List<IBenchmarkEventHandler>();
+        private readonly List<IEventHandler> eventHandlers = new List<IEventHandler>();
         private readonly List<IColumnHidingRule> columnHidingRules = new List<IColumnHidingRule>();
 
         public IEnumerable<IColumnProvider> GetColumnProviders() => columnProviders;
@@ -46,7 +47,7 @@ public class ManualConfig : IConfig
         public IEnumerable<HardwareCounter> GetHardwareCounters() => hardwareCounters;
         public IEnumerable<IFilter> GetFilters() => filters;
         public IEnumerable<BenchmarkLogicalGroupRule> GetLogicalGroupRules() => logicalGroupRules;
-        public IEnumerable<IBenchmarkEventHandler> GetEventHandlers() => eventHandlers;
+        public IEnumerable<IEventHandler> GetEventHandlers() => eventHandlers;
         public IEnumerable<IColumnHidingRule> GetColumnHidingRules() => columnHidingRules;
 
         [PublicAPI] public ConfigOptions Options { get; set; }
@@ -223,7 +224,7 @@ public ManualConfig AddLogicalGroupRules(params BenchmarkLogicalGroupRule[] rule
             return this;
         }
 
-        public ManualConfig AddEventHandler(params IBenchmarkEventHandler[] eventHandlers)
+        public ManualConfig AddEventHandler(params IEventHandler[] eventHandlers)
         {
             this.eventHandlers.AddRange(eventHandlers);
             return this;

src/BenchmarkDotNet/Running/CompositeBenchmarkEventHandler.cs → src/BenchmarkDotNet/EventHandlers/CompositeEventHandler.cs

@@ -1,16 +1,17 @@
 using System;
 using System.Collections.Generic;
 using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Toolchains.Results;
 using BenchmarkDotNet.Validators;
 
-namespace BenchmarkDotNet.Running
+namespace BenchmarkDotNet.EventHandlers
 {
-    public class CompositeBenchmarkEventHandler : IBenchmarkEventHandler
+    public class CompositeEventHandler : IEventHandler
     {
-        private readonly IReadOnlyCollection<IBenchmarkEventHandler> eventHandlers;
+        private readonly IReadOnlyCollection<IEventHandler> eventHandlers;
 
-        public CompositeBenchmarkEventHandler(IReadOnlyCollection<IBenchmarkEventHandler> eventHandlers)
+        public CompositeEventHandler(IReadOnlyCollection<IEventHandler> eventHandlers)
         {
             this.eventHandlers = eventHandlers;
         }

src/BenchmarkDotNet/Running/BenchmarkEventHandlerBase.cs → src/BenchmarkDotNet/EventHandlers/EventHandlerBase.cs

@@ -1,12 +1,13 @@
 using System;
 using System.Collections.Generic;
 using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Toolchains.Results;
 using BenchmarkDotNet.Validators;
 
-namespace BenchmarkDotNet.Running
+namespace BenchmarkDotNet.EventHandlers
 {
-    public class BenchmarkEventHandlerBase : IBenchmarkEventHandler
+    public class EventHandlerBase : IEventHandler
     {
         public virtual void HandleBuildFailed(BenchmarkCase benchmarkCase, BuildResult buildResult)
         {

src/BenchmarkDotNet/Running/IBenchmarkEventHandler.cs → src/BenchmarkDotNet/EventHandlers/IEventHandler.cs

@@ -1,12 +1,13 @@
 using System;
 using System.Collections.Generic;
 using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Toolchains.Results;
 using BenchmarkDotNet.Validators;
 
-namespace BenchmarkDotNet.Running
+namespace BenchmarkDotNet.EventHandlers
 {
-    public interface IBenchmarkEventHandler
+    public interface IEventHandler
     {
         void HandleStartValidationStage();
         void HandleUnsupportedBenchmark(BenchmarkCase benchmarkCase);

src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs

@@ -11,6 +11,7 @@
 using BenchmarkDotNet.Diagnosers;
 using BenchmarkDotNet.Engines;
 using BenchmarkDotNet.Environments;
+using BenchmarkDotNet.EventHandlers;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Extensions;
 using BenchmarkDotNet.Helpers;
@@ -64,6 +65,14 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos)
 
                 validationErrors.AddRange(Validate(supportedBenchmarks));
 
+                foreach (var validationError in validationErrors)
+                {
+                    var validationErrorEventHandler = validationError.BenchmarkCase != null
+                        ? validationError.BenchmarkCase.Config.GetCompositeEventHandler()
+                        : eventHandler;
+                    validationErrorEventHandler.HandleValidationError(validationError);
+                }
+
                 PrintValidationErrors(compositeLogger, validationErrors);
 
                 if (validationErrors.Any(validationError => validationError.IsCritical))
@@ -82,6 +91,17 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos)
 
                 var buildPartitions = BenchmarkPartitioner.CreateForBuild(supportedBenchmarks, resolver);
                 var buildResults = BuildInParallel(compositeLogger, rootArtifactsFolderPath, buildPartitions, in globalChronometer);
+
+                foreach (var buildPartition in buildPartitions)
+                {
+                    var buildResult = buildResults[buildPartition];
+                    if (!buildResult.IsBuildSuccess)
+                    {
+                        foreach (var benchmark in buildPartition.Benchmarks)
+                            benchmark.Config.GetCompositeEventHandler().HandleBuildFailed(benchmark.BenchmarkCase, buildResult);
+                    }
+                }
+
                 var allBuildsHaveFailed = buildResults.Values.All(buildResult => !buildResult.IsBuildSuccess);
 
                 eventHandler.HandleStartRunStage();
@@ -356,8 +376,6 @@ private static ImmutableArray<ValidationError> Validate(params BenchmarkRunInfo[
                 foreach (var error in compatibilityValidationErrors)
                 {
                     validationErrors.Add(error);
-                    foreach (var eventHandler in benchmarks.SelectMany(b => b.Config.GetEventHandlers()).Distinct())
-                        eventHandler.HandleValidationError(error);
                 }
             }
 
@@ -369,7 +387,6 @@ private static ImmutableArray<ValidationError> Validate(params BenchmarkRunInfo[
                 foreach (var error in validator.Validate(new ValidationParameters(benchmarkRunInfo.BenchmarksCases, config)))
                 {
                     validationErrors.Add(error);
-                    eventHandler.HandleValidationError(error);
                 }
             }
 
@@ -406,16 +423,6 @@ private static Dictionary<BuildPartition, BuildResult> BuildInParallel(ILogger l
 
             logger.WriteLineHeader($"// ***** Done, took {GetFormattedDifference(afterParallelBuild, afterSequentialBuild)}   *****");
 
-            foreach (var buildPartition in buildPartitions)
-            {
-                var buildResult = buildResults[buildPartition];
-                if (!buildResult.IsBuildSuccess)
-                {
-                    foreach (var benchmark in buildPartition.Benchmarks)
-                        benchmark.Config.GetCompositeEventHandler().HandleBuildFailed(benchmark.BenchmarkCase, buildResult);
-                }
-            }
-
             return buildResults;
 
             static string GetFormattedDifference(ClockSpan before, ClockSpan after)
@@ -685,14 +692,14 @@ void AddLogger(ILogger logger)
             return new CompositeLogger(loggers.Values.ToImmutableHashSet());
         }
 
-        private static IBenchmarkEventHandler CreateCompositeEventHandler(BenchmarkRunInfo[] benchmarkRunInfos)
+        private static IEventHandler CreateCompositeEventHandler(BenchmarkRunInfo[] benchmarkRunInfos)
         {
-            var eventHandlers = new HashSet<IBenchmarkEventHandler>();
+            var eventHandlers = new HashSet<IEventHandler>();
 
             foreach (var info in benchmarkRunInfos)
                 eventHandlers.AddRange(info.Config.GetEventHandlers());
 
-            return new CompositeBenchmarkEventHandler(eventHandlers);
+            return new CompositeEventHandler(eventHandlers);
         }
 
         private static void Cleanup(ILogger logger, HashSet<string> artifactsToCleanup)