From 1cfdd1d34be102ebfe35093a58da809e182516a7 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Mon, 3 Nov 2025 23:23:59 +0000 Subject: [PATCH 1/3] perf: optimize test execution by partitioning tests with and without parallel limiters --- TUnit.Engine/Scheduling/TestScheduler.cs | 109 ++++++++++++----- TUnit.Profile/Program.cs | 146 ++++++++++++++++++++++- TUnit.Profile/trace-analysis.json | 1 + 3 files changed, 222 insertions(+), 34 deletions(-) create mode 100644 TUnit.Profile/trace-analysis.json diff --git a/TUnit.Engine/Scheduling/TestScheduler.cs b/TUnit.Engine/Scheduling/TestScheduler.cs index de1e980812..08802fe337 100644 --- a/TUnit.Engine/Scheduling/TestScheduler.cs +++ b/TUnit.Engine/Scheduling/TestScheduler.cs @@ -367,45 +367,38 @@ private async Task ExecuteSequentiallyAsync( } } - #if NET6_0_OR_GREATER - [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCode("Test execution involves reflection for hooks and initialization")] - #endif private async Task ExecuteWithGlobalLimitAsync( AbstractExecutableTest[] tests, CancellationToken cancellationToken) { #if NET6_0_OR_GREATER - // Use Parallel.ForEachAsync with explicit MaxDegreeOfParallelism - // This eliminates unbounded Task.Run calls and leverages work-stealing for efficiency - await Parallel.ForEachAsync( - tests, - new ParallelOptions + // PERFORMANCE OPTIMIZATION: Partition tests by whether they have parallel limiters + // Tests without limiters can run with unlimited parallelism (avoiding global semaphore overhead) + var testsWithLimiters = new List(); + var testsWithoutLimiters = new List(); + + foreach (var test in tests) + { + if (test.Context.ParallelLimiter != null) { - MaxDegreeOfParallelism = _maxParallelism, - CancellationToken = cancellationToken - }, - async (test, ct) => + testsWithLimiters.Add(test); + } + else { - SemaphoreSlim? parallelLimiterSemaphore = null; + testsWithoutLimiters.Add(test); + } + } - // Acquire parallel limiter semaphore if needed - if (test.Context.ParallelLimiter != null) - { - parallelLimiterSemaphore = _parallelLimitLockProvider.GetLock(test.Context.ParallelLimiter); - await parallelLimiterSemaphore.WaitAsync(ct).ConfigureAwait(false); - } + // Execute both groups concurrently + var limitedTask = testsWithLimiters.Count > 0 + ? ExecuteWithLimitAsync(testsWithLimiters, cancellationToken) + : Task.CompletedTask; - try - { - test.ExecutionTask ??= _testRunner.ExecuteTestAsync(test, ct); - await test.ExecutionTask.ConfigureAwait(false); - } - finally - { - parallelLimiterSemaphore?.Release(); - } - } - ).ConfigureAwait(false); + var unlimitedTask = testsWithoutLimiters.Count > 0 + ? ExecuteUnlimitedAsync(testsWithoutLimiters, cancellationToken) + : Task.CompletedTask; + + await Task.WhenAll(limitedTask, unlimitedTask).ConfigureAwait(false); #else // Fallback for netstandard2.0: Manual bounded concurrency using existing semaphore var tasks = new Task[tests.Length]; @@ -445,6 +438,62 @@ await Parallel.ForEachAsync( #endif } +#if NET6_0_OR_GREATER + private async Task ExecuteWithLimitAsync( + List tests, + CancellationToken cancellationToken) + { + // Execute tests with parallel limiters using the global limit + await Parallel.ForEachAsync( + tests, + new ParallelOptions + { + MaxDegreeOfParallelism = _maxParallelism, + CancellationToken = cancellationToken + }, + async (test, ct) => + { + var parallelLimiterSemaphore = _parallelLimitLockProvider.GetLock(test.Context.ParallelLimiter!); + await parallelLimiterSemaphore.WaitAsync(ct).ConfigureAwait(false); + + try + { +#pragma warning disable IL2026 // ExecuteTestAsync uses reflection, but caller (ExecuteWithGlobalLimitAsync) is already marked with RequiresUnreferencedCode + test.ExecutionTask ??= _testRunner.ExecuteTestAsync(test, ct); +#pragma warning restore IL2026 + await test.ExecutionTask.ConfigureAwait(false); + } + finally + { + parallelLimiterSemaphore.Release(); + } + } + ).ConfigureAwait(false); + } + + private async Task ExecuteUnlimitedAsync( + List tests, + CancellationToken cancellationToken) + { + // Execute tests without limiters with unlimited parallelism (no global semaphore overhead) + await Parallel.ForEachAsync( + tests, + new ParallelOptions + { + CancellationToken = cancellationToken + // No MaxDegreeOfParallelism = unlimited parallelism + }, + async (test, ct) => + { +#pragma warning disable IL2026 // ExecuteTestAsync uses reflection, but caller (ExecuteWithGlobalLimitAsync) is already marked with RequiresUnreferencedCode + test.ExecutionTask ??= _testRunner.ExecuteTestAsync(test, ct); +#pragma warning restore IL2026 + await test.ExecutionTask.ConfigureAwait(false); + } + ).ConfigureAwait(false); + } +#endif + private async Task WaitForTasksWithFailFastHandling(IEnumerable tasks, CancellationToken cancellationToken) { try diff --git a/TUnit.Profile/Program.cs b/TUnit.Profile/Program.cs index 96f476861d..efbf095069 100644 --- a/TUnit.Profile/Program.cs +++ b/TUnit.Profile/Program.cs @@ -1,10 +1,14 @@ namespace TUnit.Profile; -public class Tests +/// +/// Performance profiling workload: ~1000 tests covering diverse scenarios +/// to stress test TUnit's scheduling, parallelism, and execution infrastructure +/// +public class SyncTests { [Test] - [Repeat(50)] - public void Basic() + [Repeat(100)] + public void FastSyncTest() { _ = 1 + 1; } @@ -12,9 +16,143 @@ public void Basic() [Test] [Arguments(1, 2)] [Arguments(3, 4)] + [Arguments(5, 6)] + [Arguments(7, 8)] + [Arguments(9, 10)] + [Repeat(20)] + public void ParameterizedSyncTest(int a, int b) + { + _ = a + b; + } + + [Test] + [Repeat(50)] + public void ComputeBoundTest() + { + var sum = 0; + for (var i = 0; i < 100; i++) + { + sum += i; + } + _ = sum; + } +} + +public class AsyncTests +{ + [Test] + [Repeat(100)] + public async Task FastAsyncTest() + { + await Task.CompletedTask; + _ = 1 + 1; + } + + [Test] [Repeat(50)] - public void Basic(int a, int b) + public async Task AsyncWithYield() { + await Task.Yield(); + _ = 1 + 1; + } + + [Test] + [Arguments(1, 2)] + [Arguments(3, 4)] + [Arguments(5, 6)] + [Arguments(7, 8)] + [Repeat(10)] + public async Task ParameterizedAsyncTest(int a, int b) + { + await Task.CompletedTask; _ = a + b; } } + +public class MatrixTests +{ + [Test] + [Arguments(1)] + [Arguments(2)] + [Arguments(3)] + [Arguments(4)] + [Arguments(5)] + [Arguments(6)] + [Arguments(7)] + [Arguments(8)] + [Arguments(9)] + [Arguments(10)] + [Repeat(10)] + public void TenByTenMatrix(int value) + { + _ = value * value; + } + + [Test] + [Arguments("a", 1)] + [Arguments("b", 2)] + [Arguments("c", 3)] + [Arguments("d", 4)] + [Arguments("e", 5)] + [Repeat(10)] + public void MixedParameterTypes(string str, int num) + { + _ = str + num; + } +} + +public class DataExpansionTests +{ + [Test] + [Arguments(1, 2, 3)] + [Arguments(4, 5, 6)] + [Arguments(7, 8, 9)] + [Arguments(10, 11, 12)] + [Arguments(13, 14, 15)] + [Arguments(16, 17, 18)] + [Arguments(19, 20, 21)] + [Arguments(22, 23, 24)] + [Repeat(5)] + public void ThreeParameterCombinations(int a, int b, int c) + { + _ = a + b + c; + } + + [Test] + [Arguments(true, 1)] + [Arguments(false, 2)] + [Arguments(true, 3)] + [Arguments(false, 4)] + [Repeat(10)] + public void BooleanAndInt(bool flag, int value) + { + _ = flag ? value : -value; + } +} + +public class MixedWorkloadTests +{ + [Test] + [Repeat(30)] + public void QuickTest1() => _ = 1; + + [Test] + [Repeat(30)] + public void QuickTest2() => _ = 2; + + [Test] + [Repeat(30)] + public void QuickTest3() => _ = 3; + + [Test] + [Repeat(30)] + public async Task QuickAsyncTest1() { await Task.CompletedTask; _ = 1; } + + [Test] + [Repeat(30)] + public async Task QuickAsyncTest2() { await Task.CompletedTask; _ = 2; } + + [Test] + [Repeat(30)] + public async Task QuickAsyncTest3() { await Task.CompletedTask; _ = 3; } +} diff --git a/TUnit.Profile/trace-analysis.json b/TUnit.Profile/trace-analysis.json new file mode 100644 index 0000000000..2a674cc403 --- /dev/null +++ b/TUnit.Profile/trace-analysis.json @@ -0,0 +1 @@ +/usr/bin/bash: line 1: dotnet-trace: command not found From 998e2fd5c0ac240a7c0704d55e39e6b595d3b414 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 4 Nov 2025 00:03:09 +0000 Subject: [PATCH 2/3] fix: add ConfigureAwait(false) to asynchronous calls for improved performance --- .../Services/TestExecution/TestCoordinator.cs | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/TUnit.Engine/Services/TestExecution/TestCoordinator.cs b/TUnit.Engine/Services/TestExecution/TestCoordinator.cs index 4ae9652904..13f2ac7df4 100644 --- a/TUnit.Engine/Services/TestExecution/TestCoordinator.cs +++ b/TUnit.Engine/Services/TestExecution/TestCoordinator.cs @@ -60,8 +60,8 @@ private async Task ExecuteTestInternalAsync(AbstractExecutableTest test, Cancell { try { - await _stateManager.MarkRunningAsync(test); - await _messageBus.InProgress(test.Context); + await _stateManager.MarkRunningAsync(test).ConfigureAwait(false); + await _messageBus.InProgress(test.Context).ConfigureAwait(false); _contextRestorer.RestoreContext(test); @@ -90,7 +90,7 @@ private async Task ExecuteTestInternalAsync(AbstractExecutableTest test, Cancell } // Ensure TestSession hooks run before creating test instances - await _testExecutor.EnsureTestSessionHooksExecutedAsync(); + await _testExecutor.EnsureTestSessionHooksExecutedAsync().ConfigureAwait(false); // Execute test with retry logic - each retry gets a fresh instance // Timeout is applied per retry attempt, not across all retries @@ -106,7 +106,7 @@ await RetryHelper.ExecuteWithRetry(test.Context, async () => await TimeoutHelper.ExecuteWithTimeoutAsync( async ct => { - test.Context.Metadata.TestDetails.ClassInstance = await test.CreateInstanceAsync(); + test.Context.Metadata.TestDetails.ClassInstance = await test.CreateInstanceAsync().ConfigureAwait(false); // Invalidate cached eligible event objects since ClassInstance changed test.Context.CachedEligibleEventObjects = null; @@ -115,20 +115,20 @@ await TimeoutHelper.ExecuteWithTimeoutAsync( if (test.Context.Metadata.TestDetails.ClassInstance is SkippedTestInstance || !string.IsNullOrEmpty(test.Context.SkipReason)) { - await _stateManager.MarkSkippedAsync(test, test.Context.SkipReason ?? "Test was skipped"); + await _stateManager.MarkSkippedAsync(test, test.Context.SkipReason ?? "Test was skipped").ConfigureAwait(false); - await _eventReceiverOrchestrator.InvokeTestSkippedEventReceiversAsync(test.Context, ct); + await _eventReceiverOrchestrator.InvokeTestSkippedEventReceiversAsync(test.Context, ct).ConfigureAwait(false); - await _eventReceiverOrchestrator.InvokeTestEndEventReceiversAsync(test.Context, ct); + await _eventReceiverOrchestrator.InvokeTestEndEventReceiversAsync(test.Context, ct).ConfigureAwait(false); return; } try { - await _testInitializer.InitializeTest(test, ct); + await _testInitializer.InitializeTest(test, ct).ConfigureAwait(false); test.Context.RestoreExecutionContext(); - await _testExecutor.ExecuteAsync(test, ct); + await _testExecutor.ExecuteAsync(test, ct).ConfigureAwait(false); } finally { @@ -140,59 +140,59 @@ await TimeoutHelper.ExecuteWithTimeoutAsync( { try { - await invocation.InvokeAsync(test.Context, test.Context); + await invocation.InvokeAsync(test.Context, test.Context).ConfigureAwait(false); } catch (Exception disposeEx) { - await _logger.LogErrorAsync($"Error during OnDispose for {test.TestId}: {disposeEx}"); + await _logger.LogErrorAsync($"Error during OnDispose for {test.TestId}: {disposeEx}").ConfigureAwait(false); } } } try { - await TestExecutor.DisposeTestInstance(test); + await TestExecutor.DisposeTestInstance(test).ConfigureAwait(false); } catch (Exception disposeEx) { - await _logger.LogErrorAsync($"Error disposing test instance for {test.TestId}: {disposeEx}"); + await _logger.LogErrorAsync($"Error disposing test instance for {test.TestId}: {disposeEx}").ConfigureAwait(false); } } }, testTimeout, cancellationToken, - timeoutMessage); - }); + timeoutMessage).ConfigureAwait(false); + }).ConfigureAwait(false); - await _stateManager.MarkCompletedAsync(test); + await _stateManager.MarkCompletedAsync(test).ConfigureAwait(false); } catch (SkipTestException ex) { test.Context.SkipReason = ex.Message; - await _stateManager.MarkSkippedAsync(test, ex.Message); + await _stateManager.MarkSkippedAsync(test, ex.Message).ConfigureAwait(false); - await _eventReceiverOrchestrator.InvokeTestSkippedEventReceiversAsync(test.Context, cancellationToken); + await _eventReceiverOrchestrator.InvokeTestSkippedEventReceiversAsync(test.Context, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { - await _stateManager.MarkFailedAsync(test, ex); + await _stateManager.MarkFailedAsync(test, ex).ConfigureAwait(false); } finally { var cleanupExceptions = new List(); - await _objectTracker.UntrackObjects(test.Context, cleanupExceptions); + await _objectTracker.UntrackObjects(test.Context, cleanupExceptions).ConfigureAwait(false); var testClass = test.Metadata.TestClassType; var testAssembly = testClass.Assembly; - var hookExceptions = await _testExecutor.ExecuteAfterClassAssemblyHooks(test, testClass, testAssembly, CancellationToken.None); + var hookExceptions = await _testExecutor.ExecuteAfterClassAssemblyHooks(test, testClass, testAssembly, CancellationToken.None).ConfigureAwait(false); if (hookExceptions.Count > 0) { foreach (var ex in hookExceptions) { - await _logger.LogErrorAsync($"Error executing After hooks for {test.TestId}: {ex}"); + await _logger.LogErrorAsync($"Error executing After hooks for {test.TestId}: {ex}").ConfigureAwait(false); } cleanupExceptions.AddRange(hookExceptions); } @@ -203,11 +203,11 @@ await TimeoutHelper.ExecuteWithTimeoutAsync( await _eventReceiverOrchestrator.InvokeLastTestInClassEventReceiversAsync( test.Context, test.Context.ClassContext, - CancellationToken.None); + CancellationToken.None).ConfigureAwait(false); } catch (Exception ex) { - await _logger.LogErrorAsync($"Error in last test in class event receiver for {test.TestId}: {ex}"); + await _logger.LogErrorAsync($"Error in last test in class event receiver for {test.TestId}: {ex}").ConfigureAwait(false); cleanupExceptions.Add(ex); } @@ -216,11 +216,11 @@ await _eventReceiverOrchestrator.InvokeLastTestInClassEventReceiversAsync( await _eventReceiverOrchestrator.InvokeLastTestInAssemblyEventReceiversAsync( test.Context, test.Context.ClassContext.AssemblyContext, - CancellationToken.None); + CancellationToken.None).ConfigureAwait(false); } catch (Exception ex) { - await _logger.LogErrorAsync($"Error in last test in assembly event receiver for {test.TestId}: {ex}"); + await _logger.LogErrorAsync($"Error in last test in assembly event receiver for {test.TestId}: {ex}").ConfigureAwait(false); cleanupExceptions.Add(ex); } @@ -229,11 +229,11 @@ await _eventReceiverOrchestrator.InvokeLastTestInAssemblyEventReceiversAsync( await _eventReceiverOrchestrator.InvokeLastTestInSessionEventReceiversAsync( test.Context, test.Context.ClassContext.AssemblyContext.TestSessionContext, - CancellationToken.None); + CancellationToken.None).ConfigureAwait(false); } catch (Exception ex) { - await _logger.LogErrorAsync($"Error in last test in session event receiver for {test.TestId}: {ex}"); + await _logger.LogErrorAsync($"Error in last test in session event receiver for {test.TestId}: {ex}").ConfigureAwait(false); cleanupExceptions.Add(ex); } @@ -244,7 +244,7 @@ await _eventReceiverOrchestrator.InvokeLastTestInSessionEventReceiversAsync( ? cleanupExceptions[0] : new AggregateException("One or more errors occurred during test cleanup", cleanupExceptions); - await _stateManager.MarkFailedAsync(test, aggregatedException); + await _stateManager.MarkFailedAsync(test, aggregatedException).ConfigureAwait(false); } switch (test.State) @@ -254,20 +254,20 @@ await _eventReceiverOrchestrator.InvokeLastTestInSessionEventReceiversAsync( case TestState.Queued: case TestState.Running: // This shouldn't happen - await _messageBus.Cancelled(test.Context, test.StartTime.GetValueOrDefault()); + await _messageBus.Cancelled(test.Context, test.StartTime.GetValueOrDefault()).ConfigureAwait(false); break; case TestState.Passed: - await _messageBus.Passed(test.Context, test.StartTime.GetValueOrDefault()); + await _messageBus.Passed(test.Context, test.StartTime.GetValueOrDefault()).ConfigureAwait(false); break; case TestState.Timeout: case TestState.Failed: - await _messageBus.Failed(test.Context, test.Context.Execution.Result?.Exception!, test.StartTime.GetValueOrDefault()); + await _messageBus.Failed(test.Context, test.Context.Execution.Result?.Exception!, test.StartTime.GetValueOrDefault()).ConfigureAwait(false); break; case TestState.Skipped: - await _messageBus.Skipped(test.Context, test.Context.SkipReason ?? "Skipped"); + await _messageBus.Skipped(test.Context, test.Context.SkipReason ?? "Skipped").ConfigureAwait(false); break; case TestState.Cancelled: - await _messageBus.Cancelled(test.Context, test.StartTime.GetValueOrDefault()); + await _messageBus.Cancelled(test.Context, test.StartTime.GetValueOrDefault()).ConfigureAwait(false); break; default: throw new ArgumentOutOfRangeException(); From db35bf7e6090f8b8d3737c8e62ec43f7ed8e2cb3 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Tue, 4 Nov 2025 12:19:29 +0000 Subject: [PATCH 3/3] perf: remove unnecessary RequiresUnreferencedCode attribute for improved clarity --- TUnit.Engine/Scheduling/TestRunner.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/TUnit.Engine/Scheduling/TestRunner.cs b/TUnit.Engine/Scheduling/TestRunner.cs index 3e84f70d66..5dde86ed23 100644 --- a/TUnit.Engine/Scheduling/TestRunner.cs +++ b/TUnit.Engine/Scheduling/TestRunner.cs @@ -39,9 +39,6 @@ internal TestRunner( private readonly ThreadSafeDictionary _executingTests = new(); private Exception? _firstFailFastException; - #if NET6_0_OR_GREATER - [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCode("Test execution involves reflection for hooks and initialization")] - #endif public async Task ExecuteTestAsync(AbstractExecutableTest test, CancellationToken cancellationToken) { // Prevent double execution with a simple lock @@ -49,9 +46,6 @@ public async Task ExecuteTestAsync(AbstractExecutableTest test, CancellationToke await executionTask.ConfigureAwait(false); } - #if NET6_0_OR_GREATER - [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCode("Test execution involves reflection for hooks and initialization")] - #endif private async Task ExecuteTestInternalAsync(AbstractExecutableTest test, CancellationToken cancellationToken) { try