Skip to content

perf: optimize test execution by partitioning tests #3676

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
thomhurst merged 3 commits into from
Nov 4, 2025
Merged

perf: optimize test execution by partitioning tests #3676

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1cfdd1d
perf: optimize test execution by partitioning tests with and without …
thomhurst Nov 3, 2025
998e2fd
fix: add ConfigureAwait(false) to asynchronous calls for improved per…
thomhurst Nov 4, 2025
db35bf7
perf: remove unnecessary RequiresUnreferencedCode attribute for impro…
thomhurst Nov 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions TUnit.Engine/Scheduling/TestRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,19 +39,13 @@ internal TestRunner(
private readonly ThreadSafeDictionary<string, Task> _executingTests = new();
private Exception? _firstFailFastException;

#if NET6_0_OR_GREATER
[System.Diagnostics.CodeAnalysis.RequiresUnreferencedCode("Test execution involves reflection for hooks and initialization")]
#endif
public async Task ExecuteTestAsync(AbstractExecutableTest test, CancellationToken cancellationToken)
{
// Prevent double execution with a simple lock
var executionTask = _executingTests.GetOrAdd(test.TestId, _ => ExecuteTestInternalAsync(test, cancellationToken));
await executionTask.ConfigureAwait(false);
}

#if NET6_0_OR_GREATER
[System.Diagnostics.CodeAnalysis.RequiresUnreferencedCode("Test execution involves reflection for hooks and initialization")]
#endif
private async Task ExecuteTestInternalAsync(AbstractExecutableTest test, CancellationToken cancellationToken)
{
try
Expand Down
109 changes: 79 additions & 30 deletions TUnit.Engine/Scheduling/TestScheduler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -367,45 +367,38 @@ private async Task ExecuteSequentiallyAsync(
}
}

#if NET6_0_OR_GREATER
[System.Diagnostics.CodeAnalysis.RequiresUnreferencedCode("Test execution involves reflection for hooks and initialization")]
#endif
private async Task ExecuteWithGlobalLimitAsync(
AbstractExecutableTest[] tests,
CancellationToken cancellationToken)
{
#if NET6_0_OR_GREATER
// Use Parallel.ForEachAsync with explicit MaxDegreeOfParallelism
// This eliminates unbounded Task.Run calls and leverages work-stealing for efficiency
await Parallel.ForEachAsync(
tests,
new ParallelOptions
// PERFORMANCE OPTIMIZATION: Partition tests by whether they have parallel limiters
// Tests without limiters can run with unlimited parallelism (avoiding global semaphore overhead)
var testsWithLimiters = new List<AbstractExecutableTest>();
var testsWithoutLimiters = new List<AbstractExecutableTest>();

foreach (var test in tests)
{
if (test.Context.ParallelLimiter != null)
{
MaxDegreeOfParallelism = _maxParallelism,
CancellationToken = cancellationToken
},
async (test, ct) =>
testsWithLimiters.Add(test);
}
else
{
SemaphoreSlim? parallelLimiterSemaphore = null;
testsWithoutLimiters.Add(test);
}
}

// Acquire parallel limiter semaphore if needed
if (test.Context.ParallelLimiter != null)
{
parallelLimiterSemaphore = _parallelLimitLockProvider.GetLock(test.Context.ParallelLimiter);
await parallelLimiterSemaphore.WaitAsync(ct).ConfigureAwait(false);
}
// Execute both groups concurrently
var limitedTask = testsWithLimiters.Count > 0
? ExecuteWithLimitAsync(testsWithLimiters, cancellationToken)
: Task.CompletedTask;

try
{
test.ExecutionTask ??= _testRunner.ExecuteTestAsync(test, ct);
await test.ExecutionTask.ConfigureAwait(false);
}
finally
{
parallelLimiterSemaphore?.Release();
}
}
).ConfigureAwait(false);
var unlimitedTask = testsWithoutLimiters.Count > 0
? ExecuteUnlimitedAsync(testsWithoutLimiters, cancellationToken)
: Task.CompletedTask;

await Task.WhenAll(limitedTask, unlimitedTask).ConfigureAwait(false);
#else
// Fallback for netstandard2.0: Manual bounded concurrency using existing semaphore
var tasks = new Task[tests.Length];
Expand Down Expand Up @@ -445,6 +438,62 @@ await Parallel.ForEachAsync(
#endif
}

#if NET6_0_OR_GREATER
private async Task ExecuteWithLimitAsync(
List<AbstractExecutableTest> tests,
CancellationToken cancellationToken)
{
// Execute tests with parallel limiters using the global limit
await Parallel.ForEachAsync(
tests,
new ParallelOptions
{
MaxDegreeOfParallelism = _maxParallelism,
CancellationToken = cancellationToken
},
async (test, ct) =>
{
var parallelLimiterSemaphore = _parallelLimitLockProvider.GetLock(test.Context.ParallelLimiter!);
await parallelLimiterSemaphore.WaitAsync(ct).ConfigureAwait(false);

try
{
#pragma warning disable IL2026 // ExecuteTestAsync uses reflection, but caller (ExecuteWithGlobalLimitAsync) is already marked with RequiresUnreferencedCode
test.ExecutionTask ??= _testRunner.ExecuteTestAsync(test, ct);
#pragma warning restore IL2026
await test.ExecutionTask.ConfigureAwait(false);
}
finally
{
parallelLimiterSemaphore.Release();
}
}
).ConfigureAwait(false);
}

private async Task ExecuteUnlimitedAsync(
List<AbstractExecutableTest> tests,
CancellationToken cancellationToken)
{
// Execute tests without limiters with unlimited parallelism (no global semaphore overhead)
await Parallel.ForEachAsync(
tests,
new ParallelOptions
{
CancellationToken = cancellationToken
// No MaxDegreeOfParallelism = unlimited parallelism
},
async (test, ct) =>
{
#pragma warning disable IL2026 // ExecuteTestAsync uses reflection, but caller (ExecuteWithGlobalLimitAsync) is already marked with RequiresUnreferencedCode
test.ExecutionTask ??= _testRunner.ExecuteTestAsync(test, ct);
#pragma warning restore IL2026
await test.ExecutionTask.ConfigureAwait(false);
}
).ConfigureAwait(false);
}
#endif

private async Task WaitForTasksWithFailFastHandling(IEnumerable<Task> tasks, CancellationToken cancellationToken)
{
try
Expand Down
68 changes: 34 additions & 34 deletions TUnit.Engine/Services/TestExecution/TestCoordinator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ private async Task ExecuteTestInternalAsync(AbstractExecutableTest test, Cancell
{
try
{
await _stateManager.MarkRunningAsync(test);
await _messageBus.InProgress(test.Context);
await _stateManager.MarkRunningAsync(test).ConfigureAwait(false);
await _messageBus.InProgress(test.Context).ConfigureAwait(false);

_contextRestorer.RestoreContext(test);

Expand Down Expand Up @@ -90,7 +90,7 @@ private async Task ExecuteTestInternalAsync(AbstractExecutableTest test, Cancell
}

// Ensure TestSession hooks run before creating test instances
await _testExecutor.EnsureTestSessionHooksExecutedAsync();
await _testExecutor.EnsureTestSessionHooksExecutedAsync().ConfigureAwait(false);

// Execute test with retry logic - each retry gets a fresh instance
// Timeout is applied per retry attempt, not across all retries
Expand All @@ -106,7 +106,7 @@ await RetryHelper.ExecuteWithRetry(test.Context, async () =>
await TimeoutHelper.ExecuteWithTimeoutAsync(
async ct =>
{
test.Context.Metadata.TestDetails.ClassInstance = await test.CreateInstanceAsync();
test.Context.Metadata.TestDetails.ClassInstance = await test.CreateInstanceAsync().ConfigureAwait(false);

// Invalidate cached eligible event objects since ClassInstance changed
test.Context.CachedEligibleEventObjects = null;
Expand All @@ -115,20 +115,20 @@ await TimeoutHelper.ExecuteWithTimeoutAsync(
if (test.Context.Metadata.TestDetails.ClassInstance is SkippedTestInstance ||
!string.IsNullOrEmpty(test.Context.SkipReason))
{
await _stateManager.MarkSkippedAsync(test, test.Context.SkipReason ?? "Test was skipped");
await _stateManager.MarkSkippedAsync(test, test.Context.SkipReason ?? "Test was skipped").ConfigureAwait(false);

await _eventReceiverOrchestrator.InvokeTestSkippedEventReceiversAsync(test.Context, ct);
await _eventReceiverOrchestrator.InvokeTestSkippedEventReceiversAsync(test.Context, ct).ConfigureAwait(false);

await _eventReceiverOrchestrator.InvokeTestEndEventReceiversAsync(test.Context, ct);
await _eventReceiverOrchestrator.InvokeTestEndEventReceiversAsync(test.Context, ct).ConfigureAwait(false);

return;
}

try
{
await _testInitializer.InitializeTest(test, ct);
await _testInitializer.InitializeTest(test, ct).ConfigureAwait(false);
test.Context.RestoreExecutionContext();
await _testExecutor.ExecuteAsync(test, ct);
await _testExecutor.ExecuteAsync(test, ct).ConfigureAwait(false);
}
finally
{
Expand All @@ -140,59 +140,59 @@ await TimeoutHelper.ExecuteWithTimeoutAsync(
{
try
{
await invocation.InvokeAsync(test.Context, test.Context);
await invocation.InvokeAsync(test.Context, test.Context).ConfigureAwait(false);
}
catch (Exception disposeEx)
{
await _logger.LogErrorAsync($"Error during OnDispose for {test.TestId}: {disposeEx}");
await _logger.LogErrorAsync($"Error during OnDispose for {test.TestId}: {disposeEx}").ConfigureAwait(false);
}
}
}

try
{
await TestExecutor.DisposeTestInstance(test);
await TestExecutor.DisposeTestInstance(test).ConfigureAwait(false);
}
catch (Exception disposeEx)
{
await _logger.LogErrorAsync($"Error disposing test instance for {test.TestId}: {disposeEx}");
await _logger.LogErrorAsync($"Error disposing test instance for {test.TestId}: {disposeEx}").ConfigureAwait(false);
}
}
},
testTimeout,
cancellationToken,
timeoutMessage);
});
timeoutMessage).ConfigureAwait(false);
}).ConfigureAwait(false);

await _stateManager.MarkCompletedAsync(test);
await _stateManager.MarkCompletedAsync(test).ConfigureAwait(false);

}
catch (SkipTestException ex)
{
test.Context.SkipReason = ex.Message;
await _stateManager.MarkSkippedAsync(test, ex.Message);
await _stateManager.MarkSkippedAsync(test, ex.Message).ConfigureAwait(false);

await _eventReceiverOrchestrator.InvokeTestSkippedEventReceiversAsync(test.Context, cancellationToken);
await _eventReceiverOrchestrator.InvokeTestSkippedEventReceiversAsync(test.Context, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
await _stateManager.MarkFailedAsync(test, ex);
await _stateManager.MarkFailedAsync(test, ex).ConfigureAwait(false);
}
finally
{
var cleanupExceptions = new List<Exception>();

await _objectTracker.UntrackObjects(test.Context, cleanupExceptions);
await _objectTracker.UntrackObjects(test.Context, cleanupExceptions).ConfigureAwait(false);

var testClass = test.Metadata.TestClassType;
var testAssembly = testClass.Assembly;
var hookExceptions = await _testExecutor.ExecuteAfterClassAssemblyHooks(test, testClass, testAssembly, CancellationToken.None);
var hookExceptions = await _testExecutor.ExecuteAfterClassAssemblyHooks(test, testClass, testAssembly, CancellationToken.None).ConfigureAwait(false);

if (hookExceptions.Count > 0)
{
foreach (var ex in hookExceptions)
{
await _logger.LogErrorAsync($"Error executing After hooks for {test.TestId}: {ex}");
await _logger.LogErrorAsync($"Error executing After hooks for {test.TestId}: {ex}").ConfigureAwait(false);
}
cleanupExceptions.AddRange(hookExceptions);
}
Expand All @@ -203,11 +203,11 @@ await TimeoutHelper.ExecuteWithTimeoutAsync(
await _eventReceiverOrchestrator.InvokeLastTestInClassEventReceiversAsync(
test.Context,
test.Context.ClassContext,
CancellationToken.None);
CancellationToken.None).ConfigureAwait(false);
}
catch (Exception ex)
{
await _logger.LogErrorAsync($"Error in last test in class event receiver for {test.TestId}: {ex}");
await _logger.LogErrorAsync($"Error in last test in class event receiver for {test.TestId}: {ex}").ConfigureAwait(false);
cleanupExceptions.Add(ex);
}

Expand All @@ -216,11 +216,11 @@ await _eventReceiverOrchestrator.InvokeLastTestInClassEventReceiversAsync(
await _eventReceiverOrchestrator.InvokeLastTestInAssemblyEventReceiversAsync(
test.Context,
test.Context.ClassContext.AssemblyContext,
CancellationToken.None);
CancellationToken.None).ConfigureAwait(false);
}
catch (Exception ex)
{
await _logger.LogErrorAsync($"Error in last test in assembly event receiver for {test.TestId}: {ex}");
await _logger.LogErrorAsync($"Error in last test in assembly event receiver for {test.TestId}: {ex}").ConfigureAwait(false);
cleanupExceptions.Add(ex);
}

Expand All @@ -229,11 +229,11 @@ await _eventReceiverOrchestrator.InvokeLastTestInAssemblyEventReceiversAsync(
await _eventReceiverOrchestrator.InvokeLastTestInSessionEventReceiversAsync(
test.Context,
test.Context.ClassContext.AssemblyContext.TestSessionContext,
CancellationToken.None);
CancellationToken.None).ConfigureAwait(false);
}
catch (Exception ex)
{
await _logger.LogErrorAsync($"Error in last test in session event receiver for {test.TestId}: {ex}");
await _logger.LogErrorAsync($"Error in last test in session event receiver for {test.TestId}: {ex}").ConfigureAwait(false);
cleanupExceptions.Add(ex);
}

Expand All @@ -244,7 +244,7 @@ await _eventReceiverOrchestrator.InvokeLastTestInSessionEventReceiversAsync(
? cleanupExceptions[0]
: new AggregateException("One or more errors occurred during test cleanup", cleanupExceptions);

await _stateManager.MarkFailedAsync(test, aggregatedException);
await _stateManager.MarkFailedAsync(test, aggregatedException).ConfigureAwait(false);
}

switch (test.State)
Expand All @@ -254,20 +254,20 @@ await _eventReceiverOrchestrator.InvokeLastTestInSessionEventReceiversAsync(
case TestState.Queued:
case TestState.Running:
// This shouldn't happen
await _messageBus.Cancelled(test.Context, test.StartTime.GetValueOrDefault());
await _messageBus.Cancelled(test.Context, test.StartTime.GetValueOrDefault()).ConfigureAwait(false);
break;
case TestState.Passed:
await _messageBus.Passed(test.Context, test.StartTime.GetValueOrDefault());
await _messageBus.Passed(test.Context, test.StartTime.GetValueOrDefault()).ConfigureAwait(false);
break;
case TestState.Timeout:
case TestState.Failed:
await _messageBus.Failed(test.Context, test.Context.Execution.Result?.Exception!, test.StartTime.GetValueOrDefault());
await _messageBus.Failed(test.Context, test.Context.Execution.Result?.Exception!, test.StartTime.GetValueOrDefault()).ConfigureAwait(false);
break;
case TestState.Skipped:
await _messageBus.Skipped(test.Context, test.Context.SkipReason ?? "Skipped");
await _messageBus.Skipped(test.Context, test.Context.SkipReason ?? "Skipped").ConfigureAwait(false);
break;
case TestState.Cancelled:
await _messageBus.Cancelled(test.Context, test.StartTime.GetValueOrDefault());
await _messageBus.Cancelled(test.Context, test.StartTime.GetValueOrDefault()).ConfigureAwait(false);
break;
default:
throw new ArgumentOutOfRangeException();
Expand Down
Loading
Loading