diff --git a/.github/workflows/generate-readme.yml b/.github/workflows/generate-readme.yml index 08bd9e860d..9b06bcf623 100644 --- a/.github/workflows/generate-readme.yml +++ b/.github/workflows/generate-readme.yml @@ -17,11 +17,11 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v5 with: - dotnet-version: 9.0.x + dotnet-version: 10.0.x - name: Generate ReadMe uses: ./.github/actions/execute-pipeline with: categories: ReadMe admin-token: ${{ secrets.ADMIN_TOKEN }} - environment: Development \ No newline at end of file + environment: Development diff --git a/Directory.Packages.props b/Directory.Packages.props index 7c5d08daf0..e8332d9341 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -83,9 +83,9 @@ - - - + + + diff --git a/README.md b/README.md index 5525b39b47..f15d27f180 100644 --- a/README.md +++ b/README.md @@ -375,683 +375,145 @@ dotnet add package TUnit --prerelease ### Scenario: Building the test project -#### macos-latest - -``` - -BenchmarkDotNet v0.15.4, macOS Sequoia 15.6.1 (24G90) [Darwin 24.6.0] -Apple M1 (Virtual), 1 CPU, 3 logical and 3 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), Arm64 RyuJIT armv8.0-a - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), Arm64 RyuJIT armv8.0-a - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|------------- |-------- |--------:|---------:|---------:|--------:| -| Build_TUnit | 0.66.6 | 1.425 s | 0.0954 s | 0.2722 s | 1.384 s | -| Build_NUnit | 4.4.0 | 1.171 s | 0.0670 s | 0.1934 s | 1.150 s | -| Build_xUnit | 2.9.3 | 1.149 s | 0.0853 s | 0.2448 s | 1.101 s | -| Build_MSTest | 3.11.0 | 1.108 s | 0.0468 s | 0.1365 s | 1.103 s | -| Build_xUnit3 | 3.1.0 | 1.144 s | 0.0763 s | 0.2238 s | 1.093 s | - - - -#### ubuntu-latest - ``` BenchmarkDotNet v0.15.4, Linux Ubuntu 24.04.3 LTS (Noble Numbat) AMD EPYC 7763 2.45GHz, 1 CPU, 4 logical and 2 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 +.NET SDK 10.0.100-rc.2.25502.107 + [Host] : .NET 10.0.0 (10.0.0-rc.2.25502.107, 10.0.25.50307), X64 RyuJIT x86-64-v3 + RyuJitX64 : .NET 10.0.0 (10.0.0-rc.2.25502.107, 10.0.25.50307), X64 RyuJIT x86-64-v3 -Runtime=.NET 9.0 +Job=RyuJitX64 Jit=RyuJit Platform=X64 +PowerPlanMode=00000000-0000-0000-0000-000000000000 Runtime=.NET 10.0 Concurrent=True +Server=True ``` | Method | Version | Mean | Error | StdDev | Median | |------------- |-------- |--------:|---------:|---------:|--------:| -| Build_TUnit | 0.66.6 | 1.645 s | 0.0212 s | 0.0188 s | 1.645 s | -| Build_NUnit | 4.4.0 | 1.500 s | 0.0294 s | 0.0302 s | 1.502 s | -| Build_xUnit | 2.9.3 | 1.513 s | 0.0175 s | 0.0163 s | 1.513 s | -| Build_MSTest | 3.11.0 | 1.534 s | 0.0136 s | 0.0127 s | 1.539 s | -| Build_xUnit3 | 3.1.0 | 1.496 s | 0.0247 s | 0.0219 s | 1.500 s | - - - -#### windows-latest - -``` - -BenchmarkDotNet v0.15.4, Windows 11 (10.0.26100.6584/24H2/2024Update/HudsonValley) (Hyper-V) -AMD EPYC 7763 2.44GHz, 1 CPU, 4 logical and 2 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|------------- |-------- |--------:|---------:|---------:|--------:| -| Build_TUnit | 0.66.6 | 2.134 s | 0.0440 s | 0.1275 s | 2.143 s | -| Build_NUnit | 4.4.0 | 2.085 s | 0.0360 s | 0.0319 s | 2.085 s | -| Build_xUnit | 2.9.3 | 2.055 s | 0.0406 s | 0.0865 s | 2.057 s | -| Build_MSTest | 3.11.0 | 2.160 s | 0.0431 s | 0.1142 s | 2.152 s | -| Build_xUnit3 | 3.1.0 | 2.136 s | 0.0424 s | 0.0505 s | 2.124 s | - - -### Scenario: Tests focused on assertion performance and validation - -#### macos-latest - -``` - -BenchmarkDotNet v0.15.4, macOS Sequoia 15.6.1 (24G90) [Darwin 24.6.0] -Apple M1 (Virtual), 1 CPU, 3 logical and 3 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), Arm64 RyuJIT armv8.0-a - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), Arm64 RyuJIT armv8.0-a - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |----------:|----------:|----------:|----------:| -| TUnit | 0.66.6 | 507.19 ms | 28.541 ms | 83.71 ms | 497.91 ms | -| NUnit | 4.4.0 | NA | NA | NA | NA | -| xUnit | 2.9.3 | 884.34 ms | 54.651 ms | 160.28 ms | 895.71 ms | -| MSTest | 3.11.0 | 768.27 ms | 39.207 ms | 115.60 ms | 746.33 ms | -| xUnit3 | 3.1.0 | 493.87 ms | 16.161 ms | 47.40 ms | 503.28 ms | -| TUnit_AOT | 0.66.6 | 70.85 ms | 7.276 ms | 21.34 ms | 67.87 ms | - -Benchmarks with issues: - RuntimeBenchmarks.NUnit: Job-YNJDZW(Runtime=.NET 9.0) - - - -#### ubuntu-latest - -``` - -BenchmarkDotNet v0.15.4, Linux Ubuntu 24.04.3 LTS (Noble Numbat) -AMD EPYC 7763 2.61GHz, 1 CPU, 4 logical and 2 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |----------:|----------:|----------:|----------:| -| TUnit | 0.66.6 | 487.37 ms | 3.093 ms | 2.893 ms | 487.43 ms | -| NUnit | 4.4.0 | 908.34 ms | 17.276 ms | 16.967 ms | 911.94 ms | -| xUnit | 2.9.3 | 984.77 ms | 19.259 ms | 18.015 ms | 979.06 ms | -| MSTest | 3.11.0 | 836.95 ms | 11.507 ms | 10.764 ms | 832.08 ms | -| xUnit3 | 3.1.0 | 458.16 ms | 4.159 ms | 3.890 ms | 457.83 ms | -| TUnit_AOT | 0.66.6 | 25.31 ms | 0.431 ms | 0.382 ms | 25.16 ms | - - - -#### windows-latest - -``` - -BenchmarkDotNet v0.15.4, Windows 11 (10.0.26100.6584/24H2/2024Update/HudsonValley) (Hyper-V) -AMD EPYC 7763 2.44GHz, 1 CPU, 4 logical and 2 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |------------:|----------:|----------:|------------:| -| TUnit | 0.66.6 | 533.36 ms | 3.679 ms | 3.441 ms | 532.96 ms | -| NUnit | 4.4.0 | 1,016.18 ms | 20.186 ms | 34.820 ms | 1,015.43 ms | -| xUnit | 2.9.3 | 1,068.97 ms | 20.224 ms | 16.888 ms | 1,073.74 ms | -| MSTest | 3.11.0 | 968.58 ms | 17.772 ms | 31.127 ms | 964.24 ms | -| xUnit3 | 3.1.0 | 516.29 ms | 7.819 ms | 6.931 ms | 515.39 ms | -| TUnit_AOT | 0.66.6 | 65.97 ms | 2.281 ms | 6.724 ms | 65.43 ms | +| Build_TUnit | 0.77.3 | 1.781 s | 0.0327 s | 0.0306 s | 1.775 s | +| Build_NUnit | 4.4.0 | 1.567 s | 0.0224 s | 0.0199 s | 1.572 s | +| Build_MSTest | 4.0.1 | 1.653 s | 0.0255 s | 0.0226 s | 1.657 s | +| Build_xUnit3 | 3.1.0 | 1.569 s | 0.0135 s | 0.0126 s | 1.566 s | ### Scenario: Tests running asynchronous operations and async/await patterns -#### macos-latest - -``` - -BenchmarkDotNet v0.15.4, macOS Sequoia 15.6.1 (24G90) [Darwin 24.6.0] -Apple M1 (Virtual), 1 CPU, 3 logical and 3 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), Arm64 RyuJIT armv8.0-a - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), Arm64 RyuJIT armv8.0-a - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |-----------:|---------:|----------:|-----------:| -| TUnit | 0.66.6 | 608.9 ms | 39.85 ms | 117.49 ms | 577.8 ms | -| NUnit | 4.4.0 | 1,260.3 ms | 57.30 ms | 166.24 ms | 1,274.6 ms | -| xUnit | 2.9.3 | NA | NA | NA | NA | -| MSTest | 3.11.0 | 1,050.5 ms | 41.59 ms | 119.99 ms | 1,053.4 ms | -| xUnit3 | 3.1.0 | 559.6 ms | 21.67 ms | 62.18 ms | 567.1 ms | -| TUnit_AOT | 0.66.6 | 144.6 ms | 11.89 ms | 34.88 ms | 145.0 ms | - -Benchmarks with issues: - RuntimeBenchmarks.xUnit: Job-YNJDZW(Runtime=.NET 9.0) - - - -#### ubuntu-latest - -``` - -BenchmarkDotNet v0.15.4, Linux Ubuntu 24.04.3 LTS (Noble Numbat) -AMD EPYC 7763 2.45GHz, 1 CPU, 4 logical and 2 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |----------:|----------:|----------:|----------:| -| TUnit | 0.66.6 | 447.34 ms | 3.704 ms | 3.465 ms | 447.86 ms | -| NUnit | 4.4.0 | 907.30 ms | 17.922 ms | 18.404 ms | 901.36 ms | -| xUnit | 2.9.3 | 988.70 ms | 19.061 ms | 17.830 ms | 983.11 ms | -| MSTest | 3.11.0 | 842.79 ms | 16.374 ms | 17.520 ms | 842.60 ms | -| xUnit3 | 3.1.0 | 460.86 ms | 2.968 ms | 2.777 ms | 459.81 ms | -| TUnit_AOT | 0.66.6 | 26.67 ms | 0.506 ms | 0.562 ms | 26.78 ms | - - - -#### windows-latest - -``` - -BenchmarkDotNet v0.15.4, Windows 11 (10.0.26100.6584/24H2/2024Update/HudsonValley) (Hyper-V) -AMD EPYC 7763 2.44GHz, 1 CPU, 4 logical and 2 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |------------:|----------:|----------:|------------:| -| TUnit | 0.66.6 | 494.36 ms | 7.194 ms | 6.729 ms | 490.64 ms | -| NUnit | 4.4.0 | 1,005.78 ms | 18.596 ms | 17.394 ms | 1,007.02 ms | -| xUnit | 2.9.3 | 1,096.34 ms | 20.797 ms | 22.253 ms | 1,097.98 ms | -| MSTest | 3.11.0 | 962.06 ms | 18.672 ms | 26.175 ms | 957.03 ms | -| xUnit3 | 3.1.0 | 522.65 ms | 9.988 ms | 9.343 ms | 521.69 ms | -| TUnit_AOT | 0.66.6 | 89.31 ms | 2.294 ms | 6.728 ms | 89.53 ms | - - -### Scenario: Simple tests with basic operations and assertions - -#### macos-latest - -``` - -BenchmarkDotNet v0.15.4, macOS Sequoia 15.6.1 (24G90) [Darwin 24.6.0] -Apple M1 (Virtual), 1 CPU, 3 logical and 3 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), Arm64 RyuJIT armv8.0-a - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), Arm64 RyuJIT armv8.0-a - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |----------:|----------:|---------:|----------:| -| TUnit | 0.66.6 | 413.11 ms | 29.400 ms | 85.30 ms | 389.28 ms | -| NUnit | 4.4.0 | NA | NA | NA | NA | -| xUnit | 2.9.3 | NA | NA | NA | NA | -| MSTest | 3.11.0 | NA | NA | NA | NA | -| xUnit3 | 3.1.0 | 289.63 ms | 7.455 ms | 21.51 ms | 285.72 ms | -| TUnit_AOT | 0.66.6 | 66.85 ms | 9.402 ms | 27.57 ms | 61.87 ms | - -Benchmarks with issues: - RuntimeBenchmarks.NUnit: Job-YNJDZW(Runtime=.NET 9.0) - RuntimeBenchmarks.xUnit: Job-YNJDZW(Runtime=.NET 9.0) - RuntimeBenchmarks.MSTest: Job-YNJDZW(Runtime=.NET 9.0) - - - -#### ubuntu-latest - ``` BenchmarkDotNet v0.15.4, Linux Ubuntu 24.04.3 LTS (Noble Numbat) -AMD EPYC 7763 2.45GHz, 1 CPU, 4 logical and 2 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |----------:|----------:|----------:|----------:| -| TUnit | 0.66.6 | 466.81 ms | 5.399 ms | 4.786 ms | 466.44 ms | -| NUnit | 4.4.0 | 919.54 ms | 16.718 ms | 15.638 ms | 919.80 ms | -| xUnit | 2.9.3 | 994.05 ms | 10.533 ms | 9.337 ms | 992.55 ms | -| MSTest | 3.11.0 | 853.21 ms | 16.534 ms | 17.691 ms | 854.80 ms | -| xUnit3 | 3.1.0 | 459.93 ms | 3.301 ms | 2.926 ms | 458.96 ms | -| TUnit_AOT | 0.66.6 | 25.34 ms | 0.444 ms | 0.415 ms | 25.14 ms | - - - -#### windows-latest - -``` - -BenchmarkDotNet v0.15.4, Windows 11 (10.0.26100.6584/24H2/2024Update/HudsonValley) (Hyper-V) -AMD EPYC 7763 2.44GHz, 1 CPU, 4 logical and 2 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 +AMD EPYC 7763 2.60GHz, 1 CPU, 4 logical and 2 physical cores +.NET SDK 10.0.100-rc.2.25502.107 + [Host] : .NET 10.0.0 (10.0.0-rc.2.25502.107, 10.0.25.50307), X64 RyuJIT x86-64-v3 + RyuJitX64 : .NET 10.0.0 (10.0.0-rc.2.25502.107, 10.0.25.50307), X64 RyuJIT x86-64-v3 -Runtime=.NET 9.0 +Job=RyuJitX64 Jit=RyuJit Platform=X64 +PowerPlanMode=00000000-0000-0000-0000-000000000000 Runtime=.NET 10.0 Concurrent=True +Server=True ``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |------------:|----------:|----------:|------------:| -| TUnit | 0.66.6 | 517.27 ms | 4.416 ms | 4.130 ms | 518.39 ms | -| NUnit | 4.4.0 | 1,018.48 ms | 20.084 ms | 19.725 ms | 1,014.70 ms | -| xUnit | 2.9.3 | 1,079.55 ms | 19.399 ms | 18.145 ms | 1,079.08 ms | -| MSTest | 3.11.0 | 950.05 ms | 18.521 ms | 21.329 ms | 953.91 ms | -| xUnit3 | 3.1.0 | 509.16 ms | 6.805 ms | 6.365 ms | 507.00 ms | -| TUnit_AOT | 0.66.6 | 66.58 ms | 1.505 ms | 4.414 ms | 65.67 ms | +| Method | Version | Mean | Error | StdDev | Median | +|---------- |-------- |----------:|---------:|----------:|----------:| +| TUnit | 0.77.3 | 478.94 ms | 5.325 ms | 4.981 ms | 479.13 ms | +| NUnit | 4.4.0 | 526.96 ms | 9.078 ms | 8.048 ms | 527.32 ms | +| MSTest | 4.0.1 | 503.03 ms | 9.642 ms | 12.872 ms | 499.80 ms | +| xUnit3 | 3.1.0 | 501.98 ms | 6.870 ms | 6.426 ms | 500.00 ms | +| TUnit_AOT | 0.77.3 | 34.13 ms | 0.487 ms | 0.406 ms | 33.97 ms | ### Scenario: Parameterized tests with multiple test cases using data attributes -#### macos-latest - ``` -BenchmarkDotNet v0.15.4, macOS Sequoia 15.6.1 (24G90) [Darwin 24.6.0] -Apple M1 (Virtual), 1 CPU, 3 logical and 3 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), Arm64 RyuJIT armv8.0-a - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), Arm64 RyuJIT armv8.0-a +BenchmarkDotNet v0.15.4, Linux Ubuntu 24.04.3 LTS (Noble Numbat) +AMD EPYC 7763 2.75GHz, 1 CPU, 4 logical and 2 physical cores +.NET SDK 10.0.100-rc.2.25502.107 + [Host] : .NET 10.0.0 (10.0.0-rc.2.25502.107, 10.0.25.50307), X64 RyuJIT x86-64-v3 + RyuJitX64 : .NET 10.0.0 (10.0.0-rc.2.25502.107, 10.0.25.50307), X64 RyuJIT x86-64-v3 -Runtime=.NET 9.0 +Job=RyuJitX64 Jit=RyuJit Platform=X64 +PowerPlanMode=00000000-0000-0000-0000-000000000000 Runtime=.NET 10.0 Concurrent=True +Server=True ``` | Method | Version | Mean | Error | StdDev | Median | |---------- |-------- |----------:|----------:|---------:|----------:| -| TUnit | 0.66.6 | 408.74 ms | 18.928 ms | 55.21 ms | 396.30 ms | -| NUnit | 4.4.0 | NA | NA | NA | NA | -| xUnit | 2.9.3 | NA | NA | NA | NA | -| MSTest | 3.11.0 | 659.03 ms | 22.634 ms | 66.38 ms | 666.53 ms | -| xUnit3 | 3.1.0 | 377.11 ms | 11.684 ms | 33.90 ms | 379.14 ms | -| TUnit_AOT | 0.66.6 | 44.69 ms | 3.968 ms | 11.39 ms | 41.36 ms | - -Benchmarks with issues: - RuntimeBenchmarks.NUnit: Job-YNJDZW(Runtime=.NET 9.0) - RuntimeBenchmarks.xUnit: Job-YNJDZW(Runtime=.NET 9.0) - +| TUnit | 0.77.3 | 485.31 ms | 3.767 ms | 3.524 ms | 484.81 ms | +| NUnit | 4.4.0 | 687.51 ms | 11.332 ms | 8.847 ms | 683.38 ms | +| MSTest | 4.0.1 | 689.18 ms | 8.267 ms | 7.329 ms | 688.63 ms | +| xUnit3 | 3.1.0 | 502.63 ms | 2.875 ms | 2.690 ms | 502.82 ms | +| TUnit_AOT | 0.77.3 | 34.64 ms | 0.688 ms | 1.995 ms | 34.20 ms | -#### ubuntu-latest +### Scenario: Tests executing massively parallel workloads with CPU-bound, I/O-bound, and mixed operations ``` BenchmarkDotNet v0.15.4, Linux Ubuntu 24.04.3 LTS (Noble Numbat) AMD EPYC 7763 2.45GHz, 1 CPU, 4 logical and 2 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 +.NET SDK 10.0.100-rc.2.25502.107 + [Host] : .NET 10.0.0 (10.0.0-rc.2.25502.107, 10.0.25.50307), X64 RyuJIT x86-64-v3 + RyuJitX64 : .NET 10.0.0 (10.0.0-rc.2.25502.107, 10.0.25.50307), X64 RyuJIT x86-64-v3 -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |------------:|----------:|----------:|------------:| -| TUnit | 0.66.6 | 460.22 ms | 2.117 ms | 1.877 ms | 460.03 ms | -| NUnit | 4.4.0 | 909.38 ms | 16.413 ms | 15.352 ms | 917.73 ms | -| xUnit | 2.9.3 | 1,016.83 ms | 18.472 ms | 17.279 ms | 1,008.66 ms | -| MSTest | 3.11.0 | 866.29 ms | 16.852 ms | 18.031 ms | 865.01 ms | -| xUnit3 | 3.1.0 | 486.65 ms | 3.559 ms | 3.155 ms | 487.03 ms | -| TUnit_AOT | 0.66.6 | 28.22 ms | 0.343 ms | 0.304 ms | 28.31 ms | - - - -#### windows-latest - -``` - -BenchmarkDotNet v0.15.4, Windows 11 (10.0.26100.6584/24H2/2024Update/HudsonValley) (Hyper-V) -AMD EPYC 7763 2.44GHz, 1 CPU, 4 logical and 2 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |------------:|----------:|-----------:|------------:| -| TUnit | 0.66.6 | 550.13 ms | 10.668 ms | 10.477 ms | 545.52 ms | -| NUnit | 4.4.0 | 1,115.22 ms | 22.111 ms | 23.658 ms | 1,115.33 ms | -| xUnit | 2.9.3 | 1,209.43 ms | 35.495 ms | 100.694 ms | 1,223.99 ms | -| MSTest | 3.11.0 | 1,041.10 ms | 20.681 ms | 23.817 ms | 1,038.52 ms | -| xUnit3 | 3.1.0 | 565.34 ms | 10.258 ms | 9.596 ms | 565.75 ms | -| TUnit_AOT | 0.66.6 | 71.69 ms | 1.423 ms | 3.410 ms | 71.39 ms | - - -### Scenario: Tests utilizing class fixtures and shared test context - -#### macos-latest - -``` - -BenchmarkDotNet v0.15.4, macOS Sequoia 15.6.1 (24G90) [Darwin 24.6.0] -Apple M1 (Virtual), 1 CPU, 3 logical and 3 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), Arm64 RyuJIT armv8.0-a - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), Arm64 RyuJIT armv8.0-a - -Runtime=.NET 9.0 +Job=RyuJitX64 Jit=RyuJit Platform=X64 +PowerPlanMode=00000000-0000-0000-0000-000000000000 Runtime=.NET 10.0 Concurrent=True +Server=True ``` | Method | Version | Mean | Error | StdDev | Median | |---------- |-------- |----------:|----------:|---------:|----------:| -| TUnit | 0.66.6 | 286.49 ms | 6.152 ms | 17.75 ms | 285.41 ms | -| NUnit | 4.4.0 | NA | NA | NA | NA | -| xUnit | 2.9.3 | NA | NA | NA | NA | -| MSTest | 3.11.0 | NA | NA | NA | NA | -| xUnit3 | 3.1.0 | 375.36 ms | 22.387 ms | 66.01 ms | 362.13 ms | -| TUnit_AOT | 0.66.6 | 64.68 ms | 6.634 ms | 19.03 ms | 62.08 ms | +| TUnit | 0.77.3 | 499.03 ms | 6.589 ms | 5.841 ms | 498.34 ms | +| NUnit | 4.4.0 | 582.75 ms | 10.155 ms | 9.499 ms | 583.13 ms | +| MSTest | 4.0.1 | 560.42 ms | 8.550 ms | 7.997 ms | 560.16 ms | +| xUnit3 | 3.1.0 | 567.61 ms | 4.273 ms | 3.788 ms | 566.90 ms | +| TUnit_AOT | 0.77.3 | 39.00 ms | 0.255 ms | 0.238 ms | 39.04 ms | -Benchmarks with issues: - RuntimeBenchmarks.NUnit: Job-YNJDZW(Runtime=.NET 9.0) - RuntimeBenchmarks.xUnit: Job-YNJDZW(Runtime=.NET 9.0) - RuntimeBenchmarks.MSTest: Job-YNJDZW(Runtime=.NET 9.0) - - -#### ubuntu-latest +### Scenario: Tests with complex parameter combinations creating 25-125 test variations ``` BenchmarkDotNet v0.15.4, Linux Ubuntu 24.04.3 LTS (Noble Numbat) AMD EPYC 7763 2.45GHz, 1 CPU, 4 logical and 2 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |------------:|----------:|----------:|------------:| -| TUnit | 0.66.6 | 452.99 ms | 3.907 ms | 3.263 ms | 453.44 ms | -| NUnit | 4.4.0 | 930.95 ms | 14.776 ms | 13.821 ms | 935.97 ms | -| xUnit | 2.9.3 | 1,006.26 ms | 16.690 ms | 15.611 ms | 1,000.82 ms | -| MSTest | 3.11.0 | 850.14 ms | 16.934 ms | 18.119 ms | 852.12 ms | -| xUnit3 | 3.1.0 | 450.19 ms | 4.025 ms | 3.765 ms | 450.09 ms | -| TUnit_AOT | 0.66.6 | 38.57 ms | 1.098 ms | 3.220 ms | 38.60 ms | - - +.NET SDK 10.0.100-rc.2.25502.107 + [Host] : .NET 10.0.0 (10.0.0-rc.2.25502.107, 10.0.25.50307), X64 RyuJIT x86-64-v3 + RyuJitX64 : .NET 10.0.0 (10.0.0-rc.2.25502.107, 10.0.25.50307), X64 RyuJIT x86-64-v3 -#### windows-latest - -``` - -BenchmarkDotNet v0.15.4, Windows 11 (10.0.26100.6584/24H2/2024Update/HudsonValley) (Hyper-V) -AMD EPYC 7763 2.44GHz, 1 CPU, 4 logical and 2 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |------------:|----------:|----------:|------------:| -| TUnit | 0.66.6 | 495.87 ms | 8.424 ms | 10.653 ms | 492.34 ms | -| NUnit | 4.4.0 | 973.98 ms | 19.011 ms | 18.671 ms | 972.03 ms | -| xUnit | 2.9.3 | 1,049.78 ms | 14.531 ms | 13.593 ms | 1,044.49 ms | -| MSTest | 3.11.0 | 916.98 ms | 16.913 ms | 15.821 ms | 912.23 ms | -| xUnit3 | 3.1.0 | 486.78 ms | 3.897 ms | 3.645 ms | 486.01 ms | -| TUnit_AOT | 0.66.6 | 91.20 ms | 1.800 ms | 3.152 ms | 91.43 ms | - - -### Scenario: Tests executing in parallel to test framework parallelization - -#### macos-latest - -``` - -BenchmarkDotNet v0.15.4, macOS Sequoia 15.6.1 (24G90) [Darwin 24.6.0] -Apple M1 (Virtual), 1 CPU, 3 logical and 3 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), Arm64 RyuJIT armv8.0-a - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), Arm64 RyuJIT armv8.0-a - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |------------:|-----------:|----------:|------------:| -| TUnit | 0.66.6 | 410.22 ms | 23.733 ms | 69.23 ms | 402.84 ms | -| NUnit | 4.4.0 | 1,342.07 ms | 95.053 ms | 274.25 ms | 1,303.84 ms | -| xUnit | 2.9.3 | 1,328.93 ms | 101.864 ms | 300.35 ms | 1,301.91 ms | -| MSTest | 3.11.0 | NA | NA | NA | NA | -| xUnit3 | 3.1.0 | 470.25 ms | 20.347 ms | 58.38 ms | 468.24 ms | -| TUnit_AOT | 0.66.6 | 58.77 ms | 7.644 ms | 21.93 ms | 53.57 ms | - -Benchmarks with issues: - RuntimeBenchmarks.MSTest: Job-YNJDZW(Runtime=.NET 9.0) - - - -#### ubuntu-latest - -``` - -BenchmarkDotNet v0.15.4, Linux Ubuntu 24.04.3 LTS (Noble Numbat) -AMD EPYC 7763 2.45GHz, 1 CPU, 4 logical and 2 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - -Runtime=.NET 9.0 +Job=RyuJitX64 Jit=RyuJit Platform=X64 +PowerPlanMode=00000000-0000-0000-0000-000000000000 Runtime=.NET 10.0 Concurrent=True +Server=True ``` | Method | Version | Mean | Error | StdDev | Median | |---------- |-------- |----------:|----------:|----------:|----------:| -| TUnit | 0.66.6 | 442.19 ms | 1.731 ms | 1.534 ms | 441.97 ms | -| NUnit | 4.4.0 | 918.06 ms | 17.680 ms | 17.364 ms | 921.88 ms | -| xUnit | 2.9.3 | 983.91 ms | 18.873 ms | 17.654 ms | 979.36 ms | -| MSTest | 3.11.0 | 838.79 ms | 14.777 ms | 13.823 ms | 844.27 ms | -| xUnit3 | 3.1.0 | 449.58 ms | 2.903 ms | 2.573 ms | 449.82 ms | -| TUnit_AOT | 0.66.6 | 24.97 ms | 0.208 ms | 0.184 ms | 24.95 ms | +| TUnit | 0.77.3 | 509.47 ms | 6.776 ms | 6.007 ms | 508.85 ms | +| NUnit | 4.4.0 | 596.89 ms | 11.807 ms | 16.934 ms | 594.68 ms | +| MSTest | 4.0.1 | 610.32 ms | 11.764 ms | 11.004 ms | 613.12 ms | +| xUnit3 | 3.1.0 | 533.11 ms | 3.656 ms | 3.053 ms | 533.28 ms | +| TUnit_AOT | 0.77.3 | 37.61 ms | 0.499 ms | 0.467 ms | 37.68 ms | - -#### windows-latest +### Scenario: Large-scale parameterized tests with 100+ test cases testing framework scalability ``` -BenchmarkDotNet v0.15.4, Windows 11 (10.0.26100.6584/24H2/2024Update/HudsonValley) (Hyper-V) -AMD EPYC 7763 2.44GHz, 1 CPU, 4 logical and 2 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |------------:|----------:|----------:|------------:| -| TUnit | 0.66.6 | 510.80 ms | 3.865 ms | 3.616 ms | 511.67 ms | -| NUnit | 4.4.0 | 1,085.21 ms | 21.543 ms | 37.161 ms | 1,079.30 ms | -| xUnit | 2.9.3 | 1,125.35 ms | 22.326 ms | 27.418 ms | 1,124.00 ms | -| MSTest | 3.11.0 | 979.31 ms | 15.894 ms | 14.867 ms | 979.73 ms | -| xUnit3 | 3.1.0 | 526.16 ms | 5.769 ms | 5.396 ms | 526.42 ms | -| TUnit_AOT | 0.66.6 | 65.78 ms | 1.726 ms | 5.063 ms | 65.93 ms | - - -### Scenario: A test that takes 50ms to execute, repeated 100 times - -#### macos-latest - -``` - -BenchmarkDotNet v0.15.4, macOS Sequoia 15.6.1 (24G90) [Darwin 24.6.0] -Apple M1 (Virtual), 1 CPU, 3 logical and 3 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), Arm64 RyuJIT armv8.0-a - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), Arm64 RyuJIT armv8.0-a +BenchmarkDotNet v0.15.4, Linux Ubuntu 24.04.3 LTS (Noble Numbat) +AMD EPYC 7763 2.74GHz, 1 CPU, 4 logical and 2 physical cores +.NET SDK 10.0.100-rc.2.25502.107 + [Host] : .NET 10.0.0 (10.0.0-rc.2.25502.107, 10.0.25.50307), X64 RyuJIT x86-64-v3 + RyuJitX64 : .NET 10.0.0 (10.0.0-rc.2.25502.107, 10.0.25.50307), X64 RyuJIT x86-64-v3 -Runtime=.NET 9.0 +Job=RyuJitX64 Jit=RyuJit Platform=X64 +PowerPlanMode=00000000-0000-0000-0000-000000000000 Runtime=.NET 10.0 Concurrent=True +Server=True ``` | Method | Version | Mean | Error | StdDev | Median | |---------- |-------- |----------:|----------:|----------:|----------:| -| TUnit | 0.66.6 | 338.92 ms | 19.812 ms | 58.105 ms | 325.53 ms | -| NUnit | 4.4.0 | 574.90 ms | 9.258 ms | 7.228 ms | 572.83 ms | -| xUnit | 2.9.3 | NA | NA | NA | NA | -| MSTest | 3.11.0 | NA | NA | NA | NA | -| xUnit3 | 3.1.0 | 324.91 ms | 10.115 ms | 29.507 ms | 318.54 ms | -| TUnit_AOT | 0.66.6 | 60.53 ms | 8.531 ms | 24.613 ms | 56.15 ms | - -Benchmarks with issues: - RuntimeBenchmarks.xUnit: Job-YNJDZW(Runtime=.NET 9.0) - RuntimeBenchmarks.MSTest: Job-YNJDZW(Runtime=.NET 9.0) - - - -#### ubuntu-latest - -``` - -BenchmarkDotNet v0.15.4, Linux Ubuntu 24.04.3 LTS (Noble Numbat) -AMD EPYC 7763 2.90GHz, 1 CPU, 4 logical and 2 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |------------:|----------:|----------:|------------:| -| TUnit | 0.66.6 | 486.94 ms | 3.897 ms | 3.645 ms | 487.44 ms | -| NUnit | 4.4.0 | 925.61 ms | 17.307 ms | 16.189 ms | 922.59 ms | -| xUnit | 2.9.3 | 1,085.95 ms | 20.200 ms | 17.906 ms | 1,082.26 ms | -| MSTest | 3.11.0 | 858.03 ms | 16.000 ms | 14.966 ms | 856.06 ms | -| xUnit3 | 3.1.0 | 491.88 ms | 4.490 ms | 3.980 ms | 489.95 ms | -| TUnit_AOT | 0.66.6 | 40.90 ms | 0.680 ms | 0.636 ms | 40.67 ms | - - - -#### windows-latest - -``` - -BenchmarkDotNet v0.15.4, Windows 11 (10.0.26100.6584/24H2/2024Update/HudsonValley) (Hyper-V) -AMD EPYC 7763 2.44GHz, 1 CPU, 4 logical and 2 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |------------:|----------:|----------:|------------:| -| TUnit | 0.66.6 | 560.34 ms | 10.543 ms | 9.862 ms | 559.08 ms | -| NUnit | 4.4.0 | 1,072.50 ms | 20.980 ms | 45.608 ms | 1,074.71 ms | -| xUnit | 2.9.3 | 1,232.15 ms | 24.387 ms | 26.094 ms | 1,232.77 ms | -| MSTest | 3.11.0 | 994.43 ms | 19.847 ms | 54.995 ms | 991.14 ms | -| xUnit3 | 3.1.0 | 580.80 ms | 11.371 ms | 15.180 ms | 581.16 ms | -| TUnit_AOT | 0.66.6 | 81.43 ms | 2.104 ms | 6.203 ms | 82.90 ms | - - -### Scenario: Tests with setup and teardown lifecycle methods - -#### macos-latest - -``` - -BenchmarkDotNet v0.15.4, macOS Sequoia 15.6.1 (24G90) [Darwin 24.6.0] -Apple M1 (Virtual), 1 CPU, 3 logical and 3 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), Arm64 RyuJIT armv8.0-a - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), Arm64 RyuJIT armv8.0-a - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |------------:|----------:|----------:|------------:| -| TUnit | 0.66.6 | 525.73 ms | 37.455 ms | 108.66 ms | 518.72 ms | -| NUnit | 4.4.0 | 1,163.42 ms | 50.082 ms | 143.69 ms | 1,154.49 ms | -| xUnit | 2.9.3 | 1,063.35 ms | 38.811 ms | 113.83 ms | 1,063.37 ms | -| MSTest | 3.11.0 | 864.08 ms | 79.507 ms | 234.43 ms | 768.70 ms | -| xUnit3 | 3.1.0 | 389.16 ms | 18.371 ms | 53.59 ms | 391.37 ms | -| TUnit_AOT | 0.66.6 | 50.16 ms | 6.305 ms | 18.19 ms | 48.60 ms | - - - -#### ubuntu-latest - -``` - -BenchmarkDotNet v0.15.4, Linux Ubuntu 24.04.3 LTS (Noble Numbat) -AMD EPYC 7763 3.14GHz, 1 CPU, 4 logical and 2 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |------------:|----------:|----------:|------------:| -| TUnit | 0.66.6 | 467.37 ms | 2.088 ms | 1.953 ms | 467.00 ms | -| NUnit | 4.4.0 | 937.52 ms | 18.192 ms | 19.465 ms | 935.06 ms | -| xUnit | 2.9.3 | 1,033.10 ms | 18.060 ms | 16.894 ms | 1,028.90 ms | -| MSTest | 3.11.0 | 885.00 ms | 17.336 ms | 17.026 ms | 882.50 ms | -| xUnit3 | 3.1.0 | 465.87 ms | 3.757 ms | 3.331 ms | 465.39 ms | -| TUnit_AOT | 0.66.6 | 26.79 ms | 0.208 ms | 0.174 ms | 26.82 ms | - - - -#### windows-latest - -``` - -BenchmarkDotNet v0.15.4, Windows 11 (10.0.26100.6584/24H2/2024Update/HudsonValley) (Hyper-V) -AMD EPYC 7763 2.44GHz, 1 CPU, 4 logical and 2 physical cores -.NET SDK 9.0.305 - [Host] : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - Job-YNJDZW : .NET 9.0.9 (9.0.9, 9.0.925.41916), X64 RyuJIT x86-64-v3 - -Runtime=.NET 9.0 - -``` -| Method | Version | Mean | Error | StdDev | Median | -|---------- |-------- |------------:|----------:|----------:|------------:| -| TUnit | 0.66.6 | 527.84 ms | 10.425 ms | 24.160 ms | 521.87 ms | -| NUnit | 4.4.0 | 1,069.13 ms | 21.368 ms | 22.864 ms | 1,069.02 ms | -| xUnit | 2.9.3 | 1,152.17 ms | 22.381 ms | 29.102 ms | 1,160.05 ms | -| MSTest | 3.11.0 | 994.82 ms | 19.730 ms | 21.929 ms | 994.32 ms | -| xUnit3 | 3.1.0 | 544.24 ms | 10.540 ms | 14.071 ms | 543.06 ms | -| TUnit_AOT | 0.66.6 | 68.69 ms | 2.349 ms | 6.815 ms | 68.38 ms | +| TUnit | 0.77.3 | 507.33 ms | 5.869 ms | 5.490 ms | 509.06 ms | +| NUnit | 4.4.0 | 613.34 ms | 4.137 ms | 3.455 ms | 612.68 ms | +| MSTest | 4.0.1 | 632.89 ms | 11.515 ms | 10.208 ms | 635.09 ms | +| xUnit3 | 3.1.0 | 510.91 ms | 3.648 ms | 3.413 ms | 510.42 ms | +| TUnit_AOT | 0.77.3 | 56.39 ms | 1.220 ms | 3.597 ms | 56.85 ms | diff --git a/TUnit.Core/TestContext.cs b/TUnit.Core/TestContext.cs index 125a72f85d..b5acc046c2 100644 --- a/TUnit.Core/TestContext.cs +++ b/TUnit.Core/TestContext.cs @@ -242,6 +242,15 @@ public string GetDisplayName() return _cachedDisplayName; } + /// + /// Clears the cached display name, forcing it to be recomputed on next access. + /// This is called after discovery event receivers run to ensure custom argument formatters are applied. + /// + internal void InvalidateDisplayNameCache() + { + _cachedDisplayName = null; + } + public Dictionary ObjectBag => _testBuilderContext.ObjectBag; public bool ReportResult { get; set; } = true; diff --git a/TUnit.Engine/Building/TestBuilder.cs b/TUnit.Engine/Building/TestBuilder.cs index 0545329b93..11ed1b0075 100644 --- a/TUnit.Engine/Building/TestBuilder.cs +++ b/TUnit.Engine/Building/TestBuilder.cs @@ -802,6 +802,11 @@ public async Task BuildTestAsync(TestMetadata metadata, await InvokeDiscoveryEventReceiversAsync(context); + // Clear the cached display name after discovery events + // This ensures that ArgumentDisplayFormatterAttribute and similar attributes + // have a chance to register their formatters before the display name is finalized + context.InvalidateDisplayNameCache(); + return test; } diff --git a/TUnit.Pipeline/Modules/GenerateReadMeModule.cs b/TUnit.Pipeline/Modules/GenerateReadMeModule.cs index 4fa520a012..45b8f5f4fb 100644 --- a/TUnit.Pipeline/Modules/GenerateReadMeModule.cs +++ b/TUnit.Pipeline/Modules/GenerateReadMeModule.cs @@ -58,18 +58,28 @@ public class GenerateReadMeModule : Module var fileContents = new StringBuilder(); - // Grouping is by Scenario - foreach (var groupedArtifacts in artifacts.Artifacts + // Expected artifact name pattern: ubuntu_markdown_{build_time|run_time_{class}} + // Example: ubuntu_markdown_run_time_AsyncTests, ubuntu_markdown_build_time + var benchmarkArtifacts = artifacts.Artifacts + .Where(x => x.Name.StartsWith("ubuntu_markdown_")) + .ToList(); + + if (benchmarkArtifacts.Count == 0) + { + context.Logger.LogWarning("No benchmark markdown artifacts found."); + return null; + } + + // Grouping is by Scenario (e.g., "markdown_run_time_AsyncTests" or "markdown_build_time") + foreach (var groupedArtifacts in benchmarkArtifacts .OrderBy(x => x.Name) - .GroupBy(x => x.Name.Split("_", 2)[1])) + .GroupBy(x => x.Name.Substring("ubuntu_".Length))) { fileContents.AppendLine($"### Scenario: {GetScenario(groupedArtifacts.Key)}"); foreach (var artifact in groupedArtifacts.OrderBy(x => x.Name)) { - var operatingSystem = artifact.Name.Split("_")[0]; - - context.Logger.LogInformation("Processing artifact: {ArtifactName} for OS: {OperatingSystem}", artifact.Name, operatingSystem); + context.Logger.LogInformation("Processing artifact: {ArtifactName}", artifact.Name); var stream = await context.GitHub().Client.Actions.Artifacts.DownloadArtifact( context.GitHub().RepositoryInfo.Owner, @@ -86,13 +96,11 @@ public class GenerateReadMeModule : Module var contents = await markdownFile.ReadAsync(cancellationToken); - fileContents.AppendLine(); - fileContents.AppendLine($"#### {operatingSystem}"); fileContents.AppendLine(); fileContents.AppendLine(contents); fileContents.AppendLine(); - context.Logger.LogInformation("Added contents from {MarkdownFile} for OS: {OperatingSystem}", markdownFile.Name, operatingSystem); + context.Logger.LogInformation("Added contents from {MarkdownFile}", markdownFile.Name); } } @@ -118,14 +126,11 @@ private string GetScenario(string fileName) return fileName.Split("_").Last() switch { - "AssertionTests" => "Tests focused on assertion performance and validation", "AsyncTests" => "Tests running asynchronous operations and async/await patterns", - "BasicTests" => "Simple tests with basic operations and assertions", "DataDrivenTests" => "Parameterized tests with multiple test cases using data attributes", - "FixtureTests" => "Tests utilizing class fixtures and shared test context", - "ParallelTests" => "Tests executing in parallel to test framework parallelization", - "RepeatTests" => "A test that takes 50ms to execute, repeated 100 times", - "SetupTeardownTests" => "Tests with setup and teardown lifecycle methods", + "MassiveParallelTests" => "Tests executing massively parallel workloads with CPU-bound, I/O-bound, and mixed operations", + "MatrixTests" => "Tests with complex parameter combinations creating 25-125 test variations", + "ScaleTests" => "Large-scale parameterized tests with 100+ test cases testing framework scalability", _ => throw new ArgumentException($"Unknown class name: {fileName}", nameof(fileName)) }; } diff --git a/TUnit.Templates/content/TUnit.AspNet.FSharp/TestProject/TestProject.fsproj b/TUnit.Templates/content/TUnit.AspNet.FSharp/TestProject/TestProject.fsproj index 57e512f7fa..2b80e58a62 100644 --- a/TUnit.Templates/content/TUnit.AspNet.FSharp/TestProject/TestProject.fsproj +++ b/TUnit.Templates/content/TUnit.AspNet.FSharp/TestProject/TestProject.fsproj @@ -10,8 +10,8 @@ - - + + diff --git a/TUnit.Templates/content/TUnit.AspNet/TestProject/TestProject.csproj b/TUnit.Templates/content/TUnit.AspNet/TestProject/TestProject.csproj index 3798f772a3..08d17501b4 100644 --- a/TUnit.Templates/content/TUnit.AspNet/TestProject/TestProject.csproj +++ b/TUnit.Templates/content/TUnit.AspNet/TestProject/TestProject.csproj @@ -9,7 +9,7 @@ - + diff --git a/TUnit.Templates/content/TUnit.Aspire.Starter/ExampleNamespace.TestProject/ExampleNamespace.TestProject.csproj b/TUnit.Templates/content/TUnit.Aspire.Starter/ExampleNamespace.TestProject/ExampleNamespace.TestProject.csproj index 45fb1fee62..dce660bd96 100644 --- a/TUnit.Templates/content/TUnit.Aspire.Starter/ExampleNamespace.TestProject/ExampleNamespace.TestProject.csproj +++ b/TUnit.Templates/content/TUnit.Aspire.Starter/ExampleNamespace.TestProject/ExampleNamespace.TestProject.csproj @@ -11,7 +11,7 @@ - + diff --git a/TUnit.Templates/content/TUnit.Aspire.Test/ExampleNamespace.csproj b/TUnit.Templates/content/TUnit.Aspire.Test/ExampleNamespace.csproj index c7633ba283..95d7a78f4c 100644 --- a/TUnit.Templates/content/TUnit.Aspire.Test/ExampleNamespace.csproj +++ b/TUnit.Templates/content/TUnit.Aspire.Test/ExampleNamespace.csproj @@ -10,7 +10,7 @@ - + diff --git a/TUnit.Templates/content/TUnit.FSharp/TestProject.fsproj b/TUnit.Templates/content/TUnit.FSharp/TestProject.fsproj index 6bbee956ef..3ee8ed4d7c 100644 --- a/TUnit.Templates/content/TUnit.FSharp/TestProject.fsproj +++ b/TUnit.Templates/content/TUnit.FSharp/TestProject.fsproj @@ -10,8 +10,8 @@ - - + + diff --git a/TUnit.Templates/content/TUnit.Playwright/TestProject.csproj b/TUnit.Templates/content/TUnit.Playwright/TestProject.csproj index c6051ab96b..8495b0595a 100644 --- a/TUnit.Templates/content/TUnit.Playwright/TestProject.csproj +++ b/TUnit.Templates/content/TUnit.Playwright/TestProject.csproj @@ -8,7 +8,7 @@ - + diff --git a/TUnit.Templates/content/TUnit.VB/TestProject.vbproj b/TUnit.Templates/content/TUnit.VB/TestProject.vbproj index 75bda64de7..a347213711 100644 --- a/TUnit.Templates/content/TUnit.VB/TestProject.vbproj +++ b/TUnit.Templates/content/TUnit.VB/TestProject.vbproj @@ -8,6 +8,6 @@ - + diff --git a/TUnit.Templates/content/TUnit/TestProject.csproj b/TUnit.Templates/content/TUnit/TestProject.csproj index 5f40856c51..e7b61c658b 100644 --- a/TUnit.Templates/content/TUnit/TestProject.csproj +++ b/TUnit.Templates/content/TUnit/TestProject.csproj @@ -8,7 +8,7 @@ - + \ No newline at end of file diff --git a/TUnit.TestProject/ArgumentDisplayFormatterTests.cs b/TUnit.TestProject/ArgumentDisplayFormatterTests.cs new file mode 100644 index 0000000000..c02c7f0d14 --- /dev/null +++ b/TUnit.TestProject/ArgumentDisplayFormatterTests.cs @@ -0,0 +1,72 @@ +using TUnit.TestProject.Attributes; + +namespace TUnit.TestProject; + +[EngineTest(ExpectedResult.Pass)] +public class ArgumentDisplayFormatterTests +{ + [Test] + [MethodDataSource(nameof(Data1))] + [ArgumentDisplayFormatter] + public async Task FormatterShouldBeAppliedToMethodDataSource(Foo foo) + { + // Verify the formatter was applied by checking the display name + var displayName = TestContext.Current!.GetDisplayName(); + await Assert.That(displayName).IsEqualTo("FormatterShouldBeAppliedToMethodDataSource(FooFormatterValue)"); + } + + [Test] + [Arguments(1, 2, 3)] + [ArgumentDisplayFormatter] + public async Task FormatterShouldBeAppliedToArguments(int a, int b, int c) + { + // Verify the formatter was applied by checking the display name + var displayName = TestContext.Current!.GetDisplayName(); + await Assert.That(displayName).IsEqualTo("FormatterShouldBeAppliedToArguments(INT:1, INT:2, INT:3)"); + } + + [Test] + [MethodDataSource(nameof(DataWithException))] + [ArgumentDisplayFormatter] + public async Task FormatterShouldPreventExceptionInToString(Bar bar) + { + // The Bar.ToString() throws, but the formatter should prevent that + var displayName = TestContext.Current!.GetDisplayName(); + await Assert.That(displayName).IsEqualTo("FormatterShouldPreventExceptionInToString(BarFormatterValue)"); + } + + public static IEnumerable Data1() => [new Foo()]; + + public static IEnumerable DataWithException() => [new Bar()]; +} + +public class Foo +{ + public override string ToString() => throw new Exception("Foo.ToString should not be called"); +} + +public class Bar +{ + public override string ToString() => throw new Exception("Bar.ToString should not be called"); +} + +public class FooFormatter : ArgumentDisplayFormatter +{ + public override bool CanHandle(object? value) => value is Foo; + + public override string FormatValue(object? value) => "FooFormatterValue"; +} + +public class BarFormatter : ArgumentDisplayFormatter +{ + public override bool CanHandle(object? value) => value is Bar; + + public override string FormatValue(object? value) => "BarFormatterValue"; +} + +public class IntFormatter : ArgumentDisplayFormatter +{ + public override bool CanHandle(object? value) => value is int; + + public override string FormatValue(object? value) => $"INT:{value}"; +} diff --git a/docs/docs/customization-extensibility/logging.md b/docs/docs/customization-extensibility/logging.md index 619870a05f..90c072ece3 100644 --- a/docs/docs/customization-extensibility/logging.md +++ b/docs/docs/customization-extensibility/logging.md @@ -20,3 +20,12 @@ If you want to override this, you can inherit from `TUnitLogger` or `DefaultLogg return logLevel >= LogLevel.Error; } ``` + +## Log Level Command Line +If you are executing tests via the command line, you can set the log level via the `--log-level` argument: + +``` +dotnet run --log-level Warning +``` + +The above will show only logs that are `Warning` or higher (e.g. `Error`, `Critical`) while executing the test. diff --git a/docs/sidebars.ts b/docs/sidebars.ts index 64b47e48ca..04ae12559b 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -147,6 +147,15 @@ const sidebars: SidebarsConfig = { 'reference/test-configuration', ], }, + { + type: 'category', + label: 'Advanced', + items: [ + 'advanced/exception-handling', + 'advanced/extension-points', + 'advanced/performance-best-practices', + ], + }, { type: 'category', label: 'Migration Guides', diff --git a/global.json b/global.json index 2e2218a904..08c5374eff 100644 --- a/global.json +++ b/global.json @@ -1,7 +1,8 @@ { "sdk": { - "version": "10.0.100-rc.1.25451.107", - "rollForward": "latestFeature" + "version": "9.0.305", + "rollForward": "latestMajor", + "allowPrerelease": true }, "test": { "runner": "Microsoft.Testing.Platform"